support logging request logs and database query logs to separate files, and support rotating log files

conf/ezbookkeeping.ini

@@ -92,6 +92,21 @@ level = info
 # For "file" mode only, log file path (relative or absolute path)
 log_path = log/ezbookkeeping.log
 
+# For "file" only, request log file path (relative or absolute path). Leave blank if you want to write request log in default log file
+request_log_path =
+
+# For "file" only, query log file path (relative or absolute path). Leave blank if you want to write query log in default log file
+query_log_path =
+
+# For "file" only, whether rotate the log files
+log_file_rotate = false
+
+# For "file" only, maximum size (1 - 4294967295 bytes) of the log file before it gets rotated
+log_file_max_size = 104857600
+
+# For "file" only, maximum number of days to retain old log files. Set to 0 to retain all logs
+log_file_max_days = 7
+
 [storage]
 # Object storage type, supports "local_filesystem" and "minio" currently
 type = local_filesystem

pkg/errs/error.go

@@ -15,6 +15,7 @@ const (
 	SystemSubcategorySetting  = 1
 	SystemSubcategoryDatabase = 2
 	SystemSubcategoryMail     = 3
+	SystemSubcategoryLogging  = 4
 )
 
 // Sub categories of normal error
@@ -75,6 +76,15 @@ func NewNormalError(subCategory int32, index int32, httpStatusCode int, message
 	return New(CATEGORY_NORMAL, subCategory, index, httpStatusCode, message)
 }
 
+// NewLoggingError returns a new logging error instance
+func NewLoggingError(message string, err ...error) *Error {
+	return New(ErrLoggingError.Category,
+		ErrLoggingError.SubCategory,
+		ErrLoggingError.Index,
+		ErrLoggingError.HttpStatusCode,
+		message, err...)
+}
+
 // NewIncompleteOrIncorrectSubmissionError returns a new incomplete or incorrect submission error instance
 func NewIncompleteOrIncorrectSubmissionError(err error) *Error {
 	return New(ErrIncompleteOrIncorrectSubmission.Category,

pkg/errs/logging.go

@@ -0,0 +1,10 @@
+package errs
+
+import (
+	"net/http"
+)
+
+// Error codes related to logging
+var (
+	ErrLoggingError = NewSystemError(SystemSubcategoryLogging, 0, http.StatusInternalServerError, "logging error")
+)

pkg/log/logger.go

@@ -41,37 +41,67 @@ func init() {
 // SetLoggerConfiguration sets the logger according to the config
 func SetLoggerConfiguration(config *settings.Config, isDisableBootLog bool) error {
 	var bootWriters []io.Writer
-	var writers []io.Writer
+	var defaultWriters []io.Writer
+	var requestWriters []io.Writer
+	var queryWriters []io.Writer
 
 	if !isDisableBootLog {
 		bootWriters = append(bootWriters, os.Stdout)
 	}
 
 	if config.EnableConsoleLog {
-		writers = append(writers, os.Stdout)
+		defaultWriters = append(defaultWriters, os.Stdout)
+		requestWriters = append(requestWriters, os.Stdout)
+		queryWriters = append(queryWriters, os.Stdout)
 	}
 
 	if config.EnableFileLog {
-		logFile, err := os.OpenFile(config.FileLogPath, os.O_CREATE|os.O_WRONLY, 0666)
+		defaultWriter, err := NewRotateFileWriter(config.FileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays)
 
 		if err != nil {
 			return err
 		}
 
 		if !isDisableBootLog {
-			bootWriters = append(bootWriters, logFile)
+			bootWriters = append(bootWriters, defaultWriter)
 		}
 
-		writers = append(writers, logFile)
+		defaultWriters = append(defaultWriters, defaultWriter)
+
+		if config.RequestFileLogPath != "" && config.RequestFileLogPath != config.FileLogPath {
+			requestWriter, err := NewRotateFileWriter(config.RequestFileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays)
+
+			if err != nil {
+				return err
+			}
+
+			requestWriters = append(requestWriters, requestWriter)
+		} else {
+			requestWriters = append(requestWriters, defaultWriter)
+		}
+
+		if config.QueryFileLogPath != "" && config.QueryFileLogPath != config.FileLogPath {
+			queryWriter, err := NewRotateFileWriter(config.QueryFileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays)
+
+			if err != nil {
+				return err
+			}
+
+			queryWriters = append(queryWriters, queryWriter)
+		} else {
+			queryWriters = append(queryWriters, defaultWriter)
+		}
 	}
 
 	bootMultipleWriter := io.MultiWriter(bootWriters...)
-	multipleWriter := io.MultiWriter(writers...)
+	defaultMultipleWriter := io.MultiWriter(defaultWriters...)
+	requestMultipleWriter := io.MultiWriter(requestWriters...)
+	queryMultipleWriter := io.MultiWriter(queryWriters...)
 
 	bootLogger.SetOutput(bootMultipleWriter)
-	defaultLogger.SetOutput(multipleWriter)
+	defaultLogger.SetOutput(defaultMultipleWriter)
-	requestLogger.SetOutput(multipleWriter)
+	requestLogger.SetOutput(requestMultipleWriter)
-	sqlQueryLogger.SetOutput(multipleWriter)
+	sqlQueryLogger.SetOutput(queryMultipleWriter)
 
 	if config.LogLevel == settings.LOGLEVEL_DEBUG {
 		bootLogger.SetLevel(logrus.DebugLevel)

pkg/log/rotate_file_writer.go

@@ -0,0 +1,185 @@
+package log
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mayswind/ezbookkeeping/pkg/errs"
+)
+
+const (
+	logRotateSuffixDateFormat = "20060102150405"
+)
+
+type RotateFileWriter struct {
+	EnableRotate bool
+	MaxFileSize  int64
+	MaxFileDays  uint32
+
+	filePath  string
+	file      *os.File
+	totalSize int64
+
+	mutex                 sync.Mutex
+	lastRemoveOldFilesDay int
+}
+
+// NewRotateFileWriter returns a new rotate file writer
+func NewRotateFileWriter(filePath string, enableRotate bool, maxFileSize int64, maxFileDays uint32) (*RotateFileWriter, error) {
+	writer := &RotateFileWriter{
+		EnableRotate: enableRotate,
+		MaxFileSize:  maxFileSize,
+		MaxFileDays:  maxFileDays,
+		filePath:     filePath,
+		totalSize:    0,
+	}
+
+	err := writer.openFile()
+
+	if err != nil {
+		return nil, err
+	}
+
+	return writer, nil
+}
+
+// Write does log data to specified file
+func (w *RotateFileWriter) Write(p []byte) (n int, err error) {
+	dataSize := int64(len(p))
+
+	if w.EnableRotate && w.totalSize > 0 && w.totalSize+dataSize >= w.MaxFileSize {
+		w.mutex.Lock()
+		defer w.mutex.Unlock()
+
+		if w.EnableRotate && w.totalSize > 0 && w.totalSize+dataSize >= w.MaxFileSize {
+			err := w.rotateFile()
+
+			if err != nil {
+				BootErrorf("[rotate_file_writer.Write] cannot rotate log file \"%s\", because %s", w.file.Name(), err.Error())
+				return 0, err
+			}
+		}
+	}
+
+	writeSize, err := w.file.Write(p)
+
+	if err != nil {
+		return 0, err
+	}
+
+	w.totalSize += int64(writeSize)
+
+	if w.EnableRotate {
+		today := time.Now().Day()
+
+		if today != w.lastRemoveOldFilesDay && w.MaxFileDays > 0 {
+			w.lastRemoveOldFilesDay = today
+			go w.removeOldFiles()
+		}
+	}
+
+	return writeSize, err
+}
+
+func (w *RotateFileWriter) rotateFile() error {
+	currentFileName := w.file.Name()
+	err := w.file.Close()
+
+	if err != nil {
+		return errs.NewLoggingError(fmt.Sprintf("cannot close log file \"%s\", because %s", w.file.Name(), err.Error()), err)
+	}
+
+	w.file = nil
+	archiveFileName := fmt.Sprintf("%s.%s", currentFileName, time.Now().Format(logRotateSuffixDateFormat))
+	err = os.Rename(currentFileName, archiveFileName)
+
+	if err != nil {
+		return errs.NewLoggingError(fmt.Sprintf("cannot rename log file \"%s\" to \"%s\", because %s", currentFileName, archiveFileName, err.Error()), err)
+	}
+
+	err = w.openFile()
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (w *RotateFileWriter) openFile() error {
+	if w.file != nil {
+		BootWarnf("[rotate_file_writer.removeOldFiles] cannot reopen log file \"%s\"", w.file.Name())
+		return nil
+	}
+
+	file, err := os.OpenFile(w.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+
+	if err != nil {
+		return errs.NewLoggingError(fmt.Sprintf("cannot open log file \"%s\", because %s", w.filePath, err.Error()), err)
+	}
+
+	w.file = file
+	return nil
+}
+
+func (w *RotateFileWriter) removeOldFiles() {
+	dir := filepath.Dir(w.filePath)
+	logBaseFileName := filepath.Base(w.filePath) + "."
+
+	allLogFiles, err := os.ReadDir(dir)
+
+	if err != nil {
+		return
+	}
+
+	retainMinUnixTime := int64(0)
+
+	if w.MaxFileDays > 0 {
+		retainMinUnixTime = time.Now().AddDate(0, 0, -int(w.MaxFileDays)).Unix()
+	}
+
+	for _, file := range allLogFiles {
+		if file.IsDir() {
+			continue
+		}
+
+		logFileName := filepath.Base(file.Name())
+
+		if !strings.HasPrefix(logFileName, logBaseFileName) {
+			continue
+		}
+
+		rotateDate := logFileName[len(logBaseFileName):]
+		dotIndex := strings.Index(rotateDate, ".")
+
+		if dotIndex > 0 {
+			rotateDate = rotateDate[0:dotIndex]
+		}
+
+		if len(rotateDate) != len(logRotateSuffixDateFormat) {
+			BootErrorf("[rotate_file_writer.removeOldFiles] date suffix of old log file \"%s\" is invalid", file.Name())
+			continue
+		}
+
+		rotateDateTime, err := time.ParseInLocation(logRotateSuffixDateFormat, rotateDate, time.Now().Location())
+
+		if err != nil {
+			BootErrorf("[rotate_file_writer.removeOldFiles] cannot parse rotate date of old log file \"%s\", because %s", file.Name(), err.Error())
+			continue
+		}
+
+		if rotateDateTime.Unix() >= retainMinUnixTime {
+			continue
+		}
+
+		err = os.Remove(filepath.Join(dir, file.Name()))
+
+		if err != nil {
+			BootErrorf("[rotate_file_writer.removeOldFiles] cannot remove old log file \"%s\", because %s", file.Name(), err.Error())
+		}
+	}
+}

pkg/settings/setting.go

@@ -127,6 +127,8 @@ const (
 	defaultDatabaseConnMaxLifetime uint32 = 14400
 
 	defaultLogMode        string = "console"
+	defaultLogFileMaxSize uint32 = 104857600 // 100 MB
+	defaultLogFileMaxDays uint32 = 7         // days
 
 	defaultInMemoryDuplicateCheckerCleanupInterval uint32 = 60  // 1 minutes
 	defaultDuplicateSubmissionsInterval            uint32 = 300 // 5 minutes
@@ -227,6 +229,11 @@ type Config struct {
 
 	LogLevel           Level
 	FileLogPath        string
+	RequestFileLogPath string
+	QueryFileLogPath   string
+	LogFileRotate      bool
+	LogFileMaxSize     uint32
+	LogFileMaxDays     uint32
 
 	// Storage
 	StorageType         string
@@ -566,6 +573,28 @@ func loadLogConfiguration(config *Config, configFile *ini.File, sectionName stri
 		fileLogPath := getConfigItemStringValue(configFile, sectionName, "log_path")
 		finalFileLogPath, _ := getFinalPath(config.WorkingPath, fileLogPath)
 		config.FileLogPath = finalFileLogPath
+
+		requestFileLogPath := getConfigItemStringValue(configFile, sectionName, "request_log_path")
+
+		if requestFileLogPath != "" {
+			finalRequestFileLogPath, _ := getFinalPath(config.WorkingPath, requestFileLogPath)
+			config.RequestFileLogPath = finalRequestFileLogPath
+		} else {
+			config.RequestFileLogPath = ""
+		}
+
+		queryFileLogPath := getConfigItemStringValue(configFile, sectionName, "query_log_path")
+
+		if queryFileLogPath != "" {
+			finalQueryFileLogPath, _ := getFinalPath(config.WorkingPath, queryFileLogPath)
+			config.QueryFileLogPath = finalQueryFileLogPath
+		} else {
+			config.QueryFileLogPath = ""
+		}
+
+		config.LogFileRotate = getConfigItemBoolValue(configFile, sectionName, "log_file_rotate", false)
+		config.LogFileMaxSize = getConfigItemUint32Value(configFile, sectionName, "log_file_max_size", defaultLogFileMaxSize)
+		config.LogFileMaxDays = getConfigItemUint32Value(configFile, sectionName, "log_file_max_days", defaultLogFileMaxDays)
 	}
 
 	return nil