diff --git a/conf/ezbookkeeping.ini b/conf/ezbookkeeping.ini index 805588bb..cd16d905 100644 --- a/conf/ezbookkeeping.ini +++ b/conf/ezbookkeeping.ini @@ -92,6 +92,21 @@ level = info # For "file" mode only, log file path (relative or absolute path) log_path = log/ezbookkeeping.log +# For "file" only, request log file path (relative or absolute path). Leave blank if you want to write request log in default log file +request_log_path = + +# For "file" only, query log file path (relative or absolute path). Leave blank if you want to write query log in default log file +query_log_path = + +# For "file" only, whether rotate the log files +log_file_rotate = false + +# For "file" only, maximum size (1 - 4294967295 bytes) of the log file before it gets rotated +log_file_max_size = 104857600 + +# For "file" only, maximum number of days to retain old log files. Set to 0 to retain all logs +log_file_max_days = 7 + [storage] # Object storage type, supports "local_filesystem" and "minio" currently type = local_filesystem diff --git a/pkg/errs/error.go b/pkg/errs/error.go index d529d779..e2ace3eb 100644 --- a/pkg/errs/error.go +++ b/pkg/errs/error.go @@ -15,6 +15,7 @@ const ( SystemSubcategorySetting = 1 SystemSubcategoryDatabase = 2 SystemSubcategoryMail = 3 + SystemSubcategoryLogging = 4 ) // Sub categories of normal error @@ -75,6 +76,15 @@ func NewNormalError(subCategory int32, index int32, httpStatusCode int, message return New(CATEGORY_NORMAL, subCategory, index, httpStatusCode, message) } +// NewLoggingError returns a new logging error instance +func NewLoggingError(message string, err ...error) *Error { + return New(ErrLoggingError.Category, + ErrLoggingError.SubCategory, + ErrLoggingError.Index, + ErrLoggingError.HttpStatusCode, + message, err...) +} + // NewIncompleteOrIncorrectSubmissionError returns a new incomplete or incorrect submission error instance func NewIncompleteOrIncorrectSubmissionError(err error) *Error { return New(ErrIncompleteOrIncorrectSubmission.Category, diff --git a/pkg/errs/logging.go b/pkg/errs/logging.go new file mode 100644 index 00000000..aef70624 --- /dev/null +++ b/pkg/errs/logging.go @@ -0,0 +1,10 @@ +package errs + +import ( + "net/http" +) + +// Error codes related to logging +var ( + ErrLoggingError = NewSystemError(SystemSubcategoryLogging, 0, http.StatusInternalServerError, "logging error") +) diff --git a/pkg/log/logger.go b/pkg/log/logger.go index 990e82eb..9feb7987 100644 --- a/pkg/log/logger.go +++ b/pkg/log/logger.go @@ -41,37 +41,67 @@ func init() { // SetLoggerConfiguration sets the logger according to the config func SetLoggerConfiguration(config *settings.Config, isDisableBootLog bool) error { var bootWriters []io.Writer - var writers []io.Writer + var defaultWriters []io.Writer + var requestWriters []io.Writer + var queryWriters []io.Writer if !isDisableBootLog { bootWriters = append(bootWriters, os.Stdout) } if config.EnableConsoleLog { - writers = append(writers, os.Stdout) + defaultWriters = append(defaultWriters, os.Stdout) + requestWriters = append(requestWriters, os.Stdout) + queryWriters = append(queryWriters, os.Stdout) } if config.EnableFileLog { - logFile, err := os.OpenFile(config.FileLogPath, os.O_CREATE|os.O_WRONLY, 0666) + defaultWriter, err := NewRotateFileWriter(config.FileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays) if err != nil { return err } if !isDisableBootLog { - bootWriters = append(bootWriters, logFile) + bootWriters = append(bootWriters, defaultWriter) } - writers = append(writers, logFile) + defaultWriters = append(defaultWriters, defaultWriter) + + if config.RequestFileLogPath != "" && config.RequestFileLogPath != config.FileLogPath { + requestWriter, err := NewRotateFileWriter(config.RequestFileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays) + + if err != nil { + return err + } + + requestWriters = append(requestWriters, requestWriter) + } else { + requestWriters = append(requestWriters, defaultWriter) + } + + if config.QueryFileLogPath != "" && config.QueryFileLogPath != config.FileLogPath { + queryWriter, err := NewRotateFileWriter(config.QueryFileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays) + + if err != nil { + return err + } + + queryWriters = append(queryWriters, queryWriter) + } else { + queryWriters = append(queryWriters, defaultWriter) + } } bootMultipleWriter := io.MultiWriter(bootWriters...) - multipleWriter := io.MultiWriter(writers...) + defaultMultipleWriter := io.MultiWriter(defaultWriters...) + requestMultipleWriter := io.MultiWriter(requestWriters...) + queryMultipleWriter := io.MultiWriter(queryWriters...) bootLogger.SetOutput(bootMultipleWriter) - defaultLogger.SetOutput(multipleWriter) - requestLogger.SetOutput(multipleWriter) - sqlQueryLogger.SetOutput(multipleWriter) + defaultLogger.SetOutput(defaultMultipleWriter) + requestLogger.SetOutput(requestMultipleWriter) + sqlQueryLogger.SetOutput(queryMultipleWriter) if config.LogLevel == settings.LOGLEVEL_DEBUG { bootLogger.SetLevel(logrus.DebugLevel) diff --git a/pkg/log/rotate_file_writer.go b/pkg/log/rotate_file_writer.go new file mode 100644 index 00000000..fdf59c84 --- /dev/null +++ b/pkg/log/rotate_file_writer.go @@ -0,0 +1,185 @@ +package log + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/mayswind/ezbookkeeping/pkg/errs" +) + +const ( + logRotateSuffixDateFormat = "20060102150405" +) + +type RotateFileWriter struct { + EnableRotate bool + MaxFileSize int64 + MaxFileDays uint32 + + filePath string + file *os.File + totalSize int64 + + mutex sync.Mutex + lastRemoveOldFilesDay int +} + +// NewRotateFileWriter returns a new rotate file writer +func NewRotateFileWriter(filePath string, enableRotate bool, maxFileSize int64, maxFileDays uint32) (*RotateFileWriter, error) { + writer := &RotateFileWriter{ + EnableRotate: enableRotate, + MaxFileSize: maxFileSize, + MaxFileDays: maxFileDays, + filePath: filePath, + totalSize: 0, + } + + err := writer.openFile() + + if err != nil { + return nil, err + } + + return writer, nil +} + +// Write does log data to specified file +func (w *RotateFileWriter) Write(p []byte) (n int, err error) { + dataSize := int64(len(p)) + + if w.EnableRotate && w.totalSize > 0 && w.totalSize+dataSize >= w.MaxFileSize { + w.mutex.Lock() + defer w.mutex.Unlock() + + if w.EnableRotate && w.totalSize > 0 && w.totalSize+dataSize >= w.MaxFileSize { + err := w.rotateFile() + + if err != nil { + BootErrorf("[rotate_file_writer.Write] cannot rotate log file \"%s\", because %s", w.file.Name(), err.Error()) + return 0, err + } + } + } + + writeSize, err := w.file.Write(p) + + if err != nil { + return 0, err + } + + w.totalSize += int64(writeSize) + + if w.EnableRotate { + today := time.Now().Day() + + if today != w.lastRemoveOldFilesDay && w.MaxFileDays > 0 { + w.lastRemoveOldFilesDay = today + go w.removeOldFiles() + } + } + + return writeSize, err +} + +func (w *RotateFileWriter) rotateFile() error { + currentFileName := w.file.Name() + err := w.file.Close() + + if err != nil { + return errs.NewLoggingError(fmt.Sprintf("cannot close log file \"%s\", because %s", w.file.Name(), err.Error()), err) + } + + w.file = nil + archiveFileName := fmt.Sprintf("%s.%s", currentFileName, time.Now().Format(logRotateSuffixDateFormat)) + err = os.Rename(currentFileName, archiveFileName) + + if err != nil { + return errs.NewLoggingError(fmt.Sprintf("cannot rename log file \"%s\" to \"%s\", because %s", currentFileName, archiveFileName, err.Error()), err) + } + + err = w.openFile() + + if err != nil { + return err + } + + return nil +} + +func (w *RotateFileWriter) openFile() error { + if w.file != nil { + BootWarnf("[rotate_file_writer.removeOldFiles] cannot reopen log file \"%s\"", w.file.Name()) + return nil + } + + file, err := os.OpenFile(w.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + + if err != nil { + return errs.NewLoggingError(fmt.Sprintf("cannot open log file \"%s\", because %s", w.filePath, err.Error()), err) + } + + w.file = file + return nil +} + +func (w *RotateFileWriter) removeOldFiles() { + dir := filepath.Dir(w.filePath) + logBaseFileName := filepath.Base(w.filePath) + "." + + allLogFiles, err := os.ReadDir(dir) + + if err != nil { + return + } + + retainMinUnixTime := int64(0) + + if w.MaxFileDays > 0 { + retainMinUnixTime = time.Now().AddDate(0, 0, -int(w.MaxFileDays)).Unix() + } + + for _, file := range allLogFiles { + if file.IsDir() { + continue + } + + logFileName := filepath.Base(file.Name()) + + if !strings.HasPrefix(logFileName, logBaseFileName) { + continue + } + + rotateDate := logFileName[len(logBaseFileName):] + dotIndex := strings.Index(rotateDate, ".") + + if dotIndex > 0 { + rotateDate = rotateDate[0:dotIndex] + } + + if len(rotateDate) != len(logRotateSuffixDateFormat) { + BootErrorf("[rotate_file_writer.removeOldFiles] date suffix of old log file \"%s\" is invalid", file.Name()) + continue + } + + rotateDateTime, err := time.ParseInLocation(logRotateSuffixDateFormat, rotateDate, time.Now().Location()) + + if err != nil { + BootErrorf("[rotate_file_writer.removeOldFiles] cannot parse rotate date of old log file \"%s\", because %s", file.Name(), err.Error()) + continue + } + + if rotateDateTime.Unix() >= retainMinUnixTime { + continue + } + + err = os.Remove(filepath.Join(dir, file.Name())) + + if err != nil { + BootErrorf("[rotate_file_writer.removeOldFiles] cannot remove old log file \"%s\", because %s", file.Name(), err.Error()) + } + } +} diff --git a/pkg/settings/setting.go b/pkg/settings/setting.go index b07d7252..df187af5 100644 --- a/pkg/settings/setting.go +++ b/pkg/settings/setting.go @@ -126,7 +126,9 @@ const ( defaultDatabaseMaxOpenConn uint16 = 0 defaultDatabaseConnMaxLifetime uint32 = 14400 - defaultLogMode string = "console" + defaultLogMode string = "console" + defaultLogFileMaxSize uint32 = 104857600 // 100 MB + defaultLogFileMaxDays uint32 = 7 // days defaultInMemoryDuplicateCheckerCleanupInterval uint32 = 60 // 1 minutes defaultDuplicateSubmissionsInterval uint32 = 300 // 5 minutes @@ -225,8 +227,13 @@ type Config struct { EnableConsoleLog bool EnableFileLog bool - LogLevel Level - FileLogPath string + LogLevel Level + FileLogPath string + RequestFileLogPath string + QueryFileLogPath string + LogFileRotate bool + LogFileMaxSize uint32 + LogFileMaxDays uint32 // Storage StorageType string @@ -566,6 +573,28 @@ func loadLogConfiguration(config *Config, configFile *ini.File, sectionName stri fileLogPath := getConfigItemStringValue(configFile, sectionName, "log_path") finalFileLogPath, _ := getFinalPath(config.WorkingPath, fileLogPath) config.FileLogPath = finalFileLogPath + + requestFileLogPath := getConfigItemStringValue(configFile, sectionName, "request_log_path") + + if requestFileLogPath != "" { + finalRequestFileLogPath, _ := getFinalPath(config.WorkingPath, requestFileLogPath) + config.RequestFileLogPath = finalRequestFileLogPath + } else { + config.RequestFileLogPath = "" + } + + queryFileLogPath := getConfigItemStringValue(configFile, sectionName, "query_log_path") + + if queryFileLogPath != "" { + finalQueryFileLogPath, _ := getFinalPath(config.WorkingPath, queryFileLogPath) + config.QueryFileLogPath = finalQueryFileLogPath + } else { + config.QueryFileLogPath = "" + } + + config.LogFileRotate = getConfigItemBoolValue(configFile, sectionName, "log_file_rotate", false) + config.LogFileMaxSize = getConfigItemUint32Value(configFile, sectionName, "log_file_max_size", defaultLogFileMaxSize) + config.LogFileMaxDays = getConfigItemUint32Value(configFile, sectionName, "log_file_max_days", defaultLogFileMaxDays) } return nil