support logging request logs and database query logs to separate files, and support rotating log files
This commit is contained in:
MaysWind
@@ -92,6 +92,21 @@ level = info
|
||||
# For "file" mode only, log file path (relative or absolute path)
|
||||
log_path = log/ezbookkeeping.log
|
||||
|
||||
# For "file" only, request log file path (relative or absolute path). Leave blank if you want to write request log in default log file
|
||||
request_log_path =
|
||||
|
||||
# For "file" only, query log file path (relative or absolute path). Leave blank if you want to write query log in default log file
|
||||
query_log_path =
|
||||
|
||||
# For "file" only, whether rotate the log files
|
||||
log_file_rotate = false
|
||||
|
||||
# For "file" only, maximum size (1 - 4294967295 bytes) of the log file before it gets rotated
|
||||
log_file_max_size = 104857600
|
||||
|
||||
# For "file" only, maximum number of days to retain old log files. Set to 0 to retain all logs
|
||||
log_file_max_days = 7
|
||||
|
||||
[storage]
|
||||
# Object storage type, supports "local_filesystem" and "minio" currently
|
||||
type = local_filesystem
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
SystemSubcategorySetting = 1
|
||||
SystemSubcategoryDatabase = 2
|
||||
SystemSubcategoryMail = 3
|
||||
SystemSubcategoryLogging = 4
|
||||
)
|
||||
|
||||
// Sub categories of normal error
|
||||
@@ -75,6 +76,15 @@ func NewNormalError(subCategory int32, index int32, httpStatusCode int, message
|
||||
return New(CATEGORY_NORMAL, subCategory, index, httpStatusCode, message)
|
||||
}
|
||||
|
||||
// NewLoggingError returns a new logging error instance
|
||||
func NewLoggingError(message string, err ...error) *Error {
|
||||
return New(ErrLoggingError.Category,
|
||||
ErrLoggingError.SubCategory,
|
||||
ErrLoggingError.Index,
|
||||
ErrLoggingError.HttpStatusCode,
|
||||
message, err...)
|
||||
}
|
||||
|
||||
// NewIncompleteOrIncorrectSubmissionError returns a new incomplete or incorrect submission error instance
|
||||
func NewIncompleteOrIncorrectSubmissionError(err error) *Error {
|
||||
return New(ErrIncompleteOrIncorrectSubmission.Category,
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package errs
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Error codes related to logging
|
||||
var (
|
||||
ErrLoggingError = NewSystemError(SystemSubcategoryLogging, 0, http.StatusInternalServerError, "logging error")
|
||||
)
|
||||
+39
-9
@@ -41,37 +41,67 @@ func init() {
|
||||
// SetLoggerConfiguration sets the logger according to the config
|
||||
func SetLoggerConfiguration(config *settings.Config, isDisableBootLog bool) error {
|
||||
var bootWriters []io.Writer
|
||||
var writers []io.Writer
|
||||
var defaultWriters []io.Writer
|
||||
var requestWriters []io.Writer
|
||||
var queryWriters []io.Writer
|
||||
|
||||
if !isDisableBootLog {
|
||||
bootWriters = append(bootWriters, os.Stdout)
|
||||
}
|
||||
|
||||
if config.EnableConsoleLog {
|
||||
writers = append(writers, os.Stdout)
|
||||
defaultWriters = append(defaultWriters, os.Stdout)
|
||||
requestWriters = append(requestWriters, os.Stdout)
|
||||
queryWriters = append(queryWriters, os.Stdout)
|
||||
}
|
||||
|
||||
if config.EnableFileLog {
|
||||
logFile, err := os.OpenFile(config.FileLogPath, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
defaultWriter, err := NewRotateFileWriter(config.FileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !isDisableBootLog {
|
||||
bootWriters = append(bootWriters, logFile)
|
||||
bootWriters = append(bootWriters, defaultWriter)
|
||||
}
|
||||
|
||||
writers = append(writers, logFile)
|
||||
defaultWriters = append(defaultWriters, defaultWriter)
|
||||
|
||||
if config.RequestFileLogPath != "" && config.RequestFileLogPath != config.FileLogPath {
|
||||
requestWriter, err := NewRotateFileWriter(config.RequestFileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
requestWriters = append(requestWriters, requestWriter)
|
||||
} else {
|
||||
requestWriters = append(requestWriters, defaultWriter)
|
||||
}
|
||||
|
||||
if config.QueryFileLogPath != "" && config.QueryFileLogPath != config.FileLogPath {
|
||||
queryWriter, err := NewRotateFileWriter(config.QueryFileLogPath, config.LogFileRotate, int64(config.LogFileMaxSize), config.LogFileMaxDays)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
queryWriters = append(queryWriters, queryWriter)
|
||||
} else {
|
||||
queryWriters = append(queryWriters, defaultWriter)
|
||||
}
|
||||
}
|
||||
|
||||
bootMultipleWriter := io.MultiWriter(bootWriters...)
|
||||
multipleWriter := io.MultiWriter(writers...)
|
||||
defaultMultipleWriter := io.MultiWriter(defaultWriters...)
|
||||
requestMultipleWriter := io.MultiWriter(requestWriters...)
|
||||
queryMultipleWriter := io.MultiWriter(queryWriters...)
|
||||
|
||||
bootLogger.SetOutput(bootMultipleWriter)
|
||||
defaultLogger.SetOutput(multipleWriter)
|
||||
requestLogger.SetOutput(multipleWriter)
|
||||
sqlQueryLogger.SetOutput(multipleWriter)
|
||||
defaultLogger.SetOutput(defaultMultipleWriter)
|
||||
requestLogger.SetOutput(requestMultipleWriter)
|
||||
sqlQueryLogger.SetOutput(queryMultipleWriter)
|
||||
|
||||
if config.LogLevel == settings.LOGLEVEL_DEBUG {
|
||||
bootLogger.SetLevel(logrus.DebugLevel)
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
)
|
||||
|
||||
const (
|
||||
logRotateSuffixDateFormat = "20060102150405"
|
||||
)
|
||||
|
||||
type RotateFileWriter struct {
|
||||
EnableRotate bool
|
||||
MaxFileSize int64
|
||||
MaxFileDays uint32
|
||||
|
||||
filePath string
|
||||
file *os.File
|
||||
totalSize int64
|
||||
|
||||
mutex sync.Mutex
|
||||
lastRemoveOldFilesDay int
|
||||
}
|
||||
|
||||
// NewRotateFileWriter returns a new rotate file writer
|
||||
func NewRotateFileWriter(filePath string, enableRotate bool, maxFileSize int64, maxFileDays uint32) (*RotateFileWriter, error) {
|
||||
writer := &RotateFileWriter{
|
||||
EnableRotate: enableRotate,
|
||||
MaxFileSize: maxFileSize,
|
||||
MaxFileDays: maxFileDays,
|
||||
filePath: filePath,
|
||||
totalSize: 0,
|
||||
}
|
||||
|
||||
err := writer.openFile()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return writer, nil
|
||||
}
|
||||
|
||||
// Write does log data to specified file
|
||||
func (w *RotateFileWriter) Write(p []byte) (n int, err error) {
|
||||
dataSize := int64(len(p))
|
||||
|
||||
if w.EnableRotate && w.totalSize > 0 && w.totalSize+dataSize >= w.MaxFileSize {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
|
||||
if w.EnableRotate && w.totalSize > 0 && w.totalSize+dataSize >= w.MaxFileSize {
|
||||
err := w.rotateFile()
|
||||
|
||||
if err != nil {
|
||||
BootErrorf("[rotate_file_writer.Write] cannot rotate log file \"%s\", because %s", w.file.Name(), err.Error())
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writeSize, err := w.file.Write(p)
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
w.totalSize += int64(writeSize)
|
||||
|
||||
if w.EnableRotate {
|
||||
today := time.Now().Day()
|
||||
|
||||
if today != w.lastRemoveOldFilesDay && w.MaxFileDays > 0 {
|
||||
w.lastRemoveOldFilesDay = today
|
||||
go w.removeOldFiles()
|
||||
}
|
||||
}
|
||||
|
||||
return writeSize, err
|
||||
}
|
||||
|
||||
func (w *RotateFileWriter) rotateFile() error {
|
||||
currentFileName := w.file.Name()
|
||||
err := w.file.Close()
|
||||
|
||||
if err != nil {
|
||||
return errs.NewLoggingError(fmt.Sprintf("cannot close log file \"%s\", because %s", w.file.Name(), err.Error()), err)
|
||||
}
|
||||
|
||||
w.file = nil
|
||||
archiveFileName := fmt.Sprintf("%s.%s", currentFileName, time.Now().Format(logRotateSuffixDateFormat))
|
||||
err = os.Rename(currentFileName, archiveFileName)
|
||||
|
||||
if err != nil {
|
||||
return errs.NewLoggingError(fmt.Sprintf("cannot rename log file \"%s\" to \"%s\", because %s", currentFileName, archiveFileName, err.Error()), err)
|
||||
}
|
||||
|
||||
err = w.openFile()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *RotateFileWriter) openFile() error {
|
||||
if w.file != nil {
|
||||
BootWarnf("[rotate_file_writer.removeOldFiles] cannot reopen log file \"%s\"", w.file.Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(w.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
|
||||
if err != nil {
|
||||
return errs.NewLoggingError(fmt.Sprintf("cannot open log file \"%s\", because %s", w.filePath, err.Error()), err)
|
||||
}
|
||||
|
||||
w.file = file
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *RotateFileWriter) removeOldFiles() {
|
||||
dir := filepath.Dir(w.filePath)
|
||||
logBaseFileName := filepath.Base(w.filePath) + "."
|
||||
|
||||
allLogFiles, err := os.ReadDir(dir)
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
retainMinUnixTime := int64(0)
|
||||
|
||||
if w.MaxFileDays > 0 {
|
||||
retainMinUnixTime = time.Now().AddDate(0, 0, -int(w.MaxFileDays)).Unix()
|
||||
}
|
||||
|
||||
for _, file := range allLogFiles {
|
||||
if file.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
logFileName := filepath.Base(file.Name())
|
||||
|
||||
if !strings.HasPrefix(logFileName, logBaseFileName) {
|
||||
continue
|
||||
}
|
||||
|
||||
rotateDate := logFileName[len(logBaseFileName):]
|
||||
dotIndex := strings.Index(rotateDate, ".")
|
||||
|
||||
if dotIndex > 0 {
|
||||
rotateDate = rotateDate[0:dotIndex]
|
||||
}
|
||||
|
||||
if len(rotateDate) != len(logRotateSuffixDateFormat) {
|
||||
BootErrorf("[rotate_file_writer.removeOldFiles] date suffix of old log file \"%s\" is invalid", file.Name())
|
||||
continue
|
||||
}
|
||||
|
||||
rotateDateTime, err := time.ParseInLocation(logRotateSuffixDateFormat, rotateDate, time.Now().Location())
|
||||
|
||||
if err != nil {
|
||||
BootErrorf("[rotate_file_writer.removeOldFiles] cannot parse rotate date of old log file \"%s\", because %s", file.Name(), err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if rotateDateTime.Unix() >= retainMinUnixTime {
|
||||
continue
|
||||
}
|
||||
|
||||
err = os.Remove(filepath.Join(dir, file.Name()))
|
||||
|
||||
if err != nil {
|
||||
BootErrorf("[rotate_file_writer.removeOldFiles] cannot remove old log file \"%s\", because %s", file.Name(), err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
-3
@@ -126,7 +126,9 @@ const (
|
||||
defaultDatabaseMaxOpenConn uint16 = 0
|
||||
defaultDatabaseConnMaxLifetime uint32 = 14400
|
||||
|
||||
defaultLogMode string = "console"
|
||||
defaultLogMode string = "console"
|
||||
defaultLogFileMaxSize uint32 = 104857600 // 100 MB
|
||||
defaultLogFileMaxDays uint32 = 7 // days
|
||||
|
||||
defaultInMemoryDuplicateCheckerCleanupInterval uint32 = 60 // 1 minutes
|
||||
defaultDuplicateSubmissionsInterval uint32 = 300 // 5 minutes
|
||||
@@ -225,8 +227,13 @@ type Config struct {
|
||||
EnableConsoleLog bool
|
||||
EnableFileLog bool
|
||||
|
||||
LogLevel Level
|
||||
FileLogPath string
|
||||
LogLevel Level
|
||||
FileLogPath string
|
||||
RequestFileLogPath string
|
||||
QueryFileLogPath string
|
||||
LogFileRotate bool
|
||||
LogFileMaxSize uint32
|
||||
LogFileMaxDays uint32
|
||||
|
||||
// Storage
|
||||
StorageType string
|
||||
@@ -566,6 +573,28 @@ func loadLogConfiguration(config *Config, configFile *ini.File, sectionName stri
|
||||
fileLogPath := getConfigItemStringValue(configFile, sectionName, "log_path")
|
||||
finalFileLogPath, _ := getFinalPath(config.WorkingPath, fileLogPath)
|
||||
config.FileLogPath = finalFileLogPath
|
||||
|
||||
requestFileLogPath := getConfigItemStringValue(configFile, sectionName, "request_log_path")
|
||||
|
||||
if requestFileLogPath != "" {
|
||||
finalRequestFileLogPath, _ := getFinalPath(config.WorkingPath, requestFileLogPath)
|
||||
config.RequestFileLogPath = finalRequestFileLogPath
|
||||
} else {
|
||||
config.RequestFileLogPath = ""
|
||||
}
|
||||
|
||||
queryFileLogPath := getConfigItemStringValue(configFile, sectionName, "query_log_path")
|
||||
|
||||
if queryFileLogPath != "" {
|
||||
finalQueryFileLogPath, _ := getFinalPath(config.WorkingPath, queryFileLogPath)
|
||||
config.QueryFileLogPath = finalQueryFileLogPath
|
||||
} else {
|
||||
config.QueryFileLogPath = ""
|
||||
}
|
||||
|
||||
config.LogFileRotate = getConfigItemBoolValue(configFile, sectionName, "log_file_rotate", false)
|
||||
config.LogFileMaxSize = getConfigItemUint32Value(configFile, sectionName, "log_file_max_size", defaultLogFileMaxSize)
|
||||
config.LogFileMaxDays = getConfigItemUint32Value(configFile, sectionName, "log_file_max_days", defaultLogFileMaxDays)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user