diff --git a/cmd/cron_jobs.go b/cmd/cron_jobs.go new file mode 100644 index 00000000..e5dc1fcc --- /dev/null +++ b/cmd/cron_jobs.go @@ -0,0 +1,97 @@ +package cmd + +import ( + "fmt" + + "github.com/urfave/cli/v2" + + "github.com/mayswind/ezbookkeeping/pkg/cron" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +// CronJobs represents the cron command +var CronJobs = &cli.Command{ + Name: "cron", + Usage: "ezBookkeeping cron job utilities", + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "List all enabled cron jobs", + Action: listAllCronJobs, + }, + { + Name: "run", + Usage: "Run specified cron job", + Action: runCronJob, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "name", + Aliases: []string{"n"}, + Required: true, + Usage: "Cron job name", + }, + }, + }, + }, +} + +func listAllCronJobs(c *cli.Context) error { + config, err := initializeSystem(c) + + if err != nil { + return err + } + + err = cron.InitializeCronJobSchedulerContainer(config, false) + + if err != nil { + log.BootErrorf("[cron_jobs.listAllCronJobs] initializes cron job scheduler failed, because %s", err.Error()) + return err + } + + cronJobs := cron.Container.GetAllJobs() + + if len(cronJobs) < 1 { + log.BootErrorf("[cron_jobs.listAllCronJobs] there are no enabled cron jobs") + return err + } + + for i := 0; i < len(cronJobs); i++ { + if i > 0 { + fmt.Printf("---\n") + } + + cronJob := cronJobs[i] + + fmt.Printf("[Name] %s\n", cronJob.Name) + fmt.Printf("[Description] %s\n", cronJob.Description) + fmt.Printf("[Interval] Every %s\n", cronJob.Interval) + } + + return nil +} + +func runCronJob(c *cli.Context) error { + config, err := initializeSystem(c) + + if err != nil { + return err + } + + err = cron.InitializeCronJobSchedulerContainer(config, false) + + if err != nil { + log.BootErrorf("[cron_jobs.runCronJob] initializes cron job scheduler failed, because %s", err.Error()) + return err + } + + jobName := c.String("name") + err = cron.Container.SyncRunJobNow(jobName) + + if err != nil { + log.BootErrorf("[cron_jobs.runCronJob] failed to run cron job \"%s\", because %s", jobName, err.Error()) + return err + } + + return nil +} diff --git a/cmd/webserver.go b/cmd/webserver.go index 0823f209..05c85b42 100644 --- a/cmd/webserver.go +++ b/cmd/webserver.go @@ -15,6 +15,7 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/api" "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/cron" "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/log" "github.com/mayswind/ezbookkeeping/pkg/middlewares" @@ -62,6 +63,13 @@ func startWebServer(c *cli.Context) error { return err } + err = cron.InitializeCronJobSchedulerContainer(config, true) + + if err != nil { + log.BootErrorf("[webserver.startWebServer] initializes cron job scheduler failed, because %s", err.Error()) + return err + } + serverInfo := fmt.Sprintf("current server id is %d, current instance id is %d", requestid.Container.Current.GetCurrentServerUniqId(), requestid.Container.Current.GetCurrentInstanceUniqId()) uuidServerInfo := "" if config.UuidGeneratorType == settings.InternalUuidGeneratorType { diff --git a/conf/ezbookkeeping.ini b/conf/ezbookkeeping.ini index 0e529f11..1cb5d2ce 100644 --- a/conf/ezbookkeeping.ini +++ b/conf/ezbookkeeping.ini @@ -150,6 +150,10 @@ cleanup_interval = 60 # Set to 0 to disable duplicate checker for new data submissions, default is 300 (5 minutes) duplicate_submissions_interval = 300 +[cron] +# Set to true to clean up expired tokens periodically +enable_remove_expired_tokens = true + [security] # Used for signing, you must change it to keep your user data safe before you first run ezBookkeeping secret_key = diff --git a/ezbookkeeping.go b/ezbookkeeping.go index b8c42b2f..3910394c 100644 --- a/ezbookkeeping.go +++ b/ezbookkeeping.go @@ -36,6 +36,7 @@ func main() { cmd.WebServer, cmd.Database, cmd.UserData, + cmd.CronJobs, cmd.SecurityUtils, cmd.Utilities, }, diff --git a/go.mod b/go.mod index 09239a66..2a12ae27 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/gin-contrib/cache v1.3.0 github.com/gin-contrib/gzip v1.0.1 github.com/gin-gonic/gin v1.10.0 + github.com/go-co-op/gocron/v2 v2.11.0 github.com/go-playground/validator/v10 v10.22.0 github.com/go-sql-driver/mysql v1.8.1 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -49,6 +50,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/gomodule/redigo v1.8.9 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect @@ -61,6 +63,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62 // indirect github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -69,6 +72,7 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect golang.org/x/arch v0.8.0 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect diff --git a/go.sum b/go.sum index 0cdbfee8..3028dd3c 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-co-op/gocron/v2 v2.11.0 h1:IOowNA6SzwdRFnD4/Ol3Kj6G2xKfsoiiGq2Jhhm9bvE= +github.com/go-co-op/gocron/v2 v2.11.0/go.mod h1:xY7bJxGazKam1cz04EebrlP4S9q4iWdiAylMGP3jY9w= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -67,6 +69,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -109,6 +113,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62 h1:pyecQtsPmlkCsMkYhT5iZ+sUXuwee+OvfuJjinEA3ko= github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62/go.mod h1:65XQgovT59RWatovFwnwocoUxiI/eENTnOY5GK3STuY= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -146,6 +152,8 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= diff --git a/pkg/cron/cron_container.go b/pkg/cron/cron_container.go new file mode 100644 index 00000000..df29dfc9 --- /dev/null +++ b/pkg/cron/cron_container.go @@ -0,0 +1,101 @@ +package cron + +import ( + "time" + + "github.com/go-co-op/gocron/v2" + + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/settings" +) + +// CronJobSchedulerContainer contains the current cron job scheduler +type CronJobSchedulerContainer struct { + scheduler gocron.Scheduler + allJobs []*CronJob + allJobsMap map[string]*CronJob + allGocronJobsMap map[string]gocron.Job +} + +// Initialize a cron job scheduler container singleton instance +var ( + Container = &CronJobSchedulerContainer{ + allJobsMap: make(map[string]*CronJob), + allGocronJobsMap: make(map[string]gocron.Job), + } +) + +// InitializeCronJobSchedulerContainer initializes the cron job scheduler according to the config +func InitializeCronJobSchedulerContainer(config *settings.Config, startScheduler bool) error { + var err error + + Container.scheduler, err = gocron.NewScheduler( + gocron.WithLocation(time.Local), + gocron.WithLogger(NewGocronLoggerAdapter()), + ) + + if err != nil { + return err + } + + Container.registerAllJobs(config) + + if startScheduler { + Container.scheduler.Start() + } + + return nil +} + +// GetAllJobs returns all the cron jobs +func (c *CronJobSchedulerContainer) GetAllJobs() []*CronJob { + return c.allJobs +} + +// SyncRunJobNow runs the specified cron job synchronously now +func (c *CronJobSchedulerContainer) SyncRunJobNow(jobName string) error { + if jobName == "" { + return errs.ErrCronJobNameIsEmpty + } + + job := c.allJobsMap[jobName] + + if job == nil { + return errs.ErrCronJobNotExistsOrNotEnabled + } + + gocronJob := c.allGocronJobsMap[jobName] + + if gocronJob == nil { + return errs.ErrCronJobNotExistsOrNotEnabled + } + + job.doRun() + return nil +} + +func (c *CronJobSchedulerContainer) registerAllJobs(config *settings.Config) { + if config.EnableRemoveExpiredTokens { + Container.registerIntervalJob(RemoveExpiredTokensJob) + } +} + +func (c *CronJobSchedulerContainer) registerIntervalJob(job *CronJob) { + gocronJob, err := c.scheduler.NewJob( + gocron.DurationJob(job.Interval), + gocron.NewTask(job.doRun), + gocron.WithName(job.Name), + gocron.WithSingletonMode(gocron.LimitModeReschedule), + ) + + if err == nil { + c.allJobs = append(c.allJobs, job) + c.allJobsMap[job.Name] = job + c.allGocronJobsMap[job.Name] = gocronJob + log.Infof("[cron_container.registerJob] job \"%s\" has been registered", job.Name) + log.Debugf("[cron_container.registerJob] job \"%s\" gocron id is %s", job.Name, gocronJob.ID()) + } else { + log.Errorf("[cron_container.registerJob] job \"%s\" cannot be been registered, because %s", job.Name, err.Error()) + } +} diff --git a/pkg/cron/cron_job.go b/pkg/cron/cron_job.go new file mode 100644 index 00000000..8c0bc460 --- /dev/null +++ b/pkg/cron/cron_job.go @@ -0,0 +1,49 @@ +package cron + +import ( + "fmt" + "time" + + "github.com/mayswind/ezbookkeeping/pkg/duplicatechecker" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +type CronJob struct { + Name string + Description string + Interval time.Duration + Run func() error +} + +func (c *CronJob) doRun() { + start := time.Now() + localAddr, err := utils.GetLocalIPAddressesString() + + if err != nil { + log.Warnf("[cron_job.doRun] job \"%s\" cannot get local ipv4 address, because %s", c.Name, err.Error()) + return + } + + currentInfo := fmt.Sprintf("ip: %s, startTime: %d", localAddr, time.Now().Unix()) + found, runningInfo := duplicatechecker.Container.GetOrSetCronJobRunningInfo(c.Name, currentInfo, c.Interval) + + if found { + log.Warnf("[cron_job.doRun] job \"%s\" is already running (%s)", c.Name, runningInfo) + return + } + + err = c.Run() + + duplicatechecker.Container.Current.RemoveCronJobRunningInfo(c.Name) + + now := time.Now() + + if err != nil { + log.Errorf("[cron_job.doRun] failed to run job \"%s\", because %s", c.Name, err.Error()) + return + } + + cost := now.Sub(start).Nanoseconds() / 1e6 + log.Infof("[cron_job.doRun] run job \"%s\" successfully, cost %dms", c.Name, cost) +} diff --git a/pkg/cron/cron_jobs.go b/pkg/cron/cron_jobs.go new file mode 100644 index 00000000..cc510883 --- /dev/null +++ b/pkg/cron/cron_jobs.go @@ -0,0 +1,16 @@ +package cron + +import ( + "time" + + "github.com/mayswind/ezbookkeeping/pkg/services" +) + +var RemoveExpiredTokensJob = &CronJob{ + Name: "RemoveExpiredTokens", + Description: "Periodically remove expired user tokens from the database.", + Interval: 24 * time.Hour, + Run: func() error { + return services.Tokens.DeleteAllExpiredTokens(nil) + }, +} diff --git a/pkg/cron/cron_log.go b/pkg/cron/cron_log.go new file mode 100644 index 00000000..d0d56d96 --- /dev/null +++ b/pkg/cron/cron_log.go @@ -0,0 +1,36 @@ +package cron + +import ( + "github.com/go-co-op/gocron/v2" + + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +// GocronLoggerAdapter represents the logger adapter for gocron +type GocronLoggerAdapter struct { +} + +// Debug logs debug log +func (logger GocronLoggerAdapter) Debug(msg string, args ...any) { + log.Debugf(msg, args...) +} + +// Info logs info log +func (logger GocronLoggerAdapter) Info(msg string, args ...any) { + log.Infof(msg, args...) +} + +// Warn logs warn log +func (logger GocronLoggerAdapter) Warn(msg string, args ...any) { + log.Warnf(msg, args...) +} + +// Error logs error log +func (logger GocronLoggerAdapter) Error(msg string, args ...any) { + log.Errorf(msg, args...) +} + +// NewGocronLoggerAdapter returns a new GocronLoggerAdapter instance +func NewGocronLoggerAdapter() gocron.Logger { + return GocronLoggerAdapter{} +} diff --git a/pkg/datastore/datastore.go b/pkg/datastore/datastore.go index adfa412f..73a92f6e 100644 --- a/pkg/datastore/datastore.go +++ b/pkg/datastore/datastore.go @@ -12,6 +12,16 @@ type DataStore struct { databases []*Database } +// Count returns total count of database instances +func (s *DataStore) Count() int { + return len(s.databases) +} + +// Get returns a database instance by index +func (s *DataStore) Get(index int) *Database { + return s.databases[index] +} + // Choose returns a database instance by sharding key func (s *DataStore) Choose(key int64) *Database { return s.databases[0] diff --git a/pkg/duplicatechecker/duplicate_checker.go b/pkg/duplicatechecker/duplicate_checker.go index 999b3a04..cad28377 100644 --- a/pkg/duplicatechecker/duplicate_checker.go +++ b/pkg/duplicatechecker/duplicate_checker.go @@ -1,7 +1,11 @@ package duplicatechecker +import "time" + // DuplicateChecker is common duplicate checker interface type DuplicateChecker interface { GetSubmissionRemark(checkerType DuplicateCheckerType, uid int64, identification string) (bool, string) SetSubmissionRemark(checkerType DuplicateCheckerType, uid int64, identification string, remark string) + GetOrSetCronJobRunningInfo(jobName string, runningInfo string, runningInterval time.Duration) (bool, string) + RemoveCronJobRunningInfo(jobName string) } diff --git a/pkg/duplicatechecker/duplicate_checker_container.go b/pkg/duplicatechecker/duplicate_checker_container.go index 8dd553c2..b8cb20b7 100644 --- a/pkg/duplicatechecker/duplicate_checker_container.go +++ b/pkg/duplicatechecker/duplicate_checker_container.go @@ -1,6 +1,8 @@ package duplicatechecker import ( + "time" + "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/settings" ) @@ -36,3 +38,13 @@ func (c *DuplicateCheckerContainer) GetSubmissionRemark(checkerType DuplicateChe func (c *DuplicateCheckerContainer) SetSubmissionRemark(checkerType DuplicateCheckerType, uid int64, identification string, remark string) { c.Current.SetSubmissionRemark(checkerType, uid, identification, remark) } + +// GetOrSetCronJobRunningInfo returns the running info when the cron job is running or saves the running info by the current duplicate checker +func (c *DuplicateCheckerContainer) GetOrSetCronJobRunningInfo(jobName string, runningInfo string, runningInterval time.Duration) (bool, string) { + return c.Current.GetOrSetCronJobRunningInfo(jobName, runningInfo, runningInterval) +} + +// RemoveCronJobRunningInfo removes the running info of the cron job by the current duplicate checker +func (c *DuplicateCheckerContainer) RemoveCronJobRunningInfo(jobName string) { + c.Current.RemoveCronJobRunningInfo(jobName) +} diff --git a/pkg/duplicatechecker/duplicate_checker_type.go b/pkg/duplicatechecker/duplicate_checker_type.go index 9f5e10bf..6ce03f15 100644 --- a/pkg/duplicatechecker/duplicate_checker_type.go +++ b/pkg/duplicatechecker/duplicate_checker_type.go @@ -5,9 +5,9 @@ type DuplicateCheckerType uint8 // Types of uuid const ( - DUPLICATE_CHECKER_TYPE_DEFAULT DuplicateCheckerType = 0 - DUPLICATE_CHECKER_TYPE_NEW_ACCOUNT DuplicateCheckerType = 1 - DUPLICATE_CHECKER_TYPE_NEW_CATEGORY DuplicateCheckerType = 2 - DUPLICATE_CHECKER_TYPE_NEW_TRANSACTION DuplicateCheckerType = 3 - DUPLICATE_CHECKER_TYPE_NEW_TEMPLATE DuplicateCheckerType = 4 + DUPLICATE_CHECKER_TYPE_BACKGROUND_CRON_JOB DuplicateCheckerType = 0 + DUPLICATE_CHECKER_TYPE_NEW_ACCOUNT DuplicateCheckerType = 1 + DUPLICATE_CHECKER_TYPE_NEW_CATEGORY DuplicateCheckerType = 2 + DUPLICATE_CHECKER_TYPE_NEW_TRANSACTION DuplicateCheckerType = 3 + DUPLICATE_CHECKER_TYPE_NEW_TEMPLATE DuplicateCheckerType = 4 ) diff --git a/pkg/duplicatechecker/in_memory_duplicate_checker.go b/pkg/duplicatechecker/in_memory_duplicate_checker.go index 3b619923..8b1a1531 100644 --- a/pkg/duplicatechecker/in_memory_duplicate_checker.go +++ b/pkg/duplicatechecker/in_memory_duplicate_checker.go @@ -2,6 +2,8 @@ package duplicatechecker import ( "fmt" + "sync" + "time" "github.com/patrickmn/go-cache" @@ -11,6 +13,8 @@ import ( // InMemoryDuplicateChecker represents in-memory duplicate checker type InMemoryDuplicateChecker struct { cache *cache.Cache + + mutex sync.Mutex } // NewInMemoryDuplicateChecker returns a new in-memory duplicate checker @@ -38,6 +42,33 @@ func (c *InMemoryDuplicateChecker) SetSubmissionRemark(checkerType DuplicateChec c.cache.Set(c.getCacheKey(checkerType, uid, identification), remark, cache.DefaultExpiration) } +// GetOrSetCronJobRunningInfo returns the running info when the cron job is running or saves the running info by the current duplicate checker +func (c *InMemoryDuplicateChecker) GetOrSetCronJobRunningInfo(jobName string, runningInfo string, runningInterval time.Duration) (bool, string) { + c.mutex.Lock() + defer c.mutex.Unlock() + + existedRunningInfo, found := c.cache.Get(c.getCacheKey(DUPLICATE_CHECKER_TYPE_BACKGROUND_CRON_JOB, 0, jobName)) + + if found { + return true, existedRunningInfo.(string) + } + + expiration := runningInterval + + if expiration > 1*time.Second { + expiration = expiration - 1*time.Second + } + + c.cache.Set(c.getCacheKey(DUPLICATE_CHECKER_TYPE_BACKGROUND_CRON_JOB, 0, jobName), runningInfo, expiration) + + return false, "" +} + +// RemoveCronJobRunningInfo removes the running info of the cron job by the current duplicate checker +func (c *InMemoryDuplicateChecker) RemoveCronJobRunningInfo(jobName string) { + c.cache.Delete(c.getCacheKey(DUPLICATE_CHECKER_TYPE_BACKGROUND_CRON_JOB, 0, jobName)) +} + func (c *InMemoryDuplicateChecker) getCacheKey(checkerType DuplicateCheckerType, uid int64, identification string) string { return fmt.Sprintf("%d|%d|%s", checkerType, uid, identification) } diff --git a/pkg/errs/cron.go b/pkg/errs/cron.go new file mode 100644 index 00000000..b17a3f59 --- /dev/null +++ b/pkg/errs/cron.go @@ -0,0 +1,9 @@ +package errs + +import "net/http" + +// Error codes related to cron jobs +var ( + ErrCronJobNameIsEmpty = NewSystemError(SystemSubcategoryCron, 0, http.StatusInternalServerError, "cron job name is empty") + ErrCronJobNotExistsOrNotEnabled = NewSystemError(SystemSubcategoryCron, 1, http.StatusInternalServerError, "cron job not exists or not enabled") +) diff --git a/pkg/errs/error.go b/pkg/errs/error.go index e2ace3eb..9dbab269 100644 --- a/pkg/errs/error.go +++ b/pkg/errs/error.go @@ -1,5 +1,9 @@ package errs +import ( + "strings" +) + // ErrorCategory represents error category type ErrorCategory int32 @@ -16,6 +20,7 @@ const ( SystemSubcategoryDatabase = 2 SystemSubcategoryMail = 3 SystemSubcategoryLogging = 4 + SystemSubcategoryCron = 5 ) // Sub categories of normal error @@ -44,6 +49,10 @@ type Error struct { Context any } +type MultiErrors struct { + errors []error +} + // Error returns the error message func (err *Error) Error() string { return err.Message @@ -66,6 +75,34 @@ func New(category ErrorCategory, subCategory int32, index int32, httpStatusCode } } +// Error returns the error message +func (err *MultiErrors) Error() string { + if len(err.errors) == 1 { + return err.errors[0].Error() + } + + var ret strings.Builder + var lastErrorChar byte + + ret.WriteString("multi errors: ") + + for i := 0; i < len(err.errors); i++ { + if i > 0 { + if lastErrorChar == '.' { + ret.WriteString(" ") + } else { + ret.WriteString(", ") + } + } + + errorContent := err.errors[i].Error() + lastErrorChar = errorContent[len(errorContent)-1] + ret.WriteString(errorContent) + } + + return ret.String() +} + // NewSystemError returns a new system error instance func NewSystemError(subCategory int32, index int32, httpStatusCode int, message string) *Error { return New(CATEGORY_SYSTEM, subCategory, index, httpStatusCode, message) @@ -107,6 +144,21 @@ func NewErrorWithContext(baseError *Error, context any) *Error { } } +// NewMultiErrorOrNil returns a new multi error instance +func NewMultiErrorOrNil(errors ...error) error { + count := len(errors) + + if count < 1 { + return nil + } else if count == 1 { + return errors[0] + } + + return &MultiErrors{ + errors: errors, + } +} + // Or would return the error from err parameter if the this error is defined in this project, // or return the default error func Or(err error, defaultErr *Error) *Error { diff --git a/pkg/models/token_record.go b/pkg/models/token_record.go index 17d5050b..37cc7fdc 100644 --- a/pkg/models/token_record.go +++ b/pkg/models/token_record.go @@ -7,13 +7,13 @@ const TokenMaxUserAgentLength = 255 // TokenRecord represents token data stored in database type TokenRecord struct { - Uid int64 `xorm:"PK INDEX(IDX_token_record_uid_type_expired_time)"` + Uid int64 `xorm:"PK INDEX(IDX_token_record_uid_type_expired_time) INDEX(IDX_token_record_expired_time)"` UserTokenId int64 `xorm:"PK"` TokenType core.TokenType `xorm:"INDEX(IDX_token_record_uid_type_expired_time) TINYINT NOT NULL"` Secret string `xorm:"VARCHAR(10) NOT NULL"` UserAgent string `xorm:"VARCHAR(255)"` CreatedUnixTime int64 `xorm:"PK"` - ExpiredUnixTime int64 `xorm:"INDEX(IDX_token_record_uid_type_expired_time)"` + ExpiredUnixTime int64 `xorm:"INDEX(IDX_token_record_uid_type_expired_time) INDEX(IDX_token_record_expired_time)"` LastSeenUnixTime int64 } diff --git a/pkg/services/base.go b/pkg/services/base.go index b81bb5d6..240e09fc 100644 --- a/pkg/services/base.go +++ b/pkg/services/base.go @@ -23,6 +23,16 @@ func (s *ServiceUsingDB) TokenDB(uid int64) *datastore.Database { return s.container.TokenStore.Choose(uid) } +// TokenDBByIndex returns the datastore by index +func (s *ServiceUsingDB) TokenDBByIndex(index int) *datastore.Database { + return s.container.TokenStore.Get(index) +} + +// TokenDBCount returns the count of datastores which contains user token +func (s *ServiceUsingDB) TokenDBCount() int { + return s.container.TokenStore.Count() +} + // UserDataDB returns the datastore which contains user data func (s *ServiceUsingDB) UserDataDB(uid int64) *datastore.Database { return s.container.UserDataStore.Choose(uid) diff --git a/pkg/services/tokens.go b/pkg/services/tokens.go index 16c0fd97..fd671fbb 100644 --- a/pkg/services/tokens.go +++ b/pkg/services/tokens.go @@ -207,6 +207,32 @@ func (s *TokenService) DeleteTokensByType(c *core.Context, uid int64, tokenType }) } +// DeleteAllExpiredTokens deletes all expired tokens +func (s *TokenService) DeleteAllExpiredTokens(c *core.Context) error { + var errors []error + totalCount := int64(0) + + for i := 0; i < s.TokenDBCount(); i++ { + err := s.TokenDBByIndex(i).DoTransaction(c, func(sess *xorm.Session) error { + count, err := sess.Where("expired_unix_time<=?", time.Now().Unix()).Delete(&models.TokenRecord{}) + totalCount += count + return err + }) + + if err != nil { + errors = append(errors, err) + } + } + + if totalCount > 0 { + log.Infof("[tokens.DeleteAllExpiredTokens] %d expired tokens have been deleted", totalCount) + } else if len(errors) == 0 { + log.Infof("[tokens.DeleteAllExpiredTokens] no expired tokens have been deleted") + } + + return errs.NewMultiErrorOrNil(errors...) +} + // ExistsValidTokenByType returns whether the given token type exists func (s *TokenService) ExistsValidTokenByType(c *core.Context, uid int64, tokenType core.TokenType) (bool, error) { if uid <= 0 { diff --git a/pkg/settings/setting.go b/pkg/settings/setting.go index 908da7d2..5b0612f6 100644 --- a/pkg/settings/setting.go +++ b/pkg/settings/setting.go @@ -252,6 +252,9 @@ type Config struct { DuplicateSubmissionsInterval uint32 DuplicateSubmissionsIntervalDuration time.Duration + // Cron + EnableRemoveExpiredTokens bool + // Secret SecretKeyNoSet bool SecretKey string @@ -373,6 +376,12 @@ func LoadConfiguration(configFilePath string) (*Config, error) { return nil, err } + err = loadCronConfiguration(config, cfgFile, "cron") + + if err != nil { + return nil, err + } + err = loadSecurityConfiguration(config, cfgFile, "security") if err != nil { @@ -674,6 +683,12 @@ func loadDuplicateCheckerConfiguration(config *Config, configFile *ini.File, sec return nil } +func loadCronConfiguration(config *Config, configFile *ini.File, sectionName string) error { + config.EnableRemoveExpiredTokens = getConfigItemBoolValue(configFile, sectionName, "enable_remove_expired_tokens", false) + + return nil +} + func loadSecurityConfiguration(config *Config, configFile *ini.File, sectionName string) error { config.SecretKeyNoSet = !getConfigItemIsSet(configFile, sectionName, "secret_key") config.SecretKey = getConfigItemStringValue(configFile, sectionName, "secret_key", defaultSecretKey) diff --git a/third-party-dependencies.json b/third-party-dependencies.json index 63c6478b..c54a3c3b 100644 --- a/third-party-dependencies.json +++ b/third-party-dependencies.json @@ -99,6 +99,12 @@ "url": "https://github.com/minio/minio-go", "licenseUrl": "https://github.com/minio/minio-go/blob/v7.0.74/LICENSE" }, + { + "name": "gocron", + "copyright": "Copyright (c) 2014, 辣椒面", + "url": "https://github.com/go-co-op/gocron", + "licenseUrl": "https://github.com/go-co-op/gocron/blob/v2.11.0/LICENSE" + }, { "name": "barcode", "copyright": "Copyright (c) 2014 Florian Sundermann",