diff --git a/cmd/user_data.go b/cmd/user_data.go index 99f8ff13..f33bf4bb 100644 --- a/cmd/user_data.go +++ b/cmd/user_data.go @@ -244,6 +244,31 @@ var UserData = &cli.Command{ }, }, }, + { + Name: "transaction-import", + Usage: "Import transactions to specified user", + Action: bindAction(importUserTransaction), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "username", + Aliases: []string{"n"}, + Required: true, + Usage: "Specific user name", + }, + &cli.StringFlag{ + Name: "file", + Aliases: []string{"f"}, + Required: true, + Usage: "Specific import file path (e.g. transaction.csv)", + }, + &cli.StringFlag{ + Name: "type", + Aliases: []string{"t"}, + Required: true, + Usage: "Import file type (supports \"ezbookkeeping_csv\", \"ezbookkeeping_tsv\")", + }, + }, + }, { Name: "transaction-export", Usage: "Export user all transactions to file", @@ -639,6 +664,55 @@ func exportUserTransaction(c *core.CliContext) error { return nil } +func importUserTransaction(c *core.CliContext) error { + _, err := initializeSystem(c) + + if err != nil { + return err + } + + username := c.String("username") + filePath := c.String("file") + filetype := c.String("type") + + if filePath == "" { + log.BootErrorf(c, "[user_data.importUserTransaction] import file path is not specified") + return os.ErrNotExist + } + + fileExists, err := utils.IsExists(filePath) + + if !fileExists { + log.BootErrorf(c, "[user_data.importUserTransaction] import file does not exist") + return os.ErrExist + } + + if filetype != "ezbookkeeping_csv" && filetype != "ezbookkeeping_tsv" { + log.BootErrorf(c, "[user_data.importUserTransaction] unknown file type \"%s\"", filetype) + return errs.ErrImportFileTypeNotSupported + } + + data, err := os.ReadFile(filePath) + + if err != nil { + log.BootErrorf(c, "[user_data.importUserTransaction] failed to load import file") + return err + } + + log.BootInfof(c, "[user_data.importUserTransaction] start importing transactions to user \"%s\"", username) + + err = clis.UserData.ImportTransaction(c, username, filetype, data) + + if err != nil { + log.BootErrorf(c, "[user_data.importUserTransaction] error occurs when importing user data") + return err + } + + log.BootInfof(c, "[user_data.importUserTransaction] transactions have been imported to user \"%s\"", username) + + return nil +} + func printUserInfo(user *models.User) { fmt.Printf("[Uid] %d\n", user.Uid) fmt.Printf("[Username] %s\n", user.Username) diff --git a/pkg/api/data_managements.go b/pkg/api/data_managements.go index 7f0de135..c9a5442e 100644 --- a/pkg/api/data_managements.go +++ b/pkg/api/data_managements.go @@ -20,8 +20,8 @@ const pageCountForDataExport = 1000 // DataManagementsApi represents data management api type DataManagementsApi struct { ApiUsingConfig - ezBookKeepingCsvExporter *converters.EzBookKeepingCSVFileExporter - ezBookKeepingTsvExporter *converters.EzBookKeepingTSVFileExporter + ezBookKeepingCsvExporter *converters.EzBookKeepingCSVFileConverter + ezBookKeepingTsvExporter *converters.EzBookKeepingTSVFileConverter tokens *services.TokenService users *services.UserService accounts *services.AccountService @@ -38,8 +38,8 @@ var ( ApiUsingConfig: ApiUsingConfig{ container: settings.Container, }, - ezBookKeepingCsvExporter: &converters.EzBookKeepingCSVFileExporter{}, - ezBookKeepingTsvExporter: &converters.EzBookKeepingTSVFileExporter{}, + ezBookKeepingCsvExporter: &converters.EzBookKeepingCSVFileConverter{}, + ezBookKeepingTsvExporter: &converters.EzBookKeepingTSVFileConverter{}, tokens: services.Tokens, users: services.Users, accounts: services.Accounts, diff --git a/pkg/cli/user_data.go b/pkg/cli/user_data.go index f14d4ac9..e970f18a 100644 --- a/pkg/cli/user_data.go +++ b/pkg/cli/user_data.go @@ -10,6 +10,7 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" "github.com/mayswind/ezbookkeeping/pkg/services" "github.com/mayswind/ezbookkeeping/pkg/settings" + "github.com/mayswind/ezbookkeeping/pkg/utils" "github.com/mayswind/ezbookkeeping/pkg/validators" ) @@ -19,8 +20,8 @@ const pageCountForDataExport = 1000 // UserDataCli represents user data cli type UserDataCli struct { CliUsingConfig - ezBookKeepingCsvExporter *converters.EzBookKeepingCSVFileExporter - ezBookKeepingTsvExporter *converters.EzBookKeepingTSVFileExporter + ezBookKeepingCsvExporter *converters.EzBookKeepingCSVFileConverter + ezBookKeepingTsvExporter *converters.EzBookKeepingTSVFileConverter accounts *services.AccountService transactions *services.TransactionService categories *services.TransactionCategoryService @@ -37,8 +38,8 @@ var ( CliUsingConfig: CliUsingConfig{ container: settings.Container, }, - ezBookKeepingCsvExporter: &converters.EzBookKeepingCSVFileExporter{}, - ezBookKeepingTsvExporter: &converters.EzBookKeepingTSVFileExporter{}, + ezBookKeepingCsvExporter: &converters.EzBookKeepingCSVFileConverter{}, + ezBookKeepingTsvExporter: &converters.EzBookKeepingTSVFileConverter{}, accounts: services.Accounts, transactions: services.Transactions, categories: services.TransactionCategories, @@ -662,6 +663,73 @@ func (l *UserDataCli) ExportTransaction(c *core.CliContext, username string, fil return result, nil } +func (l *UserDataCli) ImportTransaction(c *core.CliContext, username string, fileType string, data []byte) error { + if username == "" { + log.BootErrorf(c, "[user_data.ImportTransaction] user name is empty") + return errs.ErrUsernameIsEmpty + } + + var dataImporter converters.DataConverter + + if fileType == "ezbookkeeping_csv" { + dataImporter = l.ezBookKeepingCsvExporter + } else if fileType == "ezbookkeeping_tsv" { + dataImporter = l.ezBookKeepingTsvExporter + } else { + return errs.ErrImportFileTypeNotSupported + } + + user, err := l.GetUserByUsername(c, username) + + if err != nil { + log.BootErrorf(c, "[user_data.ImportTransaction] failed to get user by user name \"%s\", because %s", username, err.Error()) + return err + } + + accountMap, categoryMap, tagMap, err := l.getUserEssentialDataForImport(c, user.Uid, username) + + if err != nil { + log.BootErrorf(c, "[user_data.ImportTransaction] failed to get essential data for user \"%s\", because %s", username, err.Error()) + return err + } + + newTransactions, newAccounts, newCategories, newTags, err := dataImporter.ParseImportedData(user, data, utils.GetTimezoneOffsetMinutes(time.Local), accountMap, categoryMap, tagMap) + + if err != nil { + log.BootErrorf(c, "[user_data.ImportTransaction] failed to parse imported data for \"%s\", because %s", username, err.Error()) + return err + } + + if len(newTransactions) < 1 { + log.BootErrorf(c, "[user_data.ImportTransaction] there are no transactions in import file") + return errs.ErrOperationFailed + } + + if len(newAccounts) > 0 { + log.BootErrorf(c, "[user_data.ImportTransaction] there are %d accounts need to be created, please create them manually", len(newAccounts)) + return errs.ErrOperationFailed + } + + if len(newCategories) > 0 { + log.BootErrorf(c, "[user_data.ImportTransaction] there are %d transaction categories need to be created, please create them manually", len(newCategories)) + return errs.ErrOperationFailed + } + + if len(newTags) > 0 { + log.BootErrorf(c, "[user_data.ImportTransaction] there are %d transaction tags need to be created, please create them manually", len(newTags)) + return errs.ErrOperationFailed + } + + err = l.transactions.BatchCreateTransactions(c, user.Uid, newTransactions) + + if err != nil { + log.BootErrorf(c, "[user_data.ImportTransaction] failed to create transaction, because %s", err.Error()) + return err + } + + return nil +} + func (l *UserDataCli) getUserIdByUsername(c *core.CliContext, username string) (int64, error) { user, err := l.GetUserByUsername(c, username) @@ -718,6 +786,42 @@ func (l *UserDataCli) getUserEssentialData(c *core.CliContext, uid int64, userna return accountMap, categoryMap, tagMap, tagIndexes, tagIndexesMap, nil } +func (l *UserDataCli) getUserEssentialDataForImport(c *core.CliContext, uid int64, username string) (accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag, err error) { + if uid <= 0 { + log.BootErrorf(c, "[user_data.getUserEssentialDataForImport] user uid \"%d\" is invalid", uid) + return nil, nil, nil, errs.ErrUserIdInvalid + } + + accounts, err := l.accounts.GetAllAccountsByUid(c, uid) + + if err != nil { + log.BootErrorf(c, "[user_data.getUserEssentialDataForImport] failed to get accounts for user \"%s\", because %s", username, err.Error()) + return nil, nil, nil, err + } + + accountMap = l.accounts.GetAccountNameMapByList(accounts) + + categories, err := l.categories.GetAllCategoriesByUid(c, uid, 0, -1) + + if err != nil { + log.BootErrorf(c, "[user_data.getUserEssentialDataForImport] failed to get categories for user \"%s\", because %s", username, err.Error()) + return nil, nil, nil, err + } + + categoryMap = l.categories.GetCategoryNameMapByList(categories) + + tags, err := l.tags.GetAllTagsByUid(c, uid) + + if err != nil { + log.BootErrorf(c, "[user_data.getUserEssentialDataForImport] failed to get tags for user \"%s\", because %s", username, err.Error()) + return nil, nil, nil, err + } + + tagMap = l.tags.GetTagNameMapByList(tags) + + return accountMap, categoryMap, tagMap, nil +} + func (l *UserDataCli) checkTransactionAccount(c *core.CliContext, transaction *models.Transaction, accountMap map[int64]*models.Account, accountHasChild map[int64]bool) error { account, exists := accountMap[transaction.AccountId] diff --git a/pkg/converters/data_converter.go b/pkg/converters/data_converter.go index 98be0a47..7b2107b9 100644 --- a/pkg/converters/data_converter.go +++ b/pkg/converters/data_converter.go @@ -4,8 +4,11 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) -// DataConverter defines the structure of data exporter +// DataConverter defines the structure of data converter type DataConverter interface { // ToExportedContent returns the exported data ToExportedContent(uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) + + // ParseImportedData returns the imported data + ParseImportedData(user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) ([]*models.Transaction, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) } diff --git a/pkg/converters/ezbookkeeping_csv_file.go b/pkg/converters/ezbookkeeping_csv_file.go index 77ae74e8..a1888a1d 100644 --- a/pkg/converters/ezbookkeeping_csv_file.go +++ b/pkg/converters/ezbookkeeping_csv_file.go @@ -4,14 +4,19 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) -// EzBookKeepingCSVFileExporter defines the structure of CSV file exporter -type EzBookKeepingCSVFileExporter struct { - EzBookKeepingPlainFileExporter +// EzBookKeepingCSVFileConverter defines the structure of CSV file converter +type EzBookKeepingCSVFileConverter struct { + EzBookKeepingPlainFileConverter } const csvSeparator = "," // ToExportedContent returns the exported CSV data -func (e *EzBookKeepingCSVFileExporter) ToExportedContent(uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) { +func (e *EzBookKeepingCSVFileConverter) ToExportedContent(uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) { return e.toExportedContent(uid, csvSeparator, transactions, accountMap, categoryMap, tagMap, allTagIndexes) } + +// ParseImportedData parses transactions of ezbookkeeping CSV data +func (e *EzBookKeepingCSVFileConverter) ParseImportedData(user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) ([]*models.Transaction, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { + return e.parseImportedData(user, csvSeparator, data, defaultTimezoneOffset, accountMap, categoryMap, tagMap) +} diff --git a/pkg/converters/ezbookkeeping_plain_file.go b/pkg/converters/ezbookkeeping_plain_file.go index c6a17f7e..57c4535e 100644 --- a/pkg/converters/ezbookkeeping_plain_file.go +++ b/pkg/converters/ezbookkeeping_plain_file.go @@ -2,15 +2,18 @@ package converters import ( "fmt" + "sort" "strings" "time" + "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/models" "github.com/mayswind/ezbookkeeping/pkg/utils" + "github.com/mayswind/ezbookkeeping/pkg/validators" ) -// EzBookKeepingPlainFileExporter defines the structure of plain file exporter -type EzBookKeepingPlainFileExporter struct { +// EzBookKeepingPlainFileConverter defines the structure of plain file converter +type EzBookKeepingPlainFileConverter struct { } const lineSeparator = "\n" @@ -20,7 +23,7 @@ const headerLine = "Time,Timezone,Type,Category,Sub Category,Account,Account Cur const dataLineFormat = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" + lineSeparator // toExportedContent returns the exported plain data -func (e *EzBookKeepingPlainFileExporter) toExportedContent(uid int64, separator string, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) { +func (e *EzBookKeepingPlainFileConverter) toExportedContent(uid int64, separator string, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) { var ret strings.Builder ret.Grow(len(transactions) * 100) @@ -75,7 +78,270 @@ func (e *EzBookKeepingPlainFileExporter) toExportedContent(uid int64, separator return []byte(ret.String()), nil } -func (e *EzBookKeepingPlainFileExporter) getTransactionTypeName(transactionDbType models.TransactionDbType) string { +func (e *EzBookKeepingPlainFileConverter) parseImportedData(user *models.User, separator string, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) ([]*models.Transaction, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { + lines := strings.Split(string(data), lineSeparator) + + if len(lines) < 2 { + return nil, nil, nil, nil, errs.ErrOperationFailed + } + + headerLineItems := strings.Split(lines[0], separator) + headerItemMap := make(map[string]int) + + for i := 0; i < len(headerLineItems); i++ { + headerItemMap[headerLineItems[i]] = i + } + + timeColumnIdx, timeColumnExists := headerItemMap["Time"] + timezoneColumnIdx, timezoneColumnExists := headerItemMap["Timezone"] + typeColumnIdx, typeColumnExists := headerItemMap["Type"] + subCategoryColumnIdx, subCategoryColumnExists := headerItemMap["Sub Category"] + accountColumnIdx, accountColumnExists := headerItemMap["Account"] + accountCurrencyColumnIdx, accountCurrencyColumnExists := headerItemMap["Account Currency"] + amountColumnIdx, amountColumnExists := headerItemMap["Amount"] + account2ColumnIdx, account2ColumnExists := headerItemMap["Account2"] + account2CurrencyColumnIdx, account2CurrencyColumnExists := headerItemMap["Account2 Currency"] + amount2ColumnIdx, amount2ColumnExists := headerItemMap["Account2 Amount"] + geoLocationIdx, geoLocationExists := headerItemMap["Geographic Location"] + tagsColumnIdx, tagsColumnExists := headerItemMap["Tags"] + descriptionColumnIdx, descriptionColumnExists := headerItemMap["Description"] + + if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists || + !accountColumnExists || !amountColumnExists || !account2ColumnExists || !amount2ColumnExists { + return nil, nil, nil, nil, errs.ErrFormatInvalid + } + + if accountMap == nil { + accountMap = make(map[string]*models.Account) + } + + if categoryMap == nil { + categoryMap = make(map[string]*models.TransactionCategory) + } + + if tagMap == nil { + tagMap = make(map[string]*models.TransactionTag) + } + + allNewTransactions := make(ImportTransactionSlice, 0, len(lines)) + allNewAccounts := make([]*models.Account, 0) + allNewSubCategories := make([]*models.TransactionCategory, 0) + allNewTags := make([]*models.TransactionTag, 0) + + for i := 1; i < len(lines); i++ { + line := lines[i] + + if len(line) < 1 { + continue + } + + lineItems := strings.Split(line, separator) + + if len(lineItems) < len(headerLineItems) { + return nil, nil, nil, nil, errs.ErrFormatInvalid + } + + timezoneOffset := defaultTimezoneOffset + + if timezoneColumnExists { + transactionTimezone, err := utils.ParseFromTimezoneOffset(lineItems[timezoneColumnIdx]) + + if err != nil { + return nil, nil, nil, nil, err + } + + timezoneOffset = utils.GetTimezoneOffsetMinutes(transactionTimezone) + } + + transactionTime, err := utils.ParseFromLongDateTime(lineItems[timeColumnIdx], timezoneOffset) + + if err != nil { + return nil, nil, nil, nil, err + } + + transactionDbType, err := e.getTransactionDbType(lineItems[typeColumnIdx]) + + if err != nil { + return nil, nil, nil, nil, err + } + + categoryId := int64(0) + + if transactionDbType != models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { + transactionCategoryType, err := e.getTransactionCategoryType(transactionDbType) + + if err != nil { + return nil, nil, nil, nil, err + } + + subCategoryName := lineItems[subCategoryColumnIdx] + + if subCategoryName == "" { + return nil, nil, nil, nil, errs.ErrFormatInvalid + } + + subCategory, exists := categoryMap[subCategoryName] + + if !exists { + subCategory = e.createNewTransactionCategoryModel(user.Uid, subCategoryName, transactionCategoryType) + allNewSubCategories = append(allNewSubCategories, subCategory) + categoryMap[subCategoryName] = subCategory + } + + categoryId = subCategory.CategoryId + } + + accountName := lineItems[accountColumnIdx] + + if accountName == "" { + return nil, nil, nil, nil, errs.ErrFormatInvalid + } + + account, exists := accountMap[accountName] + + if !exists { + currency := user.DefaultCurrency + + if accountCurrencyColumnExists { + currency = lineItems[accountCurrencyColumnIdx] + + if _, ok := validators.AllCurrencyNames[currency]; !ok { + return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + } + } + + account = e.createNewAccountModel(user.Uid, accountName, currency) + allNewAccounts = append(allNewAccounts, account) + accountMap[accountName] = account + } + + if accountCurrencyColumnExists { + if account.Currency != lineItems[accountCurrencyColumnIdx] { + return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + } + } + + amount, err := utils.ParseAmount(lineItems[amountColumnIdx]) + + if err != nil { + return nil, nil, nil, nil, err + } + + relatedAccountId := int64(0) + relatedAccountAmount := int64(0) + + if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { + account2Name := lineItems[account2ColumnIdx] + + if account2Name == "" { + return nil, nil, nil, nil, errs.ErrFormatInvalid + } + + account2, exists := accountMap[account2Name] + + if !exists { + currency := user.DefaultCurrency + + if accountCurrencyColumnExists { + currency = lineItems[account2CurrencyColumnIdx] + + if _, ok := validators.AllCurrencyNames[currency]; !ok { + return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + } + } + + account2 = e.createNewAccountModel(user.Uid, account2Name, currency) + allNewAccounts = append(allNewAccounts, account2) + accountMap[account2Name] = account2 + } + + if account2CurrencyColumnExists { + if account2.Currency != lineItems[account2CurrencyColumnIdx] { + return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + } + } + + relatedAccountId = account2.AccountId + relatedAccountAmount, err = utils.ParseAmount(lineItems[amount2ColumnIdx]) + + if err != nil { + return nil, nil, nil, nil, err + } + } + + geoLongitude := float64(0) + geoLatitude := float64(0) + + if geoLocationExists { + geoLocationItems := strings.Split(lineItems[geoLocationIdx], geoLocationSeparator) + + if len(geoLocationItems) == 2 { + geoLongitude, err = utils.StringToFloat64(geoLocationItems[0]) + + if err != nil { + return nil, nil, nil, nil, err + } + + geoLatitude, err = utils.StringToFloat64(geoLocationItems[1]) + + if err != nil { + return nil, nil, nil, nil, err + } + } + } + + if tagsColumnExists { + tagNames := strings.Split(lineItems[tagsColumnIdx], transactionTagSeparator) + + for i := 0; i < len(tagNames); i++ { + tagName := tagNames[i] + + if tagName == "" { + continue + } + + tag, exists := tagMap[tagName] + + if !exists { + tag = e.createNewTransactionTagModel(user.Uid, tagName) + allNewTags = append(allNewTags, tag) + tagMap[tagName] = tag + } + } + } + + description := "" + + if descriptionColumnExists { + description = lineItems[descriptionColumnIdx] + } + + transaction := &models.Transaction{ + Uid: user.Uid, + Type: transactionDbType, + CategoryId: categoryId, + TransactionTime: utils.GetMinTransactionTimeFromUnixTime(transactionTime.Unix()), + TimezoneUtcOffset: timezoneOffset, + AccountId: account.AccountId, + Amount: amount, + HideAmount: false, + RelatedAccountId: relatedAccountId, + RelatedAccountAmount: relatedAccountAmount, + Comment: description, + GeoLongitude: geoLongitude, + GeoLatitude: geoLatitude, + CreatedIp: "127.0.0.1", + } + + allNewTransactions = append(allNewTransactions, transaction) + } + + sort.Sort(allNewTransactions) + + return allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, nil +} + +func (e *EzBookKeepingPlainFileConverter) getTransactionTypeName(transactionDbType models.TransactionDbType) string { if transactionDbType == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { return "Balance Modification" } else if transactionDbType == models.TRANSACTION_DB_TYPE_INCOME { @@ -89,7 +355,33 @@ func (e *EzBookKeepingPlainFileExporter) getTransactionTypeName(transactionDbTyp } } -func (e *EzBookKeepingPlainFileExporter) getTransactionCategoryName(categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { +func (e *EzBookKeepingPlainFileConverter) getTransactionDbType(transactionTypeName string) (models.TransactionDbType, error) { + if transactionTypeName == "Balance Modification" { + return models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, nil + } else if transactionTypeName == "Income" { + return models.TRANSACTION_DB_TYPE_INCOME, nil + } else if transactionTypeName == "Expense" { + return models.TRANSACTION_DB_TYPE_EXPENSE, nil + } else if transactionTypeName == "Transfer" { + return models.TRANSACTION_DB_TYPE_TRANSFER_OUT, nil + } else { + return 0, errs.ErrTransactionTypeInvalid + } +} + +func (e *EzBookKeepingPlainFileConverter) getTransactionCategoryType(transactionType models.TransactionDbType) (models.TransactionCategoryType, error) { + if transactionType == models.TRANSACTION_DB_TYPE_INCOME { + return models.CATEGORY_TYPE_INCOME, nil + } else if transactionType == models.TRANSACTION_DB_TYPE_EXPENSE { + return models.CATEGORY_TYPE_EXPENSE, nil + } else if transactionType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { + return models.CATEGORY_TYPE_TRANSFER, nil + } else { + return 0, errs.ErrTransactionTypeInvalid + } +} + +func (e *EzBookKeepingPlainFileConverter) getTransactionCategoryName(categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { category, exists := categoryMap[categoryId] if !exists { @@ -109,7 +401,7 @@ func (e *EzBookKeepingPlainFileExporter) getTransactionCategoryName(categoryId i return parentCategory.Name } -func (e *EzBookKeepingPlainFileExporter) getTransactionSubCategoryName(categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { +func (e *EzBookKeepingPlainFileConverter) getTransactionSubCategoryName(categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { category, exists := categoryMap[categoryId] if exists { @@ -119,7 +411,7 @@ func (e *EzBookKeepingPlainFileExporter) getTransactionSubCategoryName(categoryI } } -func (e *EzBookKeepingPlainFileExporter) getAccountName(accountId int64, accountMap map[int64]*models.Account) string { +func (e *EzBookKeepingPlainFileConverter) getAccountName(accountId int64, accountMap map[int64]*models.Account) string { account, exists := accountMap[accountId] if exists { @@ -129,7 +421,7 @@ func (e *EzBookKeepingPlainFileExporter) getAccountName(accountId int64, account } } -func (e *EzBookKeepingPlainFileExporter) getAccountCurrency(accountId int64, accountMap map[int64]*models.Account) string { +func (e *EzBookKeepingPlainFileConverter) getAccountCurrency(accountId int64, accountMap map[int64]*models.Account) string { account, exists := accountMap[accountId] if exists { @@ -139,7 +431,7 @@ func (e *EzBookKeepingPlainFileExporter) getAccountCurrency(accountId int64, acc } } -func (e *EzBookKeepingPlainFileExporter) getTags(transactionId int64, allTagIndexes map[int64][]int64, tagMap map[int64]*models.TransactionTag) string { +func (e *EzBookKeepingPlainFileConverter) getTags(transactionId int64, allTagIndexes map[int64][]int64, tagMap map[int64]*models.TransactionTag) string { tagIndexes, exists := allTagIndexes[transactionId] if !exists { @@ -166,10 +458,33 @@ func (e *EzBookKeepingPlainFileExporter) getTags(transactionId int64, allTagInde return ret.String() } -func (e *EzBookKeepingPlainFileExporter) replaceDelimiters(text string, separator string) string { +func (e *EzBookKeepingPlainFileConverter) replaceDelimiters(text string, separator string) string { text = strings.Replace(text, separator, " ", -1) text = strings.Replace(text, "\r\n", " ", -1) text = strings.Replace(text, "\n", " ", -1) return text } + +func (e *EzBookKeepingPlainFileConverter) createNewAccountModel(uid int64, accountName string, currency string) *models.Account { + return &models.Account{ + Uid: uid, + Name: accountName, + Currency: currency, + } +} + +func (e *EzBookKeepingPlainFileConverter) createNewTransactionCategoryModel(uid int64, categoryName string, transactionCategoryType models.TransactionCategoryType) *models.TransactionCategory { + return &models.TransactionCategory{ + Uid: uid, + Name: categoryName, + Type: transactionCategoryType, + } +} + +func (e *EzBookKeepingPlainFileConverter) createNewTransactionTagModel(uid int64, tagName string) *models.TransactionTag { + return &models.TransactionTag{ + Uid: uid, + Name: tagName, + } +} diff --git a/pkg/converters/ezbookkeeping_plain_file_test.go b/pkg/converters/ezbookkeeping_plain_file_test.go new file mode 100644 index 00000000..18bc6830 --- /dev/null +++ b/pkg/converters/ezbookkeeping_plain_file_test.go @@ -0,0 +1,365 @@ +package converters + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +func TestEzBookKeepingPlainFileConverterParseImportedData_MinimumValidData(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 00:00:00,Balance Modification,,Test Account,123.45,,\n"+ + "2024-09-01 01:23:45,Income,Test Category,Test Account,0.12,,\n"+ + "2024-09-01 12:34:56,Expense,Test Category2,Test Account,1.00,,\n"+ + "2024-09-01 23:59:59,Transfer,Test Category3,Test Account,0.05,Test Account2,0.05"), 0, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + assert.Equal(t, 3, len(allNewSubCategories)) + assert.Equal(t, 0, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725153825), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725235199), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Test Account2", allNewAccounts[1].Name) + assert.Equal(t, "CNY", allNewAccounts[1].Currency) + + assert.Equal(t, int64(1234567890), allNewSubCategories[0].Uid) + assert.Equal(t, "Test Category", allNewSubCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubCategories[1].Uid) + assert.Equal(t, "Test Category2", allNewSubCategories[1].Name) + + assert.Equal(t, int64(1234567890), allNewSubCategories[2].Uid) + assert.Equal(t, "Test Category3", allNewSubCategories[2].Name) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTime(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01T12:34:56,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "09/01/2024 12:34:56,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidType(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Type,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidTimezone(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,+08:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725165296), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTimezone(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Asia/Shanghai,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidSubCategoryName(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Expense,,Test Account,123.45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidAccountName(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Expense,Test Category,,123.45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Transfer,Test Category,Test Account,123.45,,123.45"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidAccountCurrency(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Balance Modification,Test Category,Test Account,USD,123.45,,,\n"+ + "2024-09-01 12:34:56,Transfer,Test Category2,Test Account,USD,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 2, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "USD", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Test Account2", allNewAccounts[1].Name) + assert.Equal(t, "EUR", allNewAccounts[1].Currency) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidAccountCurrency(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Balance Modification,,Test Account,USD,123.45,,,\n"+ + "2024-09-01 12:34:56,Transfer,Test Category3,Test Account,CNY,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Balance Modification,,Test Account,USD,123.45,,,\n"+ + "2024-09-01 12:34:56,Transfer,Test Category3,Test Account2,CNY,1.23,Test Account,EUR,1.10"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseNotSupportedCurrency(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Balance Modification,Test Category,Test Account,XXX,123.45,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Transfer,Test Category,Test Account,USD,123.45,Test Account2,XXX,123.45"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidAmount(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123 45,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Transfer,Test Category,Test Account,123.45,Test Account2,123 45"), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidGeographicLocation(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,123.45 45.56"), 0, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, 123.45, allNewTransactions[0].GeoLongitude) + assert.Equal(t, 45.56, allNewTransactions[0].GeoLatitude) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidGeographicLocation(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,1"), 0, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, float64(0), allNewTransactions[0].GeoLongitude) + assert.Equal(t, float64(0), allNewTransactions[0].GeoLatitude) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,a b"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,1 "), 0, nil, nil, nil) + assert.NotNil(t, err) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseTag(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, allNewTags, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Tags\n"+ + "2024-09-01 00:00:00,Balance Modification,,Test Account,123.45,,,foo;;bar.;#test;hello\tworld;;"), 0, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTags[0].Uid) + assert.Equal(t, "foo", allNewTags[0].Name) + + assert.Equal(t, int64(1234567890), allNewTags[1].Uid) + assert.Equal(t, "bar.", allNewTags[1].Name) + + assert.Equal(t, int64(1234567890), allNewTags[2].Uid) + assert.Equal(t, "#test", allNewTags[2].Name) + + assert.Equal(t, int64(1234567890), allNewTags[3].Uid) + assert.Equal(t, "hello\tworld", allNewTags[3].Name) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_ParseDescription(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, err := converter.parseImportedData(user, ",", []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Description\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,foo bar\t#test"), 0, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, "foo bar\t#test", allNewTransactions[0].Comment) +} + +func TestEzBookKeepingPlainFileConverterParseImportedData_MissingRequiredColumn(t *testing.T) { + converter := &EzBookKeepingPlainFileConverter{} + + user := &models.User{ + Uid: 1, + DefaultCurrency: "CNY", + } + + _, _, _, _, err := converter.parseImportedData(user, ",", []byte(""), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "+08:00,Balance Modification,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Type,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Category,Sub Category,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Category,Test Sub Category,CNY,123.45,,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Category,Test Sub Category,Test Account,CNY,,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) + + _, _, _, _, err = converter.parseImportedData(user, ",", []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,"), 0, nil, nil, nil) + assert.NotNil(t, err) +} diff --git a/pkg/converters/ezbookkeeping_tsv_file.go b/pkg/converters/ezbookkeeping_tsv_file.go index 8d613c62..3d68599e 100644 --- a/pkg/converters/ezbookkeeping_tsv_file.go +++ b/pkg/converters/ezbookkeeping_tsv_file.go @@ -4,14 +4,19 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) -// EzBookKeepingTSVFileExporter defines the structure of TSV file exporter -type EzBookKeepingTSVFileExporter struct { - EzBookKeepingPlainFileExporter +// EzBookKeepingTSVFileConverter defines the structure of TSV file converter +type EzBookKeepingTSVFileConverter struct { + EzBookKeepingPlainFileConverter } const tsvSeparator = "\t" // ToExportedContent returns the exported TSV data -func (e *EzBookKeepingTSVFileExporter) ToExportedContent(uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) { +func (e *EzBookKeepingTSVFileConverter) ToExportedContent(uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) ([]byte, error) { return e.toExportedContent(uid, tsvSeparator, transactions, accountMap, categoryMap, tagMap, allTagIndexes) } + +// ParseImportedData parses transactions of ezbookkeeping TSV data +func (e *EzBookKeepingTSVFileConverter) ParseImportedData(user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) ([]*models.Transaction, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { + return e.parseImportedData(user, tsvSeparator, data, defaultTimezoneOffset, accountMap, categoryMap, tagMap) +} diff --git a/pkg/converters/import_transactions.go b/pkg/converters/import_transactions.go new file mode 100644 index 00000000..3329e416 --- /dev/null +++ b/pkg/converters/import_transactions.go @@ -0,0 +1,29 @@ +package converters + +import "github.com/mayswind/ezbookkeeping/pkg/models" + +// ImportTransactionSlice represents the slice data structure of import transaction data +type ImportTransactionSlice []*models.Transaction + +// Len returns the count of items +func (s ImportTransactionSlice) Len() int { + return len(s) +} + +// Swap swaps two items +func (s ImportTransactionSlice) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +// Less reports whether the first item is less than the second one +func (s ImportTransactionSlice) Less(i, j int) bool { + if s[i].Type != s[j].Type && (s[i].Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE || s[j].Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE) { + if s[i].Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { + return true + } else if s[j].Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { + return false + } + } + + return s[i].TransactionTime < s[j].TransactionTime +} diff --git a/pkg/converters/import_transactions_test.go b/pkg/converters/import_transactions_test.go new file mode 100644 index 00000000..84660ecc --- /dev/null +++ b/pkg/converters/import_transactions_test.go @@ -0,0 +1,53 @@ +package converters + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/models" +) + +func TestImportTransactionSliceLess(t *testing.T) { + var transactionSlice ImportTransactionSlice + transactionSlice = append(transactionSlice, &models.Transaction{ + TransactionId: 1, + Type: models.TRANSACTION_DB_TYPE_EXPENSE, + TransactionTime: 1, + }) + transactionSlice = append(transactionSlice, &models.Transaction{ + TransactionId: 2, + Type: models.TRANSACTION_DB_TYPE_INCOME, + TransactionTime: 2, + }) + transactionSlice = append(transactionSlice, &models.Transaction{ + TransactionId: 3, + Type: models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, + TransactionTime: 10, + }) + transactionSlice = append(transactionSlice, &models.Transaction{ + TransactionId: 4, + Type: models.TRANSACTION_DB_TYPE_TRANSFER_IN, + TransactionTime: 3, + }) + transactionSlice = append(transactionSlice, &models.Transaction{ + TransactionId: 5, + Type: models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, + TransactionTime: 11, + }) + transactionSlice = append(transactionSlice, &models.Transaction{ + TransactionId: 6, + Type: models.TRANSACTION_DB_TYPE_TRANSFER_OUT, + TransactionTime: 4, + }) + + sort.Sort(transactionSlice) + + assert.Equal(t, int64(3), transactionSlice[0].TransactionId) + assert.Equal(t, int64(5), transactionSlice[1].TransactionId) + assert.Equal(t, int64(1), transactionSlice[2].TransactionId) + assert.Equal(t, int64(2), transactionSlice[3].TransactionId) + assert.Equal(t, int64(4), transactionSlice[4].TransactionId) + assert.Equal(t, int64(6), transactionSlice[5].TransactionId) +} diff --git a/pkg/errs/data_managements.go b/pkg/errs/data_managements.go index 4687e137..8bee3cfd 100644 --- a/pkg/errs/data_managements.go +++ b/pkg/errs/data_managements.go @@ -6,5 +6,7 @@ import ( // Error codes related to data management var ( - ErrDataExportNotAllowed = NewNormalError(NormalSubcategoryDataManagement, 1, http.StatusBadRequest, "data export not allowed") + ErrDataExportNotAllowed = NewNormalError(NormalSubcategoryDataManagement, 1, http.StatusBadRequest, "data export not allowed") + ErrDataImportNotAllowed = NewNormalError(NormalSubcategoryDataManagement, 2, http.StatusBadRequest, "data import not allowed") + ErrImportTooManyTransaction = NewNormalError(NormalSubcategoryDataManagement, 3, http.StatusBadRequest, "import too many transactions") ) diff --git a/pkg/errs/global.go b/pkg/errs/global.go index 9861d135..a1302c35 100644 --- a/pkg/errs/global.go +++ b/pkg/errs/global.go @@ -21,6 +21,7 @@ var ( ErrQueryItemsInvalid = NewNormalError(NormalSubcategoryGlobal, 11, http.StatusBadRequest, "query items have invalid item") ErrParameterInvalid = NewNormalError(NormalSubcategoryGlobal, 12, http.StatusBadRequest, "parameter invalid") ErrFormatInvalid = NewNormalError(NormalSubcategoryGlobal, 13, http.StatusBadRequest, "format invalid") + ErrNumberInvalid = NewNormalError(NormalSubcategoryGlobal, 14, http.StatusBadRequest, "number invalid") ) // GetParameterInvalidMessage returns specific error message for invalid parameter error diff --git a/pkg/errs/system.go b/pkg/errs/system.go index 04b6f9e8..378b782c 100644 --- a/pkg/errs/system.go +++ b/pkg/errs/system.go @@ -4,11 +4,12 @@ import "net/http" // Error codes related to transaction categories var ( - ErrSystemError = NewSystemError(SystemSubcategoryDefault, 0, http.StatusInternalServerError, "system error") - ErrApiNotFound = NewSystemError(SystemSubcategoryDefault, 1, http.StatusNotFound, "api not found") - ErrMethodNotAllowed = NewSystemError(SystemSubcategoryDefault, 2, http.StatusMethodNotAllowed, "method not allowed") - ErrNotImplemented = NewSystemError(SystemSubcategoryDefault, 3, http.StatusNotImplemented, "not implemented") - ErrSystemIsBusy = NewSystemError(SystemSubcategoryDefault, 4, http.StatusServiceUnavailable, "system is busy") - ErrNotSupported = NewSystemError(SystemSubcategoryDefault, 5, http.StatusBadRequest, "not supported") - ErrImageTypeNotSupported = NewSystemError(SystemSubcategoryDefault, 6, http.StatusBadRequest, "image type not supported") + ErrSystemError = NewSystemError(SystemSubcategoryDefault, 0, http.StatusInternalServerError, "system error") + ErrApiNotFound = NewSystemError(SystemSubcategoryDefault, 1, http.StatusNotFound, "api not found") + ErrMethodNotAllowed = NewSystemError(SystemSubcategoryDefault, 2, http.StatusMethodNotAllowed, "method not allowed") + ErrNotImplemented = NewSystemError(SystemSubcategoryDefault, 3, http.StatusNotImplemented, "not implemented") + ErrSystemIsBusy = NewSystemError(SystemSubcategoryDefault, 4, http.StatusServiceUnavailable, "system is busy") + ErrNotSupported = NewSystemError(SystemSubcategoryDefault, 5, http.StatusBadRequest, "not supported") + ErrImageTypeNotSupported = NewSystemError(SystemSubcategoryDefault, 6, http.StatusBadRequest, "image type not supported") + ErrImportFileTypeNotSupported = NewSystemError(SystemSubcategoryDefault, 7, http.StatusBadRequest, "import file type not supported") ) diff --git a/pkg/models/transaction.go b/pkg/models/transaction.go index 4bf9833b..087aed84 100644 --- a/pkg/models/transaction.go +++ b/pkg/models/transaction.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "strings" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -30,6 +31,24 @@ const ( TRANSACTION_DB_TYPE_TRANSFER_IN TransactionDbType = 5 ) +// String returns a textual representation of the transaction types for db enum +func (s TransactionDbType) String() string { + switch s { + case TRANSACTION_DB_TYPE_MODIFY_BALANCE: + return "Modify Balance" + case TRANSACTION_DB_TYPE_INCOME: + return "Income" + case TRANSACTION_DB_TYPE_EXPENSE: + return "Expense" + case TRANSACTION_DB_TYPE_TRANSFER_OUT: + return "Transfer Out" + case TRANSACTION_DB_TYPE_TRANSFER_IN: + return "Transfer In" + default: + return fmt.Sprintf("Invalid(%d)", int(s)) + } +} + // Transaction represents transaction data stored in database type Transaction struct { TransactionId int64 `xorm:"PK"` diff --git a/pkg/services/accounts.go b/pkg/services/accounts.go index d92ab8c7..ff64e02b 100644 --- a/pkg/services/accounts.go +++ b/pkg/services/accounts.go @@ -463,3 +463,14 @@ func (s *AccountService) GetAccountMapByList(accounts []*models.Account) map[int } return accountMap } + +// GetAccountNameMapByList returns an account map by a list +func (s *AccountService) GetAccountNameMapByList(accounts []*models.Account) map[string]*models.Account { + accountMap := make(map[string]*models.Account) + + for i := 0; i < len(accounts); i++ { + account := accounts[i] + accountMap[account.Name] = account + } + return accountMap +} diff --git a/pkg/services/base.go b/pkg/services/base.go index ef6b7449..72195cc0 100644 --- a/pkg/services/base.go +++ b/pkg/services/base.go @@ -88,7 +88,7 @@ func (s *ServiceUsingUuid) GenerateUuid(uuidType uuid.UuidType) int64 { } // GenerateUuids generates new uuids according to given uuid type and count -func (s *ServiceUsingUuid) GenerateUuids(uuidType uuid.UuidType, count uint8) []int64 { +func (s *ServiceUsingUuid) GenerateUuids(uuidType uuid.UuidType, count uint16) []int64 { return s.container.GenerateUuids(uuidType, count) } diff --git a/pkg/services/transaction_categories.go b/pkg/services/transaction_categories.go index 44f27ba6..fdd2faf8 100644 --- a/pkg/services/transaction_categories.go +++ b/pkg/services/transaction_categories.go @@ -447,3 +447,14 @@ func (s *TransactionCategoryService) GetCategoryMapByList(categories []*models.T } return categoryMap } + +// GetCategoryNameMapByList returns a transaction category map by a list +func (s *TransactionCategoryService) GetCategoryNameMapByList(categories []*models.TransactionCategory) map[string]*models.TransactionCategory { + categoryMap := make(map[string]*models.TransactionCategory) + + for i := 0; i < len(categories); i++ { + category := categories[i] + categoryMap[category.Name] = category + } + return categoryMap +} diff --git a/pkg/services/transaction_tags.go b/pkg/services/transaction_tags.go index ee64eaf3..71fbd8b0 100644 --- a/pkg/services/transaction_tags.go +++ b/pkg/services/transaction_tags.go @@ -414,6 +414,17 @@ func (s *TransactionTagService) GetTagMapByList(tags []*models.TransactionTag) m return tagMap } +// GetTagNameMapByList returns a transaction tag map by a list +func (s *TransactionTagService) GetTagNameMapByList(tags []*models.TransactionTag) map[string]*models.TransactionTag { + tagMap := make(map[string]*models.TransactionTag) + + for i := 0; i < len(tags); i++ { + tag := tags[i] + tagMap[tag.Name] = tag + } + return tagMap +} + func (s *TransactionTagService) GetGroupedTransactionTagIds(tagIndexes []*models.TransactionTagIndex) map[int64][]int64 { allTransactionTagIds := make(map[int64][]int64) diff --git a/pkg/services/transactions.go b/pkg/services/transactions.go index 97147433..6e1373d5 100644 --- a/pkg/services/transactions.go +++ b/pkg/services/transactions.go @@ -223,7 +223,7 @@ func (s *TransactionService) CreateTransaction(c core.Context, transaction *mode needUuidCount = 2 } - uuids := s.GenerateUuids(uuid.UUID_TYPE_TRANSACTION, uint8(needUuidCount)) + uuids := s.GenerateUuids(uuid.UUID_TYPE_TRANSACTION, uint16(needUuidCount)) if len(uuids) < needUuidCount { return errs.ErrSystemIsBusy @@ -267,187 +267,75 @@ func (s *TransactionService) CreateTransaction(c core.Context, transaction *mode } return s.UserDataDB(transaction.Uid).DoTransaction(c, func(sess *xorm.Session) error { - // Get and verify source and destination account - sourceAccount, destinationAccount, err := s.getAccountModels(sess, transaction) + return s.doCreateTransaction(sess, transaction, transactionTagIndexes, tagIds, pictureIds, pictureUpdateModel) + }) +} + +// BatchCreateTransactions saves new transactions to database +func (s *TransactionService) BatchCreateTransactions(c core.Context, uid int64, transactions []*models.Transaction) error { + now := time.Now().Unix() + needUuidCount := uint16(0) + + for i := 0; i < len(transactions); i++ { + transaction := transactions[i] + + if transaction.Uid != uid { + return errs.ErrUserIdInvalid + } + + // Check whether account id is valid + err := s.isAccountIdValid(transaction) if err != nil { return err } - if sourceAccount.Hidden || (destinationAccount != nil && destinationAccount.Hidden) { - return errs.ErrCannotAddTransactionToHiddenAccount - } - - if sourceAccount.Type == models.ACCOUNT_TYPE_MULTI_SUB_ACCOUNTS || (destinationAccount != nil && destinationAccount.Type == models.ACCOUNT_TYPE_MULTI_SUB_ACCOUNTS) { - return errs.ErrCannotAddTransactionToParentAccount - } - - if (transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT || transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN) && - sourceAccount.Currency == destinationAccount.Currency && transaction.Amount != transaction.RelatedAccountAmount { - return errs.ErrTransactionSourceAndDestinationAmountNotEqual - } - - // Get and verify category - err = s.isCategoryValid(sess, transaction) - - if err != nil { - return err - } - - // Get and verify tags - err = s.isTagsValid(sess, transaction, transactionTagIndexes, tagIds) - - if err != nil { - return err - } - - // Get and verify pictures - err = s.isPicturesValid(sess, transaction, pictureIds) - - if err != nil { - return err - } - - // Verify balance modification transaction and calculate real amount - if transaction.Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { - otherTransactionExists, err := sess.Cols("uid", "deleted", "account_id").Where("uid=? AND deleted=? AND account_id=?", transaction.Uid, false, sourceAccount.AccountId).Limit(1).Exist(&models.Transaction{}) - - if err != nil { - return err - } else if otherTransactionExists { - return errs.ErrBalanceModificationTransactionCannotAddWhenNotEmpty - } - - transaction.RelatedAccountId = transaction.AccountId - transaction.RelatedAccountAmount = transaction.Amount - sourceAccount.Balance - } - - // Insert transaction row - var relatedTransaction *models.Transaction - if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT || transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN { - relatedTransaction = s.GetRelatedTransferTransaction(transaction) + needUuidCount += 2 + } else { + needUuidCount++ } - createdRows, err := sess.Insert(transaction) + transaction.TransactionTime = utils.GetMinTransactionTimeFromUnixTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime)) - if err != nil || createdRows < 1 { // maybe another transaction has same time - sameSecondLatestTransaction := &models.Transaction{} - minTransactionTime := utils.GetMinTransactionTimeFromUnixTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime)) - maxTransactionTime := utils.GetMaxTransactionTimeFromUnixTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime)) + transaction.CreatedUnixTime = now + transaction.UpdatedUnixTime = now + } - has, err := sess.Where("uid=? AND transaction_time>=? AND transaction_time<=?", transaction.Uid, minTransactionTime, maxTransactionTime).OrderBy("transaction_time desc").Limit(1).Get(sameSecondLatestTransaction) + if needUuidCount > uint16(65535) { + return errs.ErrImportTooManyTransaction + } - if err != nil { - return err - } else if !has { - return errs.ErrDatabaseOperationFailed - } else if sameSecondLatestTransaction.TransactionTime == maxTransactionTime-1 { - return errs.ErrTooMuchTransactionInOneSecond - } + uuids := s.GenerateUuids(uuid.UUID_TYPE_TRANSACTION, needUuidCount) + uuidIndex := 0 - transaction.TransactionTime = sameSecondLatestTransaction.TransactionTime + 1 - createdRows, err := sess.Insert(transaction) + if len(uuids) < int(needUuidCount) { + return errs.ErrSystemIsBusy + } - if err != nil { - return err - } else if createdRows < 1 { - return errs.ErrDatabaseOperationFailed - } + for i := 0; i < len(transactions); i++ { + transaction := transactions[i] + + transaction.TransactionId = uuids[uuidIndex] + uuidIndex++ + + if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT || transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN { + transaction.RelatedId = uuids[uuidIndex] + uuidIndex++ } + } - if relatedTransaction != nil { - relatedTransaction.TransactionTime = transaction.TransactionTime + 1 - - if utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime) != utils.GetUnixTimeFromTransactionTime(relatedTransaction.TransactionTime) { - return errs.ErrTooMuchTransactionInOneSecond - } - - createdRows, err := sess.Insert(relatedTransaction) - - if err != nil { - return err - } else if createdRows < 1 { - return errs.ErrDatabaseOperationFailed - } - } - - err = nil - - // Insert transaction tag index - if len(transactionTagIndexes) > 0 { - for i := 0; i < len(transactionTagIndexes); i++ { - transactionTagIndex := transactionTagIndexes[i] - transactionTagIndex.TransactionTime = transaction.TransactionTime - - _, err := sess.Insert(transactionTagIndex) - - if err != nil { - return err - } - } - } - - // Update transaction picture - if len(pictureIds) > 0 { - _, err = sess.Cols("transaction_id", "updated_unix_time").Where("uid=? AND deleted=? AND transaction_id=?", transaction.Uid, false, models.TransactionPictureNewPictureTransactionId).In("picture_id", pictureIds).Update(pictureUpdateModel) + return s.UserDataDB(uid).DoTransaction(c, func(sess *xorm.Session) error { + for i := 0; i < len(transactions); i++ { + transaction := transactions[i] + err := s.doCreateTransaction(sess, transaction, nil, nil, nil, nil) if err != nil { return err } } - // Update account table - if transaction.Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { - sourceAccount.UpdatedUnixTime = time.Now().Unix() - updatedRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance+(%d)", transaction.RelatedAccountAmount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) - - if err != nil { - return err - } else if updatedRows < 1 { - return errs.ErrDatabaseOperationFailed - } - } else if transaction.Type == models.TRANSACTION_DB_TYPE_INCOME { - sourceAccount.UpdatedUnixTime = time.Now().Unix() - updatedRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance+(%d)", transaction.Amount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) - - if err != nil { - return err - } else if updatedRows < 1 { - return errs.ErrDatabaseOperationFailed - } - } else if transaction.Type == models.TRANSACTION_DB_TYPE_EXPENSE { - sourceAccount.UpdatedUnixTime = time.Now().Unix() - updatedRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance-(%d)", transaction.Amount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) - - if err != nil { - return err - } else if updatedRows < 1 { - return errs.ErrDatabaseOperationFailed - } - } else if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { - sourceAccount.UpdatedUnixTime = time.Now().Unix() - updatedSourceRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance-(%d)", transaction.Amount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) - - if err != nil { - return err - } else if updatedSourceRows < 1 { - return errs.ErrDatabaseOperationFailed - } - - destinationAccount.UpdatedUnixTime = time.Now().Unix() - updatedDestinationRows, err := sess.ID(destinationAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance+(%d)", transaction.RelatedAccountAmount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", destinationAccount.Uid, false).Update(destinationAccount) - - if err != nil { - return err - } else if updatedDestinationRows < 1 { - return errs.ErrDatabaseOperationFailed - } - } else if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN { - return errs.ErrTransactionTypeInvalid - } - - return err + return nil }) } @@ -1587,6 +1475,190 @@ func (s *TransactionService) GetTransactionIds(transactions []*models.Transactio return transactionIds } +func (s *TransactionService) doCreateTransaction(sess *xorm.Session, transaction *models.Transaction, transactionTagIndexes []*models.TransactionTagIndex, tagIds []int64, pictureIds []int64, pictureUpdateModel *models.TransactionPictureInfo) error { + // Get and verify source and destination account + sourceAccount, destinationAccount, err := s.getAccountModels(sess, transaction) + + if err != nil { + return err + } + + if sourceAccount.Hidden || (destinationAccount != nil && destinationAccount.Hidden) { + return errs.ErrCannotAddTransactionToHiddenAccount + } + + if sourceAccount.Type == models.ACCOUNT_TYPE_MULTI_SUB_ACCOUNTS || (destinationAccount != nil && destinationAccount.Type == models.ACCOUNT_TYPE_MULTI_SUB_ACCOUNTS) { + return errs.ErrCannotAddTransactionToParentAccount + } + + if (transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT || transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN) && + sourceAccount.Currency == destinationAccount.Currency && transaction.Amount != transaction.RelatedAccountAmount { + return errs.ErrTransactionSourceAndDestinationAmountNotEqual + } + + // Get and verify category + err = s.isCategoryValid(sess, transaction) + + if err != nil { + return err + } + + // Get and verify tags + err = s.isTagsValid(sess, transaction, transactionTagIndexes, tagIds) + + if err != nil { + return err + } + + // Get and verify pictures + err = s.isPicturesValid(sess, transaction, pictureIds) + + if err != nil { + return err + } + + // Verify balance modification transaction and calculate real amount + if transaction.Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { + otherTransactionExists, err := sess.Cols("uid", "deleted", "account_id").Where("uid=? AND deleted=? AND account_id=?", transaction.Uid, false, sourceAccount.AccountId).Limit(1).Exist(&models.Transaction{}) + + if err != nil { + return err + } else if otherTransactionExists { + return errs.ErrBalanceModificationTransactionCannotAddWhenNotEmpty + } + + transaction.RelatedAccountId = transaction.AccountId + transaction.RelatedAccountAmount = transaction.Amount - sourceAccount.Balance + } + + // Insert transaction row + var relatedTransaction *models.Transaction + + if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT || transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN { + relatedTransaction = s.GetRelatedTransferTransaction(transaction) + } + + createdRows, err := sess.Insert(transaction) + + if err != nil || createdRows < 1 { // maybe another transaction has same time + sameSecondLatestTransaction := &models.Transaction{} + minTransactionTime := utils.GetMinTransactionTimeFromUnixTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime)) + maxTransactionTime := utils.GetMaxTransactionTimeFromUnixTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime)) + + has, err := sess.Where("uid=? AND transaction_time>=? AND transaction_time<=?", transaction.Uid, minTransactionTime, maxTransactionTime).OrderBy("transaction_time desc").Limit(1).Get(sameSecondLatestTransaction) + + if err != nil { + return err + } else if !has { + return errs.ErrDatabaseOperationFailed + } else if sameSecondLatestTransaction.TransactionTime == maxTransactionTime-1 { + return errs.ErrTooMuchTransactionInOneSecond + } + + transaction.TransactionTime = sameSecondLatestTransaction.TransactionTime + 1 + createdRows, err := sess.Insert(transaction) + + if err != nil { + return err + } else if createdRows < 1 { + return errs.ErrDatabaseOperationFailed + } + } + + if relatedTransaction != nil { + relatedTransaction.TransactionTime = transaction.TransactionTime + 1 + + if utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime) != utils.GetUnixTimeFromTransactionTime(relatedTransaction.TransactionTime) { + return errs.ErrTooMuchTransactionInOneSecond + } + + createdRows, err := sess.Insert(relatedTransaction) + + if err != nil { + return err + } else if createdRows < 1 { + return errs.ErrDatabaseOperationFailed + } + } + + err = nil + + // Insert transaction tag index + if len(transactionTagIndexes) > 0 { + for i := 0; i < len(transactionTagIndexes); i++ { + transactionTagIndex := transactionTagIndexes[i] + transactionTagIndex.TransactionTime = transaction.TransactionTime + + _, err := sess.Insert(transactionTagIndex) + + if err != nil { + return err + } + } + } + + // Update transaction picture + if len(pictureIds) > 0 { + _, err = sess.Cols("transaction_id", "updated_unix_time").Where("uid=? AND deleted=? AND transaction_id=?", transaction.Uid, false, models.TransactionPictureNewPictureTransactionId).In("picture_id", pictureIds).Update(pictureUpdateModel) + + if err != nil { + return err + } + } + + // Update account table + if transaction.Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { + sourceAccount.UpdatedUnixTime = time.Now().Unix() + updatedRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance+(%d)", transaction.RelatedAccountAmount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) + + if err != nil { + return err + } else if updatedRows < 1 { + return errs.ErrDatabaseOperationFailed + } + } else if transaction.Type == models.TRANSACTION_DB_TYPE_INCOME { + sourceAccount.UpdatedUnixTime = time.Now().Unix() + updatedRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance+(%d)", transaction.Amount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) + + if err != nil { + return err + } else if updatedRows < 1 { + return errs.ErrDatabaseOperationFailed + } + } else if transaction.Type == models.TRANSACTION_DB_TYPE_EXPENSE { + sourceAccount.UpdatedUnixTime = time.Now().Unix() + updatedRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance-(%d)", transaction.Amount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) + + if err != nil { + return err + } else if updatedRows < 1 { + return errs.ErrDatabaseOperationFailed + } + } else if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { + sourceAccount.UpdatedUnixTime = time.Now().Unix() + updatedSourceRows, err := sess.ID(sourceAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance-(%d)", transaction.Amount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", sourceAccount.Uid, false).Update(sourceAccount) + + if err != nil { + return err + } else if updatedSourceRows < 1 { + return errs.ErrDatabaseOperationFailed + } + + destinationAccount.UpdatedUnixTime = time.Now().Unix() + updatedDestinationRows, err := sess.ID(destinationAccount.AccountId).SetExpr("balance", fmt.Sprintf("balance+(%d)", transaction.RelatedAccountAmount)).Cols("updated_unix_time").Where("uid=? AND deleted=?", destinationAccount.Uid, false).Update(destinationAccount) + + if err != nil { + return err + } else if updatedDestinationRows < 1 { + return errs.ErrDatabaseOperationFailed + } + } else if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_IN { + return errs.ErrTransactionTypeInvalid + } + + return err +} + func (s *TransactionService) getTransactionQueryCondition(uid int64, maxTransactionTime int64, minTransactionTime int64, transactionType models.TransactionDbType, categoryIds []int64, accountIds []int64, tagIds []int64, amountFilter string, keyword string, noDuplicated bool) (string, []any) { condition := "uid=? AND deleted=?" conditionParams := make([]any, 0, 16) diff --git a/pkg/utils/converter.go b/pkg/utils/converter.go index 912236c3..b339fe56 100644 --- a/pkg/utils/converter.go +++ b/pkg/utils/converter.go @@ -1,6 +1,11 @@ package utils -import "strconv" +import ( + "strconv" + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/errs" +) // IntToString returns the textual representation of this number func IntToString(num int) string { @@ -123,3 +128,65 @@ func FormatAmount(amount int64) string { return integer + "." + decimals } + +// ParseAmount parses a textual representation of amount +func ParseAmount(amount string) (int64, error) { + if len(amount) < 1 { + return 0, nil + } + + sign := int64(1) + + if amount[0] == '-' { + amount = amount[1:] + sign = -1 + } + + if len(amount) < 1 { + return 0, errs.ErrNumberInvalid + } + + items := strings.Split(amount, ".") + + if len(items) > 2 { + return 0, errs.ErrNumberInvalid + } + + var err error + integer := int64(0) + decimals := int64(0) + + if len(items[0]) > 0 { + integer, err = StringToInt64(items[0]) + + if err != nil { + return 0, err + } + + if integer < 0 { + return 0, errs.ErrNumberInvalid + } + } + + if len(items) == 2 { + if len(items[1]) > 2 { + return 0, errs.ErrNumberInvalid + } + + decimals, err = StringToInt64(items[1]) + + if err != nil { + return 0, err + } + + if decimals < 0 { + return 0, errs.ErrNumberInvalid + } + + if len(items[1]) == 1 { + decimals = decimals * 10 + } + } + + return sign*integer*100 + sign*decimals, nil +} diff --git a/pkg/utils/converter_test.go b/pkg/utils/converter_test.go index a51e6bad..441215b6 100644 --- a/pkg/utils/converter_test.go +++ b/pkg/utils/converter_test.go @@ -172,3 +172,133 @@ func TestFormatAmount(t *testing.T) { actualValue = FormatAmount(-1234) assert.Equal(t, expectedValue, actualValue) } + +func TestParseAmount(t *testing.T) { + expectedValue := int64(0) + actualValue, err := ParseAmount("") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(0) + actualValue, err = ParseAmount("0") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(0) + actualValue, err = ParseAmount("-0") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(0) + actualValue, err = ParseAmount("0.00") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(0) + actualValue, err = ParseAmount("-0.00") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(1) + actualValue, err = ParseAmount("0.01") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-1) + actualValue, err = ParseAmount("-0.01") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(10) + actualValue, err = ParseAmount("0.10") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-10) + actualValue, err = ParseAmount("-0.10") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(10) + actualValue, err = ParseAmount("0.1") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-10) + actualValue, err = ParseAmount("-0.1") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(12) + actualValue, err = ParseAmount("0.12") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-12) + actualValue, err = ParseAmount("-0.12") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(100) + actualValue, err = ParseAmount("1") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-100) + actualValue, err = ParseAmount("-1") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(120) + actualValue, err = ParseAmount("1.2") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-120) + actualValue, err = ParseAmount("-1.2") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(123) + actualValue, err = ParseAmount("1.23") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-123) + actualValue, err = ParseAmount("-1.23") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(1234) + actualValue, err = ParseAmount("12.34") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) + + expectedValue = int64(-1234) + actualValue, err = ParseAmount("-12.34") + assert.Nil(t, err) + assert.Equal(t, expectedValue, actualValue) +} + +func TestParseAmount_InvalidAmount(t *testing.T) { + _, err := ParseAmount("-") + assert.NotNil(t, err) + + _, err = ParseAmount("--1") + assert.NotNil(t, err) + + _, err = ParseAmount("-.-1") + assert.NotNil(t, err) + + _, err = ParseAmount("0.-") + assert.NotNil(t, err) + + _, err = ParseAmount("0.-1") + assert.NotNil(t, err) + + _, err = ParseAmount("1.2.3") + assert.NotNil(t, err) + + _, err = ParseAmount("1.234") + assert.NotNil(t, err) +} diff --git a/pkg/uuid/internal_generator.go b/pkg/uuid/internal_generator.go index fa3bb966..25082875 100644 --- a/pkg/uuid/internal_generator.go +++ b/pkg/uuid/internal_generator.go @@ -60,7 +60,7 @@ func (u *InternalUuidGenerator) GenerateUuid(idType UuidType) int64 { } // GenerateUuids generates new uuids -func (u *InternalUuidGenerator) GenerateUuids(idType UuidType, count uint8) []int64 { +func (u *InternalUuidGenerator) GenerateUuids(idType UuidType, count uint16) []int64 { // 63bits = unixTime(32bits) + uuidType(4bits) + uuidServerId(8bits) + sequentialNumber(19bits) uuids := make([]int64, count) diff --git a/pkg/uuid/internal_generator_test.go b/pkg/uuid/internal_generator_test.go index c499bddb..195088e7 100644 --- a/pkg/uuid/internal_generator_test.go +++ b/pkg/uuid/internal_generator_test.go @@ -217,7 +217,7 @@ func TestGenerateUuids_Count0(t *testing.T) { func TestGenerateUuids_Count255(t *testing.T) { expectedUuidServerId := uint8(90) expectedUuidType := UUID_TYPE_TRANSACTION - expectedUuidCount := uint8(255) + expectedUuidCount := uint16(255) generator, _ := NewInternalUuidGenerator(&settings.Config{UuidServerId: expectedUuidServerId}) @@ -250,7 +250,7 @@ func TestGenerateUuids_Count255(t *testing.T) { func TestGenerateUuids_30TimesIn3Seconds(t *testing.T) { expectedUuidServerId := uint8(90) expectedUuidType := UUID_TYPE_TRANSACTION - expectedUuidCount := uint8(255) + expectedUuidCount := uint16(255) generator, _ := NewInternalUuidGenerator(&settings.Config{UuidServerId: expectedUuidServerId}) @@ -289,7 +289,7 @@ func TestGenerateUuids_30TimesIn3Seconds(t *testing.T) { func TestGenerateUuids_20000TimesConcurrent(t *testing.T) { concurrentCount := 10 - expectedUuidCount := uint8(20) + expectedUuidCount := uint16(20) generator, _ := NewInternalUuidGenerator(&settings.Config{UuidServerId: 3}) var mutex sync.Mutex var generatedIds sync.Map @@ -338,7 +338,7 @@ func TestGenerateUuids_20000TimesConcurrent(t *testing.T) { func TestGenerateUuids_1000000TimesConcurrent(t *testing.T) { concurrentCount := 10 - expectedUuidCount := uint8(250) + expectedUuidCount := uint16(250) generator, _ := NewInternalUuidGenerator(&settings.Config{UuidServerId: 3}) var mutex sync.Mutex var generatedIds sync.Map @@ -391,7 +391,7 @@ func TestGenerateUuids_1000000TimesConcurrent(t *testing.T) { func TestGenerateUuid_Over524287Times(t *testing.T) { generator, _ := NewInternalUuidGenerator(&settings.Config{UuidServerId: 1}) - onceGenerateCount := uint8(255) + onceGenerateCount := uint16(255) generationStartUnixTime := time.Now().Unix() for i := 0; i < 2057; i++ { // 2056*255=524280, 2057*255=524,535 (only can generates 524,287 uuids per second) diff --git a/pkg/uuid/uuid_container.go b/pkg/uuid/uuid_container.go index 058bd2ee..7347a184 100644 --- a/pkg/uuid/uuid_container.go +++ b/pkg/uuid/uuid_container.go @@ -33,6 +33,6 @@ func (u *UuidContainer) GenerateUuid(uuidType UuidType) int64 { } // GenerateUuids returns new uuids by the current uuid generator -func (u *UuidContainer) GenerateUuids(uuidType UuidType, count uint8) []int64 { +func (u *UuidContainer) GenerateUuids(uuidType UuidType, count uint16) []int64 { return u.Current.GenerateUuids(uuidType, count) } diff --git a/pkg/uuid/uuid_generator.go b/pkg/uuid/uuid_generator.go index 3faf6779..27ce8e7c 100644 --- a/pkg/uuid/uuid_generator.go +++ b/pkg/uuid/uuid_generator.go @@ -3,5 +3,5 @@ package uuid // UuidGenerator is common uuid generator interface type UuidGenerator interface { GenerateUuid(uuidType UuidType) int64 - GenerateUuids(uuidType UuidType, count uint8) []int64 + GenerateUuids(uuidType UuidType, count uint16) []int64 } diff --git a/src/locales/en.json b/src/locales/en.json index 72c5fd6b..71580070 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -970,6 +970,7 @@ "system is busy": "System is busy", "not supported": "Not supported", "image type not supported": "Image type is not supported", + "import file type not supported": "Import file type is not supported", "database operation failed": "Database operation failed", "SMTP server is not enabled": "SMTP server is not enabled", "incomplete or incorrect submission": "Incomplete or incorrect submission", @@ -1084,6 +1085,8 @@ "transaction tag is in use and cannot be deleted": "Transaction tag is in use and it cannot be deleted", "transaction tag index not found": "Transaction tag index is not found", "data export not allowed": "User data export is not allowed", + "data import not allowed": "User data import is not allowed", + "import too many transactions": "There are too many transactions to import", "transaction template id is invalid": "Transaction template ID is invalid", "transaction template not found": "Transaction template is not found", "transaction template type is invalid": "Transaction template type is invalid", @@ -1099,7 +1102,9 @@ "query items cannot be blank": "There are no query items", "query items too much": "There are too many query items", "query items have invalid item": "There is invalid item in query items", - "parameter invalid": "Parameter is invalid" + "parameter invalid": "Parameter is invalid", + "format invalid": "Format is invalid", + "number invalid": "Number is invalid" }, "parameter": { "id": "ID", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index 061a7e1a..3db33af5 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -970,6 +970,7 @@ "system is busy": "系统繁忙", "not supported": "不支持", "image type not supported": "图片类型不支持", + "import file type not supported": "导入文件类型不支持", "database operation failed": "数据库操作失败", "SMTP server is not enabled": "SMTP 服务器没有启用", "incomplete or incorrect submission": "提交不完整或不正确", @@ -1084,6 +1085,8 @@ "transaction tag is in use and cannot be deleted": "交易标签正在被使用，无法删除", "transaction tag index not found": "交易标签索引不存在", "data export not allowed": "不允许用户数据导出", + "data import not allowed": "不允许用户数据导入", + "import too many transactions": "导入的交易过多", "transaction template id is invalid": "交易模板ID无效", "transaction template not found": "交易模板不存在", "transaction template type is invalid": "交易模板类型无效", @@ -1099,7 +1102,9 @@ "query items cannot be blank": "请求项目不能为空", "query items too much": "请求项目过多", "query items have invalid item": "请求项目中有非法项目", - "parameter invalid": "参数错误" + "parameter invalid": "参数错误", + "format invalid": "格式错误", + "number invalid": "数字错误" }, "parameter": { "id": "ID",