From a49490baa707a1d2f06e7cc68a5ca4bd778b6d17 Mon Sep 17 00:00:00 2001 From: MaysWind Date: Tue, 24 Sep 2024 01:02:05 +0800 Subject: [PATCH] support categories with the same name but different types when import transaction --- pkg/api/transactions.go | 4 +- pkg/cli/user_data.go | 36 +++-- ...ipay_transaction_data_csv_file_importer.go | 12 +- ...transaction_data_csv_file_importer_test.go | 56 ++++---- .../base/transaction_data_converter.go | 2 +- .../data_table_transaction_data_converter.go | 91 +++++++++---- ...g_transaction_data_plain_text_converter.go | 6 +- ...nsaction_data_plain_text_converter_test.go | 126 +++++++++--------- ...oney_transaction_data_csv_file_importer.go | 22 +-- ...transaction_data_csv_file_importer_test.go | 114 ++++++++-------- ...oney_transaction_data_xls_file_importer.go | 6 +- ...transaction_data_xls_file_importer_test.go | 26 ++-- pkg/services/transaction_categories.go | 18 ++- 13 files changed, 294 insertions(+), 225 deletions(-) diff --git a/pkg/api/transactions.go b/pkg/api/transactions.go index 3d698e5b..6e264837 100644 --- a/pkg/api/transactions.go +++ b/pkg/api/transactions.go @@ -1093,7 +1093,7 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext) return nil, errs.Or(err, errs.ErrOperationFailed) } - categoryMap := a.transactionCategories.GetCategoryNameMapByList(categories) + expenseCategoryMap, incomeCategoryMap, transferCategoryMap := a.transactionCategories.GetCategoryNameMapByList(categories) tags, err := a.transactionTags.GetAllTagsByUid(c, user.Uid) @@ -1104,7 +1104,7 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext) tagMap := a.transactionTags.GetTagNameMapByList(tags) - parsedTransactions, _, _, _, err := dataImporter.ParseImportedData(c, user, fileData, utcOffset, accountMap, categoryMap, tagMap) + parsedTransactions, _, _, _, _, _, err := dataImporter.ParseImportedData(c, user, fileData, utcOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) if err != nil { log.BootErrorf(c, "[transactions.TransactionParseImportFileHandler] failed to parse imported data for user \"uid:%d\", because %s", user.Uid, err.Error()) diff --git a/pkg/cli/user_data.go b/pkg/cli/user_data.go index 027e8ce9..b12bd82c 100644 --- a/pkg/cli/user_data.go +++ b/pkg/cli/user_data.go @@ -677,14 +677,14 @@ func (l *UserDataCli) ImportTransaction(c *core.CliContext, username string, fil return err } - accountMap, categoryMap, tagMap, err := l.getUserEssentialDataForImport(c, user.Uid, username) + accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap, err := l.getUserEssentialDataForImport(c, user.Uid, username) if err != nil { log.CliErrorf(c, "[user_data.ImportTransaction] failed to get essential data for user \"%s\", because %s", username, err.Error()) return err } - parsedTransactions, newAccounts, newCategories, newTags, err := dataImporter.ParseImportedData(c, user, data, utils.GetTimezoneOffsetMinutes(time.Local), accountMap, categoryMap, tagMap) + parsedTransactions, newAccounts, newSubExpenseCategories, newSubIncomeCategories, newSubTransferCategories, newTags, err := dataImporter.ParseImportedData(c, user, data, utils.GetTimezoneOffsetMinutes(time.Local), accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) if err != nil { log.CliErrorf(c, "[user_data.ImportTransaction] failed to parse imported data for \"%s\", because %s", username, err.Error()) @@ -702,9 +702,21 @@ func (l *UserDataCli) ImportTransaction(c *core.CliContext, username string, fil return errs.ErrOperationFailed } - if len(newCategories) > 0 { - categoryNames := l.categories.GetCategoryNames(newCategories) - log.CliErrorf(c, "[user_data.ImportTransaction] there are %d transaction categories (%s) need to be created, please create them manually", len(newCategories), strings.Join(categoryNames, ",")) + if len(newSubExpenseCategories) > 0 { + categoryNames := l.categories.GetCategoryNames(newSubExpenseCategories) + log.CliErrorf(c, "[user_data.ImportTransaction] there are %d expense categories (%s) need to be created, please create them manually", len(newSubExpenseCategories), strings.Join(categoryNames, ",")) + return errs.ErrOperationFailed + } + + if len(newSubIncomeCategories) > 0 { + categoryNames := l.categories.GetCategoryNames(newSubIncomeCategories) + log.CliErrorf(c, "[user_data.ImportTransaction] there are %d income categories (%s) need to be created, please create them manually", len(newSubIncomeCategories), strings.Join(categoryNames, ",")) + return errs.ErrOperationFailed + } + + if len(newSubTransferCategories) > 0 { + categoryNames := l.categories.GetCategoryNames(newSubTransferCategories) + log.CliErrorf(c, "[user_data.ImportTransaction] there are %d transfer categories (%s) need to be created, please create them manually", len(newSubTransferCategories), strings.Join(categoryNames, ",")) return errs.ErrOperationFailed } @@ -788,17 +800,17 @@ func (l *UserDataCli) getUserEssentialData(c *core.CliContext, uid int64, userna return accountMap, categoryMap, tagMap, tagIndexes, tagIndexesMap, nil } -func (l *UserDataCli) getUserEssentialDataForImport(c *core.CliContext, uid int64, username string) (accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag, err error) { +func (l *UserDataCli) getUserEssentialDataForImport(c *core.CliContext, uid int64, username string) (accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag, err error) { if uid <= 0 { log.CliErrorf(c, "[user_data.getUserEssentialDataForImport] user uid \"%d\" is invalid", uid) - return nil, nil, nil, errs.ErrUserIdInvalid + return nil, nil, nil, nil, nil, errs.ErrUserIdInvalid } accounts, err := l.accounts.GetAllAccountsByUid(c, uid) if err != nil { log.CliErrorf(c, "[user_data.getUserEssentialDataForImport] failed to get accounts for user \"%s\", because %s", username, err.Error()) - return nil, nil, nil, err + return nil, nil, nil, nil, nil, err } accountMap = l.accounts.GetAccountNameMapByList(accounts) @@ -807,21 +819,21 @@ func (l *UserDataCli) getUserEssentialDataForImport(c *core.CliContext, uid int6 if err != nil { log.CliErrorf(c, "[user_data.getUserEssentialDataForImport] failed to get categories for user \"%s\", because %s", username, err.Error()) - return nil, nil, nil, err + return nil, nil, nil, nil, nil, err } - categoryMap = l.categories.GetCategoryNameMapByList(categories) + expenseCategoryMap, incomeCategoryMap, transferCategoryMap = l.categories.GetCategoryNameMapByList(categories) tags, err := l.tags.GetAllTagsByUid(c, uid) if err != nil { log.CliErrorf(c, "[user_data.getUserEssentialDataForImport] failed to get tags for user \"%s\", because %s", username, err.Error()) - return nil, nil, nil, err + return nil, nil, nil, nil, nil, err } tagMap = l.tags.GetTagNameMapByList(tags) - return accountMap, categoryMap, tagMap, nil + return accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap, nil } func (l *UserDataCli) checkTransactionAccount(c *core.CliContext, transaction *models.Transaction, accountMap map[int64]*models.Account, accountHasChild map[int64]bool) error { diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go index d3b4abbe..05b4a502 100644 --- a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go @@ -50,18 +50,18 @@ var ( ) // ParseImportedData returns the imported data by parsing the alipay transaction csv data -func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { +func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { enc := simplifiedchinese.GB18030 reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder()) allLines, err := c.parseAllLinesFromCsvData(ctx, reader) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, err } if len(allLines) <= 1 { log.Errorf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) - return nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile + return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile } headerLineItems := allLines[0] @@ -81,7 +81,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u if !timeColumnExists || !amountColumnExists || !statusColumnExists || !fundStatusColumnExists { log.Errorf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) - return nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } newColumns := make([]datatable.DataTableColumn, 0, 7) @@ -126,7 +126,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u if len(items) < len(headerLineItems) { log.Errorf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] cannot parse row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", i, user.Uid, len(items), len(headerLineItems)) - return nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } if items[statusColumnIdx] == alipayTransactionDataStatusSuccessName || items[statusColumnIdx] == alipayTransactionDataStatusPaymentSuccessName || items[statusColumnIdx] == alipayTransactionDataStatusRepaymentSuccessName { @@ -145,7 +145,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u alipayTransactionTypeFundStatusNameMapping, ) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, categoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } func (c *alipayTransactionDataCsvImporter) parseAllLinesFromCsvData(ctx core.Context, reader io.Reader) ([][]string, error) { diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go index 2052ce0c..31d55939 100644 --- a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go @@ -34,12 +34,14 @@ func TestAlipayCsvFileImporterParseImportedData_MinimumValidData(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte(data), 0, nil, nil, nil) + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte(data), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 3, len(allNewTransactions)) assert.Equal(t, 2, len(allNewAccounts)) - assert.Equal(t, 1, len(allNewSubCategories)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) assert.Equal(t, 0, len(allNewTags)) assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) @@ -72,8 +74,14 @@ func TestAlipayCsvFileImporterParseImportedData_MinimumValidData(t *testing.T) { assert.Equal(t, "", allNewAccounts[1].Name) assert.Equal(t, "CNY", allNewAccounts[1].Currency) - assert.Equal(t, int64(1234567890), allNewSubCategories[0].Uid) - assert.Equal(t, "", allNewSubCategories[0].Name) + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "", allNewSubTransferCategories[0].Name) } func TestAlipayCsvFileImporterParseImportedData_ParseRefundTransaction(t *testing.T) { @@ -94,7 +102,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseRefundTransaction(t *testin "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) @@ -113,7 +121,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseRefundTransaction(t *testin "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) @@ -142,7 +150,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseInvalidTime(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) data2, err := simplifiedchinese.GB18030.NewEncoder().String("支付宝交易记录明细查询\n" + @@ -154,7 +162,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseInvalidTime(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) } @@ -176,7 +184,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseInvalidType(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) } @@ -199,7 +207,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -215,7 +223,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -231,7 +239,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data3), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data3), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -248,7 +256,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data4), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data4), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -265,7 +273,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data5), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data5), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -282,7 +290,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data6), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data6), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -299,7 +307,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseAccountName(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data7), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data7), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -325,7 +333,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseDescription(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -340,7 +348,7 @@ func TestAlipayCsvFileImporterParseImportedData_ParseDescription(t *testing.T) { "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -362,10 +370,10 @@ func TestAlipayCsvFileImporterParseImportedData_MissingFileHeader(t *testing.T) "------------------------------------------------------------------------------------\n") assert.Nil(t, err) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrInvalidFileHeader.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrInvalidFileHeader.Message) } @@ -386,7 +394,7 @@ func TestAlipayCsvFileImporterParseImportedData_MissingRequiredColumn(t *testing "金额（元）,交易状态 ,资金状态 ,\n" + "0.12 ,交易成功 ,已收入 ,\n" + "------------------------------------------------------------------------------------\n") - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Amount Column @@ -397,7 +405,7 @@ func TestAlipayCsvFileImporterParseImportedData_MissingRequiredColumn(t *testing "交易创建时间 ,交易状态 ,资金状态 ,\n" + "2024-09-01 12:34:56 ,交易成功 ,已收入 ,\n" + "------------------------------------------------------------------------------------\n") - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data2), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Status Column @@ -408,7 +416,7 @@ func TestAlipayCsvFileImporterParseImportedData_MissingRequiredColumn(t *testing "交易创建时间 ,金额（元）,资金状态 ,\n" + "2024-09-01 12:34:56 ,0.12 ,已收入 ,\n" + "------------------------------------------------------------------------------------\n") - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data3), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data3), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Fund Status Column @@ -419,6 +427,6 @@ func TestAlipayCsvFileImporterParseImportedData_MissingRequiredColumn(t *testing "交易创建时间 ,金额（元）,交易状态 ,\n" + "2024-09-01 12:34:56 ,0.12 ,交易成功 ,\n" + "------------------------------------------------------------------------------------\n") - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data4), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data4), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) } diff --git a/pkg/converters/base/transaction_data_converter.go b/pkg/converters/base/transaction_data_converter.go index e4e784c5..ece463dd 100644 --- a/pkg/converters/base/transaction_data_converter.go +++ b/pkg/converters/base/transaction_data_converter.go @@ -14,7 +14,7 @@ type TransactionDataExporter interface { // TransactionDataImporter defines the structure of transaction data importer type TransactionDataImporter interface { // ParseImportedData returns the imported data - ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) + ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) } // TransactionDataConverter defines the structure of transaction data converter diff --git a/pkg/converters/datatable/data_table_transaction_data_converter.go b/pkg/converters/datatable/data_table_transaction_data_converter.go index a6a04a92..1fd2f577 100644 --- a/pkg/converters/datatable/data_table_transaction_data_converter.go +++ b/pkg/converters/datatable/data_table_transaction_data_converter.go @@ -240,16 +240,16 @@ func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder Data } // ParseImportedData returns the imported transaction data -func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, dataTable ImportedDataTable, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { +func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, dataTable ImportedDataTable, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { if dataTable.DataRowCount() < 1 { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) - return nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile + return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile } nameDbTypeMap, err := c.buildTransactionTypeNameDbTypeMap() if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, err } headerLineItems := dataTable.HeaderLineColumnNames() @@ -276,15 +276,23 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists || !accountColumnExists || !amountColumnExists || !account2ColumnExists { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) - return nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } if accountMap == nil { accountMap = make(map[string]*models.Account) } - if categoryMap == nil { - categoryMap = make(map[string]*models.TransactionCategory) + if expenseCategoryMap == nil { + expenseCategoryMap = make(map[string]*models.TransactionCategory) + } + + if incomeCategoryMap == nil { + incomeCategoryMap = make(map[string]*models.TransactionCategory) + } + + if transferCategoryMap == nil { + transferCategoryMap = make(map[string]*models.TransactionCategory) } if tagMap == nil { @@ -293,7 +301,9 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u allNewTransactions := make(models.ImportedTransactionSlice, 0, dataTable.DataRowCount()) allNewAccounts := make([]*models.Account, 0) - allNewSubCategories := make([]*models.TransactionCategory, 0) + allNewSubExpenseCategories := make([]*models.TransactionCategory, 0) + allNewSubIncomeCategories := make([]*models.TransactionCategory, 0) + allNewSubTransferCategories := make([]*models.TransactionCategory, 0) allNewTags := make([]*models.TransactionTag, 0) dataRowIterator := dataTable.DataRowIterator() @@ -310,7 +320,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if columnCount < len(headerLineItems) { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse data row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", dataRowIndex, user.Uid, columnCount, len(headerLineItems)) - return nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } timezoneOffset := defaultTimezoneOffset @@ -320,7 +330,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time zone \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(timezoneColumnIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.ErrTransactionTimeZoneInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTimeZoneInvalid } timezoneOffset = utils.GetTimezoneOffsetMinutes(transactionTimezone) @@ -330,14 +340,14 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(timeColumnIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.ErrTransactionTimeInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTimeInvalid } transactionDbType, err := c.getTransactionDbType(nameDbTypeMap, dataRow.GetData(typeColumnIdx)) if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse transaction type \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(typeColumnIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid) + return nil, nil, nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid) } categoryId := int64(0) @@ -348,19 +358,42 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse transaction category type in data row \"index:%d\" for user \"uid:%d\", because %s", dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid) + return nil, nil, nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid) } subCategoryName = dataRow.GetData(subCategoryColumnIdx) - subCategory, exists := categoryMap[subCategoryName] - if !exists { - subCategory = c.createNewTransactionCategoryModel(user.Uid, subCategoryName, transactionCategoryType) - allNewSubCategories = append(allNewSubCategories, subCategory) - categoryMap[subCategoryName] = subCategory + if transactionDbType == models.TRANSACTION_DB_TYPE_EXPENSE { + subCategory, exists := expenseCategoryMap[subCategoryName] + + if !exists { + subCategory = c.createNewTransactionCategoryModel(user.Uid, subCategoryName, transactionCategoryType) + allNewSubExpenseCategories = append(allNewSubExpenseCategories, subCategory) + expenseCategoryMap[subCategoryName] = subCategory + } + + categoryId = subCategory.CategoryId + } else if transactionDbType == models.TRANSACTION_DB_TYPE_INCOME { + subCategory, exists := incomeCategoryMap[subCategoryName] + + if !exists { + subCategory = c.createNewTransactionCategoryModel(user.Uid, subCategoryName, transactionCategoryType) + allNewSubIncomeCategories = append(allNewSubIncomeCategories, subCategory) + incomeCategoryMap[subCategoryName] = subCategory + } + + categoryId = subCategory.CategoryId + } else if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { + subCategory, exists := transferCategoryMap[subCategoryName] + + if !exists { + subCategory = c.createNewTransactionCategoryModel(user.Uid, subCategoryName, transactionCategoryType) + allNewSubTransferCategories = append(allNewSubTransferCategories, subCategory) + transferCategoryMap[subCategoryName] = subCategory + } + + categoryId = subCategory.CategoryId } - - categoryId = subCategory.CategoryId } accountName := dataRow.GetData(accountColumnIdx) @@ -371,7 +404,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if _, ok := validators.AllCurrencyNames[accountCurrency]; !ok { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] account currency \"%s\" is not supported in data row \"index:%d\" for user \"uid:%d\"", accountCurrency, dataRowIndex, user.Uid) - return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid } } @@ -386,7 +419,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if accountCurrencyColumnExists { if account.Name != "" && account.Currency != accountCurrency { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account for user \"uid:%d\"", accountCurrency, dataRowIndex, account.Currency, user.Uid) - return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid } } else if exists { accountCurrency = account.Currency @@ -396,7 +429,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(amountColumnIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.ErrAmountInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrAmountInvalid } relatedAccountId := int64(0) @@ -413,7 +446,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if _, ok := validators.AllCurrencyNames[account2Currency]; !ok { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] account2 currency \"%s\" is not supported in data row \"index:%d\" for user \"uid:%d\"", account2Currency, dataRowIndex, user.Uid) - return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid } } @@ -428,7 +461,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if account2CurrencyColumnExists { if account2.Name != "" && account2.Currency != account2Currency { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account2 for user \"uid:%d\"", account2Currency, dataRowIndex, account2.Currency, user.Uid) - return nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid } } else if exists { account2Currency = account2.Currency @@ -441,7 +474,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount2 \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(amount2ColumnIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.ErrAmountInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrAmountInvalid } } else if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { relatedAccountAmount = amount @@ -459,14 +492,14 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(geoLocationIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.ErrGeographicLocationInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrGeographicLocationInvalid } geoLatitude, err = utils.StringToFloat64(geoLocationItems[1]) if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(geoLocationIdx), dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, errs.ErrGeographicLocationInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrGeographicLocationInvalid } } } @@ -537,7 +570,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u if err != nil { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot post process data row \"index:%d\" for user \"uid:%d\", because %s", dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, err } } @@ -546,7 +579,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u sort.Sort(allNewTransactions) - return allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, nil + return allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, nil } func (c *DataTableTransactionDataImporter) buildTransactionTypeNameDbTypeMap() (map[string]models.TransactionDbType, error) { diff --git a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go index a100e2f5..4972a944 100644 --- a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go +++ b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go @@ -83,7 +83,7 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ToExportedContent(ctx c } // ParseImportedData returns the imported data by parsing the transaction plain text data -func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { +func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { dataTable, err := createNewezbookkeepingTransactionPlainTextDataTable( string(data), c.columnSeparator, @@ -91,7 +91,7 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx c ) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, err } dataTableImporter := datatable.CreateNewImporter( @@ -101,5 +101,5 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx c ezbookkeepingTagSeparator, ) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, categoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } diff --git a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go index db2abe4e..7adb4639 100644 --- a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go +++ b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go @@ -134,17 +134,19 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_MinimumValidData(t *te DefaultCurrency: "CNY", } - allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ "2024-09-01 00:00:00,Balance Modification,,Test Account,123.45,,\n"+ "2024-09-01 01:23:45,Income,Test Category,Test Account,0.12,,\n"+ "2024-09-01 12:34:56,Expense,Test Category2,Test Account,1.00,,\n"+ - "2024-09-01 23:59:59,Transfer,Test Category3,Test Account,0.05,Test Account2,0.05"), 0, nil, nil, nil) + "2024-09-01 23:59:59,Transfer,Test Category3,Test Account,0.05,Test Account2,0.05"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 4, len(allNewTransactions)) assert.Equal(t, 2, len(allNewAccounts)) - assert.Equal(t, 3, len(allNewSubCategories)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) assert.Equal(t, 0, len(allNewTags)) assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) @@ -184,14 +186,14 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_MinimumValidData(t *te assert.Equal(t, "Test Account2", allNewAccounts[1].Name) assert.Equal(t, "CNY", allNewAccounts[1].Currency) - assert.Equal(t, int64(1234567890), allNewSubCategories[0].Uid) - assert.Equal(t, "Test Category", allNewSubCategories[0].Name) + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[1].Uid) - assert.Equal(t, "Test Category2", allNewSubCategories[1].Name) + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[2].Uid) - assert.Equal(t, "Test Category3", allNewSubCategories[2].Name) + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) } func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTime(t *testing.T) { @@ -203,12 +205,12 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTime(t *te DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01T12:34:56,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01T12:34:56,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "09/01/2024 12:34:56,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "09/01/2024 12:34:56,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) } @@ -221,8 +223,8 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidType(t *te DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01 12:34:56,Type,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Type,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message) } @@ -235,8 +237,8 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidTimezone(t * DefaultCurrency: "CNY", } - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01 12:34:56,+08:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,+08:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) assert.Equal(t, int64(1725165296), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) @@ -251,8 +253,8 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTimezone(t DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01 12:34:56,Asia/Shanghai,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Asia/Shanghai,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeZoneInvalid.Message) } @@ -265,9 +267,9 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidAccountCurre DefaultCurrency: "CNY", } - allNewTransactions, allNewAccounts, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ "2024-09-01 01:23:45,Balance Modification,,Test Account,USD,123.45,,,\n"+ - "2024-09-01 12:34:56,Transfer,Test Category2,Test Account,USD,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil) + "2024-09-01 12:34:56,Transfer,Test Category2,Test Account,USD,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) @@ -292,14 +294,14 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidAccountCur DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ "2024-09-01 01:23:45,Balance Modification,,Test Account,USD,123.45,,,\n"+ - "2024-09-01 12:34:56,Transfer,Test Category3,Test Account,CNY,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil) + "2024-09-01 12:34:56,Transfer,Test Category3,Test Account,CNY,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ "2024-09-01 01:23:45,Balance Modification,,Test Account,USD,123.45,,,\n"+ - "2024-09-01 12:34:56,Transfer,Test Category3,Test Account2,CNY,1.23,Test Account,EUR,1.10"), 0, nil, nil, nil) + "2024-09-01 12:34:56,Transfer,Test Category3,Test Account2,CNY,1.23,Test Account,EUR,1.10"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) } @@ -312,12 +314,12 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseNotSupportedCurre DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ - "2024-09-01 01:23:45,Balance Modification,,Test Account,XXX,123.45,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Balance Modification,,Test Account,XXX,123.45,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ - "2024-09-01 01:23:45,Transfer,Test Category,Test Account,USD,123.45,Test Account2,XXX,123.45"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount\n"+ + "2024-09-01 01:23:45,Transfer,Test Category,Test Account,USD,123.45,Test Account2,XXX,123.45"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) } @@ -330,12 +332,12 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidAmount(t * DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123 45,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123 45,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAmountInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01 12:34:56,Transfer,Test Category,Test Account,123.45,Test Account2,123 45"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,Transfer,Test Category,Test Account,123.45,Test Account2,123 45"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAmountInvalid.Message) } @@ -348,15 +350,15 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseNoAmount2(t *test DefaultCurrency: "CNY", } - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,"), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, int64(12345), allNewTransactions[0].Amount) assert.Equal(t, int64(0), allNewTransactions[0].RelatedAccountAmount) - allNewTransactions, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2\n"+ - "2024-09-01 12:34:56,Transfer,Test Category,Test Account,123.45,Test Account2"), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2\n"+ + "2024-09-01 12:34:56,Transfer,Test Category,Test Account,123.45,Test Account2"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, int64(12345), allNewTransactions[0].Amount) @@ -372,8 +374,8 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidGeographicLo DefaultCurrency: "CNY", } - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,123.45 45.56"), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,123.45 45.56"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -390,19 +392,19 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidGeographic DefaultCurrency: "CNY", } - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,1"), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,1"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) assert.Equal(t, float64(0), allNewTransactions[0].GeoLongitude) assert.Equal(t, float64(0), allNewTransactions[0].GeoLatitude) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,a b"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,a b"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrGeographicLocationInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,1 "), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Geographic Location\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,1 "), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrGeographicLocationInvalid.Message) } @@ -415,8 +417,8 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseTag(t *testing.T) DefaultCurrency: "CNY", } - _, _, _, allNewTags, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Tags\n"+ - "2024-09-01 00:00:00,Balance Modification,,Test Account,123.45,,,foo;;bar.;#test;hello\tworld;;"), 0, nil, nil, nil) + _, _, _, _, _, allNewTags, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Tags\n"+ + "2024-09-01 00:00:00,Balance Modification,,Test Account,123.45,,,foo;;bar.;#test;hello\tworld;;"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) @@ -444,8 +446,8 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseDescription(t *te DefaultCurrency: "CNY", } - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Description\n"+ - "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,foo bar\t#test"), 0, nil, nil, nil) + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Type,Sub Category,Account,Amount,Account2,Account2 Amount,Description\n"+ + "2024-09-01 12:34:56,Expense,Test Category,Test Account,123.45,,,foo bar\t#test"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -461,7 +463,7 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_MissingFileHeader(t *t DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) } @@ -475,32 +477,32 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_MissingRequiredColumn( } // Missing Time Column - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ - "+08:00,Balance Modification,,Test Sub Category,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "+08:00,Balance Modification,,Test Sub Category,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Type Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ - "2024-09-01 00:00:00,+08:00,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Category,Sub Category,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Test Category,Test Sub Category,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Sub Category Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ - "2024-09-01 00:00:00,+08:00,Balance Modification,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Type,Account,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,Test Account,CNY,123.45,,,,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Account Name Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Category,Sub Category,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ - "2024-09-01 00:00:00,+08:00,Balance Modification,,Test Sub Category,CNY,123.45,,,,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Category,Sub Category,Account Currency,Amount,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,,Test Sub Category,CNY,123.45,,,,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Amount Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ - "2024-09-01 00:00:00,+08:00,Balance Modification,,Test Sub Category,Test Account,CNY,,,,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Account2,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,,Test Sub Category,Test Account,CNY,,,,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Account2 Name Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ - "2024-09-01 00:00:00,+08:00,Balance Modification,,Test Sub Category,Test Account,CNY,123.45,,,,,"), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Category,Sub Category,Account,Account Currency,Amount,Account2 Currency,Account2 Amount,Geographic Location,Tags,Description\n"+ + "2024-09-01 00:00:00,+08:00,Balance Modification,,Test Sub Category,Test Account,CNY,123.45,,,,,"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) } diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go index f1237734..f5706a61 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go @@ -24,22 +24,22 @@ var ( ) // ParseImportedData returns the imported data by parsing the feidee mymoney transaction csv data -func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { +func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { content := string(data) if strings.Index(content, feideeMymoneyTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom) != 0 { - return nil, nil, nil, nil, errs.ErrInvalidFileHeader + return nil, nil, nil, nil, nil, nil, errs.ErrInvalidFileHeader } allLines, err := c.parseAllLinesFromCsvData(ctx, content) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, err } if len(allLines) <= 1 { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) - return nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile + return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile } headerLineItems := allLines[0] @@ -62,7 +62,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists || !accountColumnExists || !amountColumnExists || !relatedIdColumnExists { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) - return nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } newColumns := make([]datatable.DataTableColumn, 0, 11) @@ -121,7 +121,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con if len(items) < len(headerLineItems) { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] cannot parse row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", i, user.Uid, len(items), len(headerLineItems)) - return nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } transactionType := data[datatable.DATA_TABLE_TRANSACTION_TYPE] @@ -142,7 +142,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con } else if transactionType == "转入" || transactionType == "转出" { if relatedId == "" { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] transfer transaction has blank related id in row \"index:%d\" for user \"uid:%d\"", i, user.Uid) - return nil, nil, nil, nil, errs.ErrRelatedIdCannotBeBlank + return nil, nil, nil, nil, nil, nil, errs.ErrRelatedIdCannotBeBlank } relatedData, exists := transferTransactionsMap[relatedId] @@ -168,17 +168,17 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con delete(transferTransactionsMap, relatedId) } else { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] transfer transaction type \"%s\" is not expected in row \"index:%d\" for user \"uid:%d\"", transactionType, i, user.Uid) - return nil, nil, nil, nil, errs.ErrTransactionTypeInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTypeInvalid } } else { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] cannot parse transaction type \"%s\" in row \"index:%d\" for user \"uid:%d\"", transactionType, i, user.Uid) - return nil, nil, nil, nil, errs.ErrTransactionTypeInvalid + return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTypeInvalid } } if len(transferTransactionsMap) > 0 { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] there are %d transactions (related id is %s) which don't have related records", len(transferTransactionsMap), c.getRelatedIds(transferTransactionsMap)) - return nil, nil, nil, nil, errs.ErrFoundRecordNotHasRelatedRecord + return nil, nil, nil, nil, nil, nil, errs.ErrFoundRecordNotHasRelatedRecord } dataTableImporter := datatable.CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc( @@ -187,7 +187,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con feideeMymoneyTransactionDataImporterPostProcess, ) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, categoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } func (c *feideeMymoneyTransactionDataCsvImporter) parseAllLinesFromCsvData(ctx core.Context, content string) ([][]string, error) { diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go index dff73c1f..be52d46e 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go @@ -21,7 +21,7 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi DefaultCurrency: "CNY", } - allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"123.45\",\"\",\"\"\n"+ "\"余额变更\",\"2024-09-01 01:00:00\",\"\",\"Test Account2\",\"-0.12\",\"\",\"\"\n"+ @@ -30,13 +30,15 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi "\"转出\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account\",\"0.05\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ "\"转入\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account2\",\"0.05\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ "\"转入\",\"2024-09-02 23:59:59\",\"Test Category3\",\"Test Account\",\"0.5\",\"\",\"00000000-0000-0000-0000-000000000002\"\n"+ - "\"转出\",\"2024-09-02 23:59:59\",\"Test Category3\",\"Test Account2\",\"0.5\",\"\",\"00000000-0000-0000-0000-000000000002\""), 0, nil, nil, nil) + "\"转出\",\"2024-09-02 23:59:59\",\"Test Category3\",\"Test Account2\",\"0.5\",\"\",\"00000000-0000-0000-0000-000000000002\""), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 6, len(allNewTransactions)) assert.Equal(t, 2, len(allNewAccounts)) - assert.Equal(t, 3, len(allNewSubCategories)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) assert.Equal(t, 0, len(allNewTags)) assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) @@ -91,14 +93,14 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi assert.Equal(t, "Test Account2", allNewAccounts[1].Name) assert.Equal(t, "CNY", allNewAccounts[1].Currency) - assert.Equal(t, int64(1234567890), allNewSubCategories[0].Uid) - assert.Equal(t, "Test Category", allNewSubCategories[0].Name) + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[1].Uid) - assert.Equal(t, "Test Category2", allNewSubCategories[1].Name) + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[2].Uid) - assert.Equal(t, "Test Category3", allNewSubCategories[2].Name) + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) } func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseInvalidTime(t *testing.T) { @@ -110,14 +112,14 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseInvalidTime(t *testi DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"收入\",\"2024-09-01T12:34:56\",\"Test Category\",\"Test Account\",\"0.12\",\"\",\"\""), 0, nil, nil, nil) + "\"收入\",\"2024-09-01T12:34:56\",\"Test Category\",\"Test Account\",\"0.12\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"收入\",\"09/01/2024 12:34:56\",\"Test Category\",\"Test Account\",\"0.12\",\"\",\"\""), 0, nil, nil, nil) + "\"收入\",\"09/01/2024 12:34:56\",\"Test Category\",\"Test Account\",\"0.12\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) } @@ -130,9 +132,9 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseInvalidType(t *testi DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"Type\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"0.12\",\"\",\"\""), 0, nil, nil, nil) + "\"Type\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"0.12\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message) } @@ -145,11 +147,11 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseValidAccountCurrency DefaultCurrency: "CNY", } - allNewTransactions, allNewAccounts, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"账户币种\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"USD\",\"123.45\",\"\",\"\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account\",\"USD\",\"1.23\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account2\",\"EUR\",\"1.10\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account2\",\"EUR\",\"1.10\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) @@ -174,18 +176,18 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseInvalidAccountCurren DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"账户币种\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"USD\",\"123.45\",\"\",\"\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account\",\"CNY\",\"1.23\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account2\",\"EUR\",\"1.10\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account2\",\"EUR\",\"1.10\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"账户币种\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"USD\",\"123.45\",\"\",\"\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account2\",\"CNY\",\"1.23\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account\",\"EUR\",\"1.10\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category3\",\"Test Account\",\"EUR\",\"1.10\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) } @@ -198,21 +200,21 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseNotSupportedCurrency DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"账户币种\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"XXX\",\"123.45\",\"\",\"\""), 0, nil, nil, nil) + "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"XXX\",\"123.45\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"账户币种\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"USD\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"XXX\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"XXX\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"账户币种\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"XXX\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"USD\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"USD\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) } @@ -225,21 +227,21 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseInvalidAmount(t *tes DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"123 45\",\"\",\"\""), 0, nil, nil, nil) + "\"余额变更\",\"2024-09-01 01:23:45\",\"\",\"Test Account\",\"123 45\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAmountInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"123 45\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAmountInvalid.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"转出\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"123.45\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"123 45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account2\",\"123 45\",\"\",\"00000000-0000-0000-0000-000000000001\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrAmountInvalid.Message) } @@ -252,10 +254,10 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_ParseDescription(t *testi DefaultCurrency: "CNY", } - allNewTransactions, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"支出\",\"2024-09-01 12:34:56\",\"Test Category\",\"Test Account\",\"123.45\",\"Test\n"+ - "A new line break\",\"\""), 0, nil, nil, nil) + "A new line break\",\"\""), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) @@ -271,20 +273,20 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_InvalidRelatedId(t *testi DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"转出\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account\",\"0.05\",\"\",\"\""), 0, nil, nil, nil) + "\"转出\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account\",\"0.05\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrRelatedIdCannotBeBlank.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"转入\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account\",\"0.05\",\"\",\"\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account\",\"0.05\",\"\",\"\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrRelatedIdCannotBeBlank.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ "\"转出\",\"2024-09-01 23:59:59\",\"Test Category3\",\"Test Account\",\"0.05\",\"\",\"00000000-0000-0000-0000-000000000001\"\n"+ - "\"转入\",\"2024-09-02 23:59:59\",\"Test Category3\",\"Test Account\",\"0.5\",\"\",\"00000000-0000-0000-0000-000000000002\""), 0, nil, nil, nil) + "\"转入\",\"2024-09-02 23:59:59\",\"Test Category3\",\"Test Account\",\"0.5\",\"\",\"00000000-0000-0000-0000-000000000002\""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrFoundRecordNotHasRelatedRecord.Message) } @@ -296,11 +298,11 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MissingFileHeader(t *test Uid: 1, DefaultCurrency: "CNY", } - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil) + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ + "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrInvalidFileHeader.Message) - _, _, _, _, err = converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil) + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrInvalidFileHeader.Message) } @@ -314,38 +316,38 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MissingRequiredColumn(t * } // Missing Time Column - _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil) + "\"余额变更\",\"\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Type Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"2024-09-01 00:00:00\",\"Test Category\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil) + "\"2024-09-01 00:00:00\",\"Test Category\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Sub Category Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"账户\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"2024-09-01 00:00:00\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil) + "\"余额变更\",\"2024-09-01 00:00:00\",\"Test Account\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Account Name Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"金额\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil) + "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"123.45\",\"\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Amount Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"备注\",\"关联Id\"\n"+ - "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"\",\"\"\n"), 0, nil, nil, nil) + "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) // Missing Related ID Column - _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("随手记导出文件(headers:v5;xxxxx)\n"+ "\"交易类型\",\"日期\",\"子类别\",\"账户\",\"金额\",\"备注\"\n"+ - "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"123.45\",\"\"\n"), 0, nil, nil, nil) + "\"余额变更\",\"2024-09-01 00:00:00\",\"\",\"Test Account\",\"123.45\",\"\"\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) } diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go index 663e5a2e..dbe4deb7 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go @@ -17,11 +17,11 @@ var ( ) // ParseImportedData returns the imported data by parsing the feidee mymoney transaction xls data -func (c *feideeMymoneyTransactionDataXlsImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, categoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionTag, error) { +func (c *feideeMymoneyTransactionDataXlsImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { dataTable, err := createNewFeideeMymoneyTransactionExcelFileDataTable(data) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, err } dataTableImporter := datatable.CreateNewSimpleImporterWithPostProcessFunc( @@ -30,5 +30,5 @@ func (c *feideeMymoneyTransactionDataXlsImporter) ParseImportedData(ctx core.Con feideeMymoneyTransactionDataImporterPostProcess, ) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, categoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go index f9094d73..dc56a33e 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go @@ -24,12 +24,14 @@ func TestFeideeMymoneyTransactionDataXlsImporterParseImportedData_MinimumValidDa testdata, err := os.ReadFile("../../../testdata/feidee_mymoney_test_file.xls") assert.Nil(t, err) - allNewTransactions, allNewAccounts, allNewSubCategories, allNewTags, err := converter.ParseImportedData(context, user, testdata, 0, nil, nil, nil) + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, testdata, 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 7, len(allNewTransactions)) assert.Equal(t, 2, len(allNewAccounts)) - assert.Equal(t, 5, len(allNewSubCategories)) + assert.Equal(t, 2, len(allNewSubExpenseCategories)) + assert.Equal(t, 2, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) assert.Equal(t, 0, len(allNewTags)) assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) @@ -92,18 +94,18 @@ func TestFeideeMymoneyTransactionDataXlsImporterParseImportedData_MinimumValidDa assert.Equal(t, "Test Account2", allNewAccounts[1].Name) assert.Equal(t, "CNY", allNewAccounts[1].Currency) - assert.Equal(t, int64(1234567890), allNewSubCategories[0].Uid) - assert.Equal(t, "Test Category", allNewSubCategories[0].Name) + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[1].Uid) - assert.Equal(t, "Test Category5", allNewSubCategories[1].Name) + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[1].Uid) + assert.Equal(t, "Test Category4", allNewSubExpenseCategories[1].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[2].Uid) - assert.Equal(t, "Test Category2", allNewSubCategories[2].Name) + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[3].Uid) - assert.Equal(t, "Test Category4", allNewSubCategories[3].Name) + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[1].Uid) + assert.Equal(t, "Test Category5", allNewSubIncomeCategories[1].Name) - assert.Equal(t, int64(1234567890), allNewSubCategories[4].Uid) - assert.Equal(t, "Test Category3", allNewSubCategories[4].Name) + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) } diff --git a/pkg/services/transaction_categories.go b/pkg/services/transaction_categories.go index be1e0e58..d52feaa9 100644 --- a/pkg/services/transaction_categories.go +++ b/pkg/services/transaction_categories.go @@ -449,14 +449,24 @@ func (s *TransactionCategoryService) GetCategoryMapByList(categories []*models.T } // GetCategoryNameMapByList returns a transaction category map by a list -func (s *TransactionCategoryService) GetCategoryNameMapByList(categories []*models.TransactionCategory) map[string]*models.TransactionCategory { - categoryMap := make(map[string]*models.TransactionCategory) +func (s *TransactionCategoryService) GetCategoryNameMapByList(categories []*models.TransactionCategory) (expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory) { + expenseCategoryMap = make(map[string]*models.TransactionCategory) + incomeCategoryMap = make(map[string]*models.TransactionCategory) + transferCategoryMap = make(map[string]*models.TransactionCategory) for i := 0; i < len(categories); i++ { category := categories[i] - categoryMap[category.Name] = category + + if category.Type == models.CATEGORY_TYPE_INCOME { + incomeCategoryMap[category.Name] = category + } else if category.Type == models.CATEGORY_TYPE_EXPENSE { + expenseCategoryMap[category.Name] = category + } else if category.Type == models.CATEGORY_TYPE_TRANSFER { + transferCategoryMap[category.Name] = category + } } - return categoryMap + + return expenseCategoryMap, incomeCategoryMap, transferCategoryMap } // GetCategoryNames returns a list with transaction category names from transaction category models list