From 0413f8c0aaf634eabc53f13bc3dbad52a7fca04c Mon Sep 17 00:00:00 2001 From: MaysWind Date: Sun, 13 Jul 2025 01:51:27 +0800 Subject: [PATCH] use the expense and revenue account names as category names if the transaction has not category when importing Firefly III transactions --- .../data_table_transaction_data_importer.go | 8 +- ...yiii_transaction_data_csv_file_importer.go | 40 ++++--- ...transaction_data_csv_file_importer_test.go | 7 +- .../fireflyiii_transaction_data_row_parser.go | 104 +++++++++++------- ...ash_transaction_data_file_importer_test.go | 2 +- .../gnucash/gnucash_transaction_data_table.go | 4 + .../mt/mt_transaction_data_table.go | 2 + .../ofx/ofx_transaction_data_table.go | 4 + 8 files changed, 107 insertions(+), 64 deletions(-) diff --git a/pkg/converters/converter/data_table_transaction_data_importer.go b/pkg/converters/converter/data_table_transaction_data_importer.go index e37af1ef..51182d50 100644 --- a/pkg/converters/converter/data_table_transaction_data_importer.go +++ b/pkg/converters/converter/data_table_transaction_data_importer.go @@ -188,7 +188,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u accountName := dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME) accountCurrency := user.DefaultCurrency - if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) { + if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) != "" { accountCurrency = dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) if _, ok := validators.AllCurrencyNames[accountCurrency]; !ok { @@ -205,7 +205,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u accountMap[accountName] = account } - if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) { + if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) != "" { if account.Name != "" && account.Currency != accountCurrency { log.Errorf(ctx, "[data_table_transaction_data_importer.ParseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account for user \"uid:%d\"", accountCurrency, dataRowIndex, account.Currency, user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid @@ -230,7 +230,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u account2Name = dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME) account2Currency = user.DefaultCurrency - if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) { + if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) != "" { account2Currency = dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) if _, ok := validators.AllCurrencyNames[account2Currency]; !ok { @@ -247,7 +247,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u accountMap[account2Name] = account2 } - if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) { + if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) != "" { if account2.Name != "" && account2.Currency != account2Currency { log.Errorf(ctx, "[data_table_transaction_data_importer.ParseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account2 for user \"uid:%d\"", account2Currency, dataRowIndex, account2.Currency, user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go index 1bd7db51..eaaecfd9 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go @@ -7,21 +7,24 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" "github.com/mayswind/ezbookkeeping/pkg/models" ) -var fireflyIIITransactionDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ - datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "date", - datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "type", - datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "category", - datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "source_name", - datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "currency_code", - datatable.TRANSACTION_DATA_TABLE_AMOUNT: "amount", - datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "destination_name", - datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "foreign_currency_code", - datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "foreign_amount", - datatable.TRANSACTION_DATA_TABLE_TAGS: "tags", - datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "description", +var fireflyIIITransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: true, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: true, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: true, + datatable.TRANSACTION_DATA_TABLE_TAGS: true, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true, } var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{ @@ -48,8 +51,19 @@ func (c *fireflyIIITransactionDataCsvFileImporter) ParseImportedData(ctx core.Co return nil, nil, nil, nil, nil, nil, err } + commonDataTable := datatable.CreateNewCommonDataTableFromBasicDataTable(dataTable) + + if !commonDataTable.HasColumn(fireflyIIITransactionTimeColumnName) || + !commonDataTable.HasColumn(fireflyIIITransactionTypeColumnName) || + !commonDataTable.HasColumn(fireflyIIITransactionSourceAccountColumnName) || + !commonDataTable.HasColumn(fireflyIIITransactionDestinationAccountColumnName) || + !commonDataTable.HasColumn(fireflyIIITransactionAmountColumnName) { + log.Errorf(ctx, "[fireflyiii_transaction_data_csv_file_importer.ParseImportedData] cannot parse Firefly III csv data, because missing essential columns in header row") + return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + } + transactionRowParser := createFireflyIIITransactionDataRowParser() - transactionDataTable := datatable.CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser) + transactionDataTable := datatable.CreateNewTransactionDataTableFromCommonDataTable(commonDataTable, fireflyIIITransactionSupportedColumns, transactionRowParser) dataTableImporter := converter.CreateNewImporterWithTypeNameMapping(fireflyIIITransactionTypeNameMapping, "", "", ",") return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go index 5344e499..ca9663a7 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go @@ -252,7 +252,7 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MissingFileHeader(t *testin } _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil, nil, nil) - assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) + assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) } func TestFireFlyIIICsvFileConverterParseImportedData_MissingRequiredColumn(t *testing.T) { @@ -274,11 +274,6 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MissingRequiredColumn(t *te "-123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil) assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) - // Missing Sub Category Column - _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name\n"+ - "\"Opening balance\",-123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\"\n"), 0, nil, nil, nil, nil, nil) - assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) - // Missing Account Name Column _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,destination_name,category\n"+ "\"Opening balance\",-123.45,2024-09-01T00:00:00+08:00,\"Test Account\",\n"), 0, nil, nil, nil, nil, nil) diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go index 76eadbac..ffd667f3 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go @@ -1,51 +1,77 @@ package fireflyIII import ( - "strings" - "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/models" "github.com/mayswind/ezbookkeeping/pkg/utils" ) +const fireflyIIITransactionTimeColumnName = "date" +const fireflyIIITransactionTypeColumnName = "type" +const fireflyIIITransactionCategoryColumnName = "category" +const fireflyIIITransactionSourceAccountColumnName = "source_name" +const fireflyIIITransactionCurrencyCodeColumnName = "currency_code" +const fireflyIIITransactionAmountColumnName = "amount" +const fireflyIIITransactionDestinationAccountColumnName = "destination_name" +const fireflyIIITransactionForeignCurrencyCodeColumnName = "foreign_currency_code" +const fireflyIIITransactionForeignAmountColumnName = "foreign_amount" +const fireflyIIITransactionTagsColumnName = "tags" +const fireflyIIITransactionDescriptionColumnName = "description" + // fireflyIIITransactionDataRowParser defines the structure of firefly III transaction data row parser type fireflyIIITransactionDataRowParser struct { } -// GetAddedColumns returns the added columns after converting the data row -func (p *fireflyIIITransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn { - return []datatable.TransactionDataTableColumn{ - datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE, - } -} - // Parse returns the converted transaction data row -func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { - rowData = make(map[datatable.TransactionDataTableColumn]string, len(data)) - - rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" - - for column, value := range data { - rowData[column] = value - } +func (p *fireflyIIITransactionDataRowParser) Parse(ctx core.Context, user *models.User, dataRow datatable.CommonDataTableRow, rowId string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { + rowData = make(map[datatable.TransactionDataTableColumn]string, len(fireflyIIITransactionSupportedColumns)) // parse long date time and timezone - if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" { - if strings.Index(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME], "T") <= 0 { - return nil, false, errs.ErrTransactionTimeInvalid - } + dateTime, err := utils.ParseFromLongDateTimeWithTimezoneRFC3339Format(dataRow.GetData(fireflyIIITransactionTimeColumnName)) - dateTime, err := utils.ParseFromLongDateTimeWithTimezone(strings.ReplaceAll(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME], "T", " ")) - - if err != nil { - return nil, false, errs.ErrTransactionTimeInvalid - } - - rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location()) - rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location()) + if err != nil { + return nil, false, errs.ErrTransactionTimeInvalid } + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location()) + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location()) + + // parse transaction type + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(fireflyIIITransactionTypeColumnName) + + // parse transaction category + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionCategoryColumnName) + + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { + // if the category is empty, use the source account (revenue account) name as the category name + if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" { + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName) + } + + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName) + } else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] { + // if the category is empty, use the destination account (expense account) name as the category name + if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" { + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName) + } + + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName) + } else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName) + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName) + } else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] { + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName) + } else { + return nil, false, errs.ErrTransactionTypeInvalid + } + + // parse amount + rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = dataRow.GetData(fireflyIIITransactionAmountColumnName) + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = dataRow.GetData(fireflyIIITransactionForeignAmountColumnName) + // trim trailing zero in decimal if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" { rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) @@ -71,25 +97,23 @@ func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.Transactio rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] } + // parse account currency + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = dataRow.GetData(fireflyIIITransactionCurrencyCodeColumnName) + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = dataRow.GetData(fireflyIIITransactionForeignCurrencyCodeColumnName) + // the related account currency field is foreign currency in firefly III actually - if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" { + if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" && rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] != "" { rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] } - // the destination account of modify balance transaction in firefly III is the asset account - if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] { - rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] - } - - // the destination account of income transaction in firefly III is the asset account - if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { - rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] - } + // parse tags / description + rowData[datatable.TRANSACTION_DATA_TABLE_TAGS] = dataRow.GetData(fireflyIIITransactionTagsColumnName) + rowData[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(fireflyIIITransactionDescriptionColumnName) return rowData, true, nil } // createFireflyIIITransactionDataRowParser returns firefly III transaction data row parser -func createFireflyIIITransactionDataRowParser() datatable.TransactionDataRowParser { +func createFireflyIIITransactionDataRowParser() datatable.CommonTransactionDataRowParser { return &fireflyIIITransactionDataRowParser{} } diff --git a/pkg/converters/gnucash/gnucash_transaction_data_file_importer_test.go b/pkg/converters/gnucash/gnucash_transaction_data_file_importer_test.go index df011df6..20331da3 100644 --- a/pkg/converters/gnucash/gnucash_transaction_data_file_importer_test.go +++ b/pkg/converters/gnucash/gnucash_transaction_data_file_importer_test.go @@ -818,7 +818,7 @@ func TestGnuCashTransactionDatabaseFileParseImportedData_MissingAccountRequiredN DefaultCurrency: "CNY", } - // Missing Transaction Time Node + // Missing Account Currency Node _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( "\n"+ "