zhengchen.tao/ezbookkeeping
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use the expense and revenue account names as category names if the transaction has not category when importing Firefly III transactions

Browse Source
This commit is contained in:
MaysWind
2025-07-13 01:51:27 +08:00
parent ca5c451d36
commit 0413f8c0aa
8 changed files with 107 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/converters/converter/data_table_transaction_data_importer.go
+4 -4
View File
@@ -188,7 +188,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
accountName := dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
accountCurrency := user.DefaultCurrency
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) {
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) != "" {
accountCurrency = dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY)
if _, ok := validators.AllCurrencyNames[accountCurrency]; !ok {
@@ -205,7 +205,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
accountMap[accountName] = account
}
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) {
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) != "" {
if account.Name != "" && account.Currency != accountCurrency {
log.Errorf(ctx, "[data_table_transaction_data_importer.ParseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account for user \"uid:%d\"", accountCurrency, dataRowIndex, account.Currency, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid
@@ -230,7 +230,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
account2Name = dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME)
account2Currency = user.DefaultCurrency
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) {
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) != "" {
account2Currency = dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY)
if _, ok := validators.AllCurrencyNames[account2Currency]; !ok {
@@ -247,7 +247,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
accountMap[account2Name] = account2
}
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) {
if dataTable.HasColumn(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) && dataRow.GetData(datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) != "" {
if account2.Name != "" && account2.Currency != account2Currency {
log.Errorf(ctx, "[data_table_transaction_data_importer.ParseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account2 for user \"uid:%d\"", account2Currency, dataRowIndex, account2.Currency, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid
pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go
+27 -13
View File
@@ -7,21 +7,24 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/converters/csv"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
var fireflyIIITransactionDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "date",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "type",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "category",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "source_name",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "currency_code",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "amount",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "destination_name",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "foreign_currency_code",
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "foreign_amount",
datatable.TRANSACTION_DATA_TABLE_TAGS: "tags",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "description",
var fireflyIIITransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: true,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true,
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: true,
datatable.TRANSACTION_DATA_TABLE_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_TAGS: true,
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true,
}
var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
@@ -48,8 +51,19 @@ func (c *fireflyIIITransactionDataCsvFileImporter) ParseImportedData(ctx core.Co
return nil, nil, nil, nil, nil, nil, err
}
commonDataTable := datatable.CreateNewCommonDataTableFromBasicDataTable(dataTable)
if !commonDataTable.HasColumn(fireflyIIITransactionTimeColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionTypeColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionSourceAccountColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionDestinationAccountColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionAmountColumnName) {
log.Errorf(ctx, "[fireflyiii_transaction_data_csv_file_importer.ParseImportedData] cannot parse Firefly III csv data, because missing essential columns in header row")
return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow
}
transactionRowParser := createFireflyIIITransactionDataRowParser()
transactionDataTable := datatable.CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser)
transactionDataTable := datatable.CreateNewTransactionDataTableFromCommonDataTable(commonDataTable, fireflyIIITransactionSupportedColumns, transactionRowParser)
dataTableImporter := converter.CreateNewImporterWithTypeNameMapping(fireflyIIITransactionTypeNameMapping, "", "", ",")
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go
+1 -6
View File
@@ -252,7 +252,7 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MissingFileHeader(t *testin
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
}
func TestFireFlyIIICsvFileConverterParseImportedData_MissingRequiredColumn(t *testing.T) {
@@ -274,11 +274,6 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MissingRequiredColumn(t *te
"-123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Sub Category Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name\n"+
"\"Opening balance\",-123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\"\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Account Name Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,destination_name,category\n"+
"\"Opening balance\",-123.45,2024-09-01T00:00:00+08:00,\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go
+64 -40
View File
@@ -1,51 +1,77 @@
package fireflyIII
import (
"strings"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
const fireflyIIITransactionTimeColumnName = "date"
const fireflyIIITransactionTypeColumnName = "type"
const fireflyIIITransactionCategoryColumnName = "category"
const fireflyIIITransactionSourceAccountColumnName = "source_name"
const fireflyIIITransactionCurrencyCodeColumnName = "currency_code"
const fireflyIIITransactionAmountColumnName = "amount"
const fireflyIIITransactionDestinationAccountColumnName = "destination_name"
const fireflyIIITransactionForeignCurrencyCodeColumnName = "foreign_currency_code"
const fireflyIIITransactionForeignAmountColumnName = "foreign_amount"
const fireflyIIITransactionTagsColumnName = "tags"
const fireflyIIITransactionDescriptionColumnName = "description"
// fireflyIIITransactionDataRowParser defines the structure of firefly III transaction data row parser
type fireflyIIITransactionDataRowParser struct {
}
// GetAddedColumns returns the added columns after converting the data row
func (p *fireflyIIITransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn {
return []datatable.TransactionDataTableColumn{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE,
}
}
// Parse returns the converted transaction data row
func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) {
rowData = make(map[datatable.TransactionDataTableColumn]string, len(data))
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
for column, value := range data {
rowData[column] = value
}
func (p *fireflyIIITransactionDataRowParser) Parse(ctx core.Context, user *models.User, dataRow datatable.CommonDataTableRow, rowId string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) {
rowData = make(map[datatable.TransactionDataTableColumn]string, len(fireflyIIITransactionSupportedColumns))
// parse long date time and timezone
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" {
if strings.Index(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME], "T") <= 0 {
return nil, false, errs.ErrTransactionTimeInvalid
}
dateTime, err := utils.ParseFromLongDateTimeWithTimezoneRFC3339Format(dataRow.GetData(fireflyIIITransactionTimeColumnName))
dateTime, err := utils.ParseFromLongDateTimeWithTimezone(strings.ReplaceAll(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME], "T", " "))
if err != nil {
return nil, false, errs.ErrTransactionTimeInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location())
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location())
if err != nil {
return nil, false, errs.ErrTransactionTimeInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location())
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location())
// parse transaction type
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(fireflyIIITransactionTypeColumnName)
// parse transaction category
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionCategoryColumnName)
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
// if the category is empty, use the source account (revenue account) name as the category name
if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" {
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName)
}
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
} else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] {
// if the category is empty, use the destination account (expense account) name as the category name
if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" {
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
}
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName)
} else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
} else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
} else {
return nil, false, errs.ErrTransactionTypeInvalid
}
// parse amount
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = dataRow.GetData(fireflyIIITransactionAmountColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = dataRow.GetData(fireflyIIITransactionForeignAmountColumnName)
// trim trailing zero in decimal
if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
@@ -71,25 +97,23 @@ func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.Transactio
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
}
// parse account currency
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = dataRow.GetData(fireflyIIITransactionCurrencyCodeColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = dataRow.GetData(fireflyIIITransactionForeignCurrencyCodeColumnName)
// the related account currency field is foreign currency in firefly III actually
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" && rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
}
// the destination account of modify balance transaction in firefly III is the asset account
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// the destination account of income transaction in firefly III is the asset account
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// parse tags / description
rowData[datatable.TRANSACTION_DATA_TABLE_TAGS] = dataRow.GetData(fireflyIIITransactionTagsColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(fireflyIIITransactionDescriptionColumnName)
return rowData, true, nil
}
// createFireflyIIITransactionDataRowParser returns firefly III transaction data row parser
func createFireflyIIITransactionDataRowParser() datatable.TransactionDataRowParser {
func createFireflyIIITransactionDataRowParser() datatable.CommonTransactionDataRowParser {
return &fireflyIIITransactionDataRowParser{}
}
pkg/converters/gnucash/gnucash_transaction_data_file_importer_test.go
+1 -1
View File
@@ -818,7 +818,7 @@ func TestGnuCashTransactionDatabaseFileParseImportedData_MissingAccountRequiredN
DefaultCurrency: "CNY",
}
// Missing Transaction Time Node
// Missing Account Currency Node
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"+
"<gnc-v2\n"+
pkg/converters/gnucash/gnucash_transaction_data_table.go
+4
View File
@@ -177,6 +177,8 @@ func (t *gnucashTransactionDataRowIterator) parseTransaction(ctx core.Context, u
if toAccount.Commodity != nil && toAccount.Commodity.Space == gnucashCommodityCurrencySpace {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = toAccount.Commodity.Id
} else {
return nil, false, errs.ErrAccountCurrencyInvalid
}
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = toAmount
@@ -207,6 +209,8 @@ func (t *gnucashTransactionDataRowIterator) parseTransaction(ctx core.Context, u
if fromAccount.Commodity != nil && fromAccount.Commodity.Space == gnucashCommodityCurrencySpace {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = fromAccount.Commodity.Id
} else {
return nil, false, errs.ErrAccountCurrencyInvalid
}
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = fromAmount
pkg/converters/mt/mt_transaction_data_table.go
+2
View File
@@ -122,6 +122,8 @@ func (t *mt940TransactionDataRowIterator) parseTransaction(ctx core.Context, use
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = mt940Data.OpeningBalance.Currency
} else if mt940Data.ClosingBalance != nil && mt940Data.ClosingBalance.Currency != "" {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = mt940Data.ClosingBalance.Currency
} else {
return nil, errs.ErrAccountCurrencyInvalid
}
amountValue := strings.ReplaceAll(statement.Amount, ",", ".") // decimal separator is comma in mt data
pkg/converters/ofx/ofx_transaction_data_table.go
+4
View File
@@ -148,6 +148,10 @@ func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = ofxTransaction.DefaultCurrency
}
if data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] == "" {
return nil, errs.ErrAccountCurrencyInvalid
}
if ofxTransaction.Amount == "" {
return nil, errs.ErrAmountInvalid
}