fix the incorrect transaction type and amount when importing some Firefly III data

This commit is contained in:
MaysWind
2025-07-13 16:00:53 +08:00
parent 29d14bb5ef
commit 06ef2220d6
3 changed files with 299 additions and 88 deletions
@@ -55,8 +55,10 @@ func (c *fireflyIIITransactionDataCsvFileImporter) ParseImportedData(ctx core.Co
if !commonDataTable.HasColumn(fireflyIIITransactionTimeColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionTypeColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionSourceAccountColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionDestinationAccountColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionSourceAccountNameColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionSourceAccountTypeColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionDestinationAccountNameColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionDestinationAccountTypeColumnName) ||
!commonDataTable.HasColumn(fireflyIIITransactionAmountColumnName) {
log.Errorf(ctx, "[fireflyiii_transaction_data_csv_file_importer.ParseImportedData] cannot parse Firefly III csv data, because missing essential columns in header row")
return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow
@@ -20,11 +20,11 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MinimumValidData(t *testing
DefaultCurrency: "CNY",
}
allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"\"Opening balance\",123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"+
"Deposit,0.12,2024-09-01T01:23:45+08:00,\"A revenue account\",\"Test Account\",\"Test Category\"\n"+
"Withdrawal,1.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"A expense account\",\"Test Category2\"\n"+
"Transfer,0.05,2024-09-01T23:59:59+08:00,\"Test Account\",\"Test Account2\",\"Test Category3\""), 0, nil, nil, nil, nil, nil)
allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"+
"Deposit,0.12,2024-09-01T01:23:45+08:00,\"A revenue account\",\"Revenue account\",\"Test Account\",\"Asset account\",\"Test Category\"\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category2\"\n"+
"Transfer,0.05,2024-09-01T23:59:59+08:00,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category3\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
@@ -91,16 +91,16 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidTime(t *testing
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,1.00,2024-09-01T12:34:56,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,1.00,2024-09-01 12:34:56+08:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-1.00,2024-09-01 12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidType(t *testing.T) {
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidTransactionType(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
@@ -109,11 +109,134 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidType(t *testing
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Type,123.45,2024-09-01T12:34:56+08:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
// income transactions
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Deposit,10.00,2024-09-01T12:34:56+08:00,\"A revenue account\",\"Revenue account\",\"Test Account\",\"Asset account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Deposit,10.00,2024-09-01T12:34:56+08:00,\"A revenue account\",\"Revenue account\",\"Test Account\",\"Debt\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
// expense transactions
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Debt\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
// opening balance transactions
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",10.00,2024-09-01T12:34:56+08:00,\"Initial balance\",\"Initial balance account\",\"Test Account\",\"Asset account\",\"\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
// transfer transactions
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[0].OriginalDestinationAccountName)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"Test Account2\",\"Debt\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[0].OriginalDestinationAccountName)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Deposit,10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Debt\",\"Test Account2\",\"Asset account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[0].OriginalDestinationAccountName)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Debt\",\"Test Account2\",\"Debt\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[0].OriginalDestinationAccountName)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidTransactionType(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,10.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Revenue account\",\"Test Account2\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseAccountNameAsCategoryName(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, "A expense account", allNewTransactions[0].OriginalCategoryName)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Deposit,10.00,2024-09-01T12:34:56+08:00,\"A revenue account\",\"Revenue account\",\"Test Account\",\"Asset account\",\"\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, "A revenue account", allNewTransactions[0].OriginalCategoryName)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidTimezone(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
@@ -123,20 +246,20 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidTimezone(t *testi
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,1.00,2024-09-01T12:34:56-10:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56-10:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,1.00,2024-09-01T12:34:56+00:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+00:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+12:45,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+12:45,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
@@ -151,9 +274,9 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidAccountCurrency(t
DefaultCurrency: "CNY",
}
allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,destination_name,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,USD,,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"+
"Transfer,1.23,-1.10,2024-09-01T23:59:59+08:00,USD,EUR,\"Test Account\",\"Test Account2\",\"Test Category2\""), 0, nil, nil, nil, nil, nil)
allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,USD,,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"+
"Transfer,1.23,-1.10,2024-09-01T23:59:59+08:00,USD,EUR,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category2\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
@@ -169,6 +292,45 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidAccountCurrency(t
assert.Equal(t, "EUR", allNewAccounts[1].Currency)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidForeignAmountAndCurrency(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,10.00,15.00,2024-09-01T12:34:56+08:00,USD,EUR,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, int64(1500), allNewTransactions[0].RelatedAccountAmount)
assert.Equal(t, "USD", allNewTransactions[0].OriginalSourceAccountCurrency)
assert.Equal(t, "EUR", allNewTransactions[0].OriginalDestinationAccountCurrency)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,10.00,2024-09-01T12:34:56+08:00,USD,EUR,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1000), allNewTransactions[0].Amount)
assert.Equal(t, int64(1000), allNewTransactions[0].RelatedAccountAmount)
assert.Equal(t, "USD", allNewTransactions[0].OriginalSourceAccountCurrency)
assert.Equal(t, "EUR", allNewTransactions[0].OriginalDestinationAccountCurrency)
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,10.00,2024-09-01T12:34:56+08:00,USD,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, "USD", allNewTransactions[0].OriginalSourceAccountCurrency)
assert.Equal(t, "USD", allNewTransactions[0].OriginalDestinationAccountCurrency)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidAccountCurrency(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
@@ -178,14 +340,14 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidAccountCurrency
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,destination_name,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,USD,,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"+
"Transfer,1.23,1.10,2024-09-01T23:59:59+08:00,CNY,EUR,\"Test Account\",\"Test Account2\",\"Test Category3\""), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,USD,,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"+
"Transfer,1.23,1.10,2024-09-01T23:59:59+08:00,CNY,EUR,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category3\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,destination_name,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,USD,,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"+
"Transfer,1.23,1.10,2024-09-01T23:59:59+08:00,CNY,EUR,\"Test Account2\",\"Test Account\",\"Test Category3\""), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,USD,,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\n"+
"Transfer,1.23,1.10,2024-09-01T23:59:59+08:00,CNY,EUR,\"Test Account2\",\"Asset account\",\"Test Account\",\"Asset account\",\"Test Category3\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message)
}
@@ -198,12 +360,12 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseNotSupportedCurrency(t
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,destination_name,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,XXX,,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,,2024-09-01T00:00:00+08:00,XXX,,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,destination_name,category\n"+
"Transfer,123.45,123.45,2024-09-01T23:59:59+08:00,USD,XXX,\"Test Account\",\"Test Account2\",\"Test Category2\""), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,currency_code,foreign_currency_code,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,123.45,123.45,2024-09-01T23:59:59+08:00,USD,XXX,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category2\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message)
}
@@ -216,12 +378,12 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidAmount(t *testi
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,123 45,2024-09-01T12:34:56+08:00,\"Test Account\",\"A expense account\",\"Test Category\"\n"), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-123 45,2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\"\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAmountInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,source_name,destination_name,category\n"+
"Transfer,123.45,123 45,2024-09-01T23:59:59+08:00,\"Test Account\",\"Test Account2\",\"Test Category2\""), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,foreign_amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"Transfer,123.45,123 45,2024-09-01T23:59:59+08:00,\"Test Account\",\"Asset account\",\"Test Account2\",\"Asset account\",\"Test Category2\""), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAmountInvalid.Message)
}
@@ -234,14 +396,37 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseDescription(t *testing
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,description,date,source_name,destination_name,category\n"+
"Withdrawal,123.45,\"foo bar\t#test\",2024-09-01T12:34:56+08:00,\"Test Account\",\"A expense account\",\"Test Category\"\n"), 0, nil, nil, nil, nil, nil)
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,description,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-123.45,\"foo bar\t#test\",2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\"\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, "foo bar\t#test", allNewTransactions[0].Comment)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseTags(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, allNewTags, err := converter.ParseImportedData(context, user, []byte("type,amount,tags,date,source_name,source_type,destination_name,destination_type,category\n"+
"Withdrawal,-123.45,\"tag1,tag2,tag3\",2024-09-01T12:34:56+08:00,\"Test Account\",\"Asset account\",\"A expense account\",\"Expense account\",\"Test Category\"\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, 3, len(allNewTags))
assert.Equal(t, int64(1234567890), allNewTags[0].Uid)
assert.Equal(t, "tag1", allNewTags[0].Name)
assert.Equal(t, int64(1234567890), allNewTags[1].Uid)
assert.Equal(t, "tag2", allNewTags[1].Name)
assert.Equal(t, int64(1234567890), allNewTags[2].Uid)
assert.Equal(t, "tag3", allNewTags[2].Name)
}
func TestFireFlyIIICsvFileConverterParseImportedData_MissingFileHeader(t *testing.T) {
converter := FireflyIIITransactionDataCsvFileImporter
context := core.NewNullContext()
@@ -265,13 +450,13 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MissingRequiredColumn(t *te
}
// Missing Time Column
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,source_name,destination_name,category\n"+
"\"Opening balance\",123.45,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Type Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("amount,date,source_name,destination_name,category\n"+
"123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("amount,date,source_name,source_type,destination_name,destination_type,category\n"+
"123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Account Name Column
@@ -280,12 +465,22 @@ func TestFireFlyIIICsvFileConverterParseImportedData_MissingRequiredColumn(t *te
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Amount Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,date,source_name,destination_name,category\n"+
"\"Opening balance\",2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,date,source_name,source_type,destination_name,destination_type,category\n"+
"\"Opening balance\",2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Initial balance account\",\"Test Account\",\"Asset account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Account2 Name Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,category\n"+
"\"Opening balance\",123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Source Account Type Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,destination_type,category\n"+
"\"Opening balance\",123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Test Account\",\"Asset account\",\"Asset account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Destination Account Type Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,source_type,destination_name,category\n"+
"\"Opening balance\",123.45,2024-09-01T00:00:00+08:00,\"Initial balance for \"\"Test Account\"\"\",\"Asset account\",\"Test Account\",\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
}
@@ -4,6 +4,7 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
@@ -11,15 +12,22 @@ import (
const fireflyIIITransactionTimeColumnName = "date"
const fireflyIIITransactionTypeColumnName = "type"
const fireflyIIITransactionCategoryColumnName = "category"
const fireflyIIITransactionSourceAccountColumnName = "source_name"
const fireflyIIITransactionSourceAccountNameColumnName = "source_name"
const fireflyIIITransactionSourceAccountTypeColumnName = "source_type"
const fireflyIIITransactionCurrencyCodeColumnName = "currency_code"
const fireflyIIITransactionAmountColumnName = "amount"
const fireflyIIITransactionDestinationAccountColumnName = "destination_name"
const fireflyIIITransactionDestinationAccountNameColumnName = "destination_name"
const fireflyIIITransactionDestinationAccountTypeColumnName = "destination_type"
const fireflyIIITransactionForeignCurrencyCodeColumnName = "foreign_currency_code"
const fireflyIIITransactionForeignAmountColumnName = "foreign_amount"
const fireflyIIITransactionTagsColumnName = "tags"
const fireflyIIITransactionDescriptionColumnName = "description"
const fireflyIIIAssetAccountName = "Asset account"
const fireflyIIIExpenseAccountName = "Expense account"
const fireflyIIIRevenueAccountName = "Revenue account"
const fireflyIIIDebtAccountName = "Debt"
// fireflyIIITransactionDataRowParser defines the structure of firefly III transaction data row parser
type fireflyIIITransactionDataRowParser struct {
}
@@ -38,63 +46,69 @@ func (p *fireflyIIITransactionDataRowParser) Parse(ctx core.Context, user *model
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location())
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location())
// parse transaction type
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(fireflyIIITransactionTypeColumnName)
// parse transaction type, transaction category and amount
transactionType := dataRow.GetData(fireflyIIITransactionTypeColumnName)
sourceAccountType := dataRow.GetData(fireflyIIITransactionSourceAccountTypeColumnName)
destinationAccountType := dataRow.GetData(fireflyIIITransactionDestinationAccountTypeColumnName)
amount, err := utils.ParseAmount(utils.TrimTrailingZerosInDecimal(dataRow.GetData(fireflyIIITransactionAmountColumnName)))
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
foreignAmount := amount
if dataRow.HasData(fireflyIIITransactionForeignAmountColumnName) && dataRow.GetData(fireflyIIITransactionForeignAmountColumnName) != "" {
foreignAmount, err = utils.ParseAmount(utils.TrimTrailingZerosInDecimal(dataRow.GetData(fireflyIIITransactionForeignAmountColumnName)))
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
}
// parse transaction category
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionCategoryColumnName)
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
if sourceAccountType == fireflyIIIRevenueAccountName && (destinationAccountType == fireflyIIIAssetAccountName || destinationAccountType == fireflyIIIDebtAccountName) { // income
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
// if the category is empty, use the source account (revenue account) name as the category name
if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" {
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionSourceAccountNameColumnName)
}
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
} else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountNameColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount)
} else if (sourceAccountType == fireflyIIIAssetAccountName || sourceAccountType == fireflyIIIDebtAccountName) && destinationAccountType == fireflyIIIExpenseAccountName { // expense
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
// if the category is empty, use the destination account (expense account) name as the category name
if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" {
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(fireflyIIITransactionDestinationAccountNameColumnName)
}
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName)
} else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
} else if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountNameColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
} else if transactionType == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] { // opening balance
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountColumnName)
} else {
return nil, false, errs.ErrTransactionTypeInvalid
}
// parse amount
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = dataRow.GetData(fireflyIIITransactionAmountColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = dataRow.GetData(fireflyIIITransactionForeignAmountColumnName)
// trim trailing zero in decimal
if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountNameColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount)
}
} else if (sourceAccountType == fireflyIIIAssetAccountName || sourceAccountType == fireflyIIIDebtAccountName) && (destinationAccountType == fireflyIIIAssetAccountName || destinationAccountType == fireflyIIIDebtAccountName) { // transfer
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionSourceAccountNameColumnName)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = dataRow.GetData(fireflyIIITransactionDestinationAccountNameColumnName)
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
if transactionType == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] {
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-foreignAmount)
} else {
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount)
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(foreignAmount)
}
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(amount)
} else {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
log.Errorf(ctx, "[fireflyiii_transaction_data_row_parser.Parse] cannot detect transaction type, source account type is \"%s\", destination account type is \"%s\", Firefly III transaction type is \"%s\"", sourceAccountType, destinationAccountType, transactionType)
return nil, false, errs.ErrTransactionTypeInvalid
}
// parse account currency