diff --git a/pkg/converters/ofx/ofx_transaction_data_file_importer_test.go b/pkg/converters/ofx/ofx_transaction_data_file_importer_test.go index 24fcc3af..e8b47df1 100644 --- a/pkg/converters/ofx/ofx_transaction_data_file_importer_test.go +++ b/pkg/converters/ofx/ofx_transaction_data_file_importer_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/models" "github.com/mayswind/ezbookkeeping/pkg/utils" ) @@ -22,38 +23,60 @@ func TestOFXTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte( ""+ " "+ - " "+ - " "+ - " CNY"+ - " "+ - " 123"+ - " "+ - " "+ - " "+ - " DEP"+ - " 20240901012345.000[+8:CST]"+ - " 123.45"+ - " "+ - " "+ - " CHECK"+ - " 20240901123456.000[+8:CST]"+ - " -0.12"+ - " "+ - " "+ - " XFER"+ - " 20240901235959.000[+8:CST]"+ - " -1.00"+ - " "+ - " "+ - " "+ - " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " "+ + " "+ + " CHECK"+ + " 20240901123456.000[+8:CST]"+ + " -0.12"+ + " "+ + " "+ + " XFER"+ + " 20240901235959.000[+8:CST]"+ + " -1.00"+ + " "+ + " "+ + " "+ + " "+ " "+ + " "+ + " "+ + " "+ + " USD"+ + " "+ + " 456"+ + " "+ + " "+ + " "+ + " ATM"+ + " 20240902012345.000[+8:CST]"+ + " 1.23"+ + " "+ + " "+ + " POS"+ + " 20240902123456.000[+8:CST]"+ + " -0.01"+ + " "+ + " "+ + " "+ + " "+ + " "+ ""), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) - assert.Equal(t, 3, len(allNewTransactions)) - assert.Equal(t, 2, len(allNewAccounts)) + assert.Equal(t, 5, len(allNewTransactions)) + assert.Equal(t, 3, len(allNewAccounts)) assert.Equal(t, 1, len(allNewSubExpenseCategories)) assert.Equal(t, 1, len(allNewSubIncomeCategories)) assert.Equal(t, 1, len(allNewSubTransferCategories)) @@ -83,6 +106,22 @@ func TestOFXTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) assert.Equal(t, "", allNewTransactions[2].OriginalDestinationAccountName) assert.Equal(t, "", allNewTransactions[2].OriginalCategoryName) + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[3].Type) + assert.Equal(t, int64(1725211425), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(123), allNewTransactions[3].Amount) + assert.Equal(t, "456", allNewTransactions[3].OriginalSourceAccountName) + assert.Equal(t, "USD", allNewTransactions[3].OriginalSourceAccountCurrency) + assert.Equal(t, "", allNewTransactions[3].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[4].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[4].Type) + assert.Equal(t, int64(1725251696), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime)) + assert.Equal(t, int64(1), allNewTransactions[4].Amount) + assert.Equal(t, "456", allNewTransactions[4].OriginalSourceAccountName) + assert.Equal(t, "USD", allNewTransactions[4].OriginalSourceAccountCurrency) + assert.Equal(t, "", allNewTransactions[4].OriginalCategoryName) + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) assert.Equal(t, "123", allNewAccounts[0].Name) assert.Equal(t, "CNY", allNewAccounts[0].Currency) @@ -91,6 +130,10 @@ func TestOFXTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) assert.Equal(t, "", allNewAccounts[1].Name) assert.Equal(t, "CNY", allNewAccounts[1].Currency) + assert.Equal(t, int64(1234567890), allNewAccounts[2].Uid) + assert.Equal(t, "456", allNewAccounts[2].Name) + assert.Equal(t, "USD", allNewAccounts[2].Currency) + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) assert.Equal(t, "", allNewSubExpenseCategories[0].Name) @@ -100,3 +143,501 @@ func TestOFXTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) assert.Equal(t, "", allNewSubTransferCategories[0].Name) } + +func TestOFXTransactionDataFileParseImportedData_ParseValidTransactionTime(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901"+ + " 123.45"+ + " "+ + " "+ + " DEP"+ + " 20240901123456"+ + " 123.45"+ + " "+ + " "+ + " DEP"+ + " 20240901123456.789"+ + " 123.45"+ + " "+ + " "+ + " DEP"+ + " 20240901125959.000[-3]"+ + " 123.45"+ + " "+ + " "+ + " DEP"+ + " 20240901122959.000[-3.5]"+ + " 123.45"+ + " "+ + " "+ + " DEP"+ + " 20240902030405.000[0]"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 6, len(allNewTransactions)) + + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(1725206399), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(1725206399), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime)) + assert.Equal(t, int64(1725246245), utils.GetUnixTimeFromTransactionTime(allNewTransactions[5].TransactionTime)) +} + +func TestOFXTransactionDataFileParseImportedData_ParseInvalidTransactionTime(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 2024"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 2024-09-01"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 202491"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901 12:34:56"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) +} + +func TestOFXTransactionDataFileParseImportedData_ParseAmount_CommaAsDecimalPoint(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123,45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) +} + +func TestOFXTransactionDataFileParseImportedData_ParseInvalidAmount(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123 45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} + +func TestOFXTransactionDataFileParseImportedData_ParseTransactionCurrency(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " USD"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, "USD", allNewTransactions[0].OriginalSourceAccountCurrency) +} + +func TestOFXTransactionDataFileParseImportedData_ParseDescription(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " Test"+ + " foo bar\t#test"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, "foo bar\t#test", allNewTransactions[0].Comment) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " Test"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, "Test", allNewTransactions[0].Comment) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " "+ + " Test"+ + " "+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, "Test", allNewTransactions[0].Comment) +} + +func TestOFXTransactionDataFileParseImportedData_MissingAccountFromNode(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1, + DefaultCurrency: "CNY", + } + + // Missing Posted Date Node + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrMissingAccountData.Message) +} + +func TestOFXTransactionDataFileParseImportedData_MissingCurrencyNode(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1, + DefaultCurrency: "CNY", + } + + // Missing Default Currency Node + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) +} + +func TestOFXTransactionDataFileParseImportedData_MissingTransactionRequiredNode(t *testing.T) { + converter := OFXTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1, + DefaultCurrency: "CNY", + } + + // Missing Posted Date Node + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrMissingTransactionTime.Message) + + // Missing Transaction Type Node + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " 20240901012345.000[+8:CST]"+ + " 123.45"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message) + + // Missing Amount Node + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + ""+ + " "+ + " "+ + " "+ + " CNY"+ + " "+ + " 123"+ + " "+ + " "+ + " "+ + " DEP"+ + " 20240901012345.000[+8:CST]"+ + " "+ + " "+ + " "+ + " "+ + " "+ + ""), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} diff --git a/pkg/converters/ofx/ofx_transaction_table.go b/pkg/converters/ofx/ofx_transaction_table.go index 9d16a79d..6fea776c 100644 --- a/pkg/converters/ofx/ofx_transaction_table.go +++ b/pkg/converters/ofx/ofx_transaction_table.go @@ -117,6 +117,10 @@ func (t *ofxTransactionDataRowIterator) Next(ctx core.Context, user *models.User func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user *models.User, ofxTransaction *ofxTransactionData) (map[datatable.TransactionDataTableColumn]string, error) { data := make(map[datatable.TransactionDataTableColumn]string, len(ofxTransactionSupportedColumns)) + if ofxTransaction.PostedDate == "" { + return nil, errs.ErrMissingTransactionTime + } + datetime, timezone, err := t.parseTransactionTimeAndTimeZone(ctx, ofxTransaction.PostedDate) if err != nil { @@ -126,12 +130,20 @@ func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = datetime data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = timezone + if ofxTransaction.Amount == "" { + return nil, errs.ErrAmountInvalid + } + amount, err := utils.ParseAmount(strings.ReplaceAll(ofxTransaction.Amount, ",", ".")) // ofx supports decimal point or comma to indicate the start of the fractional amount if err != nil { return nil, errs.ErrAmountInvalid } + if ofxTransaction.TransactionType == "" { + return nil, errs.ErrTransactionTypeInvalid + } + if transactionType, exists := ofxTransactionTypeMapping[ofxTransaction.TransactionType]; exists { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = utils.IntToString(int(transactionType)) @@ -150,6 +162,10 @@ func (t *ofxTransactionDataRowIterator) parseTransaction(ctx core.Context, user } } + if ofxTransaction.FromAccountId == "" { + return nil, errs.ErrMissingAccountData + } + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = ofxTransaction.FromAccountId if ofxTransaction.Currency != "" { @@ -189,12 +205,22 @@ func (t *ofxTransactionDataRowIterator) parseTransactionTimeAndTimeZone(ctx core tzOffset := ofxDefaultTimezoneOffset if len(datetime) >= 8 { // YYYYMMDD + if !utils.IsStringOnlyContainsDigits(datetime[0:8]) { + log.Errorf(ctx, "[ofx_transaction_table.parseTransactionTimeAndTimeZone] cannot parse time \"%s\", because contains non-digit character", datetime) + return "", "", errs.ErrTransactionTimeInvalid + } + year = datetime[0:4] month = datetime[4:6] day = datetime[6:8] } if len(datetime) >= 14 { // YYYYMMDDHHMMSS + if !utils.IsStringOnlyContainsDigits(datetime[8:14]) { + log.Errorf(ctx, "[ofx_transaction_table.parseTransactionTimeAndTimeZone] cannot parse time \"%s\", because contains non-digit character", datetime) + return "", "", errs.ErrTransactionTimeInvalid + } + hour = datetime[8:10] minute = datetime[10:12] second = datetime[12:14] diff --git a/pkg/utils/numbers.go b/pkg/utils/numbers.go index 42fabab6..8995d307 100644 --- a/pkg/utils/numbers.go +++ b/pkg/utils/numbers.go @@ -11,6 +11,17 @@ var ( numberPattern = regexp.MustCompile("(-?\\d+)(\\.\\d+)?") ) +// IsStringOnlyContainsDigits returns whether the specified string only contains digit characters +func IsStringOnlyContainsDigits(str string) bool { + for i := 0; i < len(str); i++ { + if str[i] < '0' || str[i] > '9' { + return false + } + } + + return true +} + // GetRandomInteger returns a random number, the max parameter represents upper limit func GetRandomInteger(max int) (int, error) { result, err := rand.Int(rand.Reader, big.NewInt(int64(max))) diff --git a/pkg/utils/numbers_test.go b/pkg/utils/numbers_test.go index 5746b7c1..221310aa 100644 --- a/pkg/utils/numbers_test.go +++ b/pkg/utils/numbers_test.go @@ -6,6 +6,17 @@ import ( "github.com/stretchr/testify/assert" ) +func TestIsStringOnlyContainsDigits(t *testing.T) { + actualValue := IsStringOnlyContainsDigits("0123456789") + assert.True(t, actualValue) + + actualValue = IsStringOnlyContainsDigits("12345a") + assert.False(t, actualValue) + + actualValue = IsStringOnlyContainsDigits("12345 ") + assert.False(t, actualValue) +} + func TestParseFirstConsecutiveNumber(t *testing.T) { expectedValue := "￥123.45" actualValue, success := ParseFirstConsecutiveNumber(expectedValue)