import transactions from iif file

This commit is contained in:
MaysWind
2024-10-27 01:22:06 +08:00
parent cfbab0432c
commit fb5484f44d
10 changed files with 1395 additions and 0 deletions
+58
View File
@@ -0,0 +1,58 @@
package iif
// iifAccountDataset defines the structure of intuit interchange format (iif) account dataset
type iifAccountDataset struct {
accountDataColumnIndexes map[string]int
accounts []*iifAccountData
}
// iifAccountData defines the structure of intuit interchange format (iif) account data
type iifAccountData struct {
dataItems []string
}
// iifTransactionDataset defines the structure of intuit interchange format (iif) transaction dataset
type iifTransactionDataset struct {
transactionDataColumnIndexes map[string]int
splitDataColumnIndexes map[string]int
transactions []*iifTransactionData
}
// iifTransactionData defines the structure of intuit interchange format (iif) transaction data
type iifTransactionData struct {
dataItems []string
splitData []*iifTransactionSplitData
}
// iifTransactionSplitData defines the structure of intuit interchange format (iif) transaction split data
type iifTransactionSplitData struct {
dataItems []string
}
func (s *iifTransactionDataset) getTransactionDataItemValue(transactionData *iifTransactionData, columnName string) (string, bool) {
if transactionData == nil {
return "", false
}
index, exists := s.transactionDataColumnIndexes[columnName]
if !exists || index < 0 || index >= len(transactionData.dataItems) {
return "", false
}
return transactionData.dataItems[index], true
}
func (s *iifTransactionDataset) getSplitDataItemValue(splitData *iifTransactionSplitData, columnName string) (string, bool) {
if splitData == nil {
return "", false
}
index, exists := s.splitDataColumnIndexes[columnName]
if !exists || index < 0 || index >= len(splitData.dataItems) {
return "", false
}
return splitData.dataItems[index], true
}
+235
View File
@@ -0,0 +1,235 @@
package iif
import (
"bytes"
"encoding/csv"
"io"
"strings"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
)
const iifAccountSampleLineSignColumnName = "!ACCNT"
const iifTransactionSampleLineSignColumnName = "!TRNS"
const iifTransactionSplitSampleLineSignColumnName = "!SPL"
const iifTransactionEndSampleLineSignColumnName = "!ENDTRNS"
const iifAccountLineSignColumnName = "ACCNT"
const iifTransactionLineSignColumnName = "TRNS"
const iifTransactionSplitLineSignColumnName = "SPL"
const iifTransactionEndLineSignColumnName = "ENDTRNS"
// iifDataReader defines the structure of intuit interchange format (iif) data reader
type iifDataReader struct {
reader *csv.Reader
}
// read returns the iif transaction dataset
func (r *iifDataReader) read(ctx core.Context) ([]*iifAccountDataset, []*iifTransactionDataset, error) {
allAccountDatasets := make([]*iifAccountDataset, 0)
allTransactionDatasets := make([]*iifTransactionDataset, 0)
currentDatasetType := ""
lastLineSign := ""
var currentAccountDataset *iifAccountDataset
var currentTransactionDataset *iifTransactionDataset
var currentTransactionData *iifTransactionData
for {
items, err := r.reader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Errorf(ctx, "[iif_data_reader.read] cannot parse tsv data, because %s", err.Error())
return nil, nil, errs.ErrInvalidIIFFile
}
if len(items) == 1 && items[0] == "" {
continue
}
if len(items[0]) < 1 {
log.Errorf(ctx, "[iif_data_reader.read] line first column is empty")
return nil, nil, errs.ErrInvalidIIFFile
}
if items[0][0] == '!' { // sample line
if lastLineSign != "" {
log.Errorf(ctx, "[iif_data_reader.read] iif missing transaction end line")
return nil, nil, errs.ErrInvalidIIFFile
}
if currentAccountDataset != nil {
allAccountDatasets = append(allAccountDatasets, currentAccountDataset)
currentAccountDataset = nil
}
if currentTransactionDataset != nil {
allTransactionDatasets = append(allTransactionDatasets, currentTransactionDataset)
currentTransactionDataset = nil
}
if items[0] == iifTransactionSplitSampleLineSignColumnName || items[0] == iifTransactionEndSampleLineSignColumnName {
log.Errorf(ctx, "[iif_data_reader.read] read transaction split sample line or transaction end sample line sign before transaction sample line sign")
return nil, nil, errs.ErrInvalidIIFFile
} else {
currentDatasetType = items[0]
lastLineSign = ""
}
}
if currentDatasetType == "" {
log.Errorf(ctx, "[iif_data_reader.read] cannot read data line before sample line")
return nil, nil, errs.ErrInvalidIIFFile
} else if currentDatasetType == iifAccountSampleLineSignColumnName {
if currentAccountDataset == nil {
currentAccountDataset, err = r.readAccountSampleLine(ctx, items)
if err != nil {
return nil, nil, err
}
} else {
if items[0] == iifAccountLineSignColumnName {
accountData := &iifAccountData{
dataItems: items,
}
currentAccountDataset.accounts = append(currentAccountDataset.accounts, accountData)
} else {
log.Errorf(ctx, "[iif_data_reader.read] iif line expected reading account sign, but actual is \"%s\"", items[0])
return nil, nil, errs.ErrInvalidIIFFile
}
}
} else if currentDatasetType == iifTransactionSampleLineSignColumnName {
if currentTransactionDataset == nil {
currentTransactionDataset, err = r.readTransactionSampleLines(ctx, items)
if err != nil {
return nil, nil, err
}
} else {
if lastLineSign == "" {
if items[0] == iifTransactionLineSignColumnName {
currentTransactionData = &iifTransactionData{
dataItems: items,
splitData: make([]*iifTransactionSplitData, 0),
}
lastLineSign = items[0]
} else {
log.Errorf(ctx, "[iif_data_reader.read] iif line expected reading transaction sign, but actual is \"%s\"", items[0])
return nil, nil, errs.ErrInvalidIIFFile
}
} else if lastLineSign == iifTransactionLineSignColumnName || lastLineSign == iifTransactionSplitLineSignColumnName {
if items[0] == iifTransactionSplitLineSignColumnName {
currentTransactionData.splitData = append(currentTransactionData.splitData, &iifTransactionSplitData{
dataItems: items,
})
lastLineSign = items[0]
} else if items[0] == iifTransactionEndLineSignColumnName {
currentTransactionDataset.transactions = append(currentTransactionDataset.transactions, currentTransactionData)
lastLineSign = ""
} else {
log.Errorf(ctx, "[iif_data_reader.read] iif line expected reading split sign or transaction end sign, but actual is \"%s\"", items[0])
return nil, nil, errs.ErrInvalidIIFFile
}
} else {
log.Errorf(ctx, "[iif_data_reader.read] iif missing transaction sample end line")
return nil, nil, errs.ErrInvalidIIFFile
}
}
}
}
if lastLineSign != "" {
log.Errorf(ctx, "[iif_data_reader.read] iif missing transaction end line")
return nil, nil, errs.ErrInvalidIIFFile
}
if currentAccountDataset != nil {
allAccountDatasets = append(allAccountDatasets, currentAccountDataset)
}
if currentTransactionDataset != nil {
allTransactionDatasets = append(allTransactionDatasets, currentTransactionDataset)
}
return allAccountDatasets, allTransactionDatasets, nil
}
func (r *iifDataReader) readAccountSampleLine(ctx core.Context, items []string) (*iifAccountDataset, error) {
accountSampleItems := items
accountDataColumnIndexes := make(map[string]int, len(accountSampleItems))
for i := 1; i < len(accountSampleItems); i++ {
columnName := accountSampleItems[i]
accountDataColumnIndexes[columnName] = i
}
return &iifAccountDataset{
accountDataColumnIndexes: accountDataColumnIndexes,
accounts: make([]*iifAccountData, 0),
}, nil
}
func (r *iifDataReader) readTransactionSampleLines(ctx core.Context, items []string) (*iifTransactionDataset, error) {
transactionSampleItems := items
transactionDataColumnIndexes := make(map[string]int, len(transactionSampleItems))
for i := 1; i < len(transactionSampleItems); i++ {
columnName := transactionSampleItems[i]
transactionDataColumnIndexes[columnName] = i
}
splitSampleItems, err := r.reader.Read()
if err == io.EOF {
log.Errorf(ctx, "[iif_data_reader.readTransactionSampleLines] expected reading transaction split sample line, but read eof")
return nil, errs.ErrInvalidIIFFile
}
if len(splitSampleItems) < 1 || splitSampleItems[0] != iifTransactionSplitSampleLineSignColumnName {
log.Errorf(ctx, "[iif_data_reader.readTransactionSampleLines] expected reading transaction split sample line, but read \"%s\"", strings.Join(splitSampleItems, "\t"))
return nil, errs.ErrInvalidIIFFile
}
splitDataColumnIndexes := make(map[string]int, len(splitSampleItems))
for i := 1; i < len(splitSampleItems); i++ {
columnName := splitSampleItems[i]
splitDataColumnIndexes[columnName] = i
}
transactionEndSampleItems, err := r.reader.Read()
if err == io.EOF {
log.Errorf(ctx, "[iif_data_reader.readTransactionSampleLines] expected reading transaction end sample line, but read eof")
return nil, errs.ErrInvalidIIFFile
}
if len(transactionEndSampleItems) < 1 || transactionEndSampleItems[0] != iifTransactionEndSampleLineSignColumnName {
log.Errorf(ctx, "[iif_data_reader.readTransactionSampleLines] expected reading transaction end sample line, but read \"%s\"", strings.Join(splitSampleItems, "\t"))
return nil, errs.ErrInvalidIIFFile
}
return &iifTransactionDataset{
transactionDataColumnIndexes: transactionDataColumnIndexes,
splitDataColumnIndexes: splitDataColumnIndexes,
transactions: make([]*iifTransactionData, 0),
}, nil
}
func createNewIifDataReader(data []byte) *iifDataReader {
reader := bytes.NewReader(data)
csvReader := csv.NewReader(reader)
csvReader.Comma = '\t'
csvReader.FieldsPerRecord = -1
return &iifDataReader{
reader: csvReader,
}
}
@@ -0,0 +1,35 @@
package iif
import (
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// iifTransactionDataFileImporter defines the structure of intuit interchange format (iif) for transaction data
type iifTransactionDataFileImporter struct{}
// Initialize an intuit interchange format (iif) file importer singleton instance
var (
IifTransactionDataFileImporter = &iifTransactionDataFileImporter{}
)
// ParseImportedData returns the imported data by parsing the intuit interchange format (iif) data
func (c *iifTransactionDataFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
iifDataReader := createNewIifDataReader(data)
accountDatasets, transactionDatasets, err := iifDataReader.read(ctx)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
transactionDataTable, err := createNewIIfTransactionDataTable(ctx, accountDatasets, transactionDatasets)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewSimpleImporter(iifTransactionTypeNameMapping)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
@@ -0,0 +1,663 @@
package iif
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
func TestIIFTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte(
"!ACCNT\tNAME\tACCNTTYPE\n"+
"ACCNT\tTest Account\tBANK\n"+
"ACCNT\tTest Account2\tBANK\n"+
"ACCNT\tTest Category\tINC\n"+
"ACCNT\tTest Category2\tEXP\n"+
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tBEGINBALCHECK\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tBEGINBALCHECK\t09/01/2024\txxx\t-123.45\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tDEPOSIT\t09/02/2024\tTest Account\t0.12\n"+
"SPL\tDEPOSIT\t09/02/2024\tTest Category\t-0.12\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tCREDIT CARD\t09/03/2024\tTest Account\t-1.00\n"+
"SPL\tCREDIT CARD\t09/03/2024\tTest Category2\t1.00\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tTRANSFER\t09/04/2024\tTest Account\t-0.05\n"+
"SPL\tTRANSFER\t09/04/2024\tTest Account2\t0.05\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/05/2024\tTest Account\t0.06\n"+
"SPL\tGENERAL JOURNAL\t09/05/2024\tTest Account2\t-0.06\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tDEPOSIT\t09/06/2024\tTest Category\t-23.45\n"+
"SPL\tDEPOSIT\t09/06/2024\tTest Account2\t23.45\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tCREDIT CARD\t09/07/2024\tTest Category2\t34.56\n"+
"SPL\tCREDIT CARD\t09/07/2024\tTest Account2\t-34.56\n"+
"ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 7, len(allNewTransactions))
assert.Equal(t, 2, len(allNewAccounts))
assert.Equal(t, 1, len(allNewSubExpenseCategories))
assert.Equal(t, 1, len(allNewSubIncomeCategories))
assert.Equal(t, 1, len(allNewSubTransferCategories))
assert.Equal(t, 0, len(allNewTags))
assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type)
assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type)
assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime))
assert.Equal(t, int64(12), allNewTransactions[1].Amount)
assert.Equal(t, "Test Account", allNewTransactions[1].OriginalSourceAccountName)
assert.Equal(t, "Test Category", allNewTransactions[1].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type)
assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime))
assert.Equal(t, int64(100), allNewTransactions[2].Amount)
assert.Equal(t, "Test Account", allNewTransactions[2].OriginalSourceAccountName)
assert.Equal(t, "Test Category2", allNewTransactions[2].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type)
assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime))
assert.Equal(t, int64(5), allNewTransactions[3].Amount)
assert.Equal(t, "Test Account", allNewTransactions[3].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[3].OriginalDestinationAccountName)
assert.Equal(t, "", allNewTransactions[3].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[4].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[4].Type)
assert.Equal(t, int64(1725494400), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime))
assert.Equal(t, int64(6), allNewTransactions[4].Amount)
assert.Equal(t, "Test Account2", allNewTransactions[4].OriginalSourceAccountName)
assert.Equal(t, "Test Account", allNewTransactions[4].OriginalDestinationAccountName)
assert.Equal(t, "", allNewTransactions[4].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[5].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[5].Type)
assert.Equal(t, int64(1725580800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[5].TransactionTime))
assert.Equal(t, int64(2345), allNewTransactions[5].Amount)
assert.Equal(t, "Test Account2", allNewTransactions[5].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[5].OriginalDestinationAccountName)
assert.Equal(t, "Test Category", allNewTransactions[5].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[6].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[6].Type)
assert.Equal(t, int64(1725667200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[6].TransactionTime))
assert.Equal(t, int64(3456), allNewTransactions[6].Amount)
assert.Equal(t, "Test Account2", allNewTransactions[6].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[6].OriginalDestinationAccountName)
assert.Equal(t, "Test Category2", allNewTransactions[6].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid)
assert.Equal(t, "Test Account", allNewAccounts[0].Name)
assert.Equal(t, "CNY", allNewAccounts[0].Currency)
assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid)
assert.Equal(t, "Test Account2", allNewAccounts[1].Name)
assert.Equal(t, "CNY", allNewAccounts[1].Currency)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid)
assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid)
assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid)
assert.Equal(t, "", allNewSubTransferCategories[0].Name)
}
func TestIIFTransactionDataFileParseImportedData_MinimumValidDataWithoutAccountData(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tDEPOSIT\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tDEPOSIT\t09/01/2024\tTest Category\t-123.45\n"+
"ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[0].Amount)
assert.Equal(t, "Test Category", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalDestinationAccountName)
assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName)
}
func TestIIFTransactionDataFileParseImportedData_MultipleDataset(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!ACCNT\tNAME\tACCNTTYPE\n"+
"ACCNT\tTest Account3\tBANK\n"+
"ACCNT\tTest Account4\tBANK\n"+
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/05/2024\tTest Account\t-0.05\n"+
"SPL\tGENERAL JOURNAL\t09/05/2024\tTest Account2\t0.05\n"+
"ENDTRNS\t\t\t\t\t\n"+
"!TRNS\tTRNSID\tTRNSTYPE\tDATE\tACCNT\tNAME\tCLASS\tAMOUNT\tDOCNUM\tMEMO\tCLEAR\tTOPRINT\tADDR5\tDUEDATE\tTERMS\n"+
"!SPL\tSPLID\tTRNSTYPE\tDATE\tACCNT\tNAME\tCLASS\tAMOUNT\tDOCNUM\tMEMO\tCLEAR\tQNTY\tREIMBEXP\tSERVICEDATE\tOTHER2\n"+
"!ENDTRNS\t\t\t\t\t\t\t\t\t\t\t\t\t\t\n"+
"TRNS\t\tTRANSFER\t09/04/2024\tTest Account3\tTest Category\tTest Class\t123.45\t\t\t\t\t\t\t\n"+
"SPL\t\tTRANSFER\t09/04/2024\tTest Account4\t\t\t-123.45\t\t\t\t\t\t\t\n"+
"ENDTRNS\t\t\t\t\t\t\t\t\t\t\t\t\t\t\n"+
"!CLASS\tNAME\tHIDDEN\n"+
"CLASS\tTest Class\tN\n"+
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tBEGINBALCHECK\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tBEGINBALCHECK\t09/01/2024\txxx\t-123.45\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tDEPOSIT\t09/02/2024\tTest Account\t0.12\n"+
"SPL\tDEPOSIT\t09/02/2024\tTest Category\t-0.12\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tCREDIT CARD\t09/03/2024\tTest Account\t-1.00\n"+
"SPL\tCREDIT CARD\t09/03/2024\tTest Category2\t1.00\n"+
"ENDTRNS\t\t\t\t\n"+
"!ACCNT\tTEST\tNAME\tACCNTTYPE\n"+
"ACCNT\t\tTest Category\tINC\n"+
"ACCNT\t\tTest Category2\tEXP\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 5, len(allNewTransactions))
assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type)
assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type)
assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime))
assert.Equal(t, int64(12), allNewTransactions[1].Amount)
assert.Equal(t, "Test Account", allNewTransactions[1].OriginalSourceAccountName)
assert.Equal(t, "Test Category", allNewTransactions[1].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type)
assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime))
assert.Equal(t, int64(100), allNewTransactions[2].Amount)
assert.Equal(t, "Test Account", allNewTransactions[2].OriginalSourceAccountName)
assert.Equal(t, "Test Category2", allNewTransactions[2].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type)
assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[3].Amount)
assert.Equal(t, "Test Account4", allNewTransactions[3].OriginalSourceAccountName)
assert.Equal(t, "Test Account3", allNewTransactions[3].OriginalDestinationAccountName)
assert.Equal(t, "Test Category", allNewTransactions[3].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[4].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[4].Type)
assert.Equal(t, int64(1725494400), utils.GetUnixTimeFromTransactionTime(allNewTransactions[4].TransactionTime))
assert.Equal(t, int64(5), allNewTransactions[4].Amount)
assert.Equal(t, "Test Account", allNewTransactions[4].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[4].OriginalDestinationAccountName)
assert.Equal(t, "", allNewTransactions[4].OriginalCategoryName)
}
func TestIIFTransactionDataFileParseImportedData_ParseCategoryAndSubCategory(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, allNewSubExpenseCategories, allNewSubIncomeCategories, _, _, err := converter.ParseImportedData(context, user, []byte(
"!ACCNT\tNAME\tACCNTTYPE\n"+
"ACCNT\tTest Parent Category:Test Category\tINC\n"+
"ACCNT\tTest Parent Category2:Test Category2\tEXP\n"+
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tDEPOSIT\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tDEPOSIT\t09/01/2024\tTest Parent Category:Test Category\t-123.45\n"+
"ENDTRNS\t\t\t\t\n"+
"TRNS\tDEPOSIT\t09/02/2024\tTest Account2\t-123.45\n"+
"SPL\tDEPOSIT\t09/02/2024\tTest Parent Category2:Test Category2\t123.45\n"+
"ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 2, len(allNewTransactions))
assert.Equal(t, 1, len(allNewSubExpenseCategories))
assert.Equal(t, 1, len(allNewSubIncomeCategories))
assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[0].Type)
assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[0].OriginalDestinationAccountName)
assert.Equal(t, "Test Category", allNewTransactions[0].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[1].Type)
assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[1].Amount)
assert.Equal(t, "Test Account2", allNewTransactions[1].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[1].OriginalDestinationAccountName)
assert.Equal(t, "Test Category2", allNewTransactions[1].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid)
assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid)
assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name)
}
func TestIIFTransactionDataFileParseImportedData_ParseNameAsTransferCategory(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, allNewSubTransferCategories, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tNAME\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tNAME\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tTRANSFER\t09/01/2024\tTest Account\tTest Category\t-123.45\n"+
"SPL\tTRANSFER\t09/01/2024\tTest Account2\t\t123.45\n"+
"ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, 1, len(allNewSubTransferCategories))
assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid)
assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[0].Type)
assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(12345), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "Test Account2", allNewTransactions[0].OriginalDestinationAccountName)
assert.Equal(t, "Test Category", allNewTransactions[0].OriginalCategoryName)
assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid)
assert.Equal(t, "Test Category", allNewSubTransferCategories[0].Name)
}
func TestIIFTransactionDataFileParseImportedData_ParseShortMonthDayFormatTime(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t9/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t9/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/2/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/2/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t9/3/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t9/3/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 3, len(allNewTransactions))
assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime))
assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime))
}
func TestIIFTransactionDataFileParseImportedData_ParseInvalidTime(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t2024/09/01\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t2024/09/01\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t9/1/24\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t9/1/24\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t2024-09-01\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t2024-09-01\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t9/24\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t9/24\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message)
}
func TestIIFTransactionDataFileParseImportedData_ParseInvalidAmount(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123 45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAmountInvalid.Message)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123 45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrAmountInvalid.Message)
}
func TestIIFTransactionDataFileParseImportedData_ParseDescription(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\tMEMO\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\tMEMO\n"+
"!ENDTRNS\t\t\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\t\"foo bar\t#test\"\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\t\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, "foo bar\t#test", allNewTransactions[0].Comment)
}
func TestIIFTransactionDataFileParseImportedData_NotSupportedToParseSplitTransaction(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-100.00\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account3\t-23.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrNotSupportedSplitTransactions.Message)
}
func TestIIFTransactionDataFileParseImportedData_InvalidDataLines(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1,
DefaultCurrency: "CNY",
}
// Missing Transaction Line
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Split Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Transaction End Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Transaction End Line (following is another header)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"!ACCNT\tNAME\tACCNTTYPE\n"+
"ACCNT\tTest Account\tBANK\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Invalid Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"TEST\t\t\t\t\t\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Repeat Transaction Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Repeat Transaction End Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tDATE\tACCNT\tAMOUNT\t\n"+
"!SPL\tDATE\tACCNT\tAMOUNT\t\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\t09/01/2024\tTest Account\t123.45\n"+
"TRNS\t09/01/2024\tTest Account\t123.45\n"+
"SPL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
}
func TestIIFTransactionDataFileParseImportedData_InvalidHeaderLines(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1,
DefaultCurrency: "CNY",
}
// Missing All Sample Lines
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Transaction Sample Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Split Sample Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Transaction End Sample Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Missing Transaction End Sample Line (following is data line)
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
// Invalid Sample Line
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!TEST\tTRNSTYPE\tDATE\tACCNT\tAMOUNT\n"+
"!ENDTRNS\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrInvalidIIFFile.Message)
}
func TestIIFTransactionDataFileParseImportedData_MissingRequiredColumn(t *testing.T) {
converter := IifTransactionDataFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1,
DefaultCurrency: "CNY",
}
// Missing Transaction Type Column
_, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(
"!TRNS\tDATE\tACCNT\tAMOUNT\t\n"+
"!SPL\tDATE\tACCNT\tAMOUNT\t\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\t09/01/2024\tTest Account\t123.45\n"+
"SPL\t09/01/2024\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Date Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tACCNT\tAMOUNT\t\n"+
"!SPL\tTRNSTYPE\tACCNT\tAMOUNT\t\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\tTest Account\t123.45\n"+
"SPL\tGENERAL JOURNAL\tTest Account2\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Account Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tAMOUNT\t\n"+
"!SPL\tTRNSTYPE\tDATE\tAMOUNT\t\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\t123.45\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\t-123.45\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
// Missing Amount Column
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(
"!TRNS\tTRNSTYPE\tDATE\tACCNT\t\n"+
"!SPL\tTRNSTYPE\tDATE\tACCNT\t\n"+
"!ENDTRNS\t\t\t\t\n"+
"TRNS\tGENERAL JOURNAL\t09/01/2024\tTest Account\n"+
"SPL\tGENERAL JOURNAL\t09/01/2024\tTest Account2\n"+
"ENDTRNS\t\t\t\t\t\n"), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
}
@@ -0,0 +1,391 @@
package iif
import (
"fmt"
"strings"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
const iifAccountNameColumnName = "NAME"
const iifAccountTypeColumnName = "ACCNTTYPE"
const iifAccountTypeIncome = "INC"
const iifAccountTypeExpense = "EXP"
const iifTransactionTypeColumnName = "TRNSTYPE"
const iifTransactionDateColumnName = "DATE"
const iifTransactionAccountNameColumnName = "ACCNT"
const iifTransactionNameColumnName = "NAME"
const iifTransactionAmountColumnName = "AMOUNT"
const iifTransactionMemoColumnName = "MEMO"
const iifTransactionTypeBeginningBalance = "BEGINBALCHECK"
const iifTransactionCategorySeparator = ":"
var iifTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true,
datatable.TRANSACTION_DATA_TABLE_CATEGORY: true,
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true,
}
var iifTransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: utils.IntToString(int(models.TRANSACTION_TYPE_MODIFY_BALANCE)),
models.TRANSACTION_TYPE_INCOME: utils.IntToString(int(models.TRANSACTION_TYPE_INCOME)),
models.TRANSACTION_TYPE_EXPENSE: utils.IntToString(int(models.TRANSACTION_TYPE_EXPENSE)),
models.TRANSACTION_TYPE_TRANSFER: utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)),
}
// iifTransactionDataTable defines the structure of intuit interchange format (iif) transaction data table
type iifTransactionDataTable struct {
incomeAccountNames map[string]bool
expenseAccountNames map[string]bool
transactionDatasets []*iifTransactionDataset
}
// iifTransactionDataRow defines the structure of intuit interchange format (iif) transaction data row
type iifTransactionDataRow struct {
dataTable *iifTransactionDataTable
finalItems map[datatable.TransactionDataTableColumn]string
}
// iifTransactionDataRowIterator defines the structure of intuit interchange format (iif) transaction data row iterator
type iifTransactionDataRowIterator struct {
dataTable *iifTransactionDataTable
currentDatasetIndex int
currentIndexInDataset int
}
// HasColumn returns whether the transaction data table has specified column
func (t *iifTransactionDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool {
_, exists := iifTransactionSupportedColumns[column]
return exists
}
// TransactionRowCount returns the total count of transaction data row
func (t *iifTransactionDataTable) TransactionRowCount() int {
totalDataRowCount := 0
for i := 0; i < len(t.transactionDatasets); i++ {
transactions := t.transactionDatasets[i]
totalDataRowCount += len(transactions.transactions)
}
return totalDataRowCount
}
// TransactionRowIterator returns the iterator of transaction data row
func (t *iifTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator {
return &iifTransactionDataRowIterator{
dataTable: t,
currentDatasetIndex: 0,
currentIndexInDataset: -1,
}
}
// IsValid returns whether this row is valid data for importing
func (r *iifTransactionDataRow) IsValid() bool {
return true
}
// GetData returns the data in the specified column type
func (r *iifTransactionDataRow) GetData(column datatable.TransactionDataTableColumn) string {
_, exists := iifTransactionSupportedColumns[column]
if exists {
return r.finalItems[column]
}
return ""
}
// HasNext returns whether the iterator does not reach the end
func (t *iifTransactionDataRowIterator) HasNext() bool {
allDatasets := t.dataTable.transactionDatasets
if t.currentDatasetIndex >= len(allDatasets) {
return false
}
currentDataset := allDatasets[t.currentDatasetIndex]
if t.currentIndexInDataset+1 < len(currentDataset.transactions) {
return true
}
for i := t.currentDatasetIndex + 1; i < len(allDatasets); i++ {
dataset := allDatasets[i]
if len(dataset.transactions) < 1 {
continue
}
return true
}
return false
}
// Next returns the next imported data row
func (t *iifTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) {
allDatasets := t.dataTable.transactionDatasets
currentIndexInDataset := t.currentIndexInDataset
for i := t.currentDatasetIndex; i < len(allDatasets); i++ {
dataset := allDatasets[i]
if currentIndexInDataset+1 < len(dataset.transactions) {
t.currentIndexInDataset++
currentIndexInDataset = t.currentIndexInDataset
break
}
t.currentDatasetIndex++
t.currentIndexInDataset = -1
currentIndexInDataset = -1
}
if t.currentDatasetIndex >= len(allDatasets) {
return nil, nil
}
currentDataset := allDatasets[t.currentDatasetIndex]
if t.currentIndexInDataset >= len(currentDataset.transactions) {
return nil, nil
}
data := currentDataset.transactions[t.currentIndexInDataset]
rowItems, err := t.parseTransaction(ctx, user, currentDataset, data)
if err != nil {
return nil, err
}
return &iifTransactionDataRow{
dataTable: t.dataTable,
finalItems: rowItems,
}, nil
}
func (t *iifTransactionDataRowIterator) parseTransaction(ctx core.Context, user *models.User, dataset *iifTransactionDataset, transactionData *iifTransactionData) (map[datatable.TransactionDataTableColumn]string, error) {
if len(transactionData.splitData) < 1 {
return nil, errs.ErrInvalidIIFFile
} else if len(transactionData.splitData) > 1 {
return nil, errs.ErrNotSupportedSplitTransactions
}
var err error
data := make(map[datatable.TransactionDataTableColumn]string, len(iifTransactionSupportedColumns))
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME], err = t.parseTransactionTime(dataset, transactionData)
if err != nil {
return nil, err
}
transactionType, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionTypeColumnName)
accountName1, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionAccountNameColumnName)
accountName2, _ := dataset.getSplitDataItemValue(transactionData.splitData[0], iifTransactionAccountNameColumnName)
amount1, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionAmountColumnName)
amount2, _ := dataset.getSplitDataItemValue(transactionData.splitData[0], iifTransactionAmountColumnName)
amountNum1, err := utils.ParseAmount(amount1)
if err != nil {
return nil, errs.ErrAmountInvalid
}
amountNum2, err := utils.ParseAmount(amount2)
if err != nil {
return nil, errs.ErrAmountInvalid
}
name, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionNameColumnName)
if transactionType == iifTransactionTypeBeginningBalance { // balance modification
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName1
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amountNum1)
} else if t.dataTable.incomeAccountNames[accountName1] || t.dataTable.incomeAccountNames[accountName2] { // income
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
categoryName := ""
accountName := ""
amountNum := int64(0)
if t.dataTable.incomeAccountNames[accountName1] && !t.dataTable.incomeAccountNames[accountName2] {
categoryName = accountName1
accountName = accountName2
amountNum = amountNum2
} else if t.dataTable.incomeAccountNames[accountName2] && !t.dataTable.incomeAccountNames[accountName1] {
categoryName = accountName2
accountName = accountName1
amountNum = amountNum1
} else {
log.Errorf(ctx, "[iif_transaction_data_table.parseTransaction] cannot parse transaction, because two accounts \"%s\" and \"%s\" are all income account", accountName1, accountName2)
return nil, errs.ErrInvalidIIFFile
}
categoryNames := strings.Split(categoryName, iifTransactionCategorySeparator)
if len(categoryNames) > 1 {
data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = categoryNames[0]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryNames[len(categoryNames)-1]
} else {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryName
}
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amountNum)
} else if t.dataTable.expenseAccountNames[accountName1] || t.dataTable.expenseAccountNames[accountName2] { // expense
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
categoryName := ""
accountName := ""
amountNum := int64(0)
if t.dataTable.expenseAccountNames[accountName1] && !t.dataTable.expenseAccountNames[accountName2] {
categoryName = accountName1
accountName = accountName2
amountNum = amountNum2
} else if t.dataTable.expenseAccountNames[accountName2] && !t.dataTable.expenseAccountNames[accountName1] {
categoryName = accountName2
accountName = accountName1
amountNum = amountNum1
} else {
log.Errorf(ctx, "[iif_transaction_data_table.parseTransaction] cannot parse transaction, because two accounts \"%s\" and \"%s\" are all expense account", accountName1, accountName2)
return nil, errs.ErrInvalidIIFFile
}
categoryNames := strings.Split(categoryName, iifTransactionCategorySeparator)
if len(categoryNames) > 1 {
data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = categoryNames[0]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryNames[len(categoryNames)-1]
} else {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryName
}
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amountNum)
} else {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = name
if amountNum1 >= 0 {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName2
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amountNum2)
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = accountName1
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(amountNum1)
} else if amountNum2 >= 0 {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName1
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amountNum1)
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = accountName2
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(amountNum2)
}
}
memo, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionMemoColumnName)
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = memo
return data, nil
}
func (t *iifTransactionDataRowIterator) parseTransactionTime(dataset *iifTransactionDataset, transactionData *iifTransactionData) (string, error) {
date, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionDateColumnName)
dateParts := strings.Split(date, "/")
if len(dateParts) != 3 {
return "", errs.ErrTransactionTimeInvalid
}
month := dateParts[0]
day := dateParts[1]
year := dateParts[2]
if len(month) < 2 {
month = "0" + month
}
if len(day) < 2 {
day = "0" + day
}
return fmt.Sprintf("%s-%s-%s 00:00:00", year, month, day), nil
}
func createNewIIfTransactionDataTable(ctx core.Context, accountDatasets []*iifAccountDataset, transactionDatasets []*iifTransactionDataset) (*iifTransactionDataTable, error) {
if len(transactionDatasets) < 1 {
return nil, errs.ErrNotFoundTransactionDataInFile
}
incomeAccountNames, expenseAccountNames := getIncomeAndExpenseAccountNameMap(accountDatasets)
for i := 0; i < len(transactionDatasets); i++ {
transactionDataset := transactionDatasets[i]
for _, requiredColumnName := range []string{
iifTransactionTypeColumnName,
iifTransactionDateColumnName,
iifTransactionAccountNameColumnName,
iifTransactionAmountColumnName,
} {
if _, exists := transactionDataset.transactionDataColumnIndexes[requiredColumnName]; !exists {
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
}
}
return &iifTransactionDataTable{
incomeAccountNames: incomeAccountNames,
expenseAccountNames: expenseAccountNames,
transactionDatasets: transactionDatasets,
}, nil
}
func getIncomeAndExpenseAccountNameMap(accountDatasets []*iifAccountDataset) (incomeAccountNames map[string]bool, expenseAccountNames map[string]bool) {
incomeAccountNames = make(map[string]bool)
expenseAccountNames = make(map[string]bool)
for i := 0; i < len(accountDatasets); i++ {
accountDataset := accountDatasets[i]
accountNameColumnIndex, accountNameColumnExists := accountDataset.accountDataColumnIndexes[iifAccountNameColumnName]
accountTypeColumnIndex, accountTypeColumnExists := accountDataset.accountDataColumnIndexes[iifAccountTypeColumnName]
if !accountNameColumnExists || accountNameColumnIndex < 0 ||
!accountTypeColumnExists || accountTypeColumnIndex < 0 {
continue
}
for j := 0; j < len(accountDataset.accounts); j++ {
items := accountDataset.accounts[j].dataItems
if accountNameColumnIndex >= len(items) ||
accountTypeColumnIndex >= len(items) {
continue
}
accountName := items[accountNameColumnIndex]
accountType := items[accountTypeColumnIndex]
if accountType == iifAccountTypeIncome {
incomeAccountNames[accountName] = true
} else if accountType == iifAccountTypeExpense {
expenseAccountNames[accountName] = true
}
}
}
return incomeAccountNames, expenseAccountNames
}
@@ -7,6 +7,7 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/converters/feidee"
"github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII"
"github.com/mayswind/ezbookkeeping/pkg/converters/gnucash"
"github.com/mayswind/ezbookkeeping/pkg/converters/iif"
"github.com/mayswind/ezbookkeeping/pkg/converters/qif"
"github.com/mayswind/ezbookkeeping/pkg/converters/wechat"
"github.com/mayswind/ezbookkeeping/pkg/errs"
@@ -35,6 +36,8 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter,
return qif.QifMonthDayYearTransactionDataImporter, nil
} else if fileType == "qif_dmy" {
return qif.QifDayMonthYearTransactionDataImporter, nil
} else if fileType == "iif" {
return iif.IifTransactionDataFileImporter, nil
} else if fileType == "gnucash" {
return gnucash.GnuCashTransactionDataImporter, nil
} else if fileType == "firefly_iii_csv" {
+1
View File
@@ -22,4 +22,5 @@ var (
ErrMissingAccountData = NewNormalError(NormalSubcategoryConverter, 15, http.StatusBadRequest, "missing account data")
ErrNotSupportedSplitTransactions = NewNormalError(NormalSubcategoryConverter, 16, http.StatusBadRequest, "not supported to import split transaction")
ErrThereAreNotSupportedTransactionType = NewNormalError(NormalSubcategoryConverter, 17, http.StatusBadRequest, "there are not supported transaction type")
ErrInvalidIIFFile = NewNormalError(NormalSubcategoryConverter, 18, http.StatusBadRequest, "invalid iif file")
)