Files
ezbookkeeping/pkg/converters/iif/iif_transaction_data_table.go
T
2025-06-20 00:55:59 +08:00

518 lines
19 KiB
Go

package iif
import (
"strings"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
const iifAccountNameColumnName = "NAME"
const iifAccountTypeColumnName = "ACCNTTYPE"
const iifAccountTypeIncome = "INC"
const iifAccountTypeExpense = "EXP"
const iifTransactionTypeColumnName = "TRNSTYPE"
const iifTransactionDateColumnName = "DATE"
const iifTransactionAccountNameColumnName = "ACCNT"
const iifTransactionNameColumnName = "NAME"
const iifTransactionAmountColumnName = "AMOUNT"
const iifTransactionMemoColumnName = "MEMO"
const iifTransactionTypeBeginningBalance = "BEGINBALCHECK"
const iifTransactionCategorySeparator = ":"
var iifTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true,
datatable.TRANSACTION_DATA_TABLE_CATEGORY: true,
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true,
}
// iifTransactionDataTable defines the structure of intuit interchange format (iif) transaction data table
type iifTransactionDataTable struct {
incomeAccountNames map[string]bool
expenseAccountNames map[string]bool
transactionDatasets []*iifTransactionDataset
}
// iifTransactionDataRow defines the structure of intuit interchange format (iif) transaction data row
type iifTransactionDataRow struct {
dataTable *iifTransactionDataTable
finalItems map[datatable.TransactionDataTableColumn]string
}
// iifTransactionDataRowIterator defines the structure of intuit interchange format (iif) transaction data row iterator
type iifTransactionDataRowIterator struct {
dataTable *iifTransactionDataTable
currentDatasetIndex int
currentIndexInDataset int
currentSplitDataIndex int
}
// HasColumn returns whether the transaction data table has specified column
func (t *iifTransactionDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool {
_, exists := iifTransactionSupportedColumns[column]
return exists
}
// TransactionRowCount returns the total count of transaction data row
func (t *iifTransactionDataTable) TransactionRowCount() int {
totalDataRowCount := 0
for i := 0; i < len(t.transactionDatasets); i++ {
datasets := t.transactionDatasets[i]
for j := 0; j < len(datasets.Transactions); j++ {
transaction := datasets.Transactions[j]
if transaction.SplitData != nil {
totalDataRowCount += len(transaction.SplitData)
}
}
}
return totalDataRowCount
}
// TransactionRowIterator returns the iterator of transaction data row
func (t *iifTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator {
return &iifTransactionDataRowIterator{
dataTable: t,
currentDatasetIndex: 0,
currentIndexInDataset: 0,
currentSplitDataIndex: -1,
}
}
// IsValid returns whether this row is valid data for importing
func (r *iifTransactionDataRow) IsValid() bool {
return true
}
// GetData returns the data in the specified column type
func (r *iifTransactionDataRow) GetData(column datatable.TransactionDataTableColumn) string {
_, exists := iifTransactionSupportedColumns[column]
if exists {
return r.finalItems[column]
}
return ""
}
// HasNext returns whether the iterator does not reach the end
func (t *iifTransactionDataRowIterator) HasNext() bool {
allDatasets := t.dataTable.transactionDatasets
if t.currentDatasetIndex >= len(allDatasets) {
return false
}
currentDataset := allDatasets[t.currentDatasetIndex]
if t.currentIndexInDataset+1 < len(currentDataset.Transactions) {
return true
} else if t.currentIndexInDataset < len(currentDataset.Transactions) &&
t.currentSplitDataIndex+1 < len(currentDataset.Transactions[t.currentIndexInDataset].SplitData) {
return true
}
for i := t.currentDatasetIndex + 1; i < len(allDatasets); i++ {
dataset := allDatasets[i]
if len(dataset.Transactions) < 1 {
continue
}
return true
}
return false
}
// Next returns the next transaction data row
func (t *iifTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) {
allDatasets := t.dataTable.transactionDatasets
for i := t.currentDatasetIndex; i < len(allDatasets); i++ {
foundNextRow := false
dataset := allDatasets[i]
for j := t.currentIndexInDataset; j < len(dataset.Transactions); j++ {
if t.currentSplitDataIndex+1 < len(dataset.Transactions[j].SplitData) {
t.currentSplitDataIndex++
foundNextRow = true
break
}
t.currentIndexInDataset++
t.currentSplitDataIndex = -1
}
if foundNextRow {
break
}
t.currentDatasetIndex++
t.currentIndexInDataset = 0
t.currentSplitDataIndex = -1
}
if t.currentDatasetIndex >= len(allDatasets) {
return nil, nil
}
currentDataset := allDatasets[t.currentDatasetIndex]
if t.currentIndexInDataset >= len(currentDataset.Transactions) {
return nil, nil
}
data := currentDataset.Transactions[t.currentIndexInDataset]
if len(data.SplitData) < 1 {
log.Errorf(ctx, "[iif_transaction_data_table.Next] cannot parsing transaction in row#%d (dataset#%d), because split data is empty", t.currentIndexInDataset, t.currentDatasetIndex)
return nil, errs.ErrInvalidIIFFile
}
if t.currentSplitDataIndex >= len(data.SplitData) {
return nil, nil
}
if len(data.SplitData) > 1 {
_, err := t.isSplitTransactionSupported(ctx, currentDataset, data)
if err != nil {
return nil, err
}
}
rowItems, err := t.parseTransaction(ctx, user, currentDataset, data, t.currentSplitDataIndex)
if err != nil {
log.Errorf(ctx, "[iif_transaction_data_table.Next] cannot parsing transaction in row#%d-split#%d (dataset#%d), because %s", t.currentIndexInDataset, t.currentSplitDataIndex, t.currentDatasetIndex, err.Error())
return nil, err
}
return &iifTransactionDataRow{
dataTable: t.dataTable,
finalItems: rowItems,
}, nil
}
func (t *iifTransactionDataRowIterator) parseTransaction(ctx core.Context, user *models.User, dataset *iifTransactionDataset, transactionData *iifTransactionData, splitDataIndex int) (map[datatable.TransactionDataTableColumn]string, error) {
var err error
data := make(map[datatable.TransactionDataTableColumn]string, len(iifTransactionSupportedColumns))
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME], err = t.parseTransactionTime(dataset, transactionData)
if err != nil {
return nil, err
}
transactionType, _ := dataset.getSplitDataItemValue(transactionData.SplitData[splitDataIndex], iifTransactionTypeColumnName)
mainAccountName, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionAccountNameColumnName)
splitAccountName, _ := dataset.getSplitDataItemValue(transactionData.SplitData[splitDataIndex], iifTransactionAccountNameColumnName)
mainAmount, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionAmountColumnName)
splitAmount, _ := dataset.getSplitDataItemValue(transactionData.SplitData[splitDataIndex], iifTransactionAmountColumnName)
mainAmountNum, err := parseAmount(mainAmount)
if err != nil {
return nil, errs.ErrAmountInvalid
}
splitAmountNum, err := parseAmount(splitAmount)
if err != nil {
return nil, errs.ErrAmountInvalid
}
if transactionType == iifTransactionTypeBeginningBalance { // balance modification
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = mainAccountName
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(mainAmountNum)
} else if (t.dataTable.incomeAccountNames[mainAccountName] && !t.dataTable.incomeAccountNames[splitAccountName] && !t.dataTable.expenseAccountNames[splitAccountName]) ||
(t.dataTable.incomeAccountNames[splitAccountName] && !t.dataTable.incomeAccountNames[mainAccountName] && !t.dataTable.expenseAccountNames[mainAccountName]) { // income
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
categoryName := ""
accountName := ""
amountNum := int64(0)
if t.dataTable.incomeAccountNames[mainAccountName] && !t.dataTable.incomeAccountNames[splitAccountName] {
categoryName = mainAccountName
accountName = splitAccountName
if len(transactionData.SplitData) > 1 {
amountNum = splitAmountNum
} else {
amountNum = -mainAmountNum
}
} else if t.dataTable.incomeAccountNames[splitAccountName] && !t.dataTable.incomeAccountNames[mainAccountName] {
categoryName = splitAccountName
accountName = mainAccountName
if len(transactionData.SplitData) > 1 {
amountNum = -splitAmountNum
} else {
amountNum = mainAmountNum
}
} else {
log.Errorf(ctx, "[iif_transaction_data_table.parseTransaction] cannot parse transaction, because main account \"%s\" and split account \"%s\" are all income account", mainAccountName, splitAccountName)
return nil, errs.ErrInvalidIIFFile
}
categoryNames := strings.Split(categoryName, iifTransactionCategorySeparator)
if len(categoryNames) > 1 {
data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = categoryNames[0]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryNames[len(categoryNames)-1]
} else {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryName
}
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amountNum)
} else if (t.dataTable.expenseAccountNames[mainAccountName] && !t.dataTable.expenseAccountNames[splitAccountName] && !t.dataTable.incomeAccountNames[splitAccountName]) ||
(t.dataTable.expenseAccountNames[splitAccountName] && !t.dataTable.expenseAccountNames[mainAccountName] && !t.dataTable.incomeAccountNames[mainAccountName]) { // expense
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
categoryName := ""
accountName := ""
amountNum := int64(0)
if t.dataTable.expenseAccountNames[mainAccountName] && !t.dataTable.expenseAccountNames[splitAccountName] {
categoryName = mainAccountName
accountName = splitAccountName
if len(transactionData.SplitData) > 1 {
amountNum = -splitAmountNum
} else {
amountNum = mainAmountNum
}
} else if t.dataTable.expenseAccountNames[splitAccountName] && !t.dataTable.expenseAccountNames[mainAccountName] {
categoryName = splitAccountName
accountName = mainAccountName
if len(transactionData.SplitData) > 1 {
amountNum = splitAmountNum
} else {
amountNum = -mainAmountNum
}
} else {
log.Errorf(ctx, "[iif_transaction_data_table.parseTransaction] cannot parse transaction, because main account \"%s\" and split account \"%s\" are all expense account", mainAccountName, splitAccountName)
return nil, errs.ErrInvalidIIFFile
}
categoryNames := strings.Split(categoryName, iifTransactionCategorySeparator)
if len(categoryNames) > 1 {
data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = categoryNames[0]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryNames[len(categoryNames)-1]
} else {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = categoryName
}
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = accountName
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amountNum)
} else {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = iifTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
amountNum := int64(0)
relatedAmountNum := int64(0)
mainAccountTransferToSplitAccount := false
if len(transactionData.SplitData) > 1 {
amountNum = splitAmountNum
relatedAmountNum = splitAmountNum
mainAccountTransferToSplitAccount = amountNum >= 0
} else {
if mainAmountNum >= 0 {
amountNum = splitAmountNum
relatedAmountNum = mainAmountNum
mainAccountTransferToSplitAccount = false
} else if splitAmountNum >= 0 {
amountNum = mainAmountNum
relatedAmountNum = splitAmountNum
mainAccountTransferToSplitAccount = true
}
}
if mainAccountTransferToSplitAccount {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = mainAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = splitAccountName
} else {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = splitAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = mainAccountName
}
if amountNum >= 0 {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amountNum)
} else {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amountNum)
}
if relatedAmountNum >= 0 {
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(relatedAmountNum)
} else {
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-relatedAmountNum)
}
}
if splitMemo, _ := dataset.getSplitDataItemValue(transactionData.SplitData[splitDataIndex], iifTransactionMemoColumnName); splitMemo != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = splitMemo
} else if memo, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionMemoColumnName); memo != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = memo
} else if splitName, _ := dataset.getSplitDataItemValue(transactionData.SplitData[splitDataIndex], iifTransactionNameColumnName); splitName != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = splitName
} else if name, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionNameColumnName); name != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = name
} else {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = ""
}
return data, nil
}
func (t *iifTransactionDataRowIterator) isSplitTransactionSupported(ctx core.Context, dataset *iifTransactionDataset, transactionData *iifTransactionData) (bool, error) {
supportSplitTransactions := true
transactionType, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionTypeColumnName)
if transactionType == iifTransactionTypeBeginningBalance { // balance modification
supportSplitTransactions = false
log.Errorf(ctx, "[iif_transaction_data_table.isSplitTransactionSupported] cannot parse split balance modification transaction#%d (dataset#%d)", t.currentIndexInDataset, t.currentDatasetIndex)
} else {
transactionAmountStr, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionAmountColumnName)
transactionAmount, err := parseAmount(transactionAmountStr)
if err != nil {
log.Errorf(ctx, "[iif_transaction_data_table.isSplitTransactionSupported] cannot parsing transaction in row#%d (dataset#%d), because transaction amount \"%s\" is invalid", t.currentIndexInDataset, t.currentDatasetIndex, transactionAmountStr)
return false, errs.ErrAmountInvalid
}
splitTotalAmount := int64(0)
for i := 0; i < len(transactionData.SplitData); i++ {
splitAmountStr, _ := dataset.getSplitDataItemValue(transactionData.SplitData[i], iifTransactionAmountColumnName)
splitAmount, err := parseAmount(splitAmountStr)
if err != nil {
log.Errorf(ctx, "[iif_transaction_data_table.isSplitTransactionSupported] cannot parsing transaction in row#%d-split#%d (dataset#%d), because split amount \"%s\" is invalid", t.currentIndexInDataset, i, t.currentDatasetIndex, splitAmountStr)
return false, errs.ErrAmountInvalid
}
splitTotalAmount += splitAmount
}
if splitTotalAmount != -transactionAmount {
supportSplitTransactions = false
log.Errorf(ctx, "[iif_transaction_data_table.isSplitTransactionSupported] cannot parse split transaction#%d (dataset#%d), because the sum amount of each split data \"%d\" not equal to the transaction amount \"%d\"", t.currentIndexInDataset, t.currentDatasetIndex, splitTotalAmount, -transactionAmount)
}
}
if len(transactionData.SplitData) > 1 && !supportSplitTransactions {
return false, errs.ErrNotSupportedSplitTransactions
}
return true, nil
}
func (t *iifTransactionDataRowIterator) parseTransactionTime(dataset *iifTransactionDataset, transactionData *iifTransactionData) (string, error) {
date, _ := dataset.getTransactionDataItemValue(transactionData, iifTransactionDateColumnName)
dateParts := strings.Split(date, "/")
if len(dateParts) != 3 {
return "", errs.ErrTransactionTimeInvalid
}
month := dateParts[0]
day := dateParts[1]
year := dateParts[2]
if utils.IsValidYearMonthDayLongOrShortDateFormat(date) && !utils.IsValidMonthDayYearLongOrShortDateFormat(date) {
year = dateParts[0]
month = dateParts[1]
day = dateParts[2]
}
return utils.FormatYearMonthDayToLongDateTime(year, month, day)
}
func createNewIIfTransactionDataTable(ctx core.Context, accountDatasets []*iifAccountDataset, transactionDatasets []*iifTransactionDataset) (*iifTransactionDataTable, error) {
if len(transactionDatasets) < 1 {
return nil, errs.ErrNotFoundTransactionDataInFile
}
incomeAccountNames, expenseAccountNames := getIncomeAndExpenseAccountNameMap(accountDatasets)
for i := 0; i < len(transactionDatasets); i++ {
transactionDataset := transactionDatasets[i]
for _, requiredColumnName := range []string{
iifTransactionDateColumnName,
iifTransactionAccountNameColumnName,
iifTransactionAmountColumnName,
} {
if _, exists := transactionDataset.TransactionDataColumnIndexes[requiredColumnName]; !exists {
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
}
}
return &iifTransactionDataTable{
incomeAccountNames: incomeAccountNames,
expenseAccountNames: expenseAccountNames,
transactionDatasets: transactionDatasets,
}, nil
}
func getIncomeAndExpenseAccountNameMap(accountDatasets []*iifAccountDataset) (incomeAccountNames map[string]bool, expenseAccountNames map[string]bool) {
incomeAccountNames = make(map[string]bool)
expenseAccountNames = make(map[string]bool)
for i := 0; i < len(accountDatasets); i++ {
accountDataset := accountDatasets[i]
accountNameColumnIndex, accountNameColumnExists := accountDataset.accountDataColumnIndexes[iifAccountNameColumnName]
accountTypeColumnIndex, accountTypeColumnExists := accountDataset.accountDataColumnIndexes[iifAccountTypeColumnName]
if !accountNameColumnExists || accountNameColumnIndex < 0 ||
!accountTypeColumnExists || accountTypeColumnIndex < 0 {
continue
}
for j := 0; j < len(accountDataset.accounts); j++ {
items := accountDataset.accounts[j].dataItems
if accountNameColumnIndex >= len(items) ||
accountTypeColumnIndex >= len(items) {
continue
}
accountName := items[accountNameColumnIndex]
accountType := items[accountTypeColumnIndex]
if accountType == iifAccountTypeIncome {
incomeAccountNames[accountName] = true
} else if accountType == iifAccountTypeExpense {
expenseAccountNames[accountName] = true
}
}
}
return incomeAccountNames, expenseAccountNames
}
func parseAmount(amount string) (int64, error) {
return utils.ParseAmount(strings.ReplaceAll(amount, ",", ""))
}