diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go index 19d966d4..aa2f8110 100644 --- a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go @@ -30,7 +30,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u enc := simplifiedchinese.GB18030 reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder()) - dataTable, err := createNewAlipayTransactionPlainTextDataTable( + transactionDataTable, err := createNewAlipayTransactionDataTable( ctx, reader, c.fileHeaderLine, @@ -43,10 +43,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u return nil, nil, nil, nil, nil, nil, err } - dataTableImporter := datatable.CreateNewSimpleImporter( - dataTable.GetDataColumnMapping(), - alipayTransactionTypeNameMapping, - ) + dataTableImporter := datatable.CreateNewSimpleImporter(alipayTransactionTypeNameMapping) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } diff --git a/pkg/converters/alipay/alipay_transaction_data_plain_text_data_table.go b/pkg/converters/alipay/alipay_transaction_data_plain_text_data_table.go index 6ef3cca2..74942005 100644 --- a/pkg/converters/alipay/alipay_transaction_data_plain_text_data_table.go +++ b/pkg/converters/alipay/alipay_transaction_data_plain_text_data_table.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "strings" - "time" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" @@ -29,14 +28,14 @@ const alipayTransactionDataProductNameTransferInText = "转入" const alipayTransactionDataProductNameTransferOutText = "转出" const alipayTransactionDataProductNameRepaymentText = "还款" -var alipayTransactionSupportedColumns = []datatable.DataTableColumn{ - datatable.DATA_TABLE_TRANSACTION_TIME, - datatable.DATA_TABLE_TRANSACTION_TYPE, - datatable.DATA_TABLE_SUB_CATEGORY, - datatable.DATA_TABLE_ACCOUNT_NAME, - datatable.DATA_TABLE_AMOUNT, - datatable.DATA_TABLE_RELATED_ACCOUNT_NAME, - datatable.DATA_TABLE_DESCRIPTION, +var alipayTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]any{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true, } // alipayTransactionColumnNames defines the structure of alipay transaction plain text header names @@ -52,8 +51,8 @@ type alipayTransactionColumnNames struct { descriptionColumnName string } -// alipayTransactionPlainTextDataTable defines the structure of alipay transaction plain text data table -type alipayTransactionPlainTextDataTable struct { +// alipayTransactionDataTable defines the structure of alipay transaction plain text data table +type alipayTransactionDataTable struct { allOriginalLines [][]string originalHeaderLineColumnNames []string originalTimeColumnIndex int @@ -67,22 +66,28 @@ type alipayTransactionPlainTextDataTable struct { originalDescriptionColumnIndex int } -// alipayTransactionPlainTextDataRow defines the structure of alipay transaction plain text data row -type alipayTransactionPlainTextDataRow struct { - dataTable *alipayTransactionPlainTextDataTable +// alipayTransactionDataRow defines the structure of alipay transaction plain text data row +type alipayTransactionDataRow struct { + dataTable *alipayTransactionDataTable isValid bool originalItems []string - finalItems map[datatable.DataTableColumn]string + finalItems map[datatable.TransactionDataTableColumn]string } -// alipayTransactionPlainTextDataRowIterator defines the structure of alipay transaction plain text data row iterator -type alipayTransactionPlainTextDataRowIterator struct { - dataTable *alipayTransactionPlainTextDataTable +// alipayTransactionDataRowIterator defines the structure of alipay transaction plain text data row iterator +type alipayTransactionDataRowIterator struct { + dataTable *alipayTransactionDataTable currentIndex int } -// DataRowCount returns the total count of data row -func (t *alipayTransactionPlainTextDataTable) DataRowCount() int { +// HasColumn returns whether the transaction data table has specified column +func (t *alipayTransactionDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool { + _, exists := alipayTransactionSupportedColumns[column] + return exists +} + +// TransactionRowCount returns the total count of transaction data row +func (t *alipayTransactionDataTable) TransactionRowCount() int { if len(t.allOriginalLines) < 1 { return 0 } @@ -90,77 +95,39 @@ func (t *alipayTransactionPlainTextDataTable) DataRowCount() int { return len(t.allOriginalLines) - 1 } -// GetDataColumnMapping returns data column map for data importer -func (t *alipayTransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string { - dataColumnMapping := make(map[datatable.DataTableColumn]string, len(alipayTransactionSupportedColumns)) - - for i := 0; i < len(alipayTransactionSupportedColumns); i++ { - column := alipayTransactionSupportedColumns[i] - dataColumnMapping[column] = utils.IntToString(int(column)) - } - - return dataColumnMapping -} - -// HeaderLineColumnNames returns the header column name list -func (t *alipayTransactionPlainTextDataTable) HeaderLineColumnNames() []string { - columnIndexes := make([]string, len(alipayTransactionSupportedColumns)) - - for i := 0; i < len(alipayTransactionSupportedColumns); i++ { - columnIndexes[i] = utils.IntToString(int(alipayTransactionSupportedColumns[i])) - } - - return columnIndexes -} - -// DataRowIterator returns the iterator of data row -func (t *alipayTransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &alipayTransactionPlainTextDataRowIterator{ +// TransactionRowIterator returns the iterator of transaction data row +func (t *alipayTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator { + return &alipayTransactionDataRowIterator{ dataTable: t, currentIndex: 0, } } -// IsValid returns whether this row contains valid data for importing -func (r *alipayTransactionPlainTextDataRow) IsValid() bool { +// IsValid returns whether this row is valid data for importing +func (r *alipayTransactionDataRow) IsValid() bool { return r.isValid } -// ColumnCount returns the total count of column in this data row -func (r *alipayTransactionPlainTextDataRow) ColumnCount() int { - return len(alipayTransactionSupportedColumns) -} +// GetData returns the data in the specified column type +func (r *alipayTransactionDataRow) GetData(column datatable.TransactionDataTableColumn) string { + _, exists := alipayTransactionSupportedColumns[column] -// GetData returns the data in the specified column index -func (r *alipayTransactionPlainTextDataRow) GetData(columnIndex int) string { - if columnIndex >= len(alipayTransactionSupportedColumns) { + if !exists { return "" } - dataColumn := alipayTransactionSupportedColumns[columnIndex] - - return r.finalItems[dataColumn] -} - -// GetTime returns the time in the specified column index -func (r *alipayTransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) { - return utils.ParseFromLongDateTime(r.GetData(columnIndex), timezoneOffset) -} - -// GetTimezoneOffset returns the time zone offset in the specified column index -func (r *alipayTransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) { - return nil, errs.ErrNotSupported + return r.finalItems[column] } // HasNext returns whether the iterator does not reach the end -func (t *alipayTransactionPlainTextDataRowIterator) HasNext() bool { +func (t *alipayTransactionDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allOriginalLines) } // Next returns the next imported data row -func (t *alipayTransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow { +func (t *alipayTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) { - return nil + return nil, nil } t.currentIndex++ @@ -187,7 +154,7 @@ func (t *alipayTransactionPlainTextDataRowIterator) Next(ctx core.Context, user isValid = false } - var finalItems map[datatable.DataTableColumn]string + var finalItems map[datatable.TransactionDataTableColumn]string var errMsg string if isValid { @@ -199,37 +166,37 @@ func (t *alipayTransactionPlainTextDataRowIterator) Next(ctx core.Context, user } } - return &alipayTransactionPlainTextDataRow{ + return &alipayTransactionDataRow{ dataTable: t.dataTable, isValid: isValid, originalItems: rowItems, finalItems: finalItems, - } + }, nil } -func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.DataTableColumn]string, string) { - data := make(map[datatable.DataTableColumn]string, 7) +func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.TransactionDataTableColumn]string, string) { + data := make(map[datatable.TransactionDataTableColumn]string, len(alipayTransactionSupportedColumns)) if t.originalTimeColumnIndex >= 0 && t.originalTimeColumnIndex < len(items) { - data[datatable.DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex] } if t.originalCategoryColumnIndex >= 0 && t.originalCategoryColumnIndex < len(items) { - data[datatable.DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex] + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex] } else { - data[datatable.DATA_TABLE_SUB_CATEGORY] = "" + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" } if t.originalAmountColumnIndex >= 0 && t.originalAmountColumnIndex < len(items) { - data[datatable.DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex] } if t.originalDescriptionColumnIndex >= 0 && t.originalDescriptionColumnIndex < len(items) && items[t.originalDescriptionColumnIndex] != "" { - data[datatable.DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex] + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex] } else if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) && items[t.originalProductNameColumnIndex] != "" { - data[datatable.DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex] + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex] } else { - data[datatable.DATA_TABLE_DESCRIPTION] = "" + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = "" } relatedAccountName := "" @@ -253,7 +220,7 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont localeTextItems := locales.GetLocaleTextItems(locale) if t.originalTypeColumnIndex >= 0 && t.originalTypeColumnIndex < len(items) { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex] if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { if statusName == alipayTransactionDataStatusClosedName { @@ -261,11 +228,11 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont } if statusName == alipayTransactionDataStatusSuccessName { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" } else { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" } } else if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { if statusName == alipayTransactionDataStatusClosedName { @@ -284,42 +251,42 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont } if statusName == alipayTransactionDataStatusRefundSuccessName { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] - data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" } else { if strings.Index(productName, alipayTransactionDataProductNameRechargePrefix) == 0 { // transfer to alipay wallet - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay } else if strings.Index(productName, alipayTransactionDataProductNameCashWithdrawalPrefix) == 0 { // transfer from alipay wallet - data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName } else if strings.Index(productName, alipayTransactionDataProductNameTransferInText) >= 0 { // transfer in - data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName } else if strings.Index(productName, alipayTransactionDataProductNameTransferOutText) >= 0 { // transfer out - data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName } else if strings.Index(productName, alipayTransactionDataProductNameRepaymentText) >= 0 { // repayment - data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName } else { return nil, fmt.Sprintf("product name (\"%s\") is unknown", productName) } } } else { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" } } - if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && statusName != "" { + if data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && statusName != "" { if statusName == alipayTransactionDataStatusRefundSuccessName || statusName == alipayTransactionDataStatusTaxRefundSuccessName { - amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT]) + amount, err := utils.ParseAmount(data[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) if err == nil { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] - data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) } } } @@ -327,7 +294,7 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont return data, "" } -func createNewAlipayTransactionPlainTextDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune, originalColumnNames alipayTransactionColumnNames) (*alipayTransactionPlainTextDataTable, error) { +func createNewAlipayTransactionDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune, originalColumnNames alipayTransactionColumnNames) (*alipayTransactionDataTable, error) { allOriginalLines, err := parseAllLinesFromAlipayTransactionPlainText(ctx, reader, fileHeaderLine, dataHeaderStartContent, dataBottomEndLineRune) if err != nil { @@ -381,7 +348,7 @@ func createNewAlipayTransactionPlainTextDataTable(ctx core.Context, reader io.Re descriptionColumnIdx = -1 } - return &alipayTransactionPlainTextDataTable{ + return &alipayTransactionDataTable{ allOriginalLines: allOriginalLines, originalHeaderLineColumnNames: originalHeaderItems, originalTimeColumnIndex: timeColumnIdx, diff --git a/pkg/converters/datatable/data_table.go b/pkg/converters/datatable/data_table.go deleted file mode 100644 index effdd0ee..00000000 --- a/pkg/converters/datatable/data_table.go +++ /dev/null @@ -1,56 +0,0 @@ -package datatable - -import ( - "time" - - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/models" -) - -// ImportedDataTable defines the structure of imported data table -type ImportedDataTable interface { - // DataRowCount returns the total count of data row - DataRowCount() int - - // HeaderLineColumnNames returns the header column name list - HeaderLineColumnNames() []string - - // DataRowIterator returns the iterator of data row - DataRowIterator() ImportedDataRowIterator -} - -// ImportedDataRow defines the structure of imported data row -type ImportedDataRow interface { - // IsValid returns whether this row contains valid data for importing - IsValid() bool - - // ColumnCount returns the total count of column in this data row - ColumnCount() int - - // GetData returns the data in the specified column index - GetData(columnIndex int) string - - // GetTime returns the time in the specified column index - GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) - - // GetTimezoneOffset returns the time zone offset in the specified column index - GetTimezoneOffset(columnIndex int) (*time.Location, error) -} - -// ImportedDataRowIterator defines the structure of imported data row iterator -type ImportedDataRowIterator interface { - // HasNext returns whether the iterator does not reach the end - HasNext() bool - - // Next returns the next imported data row - Next(ctx core.Context, user *models.User) ImportedDataRow -} - -// DataTableBuilder defines the structure of data table builder -type DataTableBuilder interface { - // AppendTransaction appends the specified transaction to data builder - AppendTransaction(data map[DataTableColumn]string) - - // ReplaceDelimiters returns the text after removing the delimiters - ReplaceDelimiters(text string) string -} diff --git a/pkg/converters/datatable/data_table_transaction_data_converter.go b/pkg/converters/datatable/data_table_transaction_data_converter.go index 862e749b..3f80464f 100644 --- a/pkg/converters/datatable/data_table_transaction_data_converter.go +++ b/pkg/converters/datatable/data_table_transaction_data_converter.go @@ -14,30 +14,8 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/validators" ) -// DataTableColumn represents the data column type of data table -type DataTableColumn byte - -// Data table columns -const ( - DATA_TABLE_TRANSACTION_TIME DataTableColumn = 1 - DATA_TABLE_TRANSACTION_TIMEZONE DataTableColumn = 2 - DATA_TABLE_TRANSACTION_TYPE DataTableColumn = 3 - DATA_TABLE_CATEGORY DataTableColumn = 4 - DATA_TABLE_SUB_CATEGORY DataTableColumn = 5 - DATA_TABLE_ACCOUNT_NAME DataTableColumn = 6 - DATA_TABLE_ACCOUNT_CURRENCY DataTableColumn = 7 - DATA_TABLE_AMOUNT DataTableColumn = 8 - DATA_TABLE_RELATED_ACCOUNT_NAME DataTableColumn = 9 - DATA_TABLE_RELATED_ACCOUNT_CURRENCY DataTableColumn = 10 - DATA_TABLE_RELATED_AMOUNT DataTableColumn = 11 - DATA_TABLE_GEOGRAPHIC_LOCATION DataTableColumn = 12 - DATA_TABLE_TAGS DataTableColumn = 13 - DATA_TABLE_DESCRIPTION DataTableColumn = 14 -) - // DataTableTransactionDataExporter defines the structure of plain text data table exporter for transaction data type DataTableTransactionDataExporter struct { - dataColumnMapping map[DataTableColumn]string transactionTypeMapping map[models.TransactionType]string geoLocationSeparator string transactionTagSeparator string @@ -45,20 +23,14 @@ type DataTableTransactionDataExporter struct { // DataTableTransactionDataImporter defines the structure of plain text data table importer for transaction data type DataTableTransactionDataImporter struct { - dataColumnMapping map[DataTableColumn]string transactionTypeMapping map[models.TransactionType]string geoLocationSeparator string transactionTagSeparator string - postProcessFunc DataTableTransactionDataImporterPostProcessFunc } -// DataTableTransactionDataImporterPostProcessFunc represents item post process function of DataTableTransactionDataImporter -type DataTableTransactionDataImporterPostProcessFunc func(core.Context, *models.ImportTransaction) error - // CreateNewExporter returns a new data table transaction data exporter according to the specified arguments -func CreateNewExporter(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataExporter { +func CreateNewExporter(transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataExporter { return &DataTableTransactionDataExporter{ - dataColumnMapping: dataColumnMapping, transactionTypeMapping: transactionTypeMapping, geoLocationSeparator: geoLocationSeparator, transactionTagSeparator: transactionTagSeparator, @@ -66,9 +38,8 @@ func CreateNewExporter(dataColumnMapping map[DataTableColumn]string, transaction } // CreateNewImporter returns a new data table transaction data importer according to the specified arguments -func CreateNewImporter(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataImporter { +func CreateNewImporter(transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataImporter { return &DataTableTransactionDataImporter{ - dataColumnMapping: dataColumnMapping, transactionTypeMapping: transactionTypeMapping, geoLocationSeparator: geoLocationSeparator, transactionTagSeparator: transactionTagSeparator, @@ -76,41 +47,14 @@ func CreateNewImporter(dataColumnMapping map[DataTableColumn]string, transaction } // CreateNewSimpleImporter returns a new data table transaction data importer according to the specified arguments -func CreateNewSimpleImporter(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string) *DataTableTransactionDataImporter { +func CreateNewSimpleImporter(transactionTypeMapping map[models.TransactionType]string) *DataTableTransactionDataImporter { return &DataTableTransactionDataImporter{ - dataColumnMapping: dataColumnMapping, transactionTypeMapping: transactionTypeMapping, } } -// CreateNewSimpleImporterWithPostProcessFunc returns a new data table transaction data importer according to the specified arguments -func CreateNewSimpleImporterWithPostProcessFunc(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string, postProcessFunc DataTableTransactionDataImporterPostProcessFunc) *DataTableTransactionDataImporter { - return &DataTableTransactionDataImporter{ - dataColumnMapping: dataColumnMapping, - transactionTypeMapping: transactionTypeMapping, - postProcessFunc: postProcessFunc, - } -} - -// CreateNewSimpleImporterFromWritableDataTable returns a new data table transaction data importer according to the specified arguments -func CreateNewSimpleImporterFromWritableDataTable(writableDataTable *WritableDataTable, transactionTypeMapping map[models.TransactionType]string) *DataTableTransactionDataImporter { - return &DataTableTransactionDataImporter{ - dataColumnMapping: writableDataTable.GetDataColumnMapping(), - transactionTypeMapping: transactionTypeMapping, - } -} - -// CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc returns a new data table transaction data importer according to the specified arguments -func CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc(writableDataTable *WritableDataTable, transactionTypeMapping map[models.TransactionType]string, postProcessFunc DataTableTransactionDataImporterPostProcessFunc) *DataTableTransactionDataImporter { - return &DataTableTransactionDataImporter{ - dataColumnMapping: writableDataTable.GetDataColumnMapping(), - transactionTypeMapping: transactionTypeMapping, - postProcessFunc: postProcessFunc, - } -} - // BuildExportedContent writes the exported transaction data to the data table builder -func (c *DataTableTransactionDataExporter) BuildExportedContent(ctx core.Context, dataTableBuilder DataTableBuilder, uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) error { +func (c *DataTableTransactionDataExporter) BuildExportedContent(ctx core.Context, dataTableBuilder TransactionDataTableBuilder, uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) error { for i := 0; i < len(transactions); i++ { transaction := transactions[i] @@ -118,27 +62,27 @@ func (c *DataTableTransactionDataExporter) BuildExportedContent(ctx core.Context continue } - dataRowMap := make(map[DataTableColumn]string, 15) + dataRowMap := make(map[TransactionDataTableColumn]string, 15) transactionTimeZone := time.FixedZone("Transaction Timezone", int(transaction.TimezoneUtcOffset)*60) - dataRowMap[DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime), transactionTimeZone) - dataRowMap[DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(transactionTimeZone) - dataRowMap[DATA_TABLE_TRANSACTION_TYPE] = dataTableBuilder.ReplaceDelimiters(c.getDisplayTransactionTypeName(transaction.Type)) - dataRowMap[DATA_TABLE_CATEGORY] = c.getExportedTransactionCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap) - dataRowMap[DATA_TABLE_SUB_CATEGORY] = c.getExportedTransactionSubCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap) - dataRowMap[DATA_TABLE_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.AccountId, accountMap) - dataRowMap[DATA_TABLE_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.AccountId, accountMap) - dataRowMap[DATA_TABLE_AMOUNT] = utils.FormatAmount(transaction.Amount) + dataRowMap[TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime), transactionTimeZone) + dataRowMap[TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(transactionTimeZone) + dataRowMap[TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataTableBuilder.ReplaceDelimiters(c.getDisplayTransactionTypeName(transaction.Type)) + dataRowMap[TRANSACTION_DATA_TABLE_CATEGORY] = c.getExportedTransactionCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap) + dataRowMap[TRANSACTION_DATA_TABLE_SUB_CATEGORY] = c.getExportedTransactionSubCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap) + dataRowMap[TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.AccountId, accountMap) + dataRowMap[TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.AccountId, accountMap) + dataRowMap[TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(transaction.Amount) if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { - dataRowMap[DATA_TABLE_RELATED_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.RelatedAccountId, accountMap) - dataRowMap[DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.RelatedAccountId, accountMap) - dataRowMap[DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(transaction.RelatedAccountAmount) + dataRowMap[TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.RelatedAccountId, accountMap) + dataRowMap[TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.RelatedAccountId, accountMap) + dataRowMap[TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(transaction.RelatedAccountAmount) } - dataRowMap[DATA_TABLE_GEOGRAPHIC_LOCATION] = c.getExportedGeographicLocation(transaction) - dataRowMap[DATA_TABLE_TAGS] = c.getExportedTags(dataTableBuilder, transaction.TransactionId, allTagIndexes, tagMap) - dataRowMap[DATA_TABLE_DESCRIPTION] = dataTableBuilder.ReplaceDelimiters(transaction.Comment) + dataRowMap[TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION] = c.getExportedGeographicLocation(transaction) + dataRowMap[TRANSACTION_DATA_TABLE_TAGS] = c.getExportedTags(dataTableBuilder, transaction.TransactionId, allTagIndexes, tagMap) + dataRowMap[TRANSACTION_DATA_TABLE_DESCRIPTION] = dataTableBuilder.ReplaceDelimiters(transaction.Comment) dataTableBuilder.AppendTransaction(dataRowMap) } @@ -162,7 +106,7 @@ func (c *DataTableTransactionDataExporter) getDisplayTransactionTypeName(transac return transactionTypeName } -func (c *DataTableTransactionDataExporter) getExportedTransactionCategoryName(dataTableBuilder DataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { +func (c *DataTableTransactionDataExporter) getExportedTransactionCategoryName(dataTableBuilder TransactionDataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { category, exists := categoryMap[categoryId] if !exists { @@ -182,7 +126,7 @@ func (c *DataTableTransactionDataExporter) getExportedTransactionCategoryName(da return dataTableBuilder.ReplaceDelimiters(parentCategory.Name) } -func (c *DataTableTransactionDataExporter) getExportedTransactionSubCategoryName(dataTableBuilder DataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { +func (c *DataTableTransactionDataExporter) getExportedTransactionSubCategoryName(dataTableBuilder TransactionDataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string { category, exists := categoryMap[categoryId] if exists { @@ -192,7 +136,7 @@ func (c *DataTableTransactionDataExporter) getExportedTransactionSubCategoryName } } -func (c *DataTableTransactionDataExporter) getExportedAccountName(dataTableBuilder DataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string { +func (c *DataTableTransactionDataExporter) getExportedAccountName(dataTableBuilder TransactionDataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string { account, exists := accountMap[accountId] if exists { @@ -202,7 +146,7 @@ func (c *DataTableTransactionDataExporter) getExportedAccountName(dataTableBuild } } -func (c *DataTableTransactionDataExporter) getAccountCurrency(dataTableBuilder DataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string { +func (c *DataTableTransactionDataExporter) getAccountCurrency(dataTableBuilder TransactionDataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string { account, exists := accountMap[accountId] if exists { @@ -220,7 +164,7 @@ func (c *DataTableTransactionDataExporter) getExportedGeographicLocation(transac return "" } -func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder DataTableBuilder, transactionId int64, allTagIndexes map[int64][]int64, tagMap map[int64]*models.TransactionTag) string { +func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder TransactionDataTableBuilder, transactionId int64, allTagIndexes map[int64][]int64, tagMap map[int64]*models.TransactionTag) string { tagIndexes, exists := allTagIndexes[transactionId] if !exists { @@ -248,8 +192,8 @@ func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder Data } // ParseImportedData returns the imported transaction data -func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, dataTable ImportedDataTable, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - if dataTable.DataRowCount() < 1 { +func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, dataTable TransactionDataTable, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { + if dataTable.TransactionRowCount() < 1 { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile } @@ -260,29 +204,12 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u return nil, nil, nil, nil, nil, nil, err } - headerLineItems := dataTable.HeaderLineColumnNames() - headerItemMap := make(map[string]int) - - for i := 0; i < len(headerLineItems); i++ { - headerItemMap[headerLineItems[i]] = i - } - - timeColumnIdx, timeColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TRANSACTION_TIME]] - timezoneColumnIdx, timezoneColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TRANSACTION_TIMEZONE]] - typeColumnIdx, typeColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TRANSACTION_TYPE]] - subCategoryColumnIdx, subCategoryColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_SUB_CATEGORY]] - accountColumnIdx, accountColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_ACCOUNT_NAME]] - accountCurrencyColumnIdx, accountCurrencyColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_ACCOUNT_CURRENCY]] - amountColumnIdx, amountColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_AMOUNT]] - account2ColumnIdx, account2ColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_RELATED_ACCOUNT_NAME]] - account2CurrencyColumnIdx, account2CurrencyColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_RELATED_ACCOUNT_CURRENCY]] - amount2ColumnIdx, amount2ColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_RELATED_AMOUNT]] - geoLocationIdx, geoLocationExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_GEOGRAPHIC_LOCATION]] - tagsColumnIdx, tagsColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TAGS]] - descriptionColumnIdx, descriptionColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_DESCRIPTION]] - - if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists || - !accountColumnExists || !amountColumnExists || !account2ColumnExists { + if !dataTable.HasColumn(TRANSACTION_DATA_TABLE_TRANSACTION_TIME) || + !dataTable.HasColumn(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) || + !dataTable.HasColumn(TRANSACTION_DATA_TABLE_SUB_CATEGORY) || + !dataTable.HasColumn(TRANSACTION_DATA_TABLE_ACCOUNT_NAME) || + !dataTable.HasColumn(TRANSACTION_DATA_TABLE_AMOUNT) || + !dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME) { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } @@ -307,59 +234,53 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u tagMap = make(map[string]*models.TransactionTag) } - allNewTransactions := make(models.ImportedTransactionSlice, 0, dataTable.DataRowCount()) + allNewTransactions := make(models.ImportedTransactionSlice, 0, dataTable.TransactionRowCount()) allNewAccounts := make([]*models.Account, 0) allNewSubExpenseCategories := make([]*models.TransactionCategory, 0) allNewSubIncomeCategories := make([]*models.TransactionCategory, 0) allNewSubTransferCategories := make([]*models.TransactionCategory, 0) allNewTags := make([]*models.TransactionTag, 0) - dataRowIterator := dataTable.DataRowIterator() + dataRowIterator := dataTable.TransactionRowIterator() dataRowIndex := 0 for dataRowIterator.HasNext() { dataRowIndex++ - dataRow := dataRowIterator.Next(ctx, user) + dataRow, err := dataRowIterator.Next(ctx, user) + + if err != nil { + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse data row \"index:%d\" for user \"uid:%d\", because %s", dataRowIndex, user.Uid, err.Error()) + return nil, nil, nil, nil, nil, nil, err + } if !dataRow.IsValid() { continue } - columnCount := dataRow.ColumnCount() - - if columnCount < 1 || (columnCount == 1 && dataRow.GetData(0) == "") { - continue - } - - if columnCount < len(headerLineItems) { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse data row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", dataRowIndex, user.Uid, columnCount, len(headerLineItems)) - return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow - } - timezoneOffset := defaultTimezoneOffset - if timezoneColumnExists { - transactionTimezone, err := dataRow.GetTimezoneOffset(timezoneColumnIdx) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE) { + transactionTimezone, err := utils.ParseFromTimezoneOffset(dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE)) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time zone \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(timezoneColumnIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time zone \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTimeZoneInvalid } timezoneOffset = utils.GetTimezoneOffsetMinutes(transactionTimezone) } - transactionTime, err := dataRow.GetTime(timeColumnIdx, timezoneOffset) + transactionTime, err := utils.ParseFromLongDateTime(dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME), timezoneOffset) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(timeColumnIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTimeInvalid } - transactionDbType, err := c.getTransactionDbType(nameDbTypeMap, dataRow.GetData(typeColumnIdx)) + transactionDbType, err := c.getTransactionDbType(nameDbTypeMap, dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse transaction type \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(typeColumnIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse transaction type \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid) } @@ -374,7 +295,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u return nil, nil, nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid) } - subCategoryName = dataRow.GetData(subCategoryColumnIdx) + subCategoryName = dataRow.GetData(TRANSACTION_DATA_TABLE_SUB_CATEGORY) if transactionDbType == models.TRANSACTION_DB_TYPE_EXPENSE { subCategory, exists := expenseCategoryMap[subCategoryName] @@ -409,11 +330,11 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u } } - accountName := dataRow.GetData(accountColumnIdx) + accountName := dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_NAME) accountCurrency := user.DefaultCurrency - if accountCurrencyColumnExists { - accountCurrency = dataRow.GetData(accountCurrencyColumnIdx) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) { + accountCurrency = dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) if _, ok := validators.AllCurrencyNames[accountCurrency]; !ok { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] account currency \"%s\" is not supported in data row \"index:%d\" for user \"uid:%d\"", accountCurrency, dataRowIndex, user.Uid) @@ -429,7 +350,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u accountMap[accountName] = account } - if accountCurrencyColumnExists { + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) { if account.Name != "" && account.Currency != accountCurrency { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account for user \"uid:%d\"", accountCurrency, dataRowIndex, account.Currency, user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid @@ -438,10 +359,10 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u accountCurrency = account.Currency } - amount, err := utils.ParseAmount(dataRow.GetData(amountColumnIdx)) + amount, err := utils.ParseAmount(dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT)) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(amountColumnIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.ErrAmountInvalid } @@ -451,11 +372,11 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u account2Currency := "" if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { - account2Name = dataRow.GetData(account2ColumnIdx) + account2Name = dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME) account2Currency = user.DefaultCurrency - if account2CurrencyColumnExists { - account2Currency = dataRow.GetData(account2CurrencyColumnIdx) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) { + account2Currency = dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) if _, ok := validators.AllCurrencyNames[account2Currency]; !ok { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] account2 currency \"%s\" is not supported in data row \"index:%d\" for user \"uid:%d\"", account2Currency, dataRowIndex, user.Uid) @@ -471,7 +392,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u accountMap[account2Name] = account2 } - if account2CurrencyColumnExists { + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) { if account2.Name != "" && account2.Currency != account2Currency { log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account2 for user \"uid:%d\"", account2Currency, dataRowIndex, account2.Currency, user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid @@ -482,11 +403,11 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u relatedAccountId = account2.AccountId - if amount2ColumnExists { - relatedAccountAmount, err = utils.ParseAmount(dataRow.GetData(amount2ColumnIdx)) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_AMOUNT) { + relatedAccountAmount, err = utils.ParseAmount(dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_AMOUNT)) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount2 \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(amount2ColumnIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount2 \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_AMOUNT), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.ErrAmountInvalid } } else if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT { @@ -497,21 +418,21 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u geoLongitude := float64(0) geoLatitude := float64(0) - if geoLocationExists { - geoLocationItems := strings.Split(dataRow.GetData(geoLocationIdx), c.geoLocationSeparator) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION) { + geoLocationItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), c.geoLocationSeparator) if len(geoLocationItems) == 2 { geoLongitude, err = utils.StringToFloat64(geoLocationItems[0]) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(geoLocationIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.ErrGeographicLocationInvalid } geoLatitude, err = utils.StringToFloat64(geoLocationItems[1]) if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(geoLocationIdx), dataRowIndex, user.Uid, err.Error()) + log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), dataRowIndex, user.Uid, err.Error()) return nil, nil, nil, nil, nil, nil, errs.ErrGeographicLocationInvalid } } @@ -520,8 +441,8 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u var tagIds []string var tagNames []string - if tagsColumnExists { - tagNameItems := strings.Split(dataRow.GetData(tagsColumnIdx), c.transactionTagSeparator) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_TAGS) { + tagNameItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS), c.transactionTagSeparator) for i := 0; i < len(tagNameItems); i++ { tagName := tagNameItems[i] @@ -548,8 +469,8 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u description := "" - if descriptionColumnExists { - description = dataRow.GetData(descriptionColumnIdx) + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_DESCRIPTION) { + description = dataRow.GetData(TRANSACTION_DATA_TABLE_DESCRIPTION) } transaction := &models.ImportTransaction{ @@ -578,15 +499,6 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u OriginalTagNames: tagNames, } - if c.postProcessFunc != nil { - err = c.postProcessFunc(ctx, transaction) - - if err != nil { - log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot post process data row \"index:%d\" for user \"uid:%d\", because %s", dataRowIndex, user.Uid, err.Error()) - return nil, nil, nil, nil, nil, nil, err - } - } - allNewTransactions = append(allNewTransactions, transaction) } diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_excel_file_data_table.go b/pkg/converters/datatable/default_excel_file_imported_data_table.go similarity index 53% rename from pkg/converters/feidee/feidee_mymoney_transaction_data_excel_file_data_table.go rename to pkg/converters/datatable/default_excel_file_imported_data_table.go index e5cb78c4..434fa525 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_excel_file_data_table.go +++ b/pkg/converters/datatable/default_excel_file_imported_data_table.go @@ -1,39 +1,34 @@ -package feidee +package datatable import ( "bytes" - "time" "github.com/shakinm/xlsReader/xls" - "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" - "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" ) -// feideeMymoneyTransactionExcelFileDataTable defines the structure of feidee mymoney transaction plain text data table -type feideeMymoneyTransactionExcelFileDataTable struct { +// DefaultExcelFileImportedDataTable defines the structure of default excel file data table +type DefaultExcelFileImportedDataTable struct { workbook *xls.Workbook headerLineColumnNames []string } -// feideeMymoneyTransactionExcelFileDataRow defines the structure of feidee mymoney transaction plain text data row -type feideeMymoneyTransactionExcelFileDataRow struct { +// DefaultExcelFileDataRow defines the structure of default excel file data table row +type DefaultExcelFileDataRow struct { sheet *xls.Sheet rowIndex int } -// feideeMymoneyTransactionExcelFileDataRowIterator defines the structure of feidee mymoney transaction plain text data row iterator -type feideeMymoneyTransactionExcelFileDataRowIterator struct { - dataTable *feideeMymoneyTransactionExcelFileDataTable +// DefaultExcelFileDataRowIterator defines the structure of default excel file data table row iterator +type DefaultExcelFileDataRowIterator struct { + dataTable *DefaultExcelFileImportedDataTable currentTableIndex int currentRowIndexInTable int } // DataRowCount returns the total count of data row -func (t *feideeMymoneyTransactionExcelFileDataTable) DataRowCount() int { +func (t *DefaultExcelFileImportedDataTable) DataRowCount() int { allSheets := t.workbook.GetSheets() totalDataRowCount := 0 @@ -50,27 +45,22 @@ func (t *feideeMymoneyTransactionExcelFileDataTable) DataRowCount() int { return totalDataRowCount } -// HeaderLineColumnNames returns the header column name list -func (t *feideeMymoneyTransactionExcelFileDataTable) HeaderLineColumnNames() []string { +// HeaderColumnNames returns the header column name list +func (t *DefaultExcelFileImportedDataTable) HeaderColumnNames() []string { return t.headerLineColumnNames } // DataRowIterator returns the iterator of data row -func (t *feideeMymoneyTransactionExcelFileDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &feideeMymoneyTransactionExcelFileDataRowIterator{ +func (t *DefaultExcelFileImportedDataTable) DataRowIterator() ImportedDataRowIterator { + return &DefaultExcelFileDataRowIterator{ dataTable: t, currentTableIndex: 0, currentRowIndexInTable: 0, } } -// IsValid returns whether this row contains valid data for importing -func (r *feideeMymoneyTransactionExcelFileDataRow) IsValid() bool { - return true -} - // ColumnCount returns the total count of column in this data row -func (r *feideeMymoneyTransactionExcelFileDataRow) ColumnCount() int { +func (r *DefaultExcelFileDataRow) ColumnCount() int { row, err := r.sheet.GetRow(r.rowIndex) if err != nil { @@ -81,7 +71,7 @@ func (r *feideeMymoneyTransactionExcelFileDataRow) ColumnCount() int { } // GetData returns the data in the specified column index -func (r *feideeMymoneyTransactionExcelFileDataRow) GetData(columnIndex int) string { +func (r *DefaultExcelFileDataRow) GetData(columnIndex int) string { row, err := r.sheet.GetRow(r.rowIndex) if err != nil { @@ -97,32 +87,8 @@ func (r *feideeMymoneyTransactionExcelFileDataRow) GetData(columnIndex int) stri return cell.GetString() } -// GetTime returns the time in the specified column index -func (r *feideeMymoneyTransactionExcelFileDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) { - str := r.GetData(columnIndex) - - if utils.IsValidLongDateTimeFormat(str) { - return utils.ParseFromLongDateTime(str, timezoneOffset) - } - - if utils.IsValidLongDateTimeWithoutSecondFormat(str) { - return utils.ParseFromLongDateTimeWithoutSecond(str, timezoneOffset) - } - - if utils.IsValidLongDateFormat(str) { - return utils.ParseFromLongDateTimeWithoutSecond(str+" 00:00", timezoneOffset) - } - - return time.Unix(0, 0), errs.ErrTransactionTimeInvalid -} - -// GetTimezoneOffset returns the time zone offset in the specified column index -func (r *feideeMymoneyTransactionExcelFileDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) { - return nil, errs.ErrNotSupported -} - // HasNext returns whether the iterator does not reach the end -func (t *feideeMymoneyTransactionExcelFileDataRowIterator) HasNext() bool { +func (t *DefaultExcelFileDataRowIterator) HasNext() bool { allSheets := t.dataTable.workbook.GetSheets() if t.currentTableIndex >= len(allSheets) { @@ -149,7 +115,7 @@ func (t *feideeMymoneyTransactionExcelFileDataRowIterator) HasNext() bool { } // Next returns the next imported data row -func (t *feideeMymoneyTransactionExcelFileDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow { +func (t *DefaultExcelFileDataRowIterator) Next() ImportedDataRow { allSheets := t.dataTable.workbook.GetSheets() currentRowIndexInTable := t.currentRowIndexInTable @@ -177,13 +143,14 @@ func (t *feideeMymoneyTransactionExcelFileDataRowIterator) Next(ctx core.Context return nil } - return &feideeMymoneyTransactionExcelFileDataRow{ + return &DefaultExcelFileDataRow{ sheet: ¤tSheet, rowIndex: t.currentRowIndexInTable, } } -func createNewFeideeMymoneyTransactionExcelFileDataTable(data []byte) (*feideeMymoneyTransactionExcelFileDataTable, error) { +// CreateNewDefaultExcelFileImportedDataTable returns default excel xls data table by file binary data +func CreateNewDefaultExcelFileImportedDataTable(data []byte) (*DefaultExcelFileImportedDataTable, error) { reader := bytes.NewReader(data) workbook, err := xls.OpenReader(reader) @@ -230,7 +197,7 @@ func createNewFeideeMymoneyTransactionExcelFileDataTable(data []byte) (*feideeMy } } - return &feideeMymoneyTransactionExcelFileDataTable{ + return &DefaultExcelFileImportedDataTable{ workbook: &workbook, headerLineColumnNames: headerRowItems, }, nil diff --git a/pkg/converters/datatable/default_plain_text_imported_data_table.go b/pkg/converters/datatable/default_plain_text_imported_data_table.go new file mode 100644 index 00000000..8a2f1fde --- /dev/null +++ b/pkg/converters/datatable/default_plain_text_imported_data_table.go @@ -0,0 +1,124 @@ +package datatable + +import ( + "encoding/csv" + "io" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +// DefaultPlainTextImportedDataTable defines the structure of default plain text data table +type DefaultPlainTextImportedDataTable struct { + allLines [][]string +} + +// DefaultPlainTextImportedDataRow defines the structure of default plain text data table row +type DefaultPlainTextImportedDataRow struct { + dataTable *DefaultPlainTextImportedDataTable + allItems []string +} + +// DefaultPlainTextImportedDataRowIterator defines the structure of default plain text data table row iterator +type DefaultPlainTextImportedDataRowIterator struct { + dataTable *DefaultPlainTextImportedDataTable + currentIndex int +} + +// DataRowCount returns the total count of data row +func (t *DefaultPlainTextImportedDataTable) DataRowCount() int { + if len(t.allLines) < 1 { + return 0 + } + + return len(t.allLines) - 1 +} + +// HeaderColumnNames returns the header column name list +func (t *DefaultPlainTextImportedDataTable) HeaderColumnNames() []string { + if len(t.allLines) < 1 { + return nil + } + + return t.allLines[0] +} + +// DataRowIterator returns the iterator of data row +func (t *DefaultPlainTextImportedDataTable) DataRowIterator() ImportedDataRowIterator { + return &DefaultPlainTextImportedDataRowIterator{ + dataTable: t, + currentIndex: 0, + } +} + +// ColumnCount returns the total count of column in this data row +func (r *DefaultPlainTextImportedDataRow) ColumnCount() int { + return len(r.allItems) +} + +// GetData returns the data in the specified column index +func (r *DefaultPlainTextImportedDataRow) GetData(columnIndex int) string { + if columnIndex >= len(r.allItems) { + return "" + } + + return r.allItems[columnIndex] +} + +// HasNext returns whether the iterator does not reach the end +func (t *DefaultPlainTextImportedDataRowIterator) HasNext() bool { + return t.currentIndex+1 < len(t.dataTable.allLines) +} + +// Next returns the next imported data row +func (t *DefaultPlainTextImportedDataRowIterator) Next() ImportedDataRow { + if t.currentIndex+1 >= len(t.dataTable.allLines) { + return nil + } + + t.currentIndex++ + + rowItems := t.dataTable.allLines[t.currentIndex] + + return &DefaultPlainTextImportedDataRow{ + dataTable: t.dataTable, + allItems: rowItems, + } +} + +// CreateNewDefaultCsvDataTable returns default csv data table by io readers +func CreateNewDefaultCsvDataTable(ctx core.Context, reader io.Reader) (*DefaultPlainTextImportedDataTable, error) { + return createNewDefaultPlainTextDataTable(ctx, reader, ',') +} + +func createNewDefaultPlainTextDataTable(ctx core.Context, reader io.Reader, comma rune) (*DefaultPlainTextImportedDataTable, error) { + csvReader := csv.NewReader(reader) + csvReader.Comma = comma + csvReader.FieldsPerRecord = -1 + + allLines := make([][]string, 0) + + for { + items, err := csvReader.Read() + + if err == io.EOF { + break + } + + if err != nil { + log.Errorf(ctx, "[Default_plain_text_imported_data_table.createNewDefaultPlainTextDataTable] cannot parse plain text data, because %s", err.Error()) + return nil, errs.ErrInvalidCSVFile + } + + if len(items) == 0 && items[0] == "" { + continue + } + + allLines = append(allLines, items) + } + + return &DefaultPlainTextImportedDataTable{ + allLines: allLines, + }, nil +} diff --git a/pkg/converters/datatable/imported_data_table.go b/pkg/converters/datatable/imported_data_table.go new file mode 100644 index 00000000..80a39ee6 --- /dev/null +++ b/pkg/converters/datatable/imported_data_table.go @@ -0,0 +1,31 @@ +package datatable + +// ImportedDataTable defines the structure of imported data table +type ImportedDataTable interface { + // DataRowCount returns the total count of data row + DataRowCount() int + + // HeaderColumnNames returns the header column name list + HeaderColumnNames() []string + + // DataRowIterator returns the iterator of data row + DataRowIterator() ImportedDataRowIterator +} + +// ImportedDataRow defines the structure of imported data row +type ImportedDataRow interface { + // ColumnCount returns the total count of column in this data row + ColumnCount() int + + // GetData returns the data in the specified column index + GetData(columnIndex int) string +} + +// ImportedDataRowIterator defines the structure of imported data row iterator +type ImportedDataRowIterator interface { + // HasNext returns whether the iterator does not reach the end + HasNext() bool + + // Next returns the next imported data row + Next() ImportedDataRow +} diff --git a/pkg/converters/datatable/imported_transaction_data_table.go b/pkg/converters/datatable/imported_transaction_data_table.go new file mode 100644 index 00000000..8bda2866 --- /dev/null +++ b/pkg/converters/datatable/imported_transaction_data_table.go @@ -0,0 +1,188 @@ +package datatable + +import ( + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" +) + +// ImportedTransactionDataTable defines the structure of imported transaction data table +type ImportedTransactionDataTable struct { + innerDataTable ImportedDataTable + dataColumnMapping map[TransactionDataTableColumn]string + dataColumnIndexes map[TransactionDataTableColumn]int + rowParser TransactionDataRowParser + addedColumns map[TransactionDataTableColumn]any +} + +// ImportedTransactionDataRow defines the structure of imported transaction data row +type ImportedTransactionDataRow struct { + transactionDataTable *ImportedTransactionDataTable + rowData map[TransactionDataTableColumn]string + rowDataValid bool +} + +// ImportedTransactionDataRowIterator defines the structure of imported transaction data row iterator +type ImportedTransactionDataRowIterator struct { + transactionDataTable *ImportedTransactionDataTable + innerIterator ImportedDataRowIterator +} + +// HasColumn returns whether the data table has specified column +func (t *ImportedTransactionDataTable) HasColumn(column TransactionDataTableColumn) bool { + index, exists := t.dataColumnIndexes[column] + + if exists && index >= 0 { + return exists + } + + if t.addedColumns != nil { + _, exists = t.addedColumns[column] + + if exists { + return exists + } + } + + return false +} + +// TransactionRowCount returns the total count of transaction data row +func (t *ImportedTransactionDataTable) TransactionRowCount() int { + return t.innerDataTable.DataRowCount() +} + +// TransactionRowIterator returns the iterator of transaction data row +func (t *ImportedTransactionDataTable) TransactionRowIterator() TransactionDataRowIterator { + return &ImportedTransactionDataRowIterator{ + transactionDataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), + } +} + +// IsValid returns whether this row is valid data for importing +func (r *ImportedTransactionDataRow) IsValid() bool { + return r.rowDataValid +} + +// GetData returns the data in the specified column type +func (r *ImportedTransactionDataRow) GetData(column TransactionDataTableColumn) string { + if !r.rowDataValid { + return "" + } + + _, exists := r.transactionDataTable.dataColumnIndexes[column] + + if exists { + return r.rowData[column] + } + + if r.transactionDataTable.addedColumns != nil { + _, exists = r.transactionDataTable.addedColumns[column] + + if exists { + return r.rowData[column] + } + } + + return "" +} + +// HasNext returns whether the iterator does not reach the end +func (t *ImportedTransactionDataRowIterator) HasNext() bool { + return t.innerIterator.HasNext() +} + +// Next returns the next transaction data row +func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) { + importedRow := t.innerIterator.Next() + + if importedRow == nil { + return nil, nil + } + + if importedRow.ColumnCount() == 1 && importedRow.GetData(0) == "" { + return &ImportedTransactionDataRow{ + transactionDataTable: t.transactionDataTable, + rowData: nil, + rowDataValid: false, + }, nil + } + + if importedRow.ColumnCount() < len(t.transactionDataTable.dataColumnIndexes) { + log.Errorf(ctx, "[imported_transaction_data_table.Next] cannot parse data row, because may missing some columns (column count %d in data row is less than header column count %d)", importedRow.ColumnCount(), len(t.transactionDataTable.dataColumnIndexes)) + return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + } + + rowData := make(map[TransactionDataTableColumn]string, len(t.transactionDataTable.dataColumnIndexes)) + rowDataValid := true + + for column, columnIndex := range t.transactionDataTable.dataColumnIndexes { + if columnIndex < 0 || columnIndex >= importedRow.ColumnCount() { + continue + } + + value := importedRow.GetData(columnIndex) + rowData[column] = value + } + + if t.transactionDataTable.rowParser != nil { + rowData, rowDataValid, err = t.transactionDataTable.rowParser.Parse(rowData) + + if err != nil { + log.Errorf(ctx, "[imported_transaction_data_table.Next] cannot parse data row, because %s", err.Error()) + return nil, err + } + } + + return &ImportedTransactionDataRow{ + transactionDataTable: t.transactionDataTable, + rowData: rowData, + rowDataValid: rowDataValid, + }, nil +} + +// CreateImportedTransactionDataTable returns transaction data table from imported data table +func CreateImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable { + return CreateImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil) +} + +// CreateImportedTransactionDataTableWithRowParser returns transaction data table from imported data table +func CreateImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable { + headerLineItems := dataTable.HeaderColumnNames() + headerItemMap := make(map[string]int, len(headerLineItems)) + + for i := 0; i < len(headerLineItems); i++ { + headerItemMap[headerLineItems[i]] = i + } + + dataColumnIndexes := make(map[TransactionDataTableColumn]int, len(headerLineItems)) + + for column, columnName := range dataColumnMapping { + columnIndex, exists := headerItemMap[columnName] + + if exists { + dataColumnIndexes[column] = columnIndex + } + } + + var addedColumns map[TransactionDataTableColumn]any + + if rowParser != nil { + addedColumnsByParser := rowParser.GetAddedColumns() + addedColumns = make(map[TransactionDataTableColumn]any, len(addedColumnsByParser)) + + for i := 0; i < len(addedColumnsByParser); i++ { + addedColumns[addedColumnsByParser[i]] = true + } + } + + return &ImportedTransactionDataTable{ + innerDataTable: dataTable, + dataColumnMapping: dataColumnMapping, + dataColumnIndexes: dataColumnIndexes, + rowParser: rowParser, + addedColumns: addedColumns, + } +} diff --git a/pkg/converters/datatable/transaction_data_table.go b/pkg/converters/datatable/transaction_data_table.go new file mode 100644 index 00000000..d79adf9a --- /dev/null +++ b/pkg/converters/datatable/transaction_data_table.go @@ -0,0 +1,75 @@ +package datatable + +import ( + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/models" +) + +// TransactionDataTable defines the structure of transaction data table +type TransactionDataTable interface { + // HasColumn returns whether the transaction data table has specified column + HasColumn(column TransactionDataTableColumn) bool + + // TransactionRowCount returns the total count of transaction data row + TransactionRowCount() int + + // TransactionRowIterator returns the iterator of transaction data row + TransactionRowIterator() TransactionDataRowIterator +} + +// TransactionDataRow defines the structure of transaction data row +type TransactionDataRow interface { + // IsValid returns whether this row is valid data for importing + IsValid() bool + + // GetData returns the data in the specified column type + GetData(column TransactionDataTableColumn) string +} + +// TransactionDataRowIterator defines the structure of transaction data row iterator +type TransactionDataRowIterator interface { + // HasNext returns whether the iterator does not reach the end + HasNext() bool + + // Next returns the next transaction data row + Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) +} + +// TransactionDataRowParser defines the structure of transaction data row parser +type TransactionDataRowParser interface { + // GetAddedColumns returns the added columns after converting the data row + GetAddedColumns() []TransactionDataTableColumn + + // Parse returns the converted transaction data row + Parse(data map[TransactionDataTableColumn]string) (rowData map[TransactionDataTableColumn]string, rowDataValid bool, err error) +} + +// TransactionDataTableBuilder defines the structure of data table builder +type TransactionDataTableBuilder interface { + // AppendTransaction appends the specified transaction to data builder + AppendTransaction(data map[TransactionDataTableColumn]string) + + // ReplaceDelimiters returns the text after removing the delimiters + ReplaceDelimiters(text string) string +} + +// TransactionDataTableColumn represents the data column type of data table +type TransactionDataTableColumn byte + +// Transaction data table columns +const ( + TRANSACTION_DATA_TABLE_TRANSACTION_TIME TransactionDataTableColumn = 1 + TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE TransactionDataTableColumn = 2 + TRANSACTION_DATA_TABLE_TRANSACTION_TYPE TransactionDataTableColumn = 3 + TRANSACTION_DATA_TABLE_CATEGORY TransactionDataTableColumn = 4 + TRANSACTION_DATA_TABLE_SUB_CATEGORY TransactionDataTableColumn = 5 + TRANSACTION_DATA_TABLE_ACCOUNT_NAME TransactionDataTableColumn = 6 + TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY TransactionDataTableColumn = 7 + TRANSACTION_DATA_TABLE_AMOUNT TransactionDataTableColumn = 8 + TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME TransactionDataTableColumn = 9 + TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY TransactionDataTableColumn = 10 + TRANSACTION_DATA_TABLE_RELATED_AMOUNT TransactionDataTableColumn = 11 + TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION TransactionDataTableColumn = 12 + TRANSACTION_DATA_TABLE_TAGS TransactionDataTableColumn = 13 + TRANSACTION_DATA_TABLE_DESCRIPTION TransactionDataTableColumn = 14 +) diff --git a/pkg/converters/datatable/writable_data_table.go b/pkg/converters/datatable/writable_data_table.go deleted file mode 100644 index cc2554d8..00000000 --- a/pkg/converters/datatable/writable_data_table.go +++ /dev/null @@ -1,152 +0,0 @@ -package datatable - -import ( - "time" - - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" -) - -// WritableDataTable defines the structure of writable data table -type WritableDataTable struct { - allData []map[DataTableColumn]string - columns []DataTableColumn -} - -// WritableDataRow defines the structure of data row of writable data table -type WritableDataRow struct { - dataTable *WritableDataTable - rowData map[DataTableColumn]string -} - -// WritableDataRowIterator defines the structure of data row iterator of writable data table -type WritableDataRowIterator struct { - dataTable *WritableDataTable - nextIndex int -} - -// Add appends a new record to data table -func (t *WritableDataTable) Add(data map[DataTableColumn]string) { - finalData := make(map[DataTableColumn]string, len(data)) - - for i := 0; i < len(t.columns); i++ { - column := t.columns[i] - - if value, exists := data[column]; exists { - finalData[column] = value - } - } - - t.allData = append(t.allData, finalData) -} - -// Get returns the record in the specified index -func (t *WritableDataTable) Get(index int) ImportedDataRow { - if index >= len(t.allData) { - return nil - } - - rowData := t.allData[index] - - return &WritableDataRow{ - dataTable: t, - rowData: rowData, - } -} - -// DataRowCount returns the total count of data row -func (t *WritableDataTable) DataRowCount() int { - return len(t.allData) -} - -// GetDataColumnMapping returns data column map for data importer -func (t *WritableDataTable) GetDataColumnMapping() map[DataTableColumn]string { - dataColumnMapping := make(map[DataTableColumn]string, len(t.columns)) - - for i := 0; i < len(t.columns); i++ { - column := t.columns[i] - dataColumnMapping[column] = utils.IntToString(int(column)) - } - - return dataColumnMapping -} - -// HeaderLineColumnNames returns the header column name list -func (t *WritableDataTable) HeaderLineColumnNames() []string { - columnIndexes := make([]string, len(t.columns)) - - for i := 0; i < len(t.columns); i++ { - columnIndexes[i] = utils.IntToString(int(t.columns[i])) - } - - return columnIndexes -} - -// DataRowIterator returns the iterator of data row -func (t *WritableDataTable) DataRowIterator() ImportedDataRowIterator { - return &WritableDataRowIterator{ - dataTable: t, - nextIndex: 0, - } -} - -// IsValid returns whether this row contains valid data for importing -func (r *WritableDataRow) IsValid() bool { - return true -} - -// ColumnCount returns the total count of column in this data row -func (r *WritableDataRow) ColumnCount() int { - return len(r.rowData) -} - -// GetData returns the data in the specified column index -func (r *WritableDataRow) GetData(columnIndex int) string { - if columnIndex >= len(r.dataTable.columns) { - return "" - } - - dataColumn := r.dataTable.columns[columnIndex] - - return r.rowData[dataColumn] -} - -// GetTime returns the time in the specified column index -func (r *WritableDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) { - return utils.ParseFromLongDateTime(r.GetData(columnIndex), timezoneOffset) -} - -// GetTimezoneOffset returns the time zone offset in the specified column index -func (r *WritableDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) { - return utils.ParseFromTimezoneOffset(r.GetData(columnIndex)) -} - -// HasNext returns whether the iterator does not reach the end -func (t *WritableDataRowIterator) HasNext() bool { - return t.nextIndex < len(t.dataTable.allData) -} - -// Next returns the next imported data row -func (t *WritableDataRowIterator) Next(ctx core.Context, user *models.User) ImportedDataRow { - if t.nextIndex >= len(t.dataTable.allData) { - return nil - } - - rowData := t.dataTable.allData[t.nextIndex] - - t.nextIndex++ - - return &WritableDataRow{ - dataTable: t.dataTable, - rowData: rowData, - } -} - -// CreateNewWritableDataTable returns a new writable data table according to the specified columns -func CreateNewWritableDataTable(columns []DataTableColumn) *WritableDataTable { - return &WritableDataTable{ - allData: make([]map[DataTableColumn]string, 0), - columns: columns, - } -} diff --git a/pkg/converters/datatable/writable_data_table_test.go b/pkg/converters/datatable/writable_data_table_test.go deleted file mode 100644 index d3266792..00000000 --- a/pkg/converters/datatable/writable_data_table_test.go +++ /dev/null @@ -1,208 +0,0 @@ -package datatable - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" -) - -func TestWritableDataTableAdd(t *testing.T) { - columns := make([]DataTableColumn, 5) - columns[0] = DATA_TABLE_TRANSACTION_TIME - columns[1] = DATA_TABLE_TRANSACTION_TYPE - columns[2] = DATA_TABLE_SUB_CATEGORY - columns[3] = DATA_TABLE_ACCOUNT_NAME - columns[4] = DATA_TABLE_AMOUNT - - writableDataTable := CreateNewWritableDataTable(columns) - - assert.Equal(t, 0, writableDataTable.DataRowCount()) - - expectedTransactionUnixTime := time.Now().Unix() - expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local) - expectedTransactionType := "Expense" - expectedSubCategory := "Test Category" - expectedAccountName := "Test Account" - expectedAmount := "123.45" - - writableDataTable.Add(map[DataTableColumn]string{ - DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime, - DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType, - DATA_TABLE_SUB_CATEGORY: expectedSubCategory, - DATA_TABLE_ACCOUNT_NAME: expectedAccountName, - DATA_TABLE_AMOUNT: expectedAmount, - }) - assert.Equal(t, 1, writableDataTable.DataRowCount()) - - dataRow := writableDataTable.Get(0) - - actualTransactionTime, err := dataRow.GetTime(0, utils.GetTimezoneOffsetMinutes(time.Local)) - assert.Nil(t, err) - - actualTransactionUnixTime := actualTransactionTime.Unix() - assert.Equal(t, expectedTransactionUnixTime, actualTransactionUnixTime) - - actualTextualTransactionTime := dataRow.GetData(0) - assert.Equal(t, expectedTextualTransactionTime, actualTextualTransactionTime) - - actualTransactionType := dataRow.GetData(1) - assert.Equal(t, expectedTransactionType, actualTransactionType) - - actualSubCategory := dataRow.GetData(2) - assert.Equal(t, expectedSubCategory, actualSubCategory) - - actualAccountName := dataRow.GetData(3) - assert.Equal(t, expectedAccountName, actualAccountName) - - actualAmount := dataRow.GetData(4) - assert.Equal(t, expectedAmount, actualAmount) -} - -func TestWritableDataTableAdd_NotExistsColumn(t *testing.T) { - columns := make([]DataTableColumn, 1) - columns[0] = DATA_TABLE_TRANSACTION_TIME - - writableDataTable := CreateNewWritableDataTable(columns) - - expectedTransactionUnixTime := time.Now().Unix() - expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local) - expectedTransactionType := "Expense" - - writableDataTable.Add(map[DataTableColumn]string{ - DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime, - DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType, - }) - assert.Equal(t, 1, writableDataTable.DataRowCount()) - - dataRow := writableDataTable.Get(0) - assert.Equal(t, 1, dataRow.ColumnCount()) -} - -func TestWritableDataTableGet_NotExistsRow(t *testing.T) { - columns := make([]DataTableColumn, 1) - columns[0] = DATA_TABLE_TRANSACTION_TIME - - writableDataTable := CreateNewWritableDataTable(columns) - assert.Equal(t, 0, writableDataTable.DataRowCount()) - - dataRow := writableDataTable.Get(0) - assert.Nil(t, dataRow) -} - -func TestWritableDataRowGetData_NotExistsColumn(t *testing.T) { - columns := make([]DataTableColumn, 1) - columns[0] = DATA_TABLE_TRANSACTION_TIME - - writableDataTable := CreateNewWritableDataTable(columns) - - expectedTransactionUnixTime := time.Now().Unix() - expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local) - - writableDataTable.Add(map[DataTableColumn]string{ - DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime, - }) - assert.Equal(t, 1, writableDataTable.DataRowCount()) - - dataRow := writableDataTable.Get(0) - assert.Equal(t, 1, dataRow.ColumnCount()) - assert.Equal(t, "", dataRow.GetData(1)) -} - -func TestWritableDataTableDataRowIterator(t *testing.T) { - columns := make([]DataTableColumn, 5) - columns[0] = DATA_TABLE_TRANSACTION_TIME - columns[1] = DATA_TABLE_TRANSACTION_TYPE - columns[2] = DATA_TABLE_SUB_CATEGORY - columns[3] = DATA_TABLE_ACCOUNT_NAME - columns[4] = DATA_TABLE_AMOUNT - - writableDataTable := CreateNewWritableDataTable(columns) - assert.Equal(t, 0, writableDataTable.DataRowCount()) - - expectedTransactionUnixTimes := make([]int64, 3) - expectedTextualTransactionTimes := make([]string, 3) - expectedTransactionTypes := make([]string, 3) - expectedSubCategories := make([]string, 3) - expectedAccountNames := make([]string, 3) - expectedAmounts := make([]string, 3) - - expectedTransactionUnixTimes[0] = time.Now().Add(-5 * time.Hour).Unix() - expectedTextualTransactionTimes[0] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[0], time.Local) - expectedTransactionTypes[0] = "Balance Modification" - expectedSubCategories[0] = "" - expectedAccountNames[0] = "Test Account" - expectedAmounts[0] = "123.45" - writableDataTable.Add(map[DataTableColumn]string{ - DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTimes[0], - DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[0], - DATA_TABLE_SUB_CATEGORY: expectedSubCategories[0], - DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[0], - DATA_TABLE_AMOUNT: expectedAmounts[0], - }) - - expectedTransactionUnixTimes[1] = time.Now().Add(-45 * time.Minute).Unix() - expectedTextualTransactionTimes[1] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[1], time.Local) - expectedTransactionTypes[1] = "Expense" - expectedSubCategories[1] = "Test Category2" - expectedAccountNames[1] = "Test Account" - expectedAmounts[1] = "-23.4" - writableDataTable.Add(map[DataTableColumn]string{ - DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTimes[1], - DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[1], - DATA_TABLE_SUB_CATEGORY: expectedSubCategories[1], - DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[1], - DATA_TABLE_AMOUNT: expectedAmounts[1], - }) - - expectedTransactionUnixTimes[2] = time.Now().Unix() - expectedTextualTransactionTimes[2] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[2], time.Local) - expectedTransactionTypes[2] = "Income" - expectedSubCategories[2] = "Test Category3" - expectedAccountNames[2] = "Test Account2" - expectedAmounts[2] = "123" - writableDataTable.Add(map[DataTableColumn]string{ - DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTimes[2], - DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[2], - DATA_TABLE_SUB_CATEGORY: expectedSubCategories[2], - DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[2], - DATA_TABLE_AMOUNT: expectedAmounts[2], - }) - assert.Equal(t, 3, writableDataTable.DataRowCount()) - - index := 0 - iterator := writableDataTable.DataRowIterator() - - for iterator.HasNext() { - dataRow := iterator.Next(core.NewNullContext(), &models.User{}) - - actualTransactionTime, err := dataRow.GetTime(0, utils.GetTimezoneOffsetMinutes(time.Local)) - assert.Nil(t, err) - - actualTransactionUnixTime := actualTransactionTime.Unix() - assert.Equal(t, expectedTransactionUnixTimes[index], actualTransactionUnixTime) - - actualTextualTransactionTime := dataRow.GetData(0) - assert.Equal(t, expectedTextualTransactionTimes[index], actualTextualTransactionTime) - - actualTransactionType := dataRow.GetData(1) - assert.Equal(t, expectedTransactionTypes[index], actualTransactionType) - - actualSubCategory := dataRow.GetData(2) - assert.Equal(t, expectedSubCategories[index], actualSubCategory) - - actualAccountName := dataRow.GetData(3) - assert.Equal(t, expectedAccountNames[index], actualAccountName) - - actualAmount := dataRow.GetData(4) - assert.Equal(t, expectedAmounts[index], actualAmount) - - index++ - } - - assert.Equal(t, 3, index) -} diff --git a/pkg/converters/datatable/writable_transaction_data_table.go b/pkg/converters/datatable/writable_transaction_data_table.go new file mode 100644 index 00000000..8d9a0e73 --- /dev/null +++ b/pkg/converters/datatable/writable_transaction_data_table.go @@ -0,0 +1,169 @@ +package datatable + +import ( + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" +) + +// WritableTransactionDataTable defines the structure of writable transaction data table +type WritableTransactionDataTable struct { + allData []map[TransactionDataTableColumn]string + supportedColumns map[TransactionDataTableColumn]any + rowParser TransactionDataRowParser + addedColumns map[TransactionDataTableColumn]any +} + +// WritableTransactionDataRow defines the structure of transaction data row of writable data table +type WritableTransactionDataRow struct { + dataTable *WritableTransactionDataTable + rowData map[TransactionDataTableColumn]string + rowDataValid bool +} + +// WritableTransactionDataRowIterator defines the structure of transaction data row iterator of writable data table +type WritableTransactionDataRowIterator struct { + dataTable *WritableTransactionDataTable + nextIndex int +} + +// Add appends a new record to data table +func (t *WritableTransactionDataTable) Add(data map[TransactionDataTableColumn]string) { + finalData := make(map[TransactionDataTableColumn]string, len(data)) + + for column, value := range data { + _, exists := t.supportedColumns[column] + + if exists { + finalData[column] = value + } + } + + t.allData = append(t.allData, finalData) +} + +// Get returns the record in the specified index +func (t *WritableTransactionDataTable) Get(index int) *WritableTransactionDataRow { + if index >= len(t.allData) { + return nil + } + + rowData := t.allData[index] + + return &WritableTransactionDataRow{ + dataTable: t, + rowData: rowData, + } +} + +// HasColumn returns whether the data table has specified column +func (t *WritableTransactionDataTable) HasColumn(column TransactionDataTableColumn) bool { + _, exists := t.supportedColumns[column] + + if exists { + return exists + } + + if t.addedColumns != nil { + _, exists = t.addedColumns[column] + + if exists { + return exists + } + } + + return false +} + +// TransactionRowCount returns the total count of transaction data row +func (t *WritableTransactionDataTable) TransactionRowCount() int { + return len(t.allData) +} + +// TransactionRowIterator returns the iterator of transaction data row +func (t *WritableTransactionDataTable) TransactionRowIterator() TransactionDataRowIterator { + return &WritableTransactionDataRowIterator{ + dataTable: t, + nextIndex: 0, + } +} + +// ColumnCount returns the total count of column in this data row +func (r *WritableTransactionDataRow) ColumnCount() int { + return len(r.rowData) +} + +// IsValid returns whether this row is valid data for importing +func (r *WritableTransactionDataRow) IsValid() bool { + return r.rowDataValid +} + +// GetData returns the data in the specified column type +func (r *WritableTransactionDataRow) GetData(column TransactionDataTableColumn) string { + return r.rowData[column] +} + +// HasNext returns whether the iterator does not reach the end +func (t *WritableTransactionDataRowIterator) HasNext() bool { + return t.nextIndex < len(t.dataTable.allData) +} + +// Next returns the next transaction data row +func (t *WritableTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) { + if t.nextIndex >= len(t.dataTable.allData) { + return nil, nil + } + + rowData := t.dataTable.allData[t.nextIndex] + rowDataValid := true + + if t.dataTable.rowParser != nil { + rowData, rowDataValid, err = t.dataTable.rowParser.Parse(rowData) + + if err != nil { + log.Errorf(ctx, "[writable_transaction_data_table.Next] cannot parse data row, because %s", err.Error()) + return nil, err + } + } + + t.nextIndex++ + + return &WritableTransactionDataRow{ + dataTable: t.dataTable, + rowData: rowData, + rowDataValid: rowDataValid, + }, nil +} + +// CreateNewWritableTransactionDataTable returns a new writable transaction data table according to the specified columns +func CreateNewWritableTransactionDataTable(columns []TransactionDataTableColumn) *WritableTransactionDataTable { + return CreateNewWritableTransactionDataTableWithRowParser(columns, nil) +} + +// CreateNewWritableTransactionDataTableWithRowParser returns a new writable transaction data table according to the specified columns +func CreateNewWritableTransactionDataTableWithRowParser(columns []TransactionDataTableColumn, rowParser TransactionDataRowParser) *WritableTransactionDataTable { + supportedColumns := make(map[TransactionDataTableColumn]any, len(columns)) + + for i := 0; i < len(columns); i++ { + column := columns[i] + supportedColumns[column] = true + } + + var addedColumns map[TransactionDataTableColumn]any + + if rowParser != nil { + addedColumnsByParser := rowParser.GetAddedColumns() + addedColumns = make(map[TransactionDataTableColumn]any, len(addedColumnsByParser)) + + for i := 0; i < len(addedColumnsByParser); i++ { + addedColumns[addedColumnsByParser[i]] = true + } + } + + return &WritableTransactionDataTable{ + allData: make([]map[TransactionDataTableColumn]string, 0), + supportedColumns: supportedColumns, + rowParser: rowParser, + addedColumns: addedColumns, + } +} diff --git a/pkg/converters/datatable/writable_transaction_data_table_test.go b/pkg/converters/datatable/writable_transaction_data_table_test.go new file mode 100644 index 00000000..19873353 --- /dev/null +++ b/pkg/converters/datatable/writable_transaction_data_table_test.go @@ -0,0 +1,196 @@ +package datatable + +import ( + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/models" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +func TestWritableDataTableAdd(t *testing.T) { + columns := make([]TransactionDataTableColumn, 5) + columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME + columns[1] = TRANSACTION_DATA_TABLE_TRANSACTION_TYPE + columns[2] = TRANSACTION_DATA_TABLE_SUB_CATEGORY + columns[3] = TRANSACTION_DATA_TABLE_ACCOUNT_NAME + columns[4] = TRANSACTION_DATA_TABLE_AMOUNT + + writableDataTable := CreateNewWritableTransactionDataTable(columns) + + assert.Equal(t, 0, writableDataTable.TransactionRowCount()) + + expectedTransactionTime := "2024-09-01 01:23:45" + expectedTransactionType := "Expense" + expectedSubCategory := "Test Category" + expectedAccountName := "Test Account" + expectedAmount := "123.45" + + writableDataTable.Add(map[TransactionDataTableColumn]string{ + TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTime, + TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType, + TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategory, + TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountName, + TRANSACTION_DATA_TABLE_AMOUNT: expectedAmount, + }) + assert.Equal(t, 1, writableDataTable.TransactionRowCount()) + + dataRow := writableDataTable.Get(0) + + actualTransactionTime := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME) + assert.Equal(t, expectedTransactionTime, actualTransactionTime) + + actualTransactionType := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) + assert.Equal(t, expectedTransactionType, actualTransactionType) + + actualSubCategory := dataRow.GetData(TRANSACTION_DATA_TABLE_SUB_CATEGORY) + assert.Equal(t, expectedSubCategory, actualSubCategory) + + actualAccountName := dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_NAME) + assert.Equal(t, expectedAccountName, actualAccountName) + + actualAmount := dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT) + assert.Equal(t, expectedAmount, actualAmount) +} + +func TestWritableDataTableAdd_NotExistsColumn(t *testing.T) { + columns := make([]TransactionDataTableColumn, 1) + columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME + + writableDataTable := CreateNewWritableTransactionDataTable(columns) + + expectedTransactionUnixTime := time.Now().Unix() + expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local) + expectedTransactionType := "Expense" + + writableDataTable.Add(map[TransactionDataTableColumn]string{ + TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime, + TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType, + }) + assert.Equal(t, 1, writableDataTable.TransactionRowCount()) + + dataRow := writableDataTable.Get(0) + assert.Equal(t, 1, dataRow.ColumnCount()) +} + +func TestWritableDataTableGet_NotExistsRow(t *testing.T) { + columns := make([]TransactionDataTableColumn, 1) + columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME + + writableDataTable := CreateNewWritableTransactionDataTable(columns) + assert.Equal(t, 0, writableDataTable.TransactionRowCount()) + + dataRow := writableDataTable.Get(0) + assert.Nil(t, dataRow) +} + +func TestWritableDataRowGetData_NotExistsColumn(t *testing.T) { + columns := make([]TransactionDataTableColumn, 1) + columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME + + writableDataTable := CreateNewWritableTransactionDataTable(columns) + + expectedTransactionUnixTime := time.Now().Unix() + expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local) + + writableDataTable.Add(map[TransactionDataTableColumn]string{ + TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime, + }) + assert.Equal(t, 1, writableDataTable.TransactionRowCount()) + + dataRow := writableDataTable.Get(0) + assert.Equal(t, 1, dataRow.ColumnCount()) + assert.Equal(t, "", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)) +} + +func TestWritableDataTableDataRowIterator(t *testing.T) { + columns := make([]TransactionDataTableColumn, 5) + columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME + columns[1] = TRANSACTION_DATA_TABLE_TRANSACTION_TYPE + columns[2] = TRANSACTION_DATA_TABLE_SUB_CATEGORY + columns[3] = TRANSACTION_DATA_TABLE_ACCOUNT_NAME + columns[4] = TRANSACTION_DATA_TABLE_AMOUNT + + writableDataTable := CreateNewWritableTransactionDataTable(columns) + assert.Equal(t, 0, writableDataTable.TransactionRowCount()) + + expectedTransactionUnixTimes := make([]int64, 3) + expectedTransactionTimes := make([]string, 3) + expectedTransactionTypes := make([]string, 3) + expectedSubCategories := make([]string, 3) + expectedAccountNames := make([]string, 3) + expectedAmounts := make([]string, 3) + + expectedTransactionUnixTimes[0] = time.Now().Add(-5 * time.Hour).Unix() + expectedTransactionTimes[0] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[0], time.Local) + expectedTransactionTypes[0] = "Balance Modification" + expectedSubCategories[0] = "" + expectedAccountNames[0] = "Test Account" + expectedAmounts[0] = "123.45" + writableDataTable.Add(map[TransactionDataTableColumn]string{ + TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTimes[0], + TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[0], + TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategories[0], + TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[0], + TRANSACTION_DATA_TABLE_AMOUNT: expectedAmounts[0], + }) + + expectedTransactionUnixTimes[1] = time.Now().Add(-45 * time.Minute).Unix() + expectedTransactionTimes[1] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[1], time.Local) + expectedTransactionTypes[1] = "Expense" + expectedSubCategories[1] = "Test Category2" + expectedAccountNames[1] = "Test Account" + expectedAmounts[1] = "-23.4" + writableDataTable.Add(map[TransactionDataTableColumn]string{ + TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTimes[1], + TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[1], + TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategories[1], + TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[1], + TRANSACTION_DATA_TABLE_AMOUNT: expectedAmounts[1], + }) + + expectedTransactionUnixTimes[2] = time.Now().Unix() + expectedTransactionTimes[2] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[2], time.Local) + expectedTransactionTypes[2] = "Income" + expectedSubCategories[2] = "Test Category3" + expectedAccountNames[2] = "Test Account2" + expectedAmounts[2] = "123" + writableDataTable.Add(map[TransactionDataTableColumn]string{ + TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTimes[2], + TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[2], + TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategories[2], + TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[2], + TRANSACTION_DATA_TABLE_AMOUNT: expectedAmounts[2], + }) + assert.Equal(t, 3, writableDataTable.TransactionRowCount()) + + index := 0 + iterator := writableDataTable.TransactionRowIterator() + + for iterator.HasNext() { + dataRow, err := iterator.Next(core.NewNullContext(), &models.User{}) + assert.Nil(t, err) + + actualTransactionTime := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME) + assert.Equal(t, expectedTransactionTimes[index], actualTransactionTime) + + actualTransactionType := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) + assert.Equal(t, expectedTransactionTypes[index], actualTransactionType) + + actualSubCategory := dataRow.GetData(TRANSACTION_DATA_TABLE_SUB_CATEGORY) + assert.Equal(t, expectedSubCategories[index], actualSubCategory) + + actualAccountName := dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_NAME) + assert.Equal(t, expectedAccountNames[index], actualAccountName) + + actualAmount := dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT) + assert.Equal(t, expectedAmounts[index], actualAmount) + + index++ + } + + assert.Equal(t, 3, index) +} diff --git a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go index 4972a944..c47a499e 100644 --- a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go +++ b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go @@ -15,21 +15,21 @@ const ezbookkeepingLineSeparator = "\n" const ezbookkeepingGeoLocationSeparator = " " const ezbookkeepingTagSeparator = ";" -var ezbookkeepingDataColumnNameMapping = map[datatable.DataTableColumn]string{ - datatable.DATA_TABLE_TRANSACTION_TIME: "Time", - datatable.DATA_TABLE_TRANSACTION_TIMEZONE: "Timezone", - datatable.DATA_TABLE_TRANSACTION_TYPE: "Type", - datatable.DATA_TABLE_CATEGORY: "Category", - datatable.DATA_TABLE_SUB_CATEGORY: "Sub Category", - datatable.DATA_TABLE_ACCOUNT_NAME: "Account", - datatable.DATA_TABLE_ACCOUNT_CURRENCY: "Account Currency", - datatable.DATA_TABLE_AMOUNT: "Amount", - datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: "Account2", - datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "Account2 Currency", - datatable.DATA_TABLE_RELATED_AMOUNT: "Account2 Amount", - datatable.DATA_TABLE_GEOGRAPHIC_LOCATION: "Geographic Location", - datatable.DATA_TABLE_TAGS: "Tags", - datatable.DATA_TABLE_DESCRIPTION: "Description", +var ezbookkeepingDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "Time", + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: "Timezone", + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "Type", + datatable.TRANSACTION_DATA_TABLE_CATEGORY: "Category", + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "Sub Category", + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "Account", + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "Account Currency", + datatable.TRANSACTION_DATA_TABLE_AMOUNT: "Amount", + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "Account2", + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "Account2 Currency", + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "Account2 Amount", + datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION: "Geographic Location", + datatable.TRANSACTION_DATA_TABLE_TAGS: "Tags", + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "Description", } var ezbookkeepingTransactionTypeNameMapping = map[models.TransactionType]string{ @@ -39,21 +39,21 @@ var ezbookkeepingTransactionTypeNameMapping = map[models.TransactionType]string{ models.TRANSACTION_TYPE_TRANSFER: "Transfer", } -var ezbookkeepingDataColumns = []datatable.DataTableColumn{ - datatable.DATA_TABLE_TRANSACTION_TIME, - datatable.DATA_TABLE_TRANSACTION_TIMEZONE, - datatable.DATA_TABLE_TRANSACTION_TYPE, - datatable.DATA_TABLE_CATEGORY, - datatable.DATA_TABLE_SUB_CATEGORY, - datatable.DATA_TABLE_ACCOUNT_NAME, - datatable.DATA_TABLE_ACCOUNT_CURRENCY, - datatable.DATA_TABLE_AMOUNT, - datatable.DATA_TABLE_RELATED_ACCOUNT_NAME, - datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY, - datatable.DATA_TABLE_RELATED_AMOUNT, - datatable.DATA_TABLE_GEOGRAPHIC_LOCATION, - datatable.DATA_TABLE_TAGS, - datatable.DATA_TABLE_DESCRIPTION, +var ezbookkeepingDataColumns = []datatable.TransactionDataTableColumn{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE, + datatable.TRANSACTION_DATA_TABLE_CATEGORY, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY, + datatable.TRANSACTION_DATA_TABLE_AMOUNT, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT, + datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION, + datatable.TRANSACTION_DATA_TABLE_TAGS, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION, } // ToExportedContent returns the exported transaction plain text data @@ -67,7 +67,6 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ToExportedContent(ctx c ) dataTableExporter := datatable.CreateNewExporter( - ezbookkeepingDataColumnNameMapping, ezbookkeepingTransactionTypeNameMapping, ezbookkeepingGeoLocationSeparator, ezbookkeepingTagSeparator, @@ -84,7 +83,7 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ToExportedContent(ctx c // ParseImportedData returns the imported data by parsing the transaction plain text data func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - dataTable, err := createNewezbookkeepingTransactionPlainTextDataTable( + dataTable, err := createNewezbookkeepingPlainTextDataTable( string(data), c.columnSeparator, ezbookkeepingLineSeparator, @@ -94,12 +93,13 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx c return nil, nil, nil, nil, nil, nil, err } + transactionDataTable := datatable.CreateImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping) + dataTableImporter := datatable.CreateNewImporter( - ezbookkeepingDataColumnNameMapping, ezbookkeepingTransactionTypeNameMapping, ezbookkeepingGeoLocationSeparator, ezbookkeepingTagSeparator, ) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } diff --git a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go index 7adb4639..9aa7df71 100644 --- a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go +++ b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go @@ -238,10 +238,22 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidTimezone(t * } allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ - "2024-09-01 12:34:56,+08:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) + "2024-09-01 12:34:56,-10:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) assert.Nil(t, err) assert.Equal(t, 1, len(allNewTransactions)) - assert.Equal(t, int64(1725165296), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,+00:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+ + "2024-09-01 12:34:56,+12:45,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) } func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTimezone(t *testing.T) { diff --git a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_data_table.go b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_data_table.go index 06da1ec5..02b10a09 100644 --- a/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_data_table.go +++ b/pkg/converters/default/ezbookkeeping_transaction_data_plain_text_data_table.go @@ -3,31 +3,27 @@ package _default import ( "fmt" "strings" - "time" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" - "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" ) -// ezBookKeepingTransactionPlainTextDataTable defines the structure of ezbookkeeping transaction plain text data table -type ezBookKeepingTransactionPlainTextDataTable struct { +// ezBookKeepingPlainTextDataTable defines the structure of ezbookkeeping plain text data table +type ezBookKeepingPlainTextDataTable struct { columnSeparator string lineSeparator string allLines []string headerLineColumnNames []string } -// ezBookKeepingTransactionPlainTextDataRow defines the structure of ezbookkeeping transaction plain text data row -type ezBookKeepingTransactionPlainTextDataRow struct { +// ezBookKeepingPlainTextDataRow defines the structure of ezbookkeeping plain text data row +type ezBookKeepingPlainTextDataRow struct { allItems []string } -// ezBookKeepingTransactionPlainTextDataRowIterator defines the structure of ezbookkeeping transaction plain text data row iterator -type ezBookKeepingTransactionPlainTextDataRowIterator struct { - dataTable *ezBookKeepingTransactionPlainTextDataTable +// ezBookKeepingPlainTextDataRowIterator defines the structure of ezbookkeeping plain text data row iterator +type ezBookKeepingPlainTextDataRowIterator struct { + dataTable *ezBookKeepingPlainTextDataTable currentIndex int } @@ -35,14 +31,14 @@ type ezBookKeepingTransactionPlainTextDataRowIterator struct { type ezBookKeepingTransactionPlainTextDataTableBuilder struct { columnSeparator string lineSeparator string - columns []datatable.DataTableColumn - dataColumnNameMapping map[datatable.DataTableColumn]string + columns []datatable.TransactionDataTableColumn + dataColumnNameMapping map[datatable.TransactionDataTableColumn]string dataLineFormat string builder *strings.Builder } // DataRowCount returns the total count of data row -func (t *ezBookKeepingTransactionPlainTextDataTable) DataRowCount() int { +func (t *ezBookKeepingPlainTextDataTable) DataRowCount() int { if len(t.allLines) < 1 { return 0 } @@ -50,31 +46,26 @@ func (t *ezBookKeepingTransactionPlainTextDataTable) DataRowCount() int { return len(t.allLines) - 1 } -// HeaderLineColumnNames returns the header column name list -func (t *ezBookKeepingTransactionPlainTextDataTable) HeaderLineColumnNames() []string { +// HeaderColumnNames returns the header column name list +func (t *ezBookKeepingPlainTextDataTable) HeaderColumnNames() []string { return t.headerLineColumnNames } // DataRowIterator returns the iterator of data row -func (t *ezBookKeepingTransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &ezBookKeepingTransactionPlainTextDataRowIterator{ +func (t *ezBookKeepingPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator { + return &ezBookKeepingPlainTextDataRowIterator{ dataTable: t, currentIndex: 0, } } -// IsValid returns whether this row contains valid data for importing -func (r *ezBookKeepingTransactionPlainTextDataRow) IsValid() bool { - return true -} - // ColumnCount returns the total count of column in this data row -func (r *ezBookKeepingTransactionPlainTextDataRow) ColumnCount() int { +func (r *ezBookKeepingPlainTextDataRow) ColumnCount() int { return len(r.allItems) } // GetData returns the data in the specified column index -func (r *ezBookKeepingTransactionPlainTextDataRow) GetData(columnIndex int) string { +func (r *ezBookKeepingPlainTextDataRow) GetData(columnIndex int) string { if columnIndex >= len(r.allItems) { return "" } @@ -82,23 +73,13 @@ func (r *ezBookKeepingTransactionPlainTextDataRow) GetData(columnIndex int) stri return r.allItems[columnIndex] } -// GetTime returns the time in the specified column index -func (r *ezBookKeepingTransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) { - return utils.ParseFromLongDateTime(r.GetData(columnIndex), timezoneOffset) -} - -// GetTimezoneOffset returns the time zone offset in the specified column index -func (r *ezBookKeepingTransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) { - return utils.ParseFromTimezoneOffset(r.GetData(columnIndex)) -} - // HasNext returns whether the iterator does not reach the end -func (t *ezBookKeepingTransactionPlainTextDataRowIterator) HasNext() bool { +func (t *ezBookKeepingPlainTextDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allLines) } // Next returns the next imported data row -func (t *ezBookKeepingTransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow { +func (t *ezBookKeepingPlainTextDataRowIterator) Next() datatable.ImportedDataRow { if t.currentIndex+1 >= len(t.dataTable.allLines) { return nil } @@ -108,13 +89,13 @@ func (t *ezBookKeepingTransactionPlainTextDataRowIterator) Next(ctx core.Context rowContent := t.dataTable.allLines[t.currentIndex] rowItems := strings.Split(rowContent, t.dataTable.columnSeparator) - return &ezBookKeepingTransactionPlainTextDataRow{ + return &ezBookKeepingPlainTextDataRow{ allItems: rowItems, } } // AppendTransaction appends the specified transaction to data builder -func (b *ezBookKeepingTransactionPlainTextDataTableBuilder) AppendTransaction(data map[datatable.DataTableColumn]string) { +func (b *ezBookKeepingTransactionPlainTextDataTableBuilder) AppendTransaction(data map[datatable.TransactionDataTableColumn]string) { dataRowParams := make([]any, len(b.columns)) for i := 0; i < len(b.columns); i++ { @@ -175,7 +156,7 @@ func (b *ezBookKeepingTransactionPlainTextDataTableBuilder) generateDataLineForm return ret.String() } -func createNewezbookkeepingTransactionPlainTextDataTable(content string, columnSeparator string, lineSeparator string) (*ezBookKeepingTransactionPlainTextDataTable, error) { +func createNewezbookkeepingPlainTextDataTable(content string, columnSeparator string, lineSeparator string) (*ezBookKeepingPlainTextDataTable, error) { allLines := strings.Split(content, lineSeparator) if len(allLines) < 2 { @@ -186,7 +167,7 @@ func createNewezbookkeepingTransactionPlainTextDataTable(content string, columnS headerLine = strings.ReplaceAll(headerLine, "\r", "") headerLineItems := strings.Split(headerLine, columnSeparator) - return &ezBookKeepingTransactionPlainTextDataTable{ + return &ezBookKeepingPlainTextDataTable{ columnSeparator: columnSeparator, lineSeparator: lineSeparator, allLines: allLines, @@ -194,7 +175,7 @@ func createNewezbookkeepingTransactionPlainTextDataTable(content string, columnS }, nil } -func createNewezbookkeepingTransactionPlainTextDataTableBuilder(transactionCount int, columns []datatable.DataTableColumn, dataColumnNameMapping map[datatable.DataTableColumn]string, columnSeparator string, lineSeparator string) *ezBookKeepingTransactionPlainTextDataTableBuilder { +func createNewezbookkeepingTransactionPlainTextDataTableBuilder(transactionCount int, columns []datatable.TransactionDataTableColumn, dataColumnNameMapping map[datatable.TransactionDataTableColumn]string, columnSeparator string, lineSeparator string) *ezBookKeepingTransactionPlainTextDataTableBuilder { var builder strings.Builder builder.Grow(transactionCount * 100) diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go index 431f1bdf..1fcceea8 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go @@ -43,7 +43,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con return nil, nil, nil, nil, nil, nil, err } - if len(allLines) <= 1 { + if len(allLines) < 2 { log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile } @@ -71,36 +71,37 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } - newColumns := make([]datatable.DataTableColumn, 0, 11) - newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TYPE) - newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TIME) + newColumns := make([]datatable.TransactionDataTableColumn, 0, 11) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME) if categoryColumnExists { - newColumns = append(newColumns, datatable.DATA_TABLE_CATEGORY) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_CATEGORY) } - newColumns = append(newColumns, datatable.DATA_TABLE_SUB_CATEGORY) - newColumns = append(newColumns, datatable.DATA_TABLE_ACCOUNT_NAME) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME) if accountCurrencyColumnExists { - newColumns = append(newColumns, datatable.DATA_TABLE_ACCOUNT_CURRENCY) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) } - newColumns = append(newColumns, datatable.DATA_TABLE_AMOUNT) - newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_ACCOUNT_NAME) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_AMOUNT) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME) if accountCurrencyColumnExists { - newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) } - newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_AMOUNT) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT) if descriptionColumnExists { - newColumns = append(newColumns, datatable.DATA_TABLE_DESCRIPTION) + newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_DESCRIPTION) } - dataTable := datatable.CreateNewWritableDataTable(newColumns) - transferTransactionsMap := make(map[string]map[datatable.DataTableColumn]string, 0) + transactionRowParser := createFeideeMymoneyTransactionDataRowParser() + dataTable := datatable.CreateNewWritableTransactionDataTableWithRowParser(newColumns, transactionRowParser) + transferTransactionsMap := make(map[string]map[datatable.TransactionDataTableColumn]string, 0) for i := 1; i < len(allLines); i++ { items := allLines[i] @@ -131,20 +132,20 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con relatedIdColumnExists, ) - transactionType := data[datatable.DATA_TABLE_TRANSACTION_TYPE] + transactionType := data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText || transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText || transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText { if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] } else if transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] } else if transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] } - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = "" - data[datatable.DATA_TABLE_RELATED_AMOUNT] = "" + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = "" + data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = "" dataTable.Add(data) } else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText || transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText { if relatedId == "" { @@ -159,18 +160,18 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con continue } - if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText && relatedData[datatable.DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferOutText { - relatedData[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] - relatedData[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.DATA_TABLE_ACCOUNT_NAME] - relatedData[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY] - relatedData[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT] + if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferOutText { + relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] + relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] + relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] + relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] dataTable.Add(relatedData) delete(transferTransactionsMap, relatedId) - } else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText && relatedData[datatable.DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferInText { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.DATA_TABLE_ACCOUNT_NAME] - data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.DATA_TABLE_ACCOUNT_CURRENCY] - data[datatable.DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.DATA_TABLE_AMOUNT] + } else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferInText { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] + data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] dataTable.Add(data) delete(transferTransactionsMap, relatedId) } else { @@ -188,11 +189,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con return nil, nil, nil, nil, nil, nil, errs.ErrFoundRecordNotHasRelatedRecord } - dataTableImporter := datatable.CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc( - dataTable, - feideeMymoneyTransactionTypeNameMapping, - feideeMymoneyTransactionDataImporterPostProcess, - ) + dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping) return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } @@ -253,40 +250,40 @@ func (c *feideeMymoneyTransactionDataCsvImporter) parseTransactionData( descriptionColumnExists bool, relatedIdColumnIdx int, relatedIdColumnExists bool, -) (map[datatable.DataTableColumn]string, string) { - data := make(map[datatable.DataTableColumn]string, 11) +) (map[datatable.TransactionDataTableColumn]string, string) { + data := make(map[datatable.TransactionDataTableColumn]string, 11) relatedId := "" if timeColumnExists && timeColumnIdx < len(items) { - data[datatable.DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx] } if typeColumnExists && typeColumnIdx < len(items) { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx] } if categoryColumnExists && categoryColumnIdx < len(items) { - data[datatable.DATA_TABLE_CATEGORY] = items[categoryColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = items[categoryColumnIdx] } if subCategoryColumnExists && subCategoryColumnIdx < len(items) { - data[datatable.DATA_TABLE_SUB_CATEGORY] = items[subCategoryColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[subCategoryColumnIdx] } if accountColumnExists && accountColumnIdx < len(items) { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = items[accountColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = items[accountColumnIdx] } if accountCurrencyColumnExists && accountCurrencyColumnIdx < len(items) { - data[datatable.DATA_TABLE_ACCOUNT_CURRENCY] = items[accountCurrencyColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = items[accountCurrencyColumnIdx] } if amountColumnExists && amountColumnIdx < len(items) { - data[datatable.DATA_TABLE_AMOUNT] = items[amountColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[amountColumnIdx] } if descriptionColumnExists && descriptionColumnIdx < len(items) { - data[datatable.DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx] + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx] } if relatedIdColumnExists && relatedIdColumnIdx < len(items) { @@ -296,7 +293,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) parseTransactionData( return data, relatedId } -func (c *feideeMymoneyTransactionDataCsvImporter) getRelatedIds(transferTransactionsMap map[string]map[datatable.DataTableColumn]string) string { +func (c *feideeMymoneyTransactionDataCsvImporter) getRelatedIds(transferTransactionsMap map[string]map[datatable.TransactionDataTableColumn]string) string { builder := strings.Builder{} for relatedId := range transferTransactionsMap { diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go index be52d46e..0dce365e 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go @@ -36,8 +36,8 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi assert.Equal(t, 6, len(allNewTransactions)) assert.Equal(t, 2, len(allNewAccounts)) - assert.Equal(t, 1, len(allNewSubExpenseCategories)) - assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 2, len(allNewSubExpenseCategories)) + assert.Equal(t, 2, len(allNewSubIncomeCategories)) assert.Equal(t, 1, len(allNewSubTransferCategories)) assert.Equal(t, 0, len(allNewTags)) @@ -94,10 +94,16 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi assert.Equal(t, "CNY", allNewAccounts[1].Currency) assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) - assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) + assert.Equal(t, "", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[1].Uid) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[1].Name) assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) - assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) + assert.Equal(t, "", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[1].Uid) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[1].Name) assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go index 9babffdd..d52bc924 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go @@ -2,19 +2,18 @@ package feidee import ( "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/models" ) -var feideeMymoneyDataColumnNameMapping = map[datatable.DataTableColumn]string{ - datatable.DATA_TABLE_TRANSACTION_TIME: "日期", - datatable.DATA_TABLE_TRANSACTION_TYPE: "交易类型", - datatable.DATA_TABLE_CATEGORY: "分类", - datatable.DATA_TABLE_SUB_CATEGORY: "子分类", - datatable.DATA_TABLE_ACCOUNT_NAME: "账户1", - datatable.DATA_TABLE_AMOUNT: "金额", - datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2", - datatable.DATA_TABLE_DESCRIPTION: "备注", +var feideeMymoneyDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "日期", + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "交易类型", + datatable.TRANSACTION_DATA_TABLE_CATEGORY: "分类", + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "子分类", + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "账户1", + datatable.TRANSACTION_DATA_TABLE_AMOUNT: "金额", + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2", + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "备注", } var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{ @@ -23,16 +22,3 @@ var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{ models.TRANSACTION_TYPE_EXPENSE: "支出", models.TRANSACTION_TYPE_TRANSFER: "转账", } - -func feideeMymoneyTransactionDataImporterPostProcess(ctx core.Context, transaction *models.ImportTransaction) error { - if transaction.Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE { - if transaction.Amount >= 0 { - transaction.Type = models.TRANSACTION_DB_TYPE_INCOME - } else if transaction.Amount < 0 { - transaction.Amount = -transaction.Amount - transaction.Type = models.TRANSACTION_DB_TYPE_EXPENSE - } - } - - return nil -} diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go new file mode 100644 index 00000000..35faa6d6 --- /dev/null +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go @@ -0,0 +1,82 @@ +package feidee + +import ( + "time" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +// feideeMymoneyTransactionDataRowParser defines the structure of feidee mymoney transaction data row parser +type feideeMymoneyTransactionDataRowParser struct { +} + +// GetAddedColumns returns the added columns after converting the data row +func (p *feideeMymoneyTransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn { + return nil +} + +// Parse returns the converted transaction data row +func (p *feideeMymoneyTransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { + rowData = make(map[datatable.TransactionDataTableColumn]string, len(data)) + + for column, value := range data { + rowData[column] = value + } + + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" { + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = p.getLongDateTime(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME]) + } + + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] { + amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) + + if err != nil { + return nil, false, errs.ErrAmountInvalid + } + + if amount >= 0 { + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] + } else { + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] + rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + } + } + + return rowData, true, nil +} + +// Parse returns the converted transaction data row +func (p *feideeMymoneyTransactionDataRowParser) getLongDateTime(str string) string { + if utils.IsValidLongDateTimeFormat(str) { + return str + } + + utcTimezone := time.UTC + utcTimezoneOffsetMinutes := utils.GetTimezoneOffsetMinutes(utcTimezone) + + if utils.IsValidLongDateTimeWithoutSecondFormat(str) { + dateTime, err := utils.ParseFromLongDateTimeWithoutSecond(str, utcTimezoneOffsetMinutes) + + if err == nil { + return utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), utcTimezone) + } + } + + if utils.IsValidLongDateFormat(str) { + dateTime, err := utils.ParseFromLongDateTimeWithoutSecond(str+" 00:00", utcTimezoneOffsetMinutes) + + if err == nil { + return utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), utcTimezone) + } + } + + return str +} + +// createFeideeMymoneyTransactionDataRowParser returns feidee mymoney transaction data row parser +func createFeideeMymoneyTransactionDataRowParser() datatable.TransactionDataRowParser { + return &feideeMymoneyTransactionDataRowParser{} +} diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go index dbe4deb7..83452058 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go @@ -18,17 +18,15 @@ var ( // ParseImportedData returns the imported data by parsing the feidee mymoney transaction xls data func (c *feideeMymoneyTransactionDataXlsImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - dataTable, err := createNewFeideeMymoneyTransactionExcelFileDataTable(data) + dataTable, err := datatable.CreateNewDefaultExcelFileImportedDataTable(data) if err != nil { return nil, nil, nil, nil, nil, nil, err } - dataTableImporter := datatable.CreateNewSimpleImporterWithPostProcessFunc( - feideeMymoneyDataColumnNameMapping, - feideeMymoneyTransactionTypeNameMapping, - feideeMymoneyTransactionDataImporterPostProcess, - ) + transactionRowParser := createFeideeMymoneyTransactionDataRowParser() + transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyDataColumnNameMapping, transactionRowParser) + dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping) - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } diff --git a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go index dc56a33e..e67d1b6a 100644 --- a/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go +++ b/pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go @@ -29,8 +29,8 @@ func TestFeideeMymoneyTransactionDataXlsImporterParseImportedData_MinimumValidDa assert.Equal(t, 7, len(allNewTransactions)) assert.Equal(t, 2, len(allNewAccounts)) - assert.Equal(t, 2, len(allNewSubExpenseCategories)) - assert.Equal(t, 2, len(allNewSubIncomeCategories)) + assert.Equal(t, 3, len(allNewSubExpenseCategories)) + assert.Equal(t, 3, len(allNewSubIncomeCategories)) assert.Equal(t, 1, len(allNewSubTransferCategories)) assert.Equal(t, 0, len(allNewTags)) @@ -95,16 +95,22 @@ func TestFeideeMymoneyTransactionDataXlsImporterParseImportedData_MinimumValidDa assert.Equal(t, "CNY", allNewAccounts[1].Currency) assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) - assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) + assert.Equal(t, "", allNewSubExpenseCategories[0].Name) assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[1].Uid) - assert.Equal(t, "Test Category4", allNewSubExpenseCategories[1].Name) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[1].Name) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[2].Uid) + assert.Equal(t, "Test Category4", allNewSubExpenseCategories[2].Name) assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) - assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) + assert.Equal(t, "", allNewSubIncomeCategories[0].Name) assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[1].Uid) - assert.Equal(t, "Test Category5", allNewSubIncomeCategories[1].Name) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[1].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[2].Uid) + assert.Equal(t, "Test Category5", allNewSubIncomeCategories[2].Name) assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go index 5b8c888e..2a9e5261 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go @@ -8,7 +8,28 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) -// fireflyiiiTransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data +var fireflyIIITransactionDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "date", + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "type", + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "category", + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "source_name", + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "currency_code", + datatable.TRANSACTION_DATA_TABLE_AMOUNT: "amount", + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "destination_name", + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "foreign_currency_code", + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "foreign_amount", + datatable.TRANSACTION_DATA_TABLE_TAGS: "tags", + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "description", +} + +var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{ + models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance", + models.TRANSACTION_TYPE_INCOME: "Deposit", + models.TRANSACTION_TYPE_EXPENSE: "Withdrawal", + models.TRANSACTION_TYPE_TRANSFER: "Transfer", +} + +// fireflyIIITransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data type fireflyIIITransactionDataCsvImporter struct{} // Initialize a firefly III transaction data csv file importer singleton instance @@ -16,25 +37,18 @@ var ( FireflyIIITransactionDataCsvImporter = &fireflyIIITransactionDataCsvImporter{} ) -// ParseImportedData returns the imported data by parsing the firefly iii transaction csv data +// ParseImportedData returns the imported data by parsing the firefly III transaction csv data func (c *fireflyIIITransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { reader := bytes.NewReader(data) - - dataTable, err := createNewFireflyIIITransactionPlainTextDataTable( - ctx, - reader, - ) + dataTable, err := datatable.CreateNewDefaultCsvDataTable(ctx, reader) if err != nil { return nil, nil, nil, nil, nil, nil, err } - dataTableImporter := datatable.CreateNewImporter( - dataTable.GetDataColumnMapping(), - fireflyIIITransactionTypeNameMapping, - "", - ",", - ) + transactionRowParser := createFireflyIIITransactionDataRowParser() + transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser) + dataTableImporter := datatable.CreateNewImporter(fireflyIIITransactionTypeNameMapping, "", ",") - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go index e4447317..689a67c4 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go @@ -114,6 +114,34 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidType(t *testing assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message) } +func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidTimezone(t *testing.T) { + converter := FireflyIIITransactionDataCsvImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+ + "Withdrawal,-1.00,2024-09-01T12:34:56-10:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+ + "Withdrawal,-1.00,2024-09-01T12:34:56+00:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+ + "Withdrawal,-1.00,2024-09-01T12:34:56+12:45,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) +} + func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidAccountCurrency(t *testing.T) { converter := FireflyIIITransactionDataCsvImporter context := core.NewNullContext() diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_plain_text_data_table.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_plain_text_data_table.go deleted file mode 100644 index b8a2cf5e..00000000 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_plain_text_data_table.go +++ /dev/null @@ -1,304 +0,0 @@ -package fireflyIII - -import ( - "encoding/csv" - "io" - "time" - - "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/errs" - "github.com/mayswind/ezbookkeeping/pkg/log" - "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" -) - -var fireflyIIITransactionSupportedColumns = []datatable.DataTableColumn{ - datatable.DATA_TABLE_TRANSACTION_TIME, - datatable.DATA_TABLE_TRANSACTION_TYPE, - datatable.DATA_TABLE_SUB_CATEGORY, - datatable.DATA_TABLE_ACCOUNT_NAME, - datatable.DATA_TABLE_ACCOUNT_CURRENCY, - datatable.DATA_TABLE_AMOUNT, - datatable.DATA_TABLE_RELATED_ACCOUNT_NAME, - datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY, - datatable.DATA_TABLE_RELATED_AMOUNT, - datatable.DATA_TABLE_TAGS, - datatable.DATA_TABLE_DESCRIPTION, -} - -var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{ - models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance", - models.TRANSACTION_TYPE_INCOME: "Deposit", - models.TRANSACTION_TYPE_EXPENSE: "Withdrawal", - models.TRANSACTION_TYPE_TRANSFER: "Transfer", -} - -// fireflyIIITransactionPlainTextDataTable defines the structure of firefly III transaction plain text data table -type fireflyIIITransactionPlainTextDataTable struct { - allOriginalLines [][]string - originalHeaderLineColumnNames []string - originalColumnIndex map[datatable.DataTableColumn]int -} - -// fireflyIIITransactionPlainTextDataRow defines the structure of firefly III transaction plain text data row -type fireflyIIITransactionPlainTextDataRow struct { - dataTable *fireflyIIITransactionPlainTextDataTable - originalItems []string - finalItems map[datatable.DataTableColumn]string -} - -// fireflyIIITransactionPlainTextDataRowIterator defines the structure of firefly III transaction plain text data row iterator -type fireflyIIITransactionPlainTextDataRowIterator struct { - dataTable *fireflyIIITransactionPlainTextDataTable - currentIndex int -} - -// DataRowCount returns the total count of data row -func (t *fireflyIIITransactionPlainTextDataTable) DataRowCount() int { - if len(t.allOriginalLines) < 1 { - return 0 - } - - return len(t.allOriginalLines) - 1 -} - -// GetDataColumnMapping returns data column map for data importer -func (t *fireflyIIITransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string { - dataColumnMapping := make(map[datatable.DataTableColumn]string, len(fireflyIIITransactionSupportedColumns)) - - for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ { - column := fireflyIIITransactionSupportedColumns[i] - dataColumnMapping[column] = utils.IntToString(int(column)) - } - - return dataColumnMapping -} - -// HeaderLineColumnNames returns the header column name list -func (t *fireflyIIITransactionPlainTextDataTable) HeaderLineColumnNames() []string { - columnIndexes := make([]string, len(fireflyIIITransactionSupportedColumns)) - - for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ { - column := fireflyIIITransactionSupportedColumns[i] - - if t.originalColumnIndex[column] >= 0 { - columnIndexes[i] = utils.IntToString(int(column)) - } else { - columnIndexes[i] = "-1" - } - } - - return columnIndexes -} - -// DataRowIterator returns the iterator of data row -func (t *fireflyIIITransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &fireflyIIITransactionPlainTextDataRowIterator{ - dataTable: t, - currentIndex: 0, - } -} - -// IsValid returns whether this row contains valid data for importing -func (r *fireflyIIITransactionPlainTextDataRow) IsValid() bool { - return true -} - -// ColumnCount returns the total count of column in this data row -func (r *fireflyIIITransactionPlainTextDataRow) ColumnCount() int { - return len(fireflyIIITransactionSupportedColumns) -} - -// GetData returns the data in the specified column index -func (r *fireflyIIITransactionPlainTextDataRow) GetData(columnIndex int) string { - if columnIndex >= len(fireflyIIITransactionSupportedColumns) { - return "" - } - - dataColumn := fireflyIIITransactionSupportedColumns[columnIndex] - - return r.finalItems[dataColumn] -} - -// GetTime returns the time in the specified column index -func (r *fireflyIIITransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) { - return utils.ParseFromLongDateTimeWithTimezone(r.GetData(columnIndex)) -} - -// GetTimezoneOffset returns the time zone offset in the specified column index -func (r *fireflyIIITransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) { - return nil, errs.ErrNotSupported -} - -// HasNext returns whether the iterator does not reach the end -func (t *fireflyIIITransactionPlainTextDataRowIterator) HasNext() bool { - return t.currentIndex+1 < len(t.dataTable.allOriginalLines) -} - -// Next returns the next imported data row -func (t *fireflyIIITransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow { - if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) { - return nil - } - - t.currentIndex++ - - rowItems := t.dataTable.allOriginalLines[t.currentIndex] - finalItems := t.dataTable.parseTransactionData(rowItems) - - return &fireflyIIITransactionPlainTextDataRow{ - dataTable: t.dataTable, - originalItems: rowItems, - finalItems: finalItems, - } -} - -func (t *fireflyIIITransactionPlainTextDataTable) parseTransactionData(items []string) map[datatable.DataTableColumn]string { - data := make(map[datatable.DataTableColumn]string, 12) - - data[datatable.DATA_TABLE_SUB_CATEGORY] = "" - - for column, index := range t.originalColumnIndex { - if index >= 0 && index < len(items) { - data[column] = items[index] - } - } - - // trim trailing zero in decimal - if data[datatable.DATA_TABLE_AMOUNT] != "" { - data[datatable.DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_AMOUNT]) - amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT]) - - if err == nil { - data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) - } - } - - if data[datatable.DATA_TABLE_RELATED_AMOUNT] != "" { - data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_RELATED_AMOUNT]) - amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_RELATED_AMOUNT]) - - if err == nil { - data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount) - } - } else { - data[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT] - } - - // the related account currency field is foreign currency in firefly iii actually - if data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" { - data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY] - } - - // the destination account of modify balance transaction in firefly iii is the asset account - if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] - } - - // the destination account of income transaction in firefly iii is the asset account - if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] - } - - return data -} - -func createNewFireflyIIITransactionPlainTextDataTable(ctx core.Context, reader io.Reader) (*fireflyIIITransactionPlainTextDataTable, error) { - allOriginalLines, err := parseAllLinesFromFireflyIIITransactionPlainText(ctx, reader) - - if err != nil { - return nil, err - } - - if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1") - return nil, errs.ErrNotFoundTransactionDataInFile - } - - originalHeaderItems := allOriginalLines[0] - originalHeaderItemMap := make(map[string]int) - - for i := 0; i < len(originalHeaderItems); i++ { - originalHeaderItemMap[originalHeaderItems[i]] = i - } - - typeColumnIdx, typeColumnExists := originalHeaderItemMap["type"] - amountColumnIdx, amountColumnExists := originalHeaderItemMap["amount"] - foreignAmountColumnIdx, foreignAmountColumnExists := originalHeaderItemMap["foreign_amount"] - currencyColumnIdx, currencyColumnExists := originalHeaderItemMap["currency_code"] - foreignCurrencyColumnIdx, foreignCurrencyColumnExists := originalHeaderItemMap["foreign_currency_code"] - descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["description"] - dateColumnIdx, dateColumnExists := originalHeaderItemMap["date"] - sourceNameColumnIdx, sourceNameColumnExists := originalHeaderItemMap["source_name"] - destinationNameColumnIdx, destinationNameColumnExists := originalHeaderItemMap["destination_name"] - categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["category"] - tagsColumnIdx, tagsColumnExists := originalHeaderItemMap["tags"] - - if !typeColumnExists || !amountColumnExists || !dateColumnExists || !sourceNameColumnExists || !destinationNameColumnExists || !categoryColumnExists { - log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse firefly III csv data, because missing essential columns in header row") - return nil, errs.ErrMissingRequiredFieldInHeaderRow - } - - if !foreignAmountColumnExists { - foreignAmountColumnIdx = -1 - } - - if !currencyColumnExists { - currencyColumnIdx = -1 - } - - if !foreignCurrencyColumnExists { - foreignCurrencyColumnIdx = -1 - } - - if !descriptionColumnExists { - descriptionColumnIdx = -1 - } - - if !tagsColumnExists { - tagsColumnIdx = -1 - } - - return &fireflyIIITransactionPlainTextDataTable{ - allOriginalLines: allOriginalLines, - originalHeaderLineColumnNames: originalHeaderItems, - originalColumnIndex: map[datatable.DataTableColumn]int{ - datatable.DATA_TABLE_TRANSACTION_TIME: dateColumnIdx, - datatable.DATA_TABLE_TRANSACTION_TYPE: typeColumnIdx, - datatable.DATA_TABLE_SUB_CATEGORY: categoryColumnIdx, - datatable.DATA_TABLE_ACCOUNT_NAME: sourceNameColumnIdx, - datatable.DATA_TABLE_ACCOUNT_CURRENCY: currencyColumnIdx, - datatable.DATA_TABLE_AMOUNT: amountColumnIdx, - datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: destinationNameColumnIdx, - datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: foreignCurrencyColumnIdx, - datatable.DATA_TABLE_RELATED_AMOUNT: foreignAmountColumnIdx, - datatable.DATA_TABLE_TAGS: tagsColumnIdx, - datatable.DATA_TABLE_DESCRIPTION: descriptionColumnIdx, - }, - }, nil -} - -func parseAllLinesFromFireflyIIITransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) { - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 - - allOriginalLines := make([][]string, 0) - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.parseAllLinesFromFireflyIIITransactionPlainText] cannot parse firefly III csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - allOriginalLines = append(allOriginalLines, items) - } - - return allOriginalLines, nil -} diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go new file mode 100644 index 00000000..227814d6 --- /dev/null +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go @@ -0,0 +1,89 @@ +package fireflyIII + +import ( + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +// fireflyIIITransactionDataRowParser defines the structure of firefly III transaction data row parser +type fireflyIIITransactionDataRowParser struct { +} + +// GetAddedColumns returns the added columns after converting the data row +func (p *fireflyIIITransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn { + return []datatable.TransactionDataTableColumn{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE, + } +} + +// Parse returns the converted transaction data row +func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { + rowData = make(map[datatable.TransactionDataTableColumn]string, len(data)) + + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" + + for column, value := range data { + rowData[column] = value + } + + // parse long date time and timezone + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" { + dateTime, err := utils.ParseFromLongDateTimeWithTimezone(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME]) + + if err != nil { + return nil, false, errs.ErrTransactionTimeInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location()) + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location()) + } + + // trim trailing zero in decimal + if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" { + rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) + amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) + + if err != nil { + return nil, false, errs.ErrAmountInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + } + + if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] != "" { + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT]) + amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT]) + + if err != nil { + return nil, false, errs.ErrAmountInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount) + } else { + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] + } + + // the related account currency field is foreign currency in firefly III actually + if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" { + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] + } + + // the destination account of modify balance transaction in firefly III is the asset account + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] { + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] + } + + // the destination account of income transaction in firefly III is the asset account + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] + } + + return rowData, true, nil +} + +// createFireflyIIITransactionDataRowParser returns firefly III transaction data row parser +func createFireflyIIITransactionDataRowParser() datatable.TransactionDataRowParser { + return &fireflyIIITransactionDataRowParser{} +}