mirror/ezbookkeeping
1
0
Fork 0
mirror of https://github.com/mayswind/ezbookkeeping.git synced 2026-05-14 06:57:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

code refactor

Browse Source
This commit is contained in:
MaysWind
2024-10-13 19:44:29 +08:00
parent 5ac9eb5d5c
commit d9b819d1a1
26 changed files with 1331 additions and 1226 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go
+3 -6
View File
@@ -30,7 +30,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u
enc := simplifiedchinese.GB18030
reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder())
dataTable, err := createNewAlipayTransactionPlainTextDataTable(
transactionDataTable, err := createNewAlipayTransactionDataTable(
ctx,
reader,
c.fileHeaderLine,
@@ -43,10 +43,7 @@ func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, u
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewSimpleImporter(
dataTable.GetDataColumnMapping(),
alipayTransactionTypeNameMapping,
)
dataTableImporter := datatable.CreateNewSimpleImporter(alipayTransactionTypeNameMapping)
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
pkg/converters/alipay/alipay_transaction_data_plain_text_data_table.go
+76 -109
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"strings"
"time"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
@@ -29,14 +28,14 @@ const alipayTransactionDataProductNameTransferInText = "转入"
const alipayTransactionDataProductNameTransferOutText = "转出"
const alipayTransactionDataProductNameRepaymentText = "还款"
var alipayTransactionSupportedColumns = []datatable.DataTableColumn{
datatable.DATA_TABLE_TRANSACTION_TIME,
datatable.DATA_TABLE_TRANSACTION_TYPE,
datatable.DATA_TABLE_SUB_CATEGORY,
datatable.DATA_TABLE_ACCOUNT_NAME,
datatable.DATA_TABLE_AMOUNT,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME,
datatable.DATA_TABLE_DESCRIPTION,
var alipayTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]any{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true,
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_AMOUNT: true,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true,
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true,
}
// alipayTransactionColumnNames defines the structure of alipay transaction plain text header names
@@ -52,8 +51,8 @@ type alipayTransactionColumnNames struct {
descriptionColumnName string
}
// alipayTransactionPlainTextDataTable defines the structure of alipay transaction plain text data table
type alipayTransactionPlainTextDataTable struct {
// alipayTransactionDataTable defines the structure of alipay transaction plain text data table
type alipayTransactionDataTable struct {
allOriginalLines [][]string
originalHeaderLineColumnNames []string
originalTimeColumnIndex int
@@ -67,22 +66,28 @@ type alipayTransactionPlainTextDataTable struct {
originalDescriptionColumnIndex int
}
// alipayTransactionPlainTextDataRow defines the structure of alipay transaction plain text data row
type alipayTransactionPlainTextDataRow struct {
dataTable *alipayTransactionPlainTextDataTable
// alipayTransactionDataRow defines the structure of alipay transaction plain text data row
type alipayTransactionDataRow struct {
dataTable *alipayTransactionDataTable
isValid bool
originalItems []string
finalItems map[datatable.DataTableColumn]string
finalItems map[datatable.TransactionDataTableColumn]string
}
// alipayTransactionPlainTextDataRowIterator defines the structure of alipay transaction plain text data row iterator
type alipayTransactionPlainTextDataRowIterator struct {
dataTable *alipayTransactionPlainTextDataTable
// alipayTransactionDataRowIterator defines the structure of alipay transaction plain text data row iterator
type alipayTransactionDataRowIterator struct {
dataTable *alipayTransactionDataTable
currentIndex int
}
// DataRowCount returns the total count of data row
func (t *alipayTransactionPlainTextDataTable) DataRowCount() int {
// HasColumn returns whether the transaction data table has specified column
func (t *alipayTransactionDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool {
_, exists := alipayTransactionSupportedColumns[column]
return exists
}
// TransactionRowCount returns the total count of transaction data row
func (t *alipayTransactionDataTable) TransactionRowCount() int {
if len(t.allOriginalLines) < 1 {
return 0
}
@@ -90,77 +95,39 @@ func (t *alipayTransactionPlainTextDataTable) DataRowCount() int {
return len(t.allOriginalLines) - 1
}
// GetDataColumnMapping returns data column map for data importer
func (t *alipayTransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string {
dataColumnMapping := make(map[datatable.DataTableColumn]string, len(alipayTransactionSupportedColumns))
for i := 0; i < len(alipayTransactionSupportedColumns); i++ {
column := alipayTransactionSupportedColumns[i]
dataColumnMapping[column] = utils.IntToString(int(column))
}
return dataColumnMapping
}
// HeaderLineColumnNames returns the header column name list
func (t *alipayTransactionPlainTextDataTable) HeaderLineColumnNames() []string {
columnIndexes := make([]string, len(alipayTransactionSupportedColumns))
for i := 0; i < len(alipayTransactionSupportedColumns); i++ {
columnIndexes[i] = utils.IntToString(int(alipayTransactionSupportedColumns[i]))
}
return columnIndexes
}
// DataRowIterator returns the iterator of data row
func (t *alipayTransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &alipayTransactionPlainTextDataRowIterator{
// TransactionRowIterator returns the iterator of transaction data row
func (t *alipayTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator {
return &alipayTransactionDataRowIterator{
dataTable: t,
currentIndex: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *alipayTransactionPlainTextDataRow) IsValid() bool {
// IsValid returns whether this row is valid data for importing
func (r *alipayTransactionDataRow) IsValid() bool {
return r.isValid
}
// ColumnCount returns the total count of column in this data row
func (r *alipayTransactionPlainTextDataRow) ColumnCount() int {
return len(alipayTransactionSupportedColumns)
}
// GetData returns the data in the specified column type
func (r *alipayTransactionDataRow) GetData(column datatable.TransactionDataTableColumn) string {
_, exists := alipayTransactionSupportedColumns[column]
// GetData returns the data in the specified column index
func (r *alipayTransactionPlainTextDataRow) GetData(columnIndex int) string {
if columnIndex >= len(alipayTransactionSupportedColumns) {
if !exists {
return ""
}
dataColumn := alipayTransactionSupportedColumns[columnIndex]
return r.finalItems[dataColumn]
}
// GetTime returns the time in the specified column index
func (r *alipayTransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
return utils.ParseFromLongDateTime(r.GetData(columnIndex), timezoneOffset)
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *alipayTransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return nil, errs.ErrNotSupported
return r.finalItems[column]
}
// HasNext returns whether the iterator does not reach the end
func (t *alipayTransactionPlainTextDataRowIterator) HasNext() bool {
func (t *alipayTransactionDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
}
// Next returns the next imported data row
func (t *alipayTransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
func (t *alipayTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) {
if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
return nil
return nil, nil
}
t.currentIndex++
@@ -187,7 +154,7 @@ func (t *alipayTransactionPlainTextDataRowIterator) Next(ctx core.Context, user
isValid = false
}
var finalItems map[datatable.DataTableColumn]string
var finalItems map[datatable.TransactionDataTableColumn]string
var errMsg string
if isValid {
@@ -199,37 +166,37 @@ func (t *alipayTransactionPlainTextDataRowIterator) Next(ctx core.Context, user
}
}
return &alipayTransactionPlainTextDataRow{
return &alipayTransactionDataRow{
dataTable: t.dataTable,
isValid: isValid,
originalItems: rowItems,
finalItems: finalItems,
}
}, nil
}
func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.DataTableColumn]string, string) {
data := make(map[datatable.DataTableColumn]string, 7)
func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.TransactionDataTableColumn]string, string) {
data := make(map[datatable.TransactionDataTableColumn]string, len(alipayTransactionSupportedColumns))
if t.originalTimeColumnIndex >= 0 && t.originalTimeColumnIndex < len(items) {
data[datatable.DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex]
}
if t.originalCategoryColumnIndex >= 0 && t.originalCategoryColumnIndex < len(items) {
data[datatable.DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex]
} else {
data[datatable.DATA_TABLE_SUB_CATEGORY] = ""
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
}
if t.originalAmountColumnIndex >= 0 && t.originalAmountColumnIndex < len(items) {
data[datatable.DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex]
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex]
}
if t.originalDescriptionColumnIndex >= 0 && t.originalDescriptionColumnIndex < len(items) && items[t.originalDescriptionColumnIndex] != "" {
data[datatable.DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex]
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex]
} else if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) && items[t.originalProductNameColumnIndex] != "" {
data[datatable.DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex]
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex]
} else {
data[datatable.DATA_TABLE_DESCRIPTION] = ""
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = ""
}
relatedAccountName := ""
@@ -253,7 +220,7 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont
localeTextItems := locales.GetLocaleTextItems(locale)
if t.originalTypeColumnIndex >= 0 && t.originalTypeColumnIndex < len(items) {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex]
if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
if statusName == alipayTransactionDataStatusClosedName {
@@ -261,11 +228,11 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont
}
if statusName == alipayTransactionDataStatusSuccessName {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
} else {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = ""
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
}
} else if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
if statusName == alipayTransactionDataStatusClosedName {
@@ -284,42 +251,42 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont
}
if statusName == alipayTransactionDataStatusRefundSuccessName {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
} else {
if strings.Index(productName, alipayTransactionDataProductNameRechargePrefix) == 0 { // transfer to alipay wallet
data[datatable.DATA_TABLE_ACCOUNT_NAME] = ""
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay
} else if strings.Index(productName, alipayTransactionDataProductNameCashWithdrawalPrefix) == 0 { // transfer from alipay wallet
data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
} else if strings.Index(productName, alipayTransactionDataProductNameTransferInText) >= 0 { // transfer in
data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
} else if strings.Index(productName, alipayTransactionDataProductNameTransferOutText) >= 0 { // transfer out
data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
} else if strings.Index(productName, alipayTransactionDataProductNameRepaymentText) >= 0 { // repayment
data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
} else {
return nil, fmt.Sprintf("product name (\"%s\") is unknown", productName)
}
}
} else {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
}
}
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && statusName != "" {
if data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && statusName != "" {
if statusName == alipayTransactionDataStatusRefundSuccessName || statusName == alipayTransactionDataStatusTaxRefundSuccessName {
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
}
}
@@ -327,7 +294,7 @@ func (t *alipayTransactionPlainTextDataTable) parseTransactionData(ctx core.Cont
return data, ""
}
func createNewAlipayTransactionPlainTextDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune, originalColumnNames alipayTransactionColumnNames) (*alipayTransactionPlainTextDataTable, error) {
func createNewAlipayTransactionDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune, originalColumnNames alipayTransactionColumnNames) (*alipayTransactionDataTable, error) {
allOriginalLines, err := parseAllLinesFromAlipayTransactionPlainText(ctx, reader, fileHeaderLine, dataHeaderStartContent, dataBottomEndLineRune)
if err != nil {
@@ -381,7 +348,7 @@ func createNewAlipayTransactionPlainTextDataTable(ctx core.Context, reader io.Re
descriptionColumnIdx = -1
}
return &alipayTransactionPlainTextDataTable{
return &alipayTransactionDataTable{
allOriginalLines: allOriginalLines,
originalHeaderLineColumnNames: originalHeaderItems,
originalTimeColumnIndex: timeColumnIdx,
pkg/converters/datatable/data_table.go
-56
View File
@@ -1,56 +0,0 @@
package datatable
import (
"time"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// ImportedDataTable defines the structure of imported data table
type ImportedDataTable interface {
// DataRowCount returns the total count of data row
DataRowCount() int
// HeaderLineColumnNames returns the header column name list
HeaderLineColumnNames() []string
// DataRowIterator returns the iterator of data row
DataRowIterator() ImportedDataRowIterator
}
// ImportedDataRow defines the structure of imported data row
type ImportedDataRow interface {
// IsValid returns whether this row contains valid data for importing
IsValid() bool
// ColumnCount returns the total count of column in this data row
ColumnCount() int
// GetData returns the data in the specified column index
GetData(columnIndex int) string
// GetTime returns the time in the specified column index
GetTime(columnIndex int, timezoneOffset int16) (time.Time, error)
// GetTimezoneOffset returns the time zone offset in the specified column index
GetTimezoneOffset(columnIndex int) (*time.Location, error)
}
// ImportedDataRowIterator defines the structure of imported data row iterator
type ImportedDataRowIterator interface {
// HasNext returns whether the iterator does not reach the end
HasNext() bool
// Next returns the next imported data row
Next(ctx core.Context, user *models.User) ImportedDataRow
}
// DataTableBuilder defines the structure of data table builder
type DataTableBuilder interface {
// AppendTransaction appends the specified transaction to data builder
AppendTransaction(data map[DataTableColumn]string)
// ReplaceDelimiters returns the text after removing the delimiters
ReplaceDelimiters(text string) string
}
pkg/converters/datatable/data_table_transaction_data_converter.go
+69 -157
View File
@@ -14,30 +14,8 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/validators"
)
// DataTableColumn represents the data column type of data table
type DataTableColumn byte
// Data table columns
const (
DATA_TABLE_TRANSACTION_TIME DataTableColumn = 1
DATA_TABLE_TRANSACTION_TIMEZONE DataTableColumn = 2
DATA_TABLE_TRANSACTION_TYPE DataTableColumn = 3
DATA_TABLE_CATEGORY DataTableColumn = 4
DATA_TABLE_SUB_CATEGORY DataTableColumn = 5
DATA_TABLE_ACCOUNT_NAME DataTableColumn = 6
DATA_TABLE_ACCOUNT_CURRENCY DataTableColumn = 7
DATA_TABLE_AMOUNT DataTableColumn = 8
DATA_TABLE_RELATED_ACCOUNT_NAME DataTableColumn = 9
DATA_TABLE_RELATED_ACCOUNT_CURRENCY DataTableColumn = 10
DATA_TABLE_RELATED_AMOUNT DataTableColumn = 11
DATA_TABLE_GEOGRAPHIC_LOCATION DataTableColumn = 12
DATA_TABLE_TAGS DataTableColumn = 13
DATA_TABLE_DESCRIPTION DataTableColumn = 14
)
// DataTableTransactionDataExporter defines the structure of plain text data table exporter for transaction data
type DataTableTransactionDataExporter struct {
dataColumnMapping map[DataTableColumn]string
transactionTypeMapping map[models.TransactionType]string
geoLocationSeparator string
transactionTagSeparator string
@@ -45,20 +23,14 @@ type DataTableTransactionDataExporter struct {
// DataTableTransactionDataImporter defines the structure of plain text data table importer for transaction data
type DataTableTransactionDataImporter struct {
dataColumnMapping map[DataTableColumn]string
transactionTypeMapping map[models.TransactionType]string
geoLocationSeparator string
transactionTagSeparator string
postProcessFunc DataTableTransactionDataImporterPostProcessFunc
}
// DataTableTransactionDataImporterPostProcessFunc represents item post process function of DataTableTransactionDataImporter
type DataTableTransactionDataImporterPostProcessFunc func(core.Context, *models.ImportTransaction) error
// CreateNewExporter returns a new data table transaction data exporter according to the specified arguments
func CreateNewExporter(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataExporter {
func CreateNewExporter(transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataExporter {
return &DataTableTransactionDataExporter{
dataColumnMapping: dataColumnMapping,
transactionTypeMapping: transactionTypeMapping,
geoLocationSeparator: geoLocationSeparator,
transactionTagSeparator: transactionTagSeparator,
@@ -66,9 +38,8 @@ func CreateNewExporter(dataColumnMapping map[DataTableColumn]string, transaction
}
// CreateNewImporter returns a new data table transaction data importer according to the specified arguments
func CreateNewImporter(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataImporter {
func CreateNewImporter(transactionTypeMapping map[models.TransactionType]string, geoLocationSeparator string, transactionTagSeparator string) *DataTableTransactionDataImporter {
return &DataTableTransactionDataImporter{
dataColumnMapping: dataColumnMapping,
transactionTypeMapping: transactionTypeMapping,
geoLocationSeparator: geoLocationSeparator,
transactionTagSeparator: transactionTagSeparator,
@@ -76,41 +47,14 @@ func CreateNewImporter(dataColumnMapping map[DataTableColumn]string, transaction
}
// CreateNewSimpleImporter returns a new data table transaction data importer according to the specified arguments
func CreateNewSimpleImporter(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string) *DataTableTransactionDataImporter {
func CreateNewSimpleImporter(transactionTypeMapping map[models.TransactionType]string) *DataTableTransactionDataImporter {
return &DataTableTransactionDataImporter{
dataColumnMapping: dataColumnMapping,
transactionTypeMapping: transactionTypeMapping,
}
}
// CreateNewSimpleImporterWithPostProcessFunc returns a new data table transaction data importer according to the specified arguments
func CreateNewSimpleImporterWithPostProcessFunc(dataColumnMapping map[DataTableColumn]string, transactionTypeMapping map[models.TransactionType]string, postProcessFunc DataTableTransactionDataImporterPostProcessFunc) *DataTableTransactionDataImporter {
return &DataTableTransactionDataImporter{
dataColumnMapping: dataColumnMapping,
transactionTypeMapping: transactionTypeMapping,
postProcessFunc: postProcessFunc,
}
}
// CreateNewSimpleImporterFromWritableDataTable returns a new data table transaction data importer according to the specified arguments
func CreateNewSimpleImporterFromWritableDataTable(writableDataTable *WritableDataTable, transactionTypeMapping map[models.TransactionType]string) *DataTableTransactionDataImporter {
return &DataTableTransactionDataImporter{
dataColumnMapping: writableDataTable.GetDataColumnMapping(),
transactionTypeMapping: transactionTypeMapping,
}
}
// CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc returns a new data table transaction data importer according to the specified arguments
func CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc(writableDataTable *WritableDataTable, transactionTypeMapping map[models.TransactionType]string, postProcessFunc DataTableTransactionDataImporterPostProcessFunc) *DataTableTransactionDataImporter {
return &DataTableTransactionDataImporter{
dataColumnMapping: writableDataTable.GetDataColumnMapping(),
transactionTypeMapping: transactionTypeMapping,
postProcessFunc: postProcessFunc,
}
}
// BuildExportedContent writes the exported transaction data to the data table builder
func (c *DataTableTransactionDataExporter) BuildExportedContent(ctx core.Context, dataTableBuilder DataTableBuilder, uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) error {
func (c *DataTableTransactionDataExporter) BuildExportedContent(ctx core.Context, dataTableBuilder TransactionDataTableBuilder, uid int64, transactions []*models.Transaction, accountMap map[int64]*models.Account, categoryMap map[int64]*models.TransactionCategory, tagMap map[int64]*models.TransactionTag, allTagIndexes map[int64][]int64) error {
for i := 0; i < len(transactions); i++ {
transaction := transactions[i]
@@ -118,27 +62,27 @@ func (c *DataTableTransactionDataExporter) BuildExportedContent(ctx core.Context
continue
}
dataRowMap := make(map[DataTableColumn]string, 15)
dataRowMap := make(map[TransactionDataTableColumn]string, 15)
transactionTimeZone := time.FixedZone("Transaction Timezone", int(transaction.TimezoneUtcOffset)*60)
dataRowMap[DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime), transactionTimeZone)
dataRowMap[DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(transactionTimeZone)
dataRowMap[DATA_TABLE_TRANSACTION_TYPE] = dataTableBuilder.ReplaceDelimiters(c.getDisplayTransactionTypeName(transaction.Type))
dataRowMap[DATA_TABLE_CATEGORY] = c.getExportedTransactionCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap)
dataRowMap[DATA_TABLE_SUB_CATEGORY] = c.getExportedTransactionSubCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap)
dataRowMap[DATA_TABLE_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.AccountId, accountMap)
dataRowMap[DATA_TABLE_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.AccountId, accountMap)
dataRowMap[DATA_TABLE_AMOUNT] = utils.FormatAmount(transaction.Amount)
dataRowMap[TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(utils.GetUnixTimeFromTransactionTime(transaction.TransactionTime), transactionTimeZone)
dataRowMap[TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(transactionTimeZone)
dataRowMap[TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataTableBuilder.ReplaceDelimiters(c.getDisplayTransactionTypeName(transaction.Type))
dataRowMap[TRANSACTION_DATA_TABLE_CATEGORY] = c.getExportedTransactionCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap)
dataRowMap[TRANSACTION_DATA_TABLE_SUB_CATEGORY] = c.getExportedTransactionSubCategoryName(dataTableBuilder, transaction.CategoryId, categoryMap)
dataRowMap[TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.AccountId, accountMap)
dataRowMap[TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.AccountId, accountMap)
dataRowMap[TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(transaction.Amount)
if transaction.Type == models.TRANSACTION_DB_TYPE_TRANSFER_OUT {
dataRowMap[DATA_TABLE_RELATED_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.RelatedAccountId, accountMap)
dataRowMap[DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.RelatedAccountId, accountMap)
dataRowMap[DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(transaction.RelatedAccountAmount)
dataRowMap[TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = c.getExportedAccountName(dataTableBuilder, transaction.RelatedAccountId, accountMap)
dataRowMap[TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = c.getAccountCurrency(dataTableBuilder, transaction.RelatedAccountId, accountMap)
dataRowMap[TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(transaction.RelatedAccountAmount)
}
dataRowMap[DATA_TABLE_GEOGRAPHIC_LOCATION] = c.getExportedGeographicLocation(transaction)
dataRowMap[DATA_TABLE_TAGS] = c.getExportedTags(dataTableBuilder, transaction.TransactionId, allTagIndexes, tagMap)
dataRowMap[DATA_TABLE_DESCRIPTION] = dataTableBuilder.ReplaceDelimiters(transaction.Comment)
dataRowMap[TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION] = c.getExportedGeographicLocation(transaction)
dataRowMap[TRANSACTION_DATA_TABLE_TAGS] = c.getExportedTags(dataTableBuilder, transaction.TransactionId, allTagIndexes, tagMap)
dataRowMap[TRANSACTION_DATA_TABLE_DESCRIPTION] = dataTableBuilder.ReplaceDelimiters(transaction.Comment)
dataTableBuilder.AppendTransaction(dataRowMap)
}
@@ -162,7 +106,7 @@ func (c *DataTableTransactionDataExporter) getDisplayTransactionTypeName(transac
return transactionTypeName
}
func (c *DataTableTransactionDataExporter) getExportedTransactionCategoryName(dataTableBuilder DataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string {
func (c *DataTableTransactionDataExporter) getExportedTransactionCategoryName(dataTableBuilder TransactionDataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string {
category, exists := categoryMap[categoryId]
if !exists {
@@ -182,7 +126,7 @@ func (c *DataTableTransactionDataExporter) getExportedTransactionCategoryName(da
return dataTableBuilder.ReplaceDelimiters(parentCategory.Name)
}
func (c *DataTableTransactionDataExporter) getExportedTransactionSubCategoryName(dataTableBuilder DataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string {
func (c *DataTableTransactionDataExporter) getExportedTransactionSubCategoryName(dataTableBuilder TransactionDataTableBuilder, categoryId int64, categoryMap map[int64]*models.TransactionCategory) string {
category, exists := categoryMap[categoryId]
if exists {
@@ -192,7 +136,7 @@ func (c *DataTableTransactionDataExporter) getExportedTransactionSubCategoryName
}
}
func (c *DataTableTransactionDataExporter) getExportedAccountName(dataTableBuilder DataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string {
func (c *DataTableTransactionDataExporter) getExportedAccountName(dataTableBuilder TransactionDataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string {
account, exists := accountMap[accountId]
if exists {
@@ -202,7 +146,7 @@ func (c *DataTableTransactionDataExporter) getExportedAccountName(dataTableBuild
}
}
func (c *DataTableTransactionDataExporter) getAccountCurrency(dataTableBuilder DataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string {
func (c *DataTableTransactionDataExporter) getAccountCurrency(dataTableBuilder TransactionDataTableBuilder, accountId int64, accountMap map[int64]*models.Account) string {
account, exists := accountMap[accountId]
if exists {
@@ -220,7 +164,7 @@ func (c *DataTableTransactionDataExporter) getExportedGeographicLocation(transac
return ""
}
func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder DataTableBuilder, transactionId int64, allTagIndexes map[int64][]int64, tagMap map[int64]*models.TransactionTag) string {
func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder TransactionDataTableBuilder, transactionId int64, allTagIndexes map[int64][]int64, tagMap map[int64]*models.TransactionTag) string {
tagIndexes, exists := allTagIndexes[transactionId]
if !exists {
@@ -248,8 +192,8 @@ func (c *DataTableTransactionDataExporter) getExportedTags(dataTableBuilder Data
}
// ParseImportedData returns the imported transaction data
func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, dataTable ImportedDataTable, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
if dataTable.DataRowCount() < 1 {
func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, dataTable TransactionDataTable, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
if dataTable.TransactionRowCount() < 1 {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile
}
@@ -260,29 +204,12 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
return nil, nil, nil, nil, nil, nil, err
}
headerLineItems := dataTable.HeaderLineColumnNames()
headerItemMap := make(map[string]int)
for i := 0; i < len(headerLineItems); i++ {
headerItemMap[headerLineItems[i]] = i
}
timeColumnIdx, timeColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TRANSACTION_TIME]]
timezoneColumnIdx, timezoneColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TRANSACTION_TIMEZONE]]
typeColumnIdx, typeColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TRANSACTION_TYPE]]
subCategoryColumnIdx, subCategoryColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_SUB_CATEGORY]]
accountColumnIdx, accountColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_ACCOUNT_NAME]]
accountCurrencyColumnIdx, accountCurrencyColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_ACCOUNT_CURRENCY]]
amountColumnIdx, amountColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_AMOUNT]]
account2ColumnIdx, account2ColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_RELATED_ACCOUNT_NAME]]
account2CurrencyColumnIdx, account2CurrencyColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_RELATED_ACCOUNT_CURRENCY]]
amount2ColumnIdx, amount2ColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_RELATED_AMOUNT]]
geoLocationIdx, geoLocationExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_GEOGRAPHIC_LOCATION]]
tagsColumnIdx, tagsColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_TAGS]]
descriptionColumnIdx, descriptionColumnExists := headerItemMap[c.dataColumnMapping[DATA_TABLE_DESCRIPTION]]
if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists ||
!accountColumnExists || !amountColumnExists || !account2ColumnExists {
if !dataTable.HasColumn(TRANSACTION_DATA_TABLE_TRANSACTION_TIME) ||
!dataTable.HasColumn(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) ||
!dataTable.HasColumn(TRANSACTION_DATA_TABLE_SUB_CATEGORY) ||
!dataTable.HasColumn(TRANSACTION_DATA_TABLE_ACCOUNT_NAME) ||
!dataTable.HasColumn(TRANSACTION_DATA_TABLE_AMOUNT) ||
!dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME) {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow
}
@@ -307,59 +234,53 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
tagMap = make(map[string]*models.TransactionTag)
}
allNewTransactions := make(models.ImportedTransactionSlice, 0, dataTable.DataRowCount())
allNewTransactions := make(models.ImportedTransactionSlice, 0, dataTable.TransactionRowCount())
allNewAccounts := make([]*models.Account, 0)
allNewSubExpenseCategories := make([]*models.TransactionCategory, 0)
allNewSubIncomeCategories := make([]*models.TransactionCategory, 0)
allNewSubTransferCategories := make([]*models.TransactionCategory, 0)
allNewTags := make([]*models.TransactionTag, 0)
dataRowIterator := dataTable.DataRowIterator()
dataRowIterator := dataTable.TransactionRowIterator()
dataRowIndex := 0
for dataRowIterator.HasNext() {
dataRowIndex++
dataRow := dataRowIterator.Next(ctx, user)
dataRow, err := dataRowIterator.Next(ctx, user)
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse data row \"index:%d\" for user \"uid:%d\", because %s", dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, err
}
if !dataRow.IsValid() {
continue
}
columnCount := dataRow.ColumnCount()
if columnCount < 1 || (columnCount == 1 && dataRow.GetData(0) == "") {
continue
}
if columnCount < len(headerLineItems) {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse data row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", dataRowIndex, user.Uid, columnCount, len(headerLineItems))
return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow
}
timezoneOffset := defaultTimezoneOffset
if timezoneColumnExists {
transactionTimezone, err := dataRow.GetTimezoneOffset(timezoneColumnIdx)
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE) {
transactionTimezone, err := utils.ParseFromTimezoneOffset(dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE))
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time zone \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(timezoneColumnIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time zone \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTimeZoneInvalid
}
timezoneOffset = utils.GetTimezoneOffsetMinutes(transactionTimezone)
}
transactionTime, err := dataRow.GetTime(timeColumnIdx, timezoneOffset)
transactionTime, err := utils.ParseFromLongDateTime(dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME), timezoneOffset)
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(timeColumnIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse time \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTimeInvalid
}
transactionDbType, err := c.getTransactionDbType(nameDbTypeMap, dataRow.GetData(typeColumnIdx))
transactionDbType, err := c.getTransactionDbType(nameDbTypeMap, dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE))
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse transaction type \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(typeColumnIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse transaction type \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid)
}
@@ -374,7 +295,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
return nil, nil, nil, nil, nil, nil, errs.Or(err, errs.ErrTransactionTypeInvalid)
}
subCategoryName = dataRow.GetData(subCategoryColumnIdx)
subCategoryName = dataRow.GetData(TRANSACTION_DATA_TABLE_SUB_CATEGORY)
if transactionDbType == models.TRANSACTION_DB_TYPE_EXPENSE {
subCategory, exists := expenseCategoryMap[subCategoryName]
@@ -409,11 +330,11 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
}
}
accountName := dataRow.GetData(accountColumnIdx)
accountName := dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
accountCurrency := user.DefaultCurrency
if accountCurrencyColumnExists {
accountCurrency = dataRow.GetData(accountCurrencyColumnIdx)
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) {
accountCurrency = dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY)
if _, ok := validators.AllCurrencyNames[accountCurrency]; !ok {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] account currency \"%s\" is not supported in data row \"index:%d\" for user \"uid:%d\"", accountCurrency, dataRowIndex, user.Uid)
@@ -429,7 +350,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
accountMap[accountName] = account
}
if accountCurrencyColumnExists {
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY) {
if account.Name != "" && account.Currency != accountCurrency {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account for user \"uid:%d\"", accountCurrency, dataRowIndex, account.Currency, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid
@@ -438,10 +359,10 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
accountCurrency = account.Currency
}
amount, err := utils.ParseAmount(dataRow.GetData(amountColumnIdx))
amount, err := utils.ParseAmount(dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT))
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(amountColumnIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.ErrAmountInvalid
}
@@ -451,11 +372,11 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
account2Currency := ""
if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT {
account2Name = dataRow.GetData(account2ColumnIdx)
account2Name = dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME)
account2Currency = user.DefaultCurrency
if account2CurrencyColumnExists {
account2Currency = dataRow.GetData(account2CurrencyColumnIdx)
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) {
account2Currency = dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY)
if _, ok := validators.AllCurrencyNames[account2Currency]; !ok {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] account2 currency \"%s\" is not supported in data row \"index:%d\" for user \"uid:%d\"", account2Currency, dataRowIndex, user.Uid)
@@ -471,7 +392,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
accountMap[account2Name] = account2
}
if account2CurrencyColumnExists {
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY) {
if account2.Name != "" && account2.Currency != account2Currency {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] currency \"%s\" in data row \"index:%d\" not equals currency \"%s\" of the account2 for user \"uid:%d\"", account2Currency, dataRowIndex, account2.Currency, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrAccountCurrencyInvalid
@@ -482,11 +403,11 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
relatedAccountId = account2.AccountId
if amount2ColumnExists {
relatedAccountAmount, err = utils.ParseAmount(dataRow.GetData(amount2ColumnIdx))
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_RELATED_AMOUNT) {
relatedAccountAmount, err = utils.ParseAmount(dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_AMOUNT))
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount2 \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(amount2ColumnIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse acmount2 \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_RELATED_AMOUNT), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.ErrAmountInvalid
}
} else if transactionDbType == models.TRANSACTION_DB_TYPE_TRANSFER_OUT {
@@ -497,21 +418,21 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
geoLongitude := float64(0)
geoLatitude := float64(0)
if geoLocationExists {
geoLocationItems := strings.Split(dataRow.GetData(geoLocationIdx), c.geoLocationSeparator)
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION) {
geoLocationItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), c.geoLocationSeparator)
if len(geoLocationItems) == 2 {
geoLongitude, err = utils.StringToFloat64(geoLocationItems[0])
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(geoLocationIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.ErrGeographicLocationInvalid
}
geoLatitude, err = utils.StringToFloat64(geoLocationItems[1])
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(geoLocationIdx), dataRowIndex, user.Uid, err.Error())
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot parse geographic location \"%s\" in data row \"index:%d\" for user \"uid:%d\", because %s", dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, errs.ErrGeographicLocationInvalid
}
}
@@ -520,8 +441,8 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
var tagIds []string
var tagNames []string
if tagsColumnExists {
tagNameItems := strings.Split(dataRow.GetData(tagsColumnIdx), c.transactionTagSeparator)
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_TAGS) {
tagNameItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS), c.transactionTagSeparator)
for i := 0; i < len(tagNameItems); i++ {
tagName := tagNameItems[i]
@@ -548,8 +469,8 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
description := ""
if descriptionColumnExists {
description = dataRow.GetData(descriptionColumnIdx)
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_DESCRIPTION) {
description = dataRow.GetData(TRANSACTION_DATA_TABLE_DESCRIPTION)
}
transaction := &models.ImportTransaction{
@@ -578,15 +499,6 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
OriginalTagNames: tagNames,
}
if c.postProcessFunc != nil {
err = c.postProcessFunc(ctx, transaction)
if err != nil {
log.Errorf(ctx, "[data_table_transaction_data_converter.parseImportedData] cannot post process data row \"index:%d\" for user \"uid:%d\", because %s", dataRowIndex, user.Uid, err.Error())
return nil, nil, nil, nil, nil, nil, err
}
}
allNewTransactions = append(allNewTransactions, transaction)
}
pkg/converters/feidee/feidee_mymoney_transaction_data_excel_file_data_table.go → pkg/converters/datatable/default_excel_file_imported_data_table.go
+21 -54
View File
@@ -1,39 +1,34 @@
package feidee
package datatable
import (
"bytes"
"time"
"github.com/shakinm/xlsReader/xls"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
// feideeMymoneyTransactionExcelFileDataTable defines the structure of feidee mymoney transaction plain text data table
type feideeMymoneyTransactionExcelFileDataTable struct {
// DefaultExcelFileImportedDataTable defines the structure of default excel file data table
type DefaultExcelFileImportedDataTable struct {
workbook *xls.Workbook
headerLineColumnNames []string
}
// feideeMymoneyTransactionExcelFileDataRow defines the structure of feidee mymoney transaction plain text data row
type feideeMymoneyTransactionExcelFileDataRow struct {
// DefaultExcelFileDataRow defines the structure of default excel file data table row
type DefaultExcelFileDataRow struct {
sheet *xls.Sheet
rowIndex int
}
// feideeMymoneyTransactionExcelFileDataRowIterator defines the structure of feidee mymoney transaction plain text data row iterator
type feideeMymoneyTransactionExcelFileDataRowIterator struct {
dataTable *feideeMymoneyTransactionExcelFileDataTable
// DefaultExcelFileDataRowIterator defines the structure of default excel file data table row iterator
type DefaultExcelFileDataRowIterator struct {
dataTable *DefaultExcelFileImportedDataTable
currentTableIndex int
currentRowIndexInTable int
}
// DataRowCount returns the total count of data row
func (t *feideeMymoneyTransactionExcelFileDataTable) DataRowCount() int {
func (t *DefaultExcelFileImportedDataTable) DataRowCount() int {
allSheets := t.workbook.GetSheets()
totalDataRowCount := 0
@@ -50,27 +45,22 @@ func (t *feideeMymoneyTransactionExcelFileDataTable) DataRowCount() int {
return totalDataRowCount
}
// HeaderLineColumnNames returns the header column name list
func (t *feideeMymoneyTransactionExcelFileDataTable) HeaderLineColumnNames() []string {
// HeaderColumnNames returns the header column name list
func (t *DefaultExcelFileImportedDataTable) HeaderColumnNames() []string {
return t.headerLineColumnNames
}
// DataRowIterator returns the iterator of data row
func (t *feideeMymoneyTransactionExcelFileDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &feideeMymoneyTransactionExcelFileDataRowIterator{
func (t *DefaultExcelFileImportedDataTable) DataRowIterator() ImportedDataRowIterator {
return &DefaultExcelFileDataRowIterator{
dataTable: t,
currentTableIndex: 0,
currentRowIndexInTable: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *feideeMymoneyTransactionExcelFileDataRow) IsValid() bool {
return true
}
// ColumnCount returns the total count of column in this data row
func (r *feideeMymoneyTransactionExcelFileDataRow) ColumnCount() int {
func (r *DefaultExcelFileDataRow) ColumnCount() int {
row, err := r.sheet.GetRow(r.rowIndex)
if err != nil {
@@ -81,7 +71,7 @@ func (r *feideeMymoneyTransactionExcelFileDataRow) ColumnCount() int {
}
// GetData returns the data in the specified column index
func (r *feideeMymoneyTransactionExcelFileDataRow) GetData(columnIndex int) string {
func (r *DefaultExcelFileDataRow) GetData(columnIndex int) string {
row, err := r.sheet.GetRow(r.rowIndex)
if err != nil {
@@ -97,32 +87,8 @@ func (r *feideeMymoneyTransactionExcelFileDataRow) GetData(columnIndex int) stri
return cell.GetString()
}
// GetTime returns the time in the specified column index
func (r *feideeMymoneyTransactionExcelFileDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
str := r.GetData(columnIndex)
if utils.IsValidLongDateTimeFormat(str) {
return utils.ParseFromLongDateTime(str, timezoneOffset)
}
if utils.IsValidLongDateTimeWithoutSecondFormat(str) {
return utils.ParseFromLongDateTimeWithoutSecond(str, timezoneOffset)
}
if utils.IsValidLongDateFormat(str) {
return utils.ParseFromLongDateTimeWithoutSecond(str+" 00:00", timezoneOffset)
}
return time.Unix(0, 0), errs.ErrTransactionTimeInvalid
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *feideeMymoneyTransactionExcelFileDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return nil, errs.ErrNotSupported
}
// HasNext returns whether the iterator does not reach the end
func (t *feideeMymoneyTransactionExcelFileDataRowIterator) HasNext() bool {
func (t *DefaultExcelFileDataRowIterator) HasNext() bool {
allSheets := t.dataTable.workbook.GetSheets()
if t.currentTableIndex >= len(allSheets) {
@@ -149,7 +115,7 @@ func (t *feideeMymoneyTransactionExcelFileDataRowIterator) HasNext() bool {
}
// Next returns the next imported data row
func (t *feideeMymoneyTransactionExcelFileDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
func (t *DefaultExcelFileDataRowIterator) Next() ImportedDataRow {
allSheets := t.dataTable.workbook.GetSheets()
currentRowIndexInTable := t.currentRowIndexInTable
@@ -177,13 +143,14 @@ func (t *feideeMymoneyTransactionExcelFileDataRowIterator) Next(ctx core.Context
return nil
}
return &feideeMymoneyTransactionExcelFileDataRow{
return &DefaultExcelFileDataRow{
sheet: &currentSheet,
rowIndex: t.currentRowIndexInTable,
}
}
func createNewFeideeMymoneyTransactionExcelFileDataTable(data []byte) (*feideeMymoneyTransactionExcelFileDataTable, error) {
// CreateNewDefaultExcelFileImportedDataTable returns default excel xls data table by file binary data
func CreateNewDefaultExcelFileImportedDataTable(data []byte) (*DefaultExcelFileImportedDataTable, error) {
reader := bytes.NewReader(data)
workbook, err := xls.OpenReader(reader)
@@ -230,7 +197,7 @@ func createNewFeideeMymoneyTransactionExcelFileDataTable(data []byte) (*feideeMy
}
}
return &feideeMymoneyTransactionExcelFileDataTable{
return &DefaultExcelFileImportedDataTable{
workbook: &workbook,
headerLineColumnNames: headerRowItems,
}, nil
pkg/converters/datatable/default_plain_text_imported_data_table.go
+124
View File
@@ -0,0 +1,124 @@
package datatable
import (
"encoding/csv"
"io"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
)
// DefaultPlainTextImportedDataTable defines the structure of default plain text data table
type DefaultPlainTextImportedDataTable struct {
allLines [][]string
}
// DefaultPlainTextImportedDataRow defines the structure of default plain text data table row
type DefaultPlainTextImportedDataRow struct {
dataTable *DefaultPlainTextImportedDataTable
allItems []string
}
// DefaultPlainTextImportedDataRowIterator defines the structure of default plain text data table row iterator
type DefaultPlainTextImportedDataRowIterator struct {
dataTable *DefaultPlainTextImportedDataTable
currentIndex int
}
// DataRowCount returns the total count of data row
func (t *DefaultPlainTextImportedDataTable) DataRowCount() int {
if len(t.allLines) < 1 {
return 0
}
return len(t.allLines) - 1
}
// HeaderColumnNames returns the header column name list
func (t *DefaultPlainTextImportedDataTable) HeaderColumnNames() []string {
if len(t.allLines) < 1 {
return nil
}
return t.allLines[0]
}
// DataRowIterator returns the iterator of data row
func (t *DefaultPlainTextImportedDataTable) DataRowIterator() ImportedDataRowIterator {
return &DefaultPlainTextImportedDataRowIterator{
dataTable: t,
currentIndex: 0,
}
}
// ColumnCount returns the total count of column in this data row
func (r *DefaultPlainTextImportedDataRow) ColumnCount() int {
return len(r.allItems)
}
// GetData returns the data in the specified column index
func (r *DefaultPlainTextImportedDataRow) GetData(columnIndex int) string {
if columnIndex >= len(r.allItems) {
return ""
}
return r.allItems[columnIndex]
}
// HasNext returns whether the iterator does not reach the end
func (t *DefaultPlainTextImportedDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allLines)
}
// Next returns the next imported data row
func (t *DefaultPlainTextImportedDataRowIterator) Next() ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allLines) {
return nil
}
t.currentIndex++
rowItems := t.dataTable.allLines[t.currentIndex]
return &DefaultPlainTextImportedDataRow{
dataTable: t.dataTable,
allItems: rowItems,
}
}
// CreateNewDefaultCsvDataTable returns default csv data table by io readers
func CreateNewDefaultCsvDataTable(ctx core.Context, reader io.Reader) (*DefaultPlainTextImportedDataTable, error) {
return createNewDefaultPlainTextDataTable(ctx, reader, ',')
}
func createNewDefaultPlainTextDataTable(ctx core.Context, reader io.Reader, comma rune) (*DefaultPlainTextImportedDataTable, error) {
csvReader := csv.NewReader(reader)
csvReader.Comma = comma
csvReader.FieldsPerRecord = -1
allLines := make([][]string, 0)
for {
items, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Errorf(ctx, "[Default_plain_text_imported_data_table.createNewDefaultPlainTextDataTable] cannot parse plain text data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
if len(items) == 0 && items[0] == "" {
continue
}
allLines = append(allLines, items)
}
return &DefaultPlainTextImportedDataTable{
allLines: allLines,
}, nil
}
pkg/converters/datatable/imported_data_table.go
+31
View File
@@ -0,0 +1,31 @@
package datatable
// ImportedDataTable defines the structure of imported data table
type ImportedDataTable interface {
// DataRowCount returns the total count of data row
DataRowCount() int
// HeaderColumnNames returns the header column name list
HeaderColumnNames() []string
// DataRowIterator returns the iterator of data row
DataRowIterator() ImportedDataRowIterator
}
// ImportedDataRow defines the structure of imported data row
type ImportedDataRow interface {
// ColumnCount returns the total count of column in this data row
ColumnCount() int
// GetData returns the data in the specified column index
GetData(columnIndex int) string
}
// ImportedDataRowIterator defines the structure of imported data row iterator
type ImportedDataRowIterator interface {
// HasNext returns whether the iterator does not reach the end
HasNext() bool
// Next returns the next imported data row
Next() ImportedDataRow
}
pkg/converters/datatable/imported_transaction_data_table.go
+188
View File
@@ -0,0 +1,188 @@
package datatable
import (
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// ImportedTransactionDataTable defines the structure of imported transaction data table
type ImportedTransactionDataTable struct {
innerDataTable ImportedDataTable
dataColumnMapping map[TransactionDataTableColumn]string
dataColumnIndexes map[TransactionDataTableColumn]int
rowParser TransactionDataRowParser
addedColumns map[TransactionDataTableColumn]any
}
// ImportedTransactionDataRow defines the structure of imported transaction data row
type ImportedTransactionDataRow struct {
transactionDataTable *ImportedTransactionDataTable
rowData map[TransactionDataTableColumn]string
rowDataValid bool
}
// ImportedTransactionDataRowIterator defines the structure of imported transaction data row iterator
type ImportedTransactionDataRowIterator struct {
transactionDataTable *ImportedTransactionDataTable
innerIterator ImportedDataRowIterator
}
// HasColumn returns whether the data table has specified column
func (t *ImportedTransactionDataTable) HasColumn(column TransactionDataTableColumn) bool {
index, exists := t.dataColumnIndexes[column]
if exists && index >= 0 {
return exists
}
if t.addedColumns != nil {
_, exists = t.addedColumns[column]
if exists {
return exists
}
}
return false
}
// TransactionRowCount returns the total count of transaction data row
func (t *ImportedTransactionDataTable) TransactionRowCount() int {
return t.innerDataTable.DataRowCount()
}
// TransactionRowIterator returns the iterator of transaction data row
func (t *ImportedTransactionDataTable) TransactionRowIterator() TransactionDataRowIterator {
return &ImportedTransactionDataRowIterator{
transactionDataTable: t,
innerIterator: t.innerDataTable.DataRowIterator(),
}
}
// IsValid returns whether this row is valid data for importing
func (r *ImportedTransactionDataRow) IsValid() bool {
return r.rowDataValid
}
// GetData returns the data in the specified column type
func (r *ImportedTransactionDataRow) GetData(column TransactionDataTableColumn) string {
if !r.rowDataValid {
return ""
}
_, exists := r.transactionDataTable.dataColumnIndexes[column]
if exists {
return r.rowData[column]
}
if r.transactionDataTable.addedColumns != nil {
_, exists = r.transactionDataTable.addedColumns[column]
if exists {
return r.rowData[column]
}
}
return ""
}
// HasNext returns whether the iterator does not reach the end
func (t *ImportedTransactionDataRowIterator) HasNext() bool {
return t.innerIterator.HasNext()
}
// Next returns the next transaction data row
func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) {
importedRow := t.innerIterator.Next()
if importedRow == nil {
return nil, nil
}
if importedRow.ColumnCount() == 1 && importedRow.GetData(0) == "" {
return &ImportedTransactionDataRow{
transactionDataTable: t.transactionDataTable,
rowData: nil,
rowDataValid: false,
}, nil
}
if importedRow.ColumnCount() < len(t.transactionDataTable.dataColumnIndexes) {
log.Errorf(ctx, "[imported_transaction_data_table.Next] cannot parse data row, because may missing some columns (column count %d in data row is less than header column count %d)", importedRow.ColumnCount(), len(t.transactionDataTable.dataColumnIndexes))
return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow
}
rowData := make(map[TransactionDataTableColumn]string, len(t.transactionDataTable.dataColumnIndexes))
rowDataValid := true
for column, columnIndex := range t.transactionDataTable.dataColumnIndexes {
if columnIndex < 0 || columnIndex >= importedRow.ColumnCount() {
continue
}
value := importedRow.GetData(columnIndex)
rowData[column] = value
}
if t.transactionDataTable.rowParser != nil {
rowData, rowDataValid, err = t.transactionDataTable.rowParser.Parse(rowData)
if err != nil {
log.Errorf(ctx, "[imported_transaction_data_table.Next] cannot parse data row, because %s", err.Error())
return nil, err
}
}
return &ImportedTransactionDataRow{
transactionDataTable: t.transactionDataTable,
rowData: rowData,
rowDataValid: rowDataValid,
}, nil
}
// CreateImportedTransactionDataTable returns transaction data table from imported data table
func CreateImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable {
return CreateImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil)
}
// CreateImportedTransactionDataTableWithRowParser returns transaction data table from imported data table
func CreateImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable {
headerLineItems := dataTable.HeaderColumnNames()
headerItemMap := make(map[string]int, len(headerLineItems))
for i := 0; i < len(headerLineItems); i++ {
headerItemMap[headerLineItems[i]] = i
}
dataColumnIndexes := make(map[TransactionDataTableColumn]int, len(headerLineItems))
for column, columnName := range dataColumnMapping {
columnIndex, exists := headerItemMap[columnName]
if exists {
dataColumnIndexes[column] = columnIndex
}
}
var addedColumns map[TransactionDataTableColumn]any
if rowParser != nil {
addedColumnsByParser := rowParser.GetAddedColumns()
addedColumns = make(map[TransactionDataTableColumn]any, len(addedColumnsByParser))
for i := 0; i < len(addedColumnsByParser); i++ {
addedColumns[addedColumnsByParser[i]] = true
}
}
return &ImportedTransactionDataTable{
innerDataTable: dataTable,
dataColumnMapping: dataColumnMapping,
dataColumnIndexes: dataColumnIndexes,
rowParser: rowParser,
addedColumns: addedColumns,
}
}
pkg/converters/datatable/transaction_data_table.go
+75
View File
@@ -0,0 +1,75 @@
package datatable
import (
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// TransactionDataTable defines the structure of transaction data table
type TransactionDataTable interface {
// HasColumn returns whether the transaction data table has specified column
HasColumn(column TransactionDataTableColumn) bool
// TransactionRowCount returns the total count of transaction data row
TransactionRowCount() int
// TransactionRowIterator returns the iterator of transaction data row
TransactionRowIterator() TransactionDataRowIterator
}
// TransactionDataRow defines the structure of transaction data row
type TransactionDataRow interface {
// IsValid returns whether this row is valid data for importing
IsValid() bool
// GetData returns the data in the specified column type
GetData(column TransactionDataTableColumn) string
}
// TransactionDataRowIterator defines the structure of transaction data row iterator
type TransactionDataRowIterator interface {
// HasNext returns whether the iterator does not reach the end
HasNext() bool
// Next returns the next transaction data row
Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error)
}
// TransactionDataRowParser defines the structure of transaction data row parser
type TransactionDataRowParser interface {
// GetAddedColumns returns the added columns after converting the data row
GetAddedColumns() []TransactionDataTableColumn
// Parse returns the converted transaction data row
Parse(data map[TransactionDataTableColumn]string) (rowData map[TransactionDataTableColumn]string, rowDataValid bool, err error)
}
// TransactionDataTableBuilder defines the structure of data table builder
type TransactionDataTableBuilder interface {
// AppendTransaction appends the specified transaction to data builder
AppendTransaction(data map[TransactionDataTableColumn]string)
// ReplaceDelimiters returns the text after removing the delimiters
ReplaceDelimiters(text string) string
}
// TransactionDataTableColumn represents the data column type of data table
type TransactionDataTableColumn byte
// Transaction data table columns
const (
TRANSACTION_DATA_TABLE_TRANSACTION_TIME TransactionDataTableColumn = 1
TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE TransactionDataTableColumn = 2
TRANSACTION_DATA_TABLE_TRANSACTION_TYPE TransactionDataTableColumn = 3
TRANSACTION_DATA_TABLE_CATEGORY TransactionDataTableColumn = 4
TRANSACTION_DATA_TABLE_SUB_CATEGORY TransactionDataTableColumn = 5
TRANSACTION_DATA_TABLE_ACCOUNT_NAME TransactionDataTableColumn = 6
TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY TransactionDataTableColumn = 7
TRANSACTION_DATA_TABLE_AMOUNT TransactionDataTableColumn = 8
TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME TransactionDataTableColumn = 9
TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY TransactionDataTableColumn = 10
TRANSACTION_DATA_TABLE_RELATED_AMOUNT TransactionDataTableColumn = 11
TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION TransactionDataTableColumn = 12
TRANSACTION_DATA_TABLE_TAGS TransactionDataTableColumn = 13
TRANSACTION_DATA_TABLE_DESCRIPTION TransactionDataTableColumn = 14
)
pkg/converters/datatable/writable_data_table.go
-152
View File
@@ -1,152 +0,0 @@
package datatable
import (
"time"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
// WritableDataTable defines the structure of writable data table
type WritableDataTable struct {
allData []map[DataTableColumn]string
columns []DataTableColumn
}
// WritableDataRow defines the structure of data row of writable data table
type WritableDataRow struct {
dataTable *WritableDataTable
rowData map[DataTableColumn]string
}
// WritableDataRowIterator defines the structure of data row iterator of writable data table
type WritableDataRowIterator struct {
dataTable *WritableDataTable
nextIndex int
}
// Add appends a new record to data table
func (t *WritableDataTable) Add(data map[DataTableColumn]string) {
finalData := make(map[DataTableColumn]string, len(data))
for i := 0; i < len(t.columns); i++ {
column := t.columns[i]
if value, exists := data[column]; exists {
finalData[column] = value
}
}
t.allData = append(t.allData, finalData)
}
// Get returns the record in the specified index
func (t *WritableDataTable) Get(index int) ImportedDataRow {
if index >= len(t.allData) {
return nil
}
rowData := t.allData[index]
return &WritableDataRow{
dataTable: t,
rowData: rowData,
}
}
// DataRowCount returns the total count of data row
func (t *WritableDataTable) DataRowCount() int {
return len(t.allData)
}
// GetDataColumnMapping returns data column map for data importer
func (t *WritableDataTable) GetDataColumnMapping() map[DataTableColumn]string {
dataColumnMapping := make(map[DataTableColumn]string, len(t.columns))
for i := 0; i < len(t.columns); i++ {
column := t.columns[i]
dataColumnMapping[column] = utils.IntToString(int(column))
}
return dataColumnMapping
}
// HeaderLineColumnNames returns the header column name list
func (t *WritableDataTable) HeaderLineColumnNames() []string {
columnIndexes := make([]string, len(t.columns))
for i := 0; i < len(t.columns); i++ {
columnIndexes[i] = utils.IntToString(int(t.columns[i]))
}
return columnIndexes
}
// DataRowIterator returns the iterator of data row
func (t *WritableDataTable) DataRowIterator() ImportedDataRowIterator {
return &WritableDataRowIterator{
dataTable: t,
nextIndex: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *WritableDataRow) IsValid() bool {
return true
}
// ColumnCount returns the total count of column in this data row
func (r *WritableDataRow) ColumnCount() int {
return len(r.rowData)
}
// GetData returns the data in the specified column index
func (r *WritableDataRow) GetData(columnIndex int) string {
if columnIndex >= len(r.dataTable.columns) {
return ""
}
dataColumn := r.dataTable.columns[columnIndex]
return r.rowData[dataColumn]
}
// GetTime returns the time in the specified column index
func (r *WritableDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
return utils.ParseFromLongDateTime(r.GetData(columnIndex), timezoneOffset)
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *WritableDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return utils.ParseFromTimezoneOffset(r.GetData(columnIndex))
}
// HasNext returns whether the iterator does not reach the end
func (t *WritableDataRowIterator) HasNext() bool {
return t.nextIndex < len(t.dataTable.allData)
}
// Next returns the next imported data row
func (t *WritableDataRowIterator) Next(ctx core.Context, user *models.User) ImportedDataRow {
if t.nextIndex >= len(t.dataTable.allData) {
return nil
}
rowData := t.dataTable.allData[t.nextIndex]
t.nextIndex++
return &WritableDataRow{
dataTable: t.dataTable,
rowData: rowData,
}
}
// CreateNewWritableDataTable returns a new writable data table according to the specified columns
func CreateNewWritableDataTable(columns []DataTableColumn) *WritableDataTable {
return &WritableDataTable{
allData: make([]map[DataTableColumn]string, 0),
columns: columns,
}
}
pkg/converters/datatable/writable_data_table_test.go
-208
View File
@@ -1,208 +0,0 @@
package datatable
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
func TestWritableDataTableAdd(t *testing.T) {
columns := make([]DataTableColumn, 5)
columns[0] = DATA_TABLE_TRANSACTION_TIME
columns[1] = DATA_TABLE_TRANSACTION_TYPE
columns[2] = DATA_TABLE_SUB_CATEGORY
columns[3] = DATA_TABLE_ACCOUNT_NAME
columns[4] = DATA_TABLE_AMOUNT
writableDataTable := CreateNewWritableDataTable(columns)
assert.Equal(t, 0, writableDataTable.DataRowCount())
expectedTransactionUnixTime := time.Now().Unix()
expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local)
expectedTransactionType := "Expense"
expectedSubCategory := "Test Category"
expectedAccountName := "Test Account"
expectedAmount := "123.45"
writableDataTable.Add(map[DataTableColumn]string{
DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime,
DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType,
DATA_TABLE_SUB_CATEGORY: expectedSubCategory,
DATA_TABLE_ACCOUNT_NAME: expectedAccountName,
DATA_TABLE_AMOUNT: expectedAmount,
})
assert.Equal(t, 1, writableDataTable.DataRowCount())
dataRow := writableDataTable.Get(0)
actualTransactionTime, err := dataRow.GetTime(0, utils.GetTimezoneOffsetMinutes(time.Local))
assert.Nil(t, err)
actualTransactionUnixTime := actualTransactionTime.Unix()
assert.Equal(t, expectedTransactionUnixTime, actualTransactionUnixTime)
actualTextualTransactionTime := dataRow.GetData(0)
assert.Equal(t, expectedTextualTransactionTime, actualTextualTransactionTime)
actualTransactionType := dataRow.GetData(1)
assert.Equal(t, expectedTransactionType, actualTransactionType)
actualSubCategory := dataRow.GetData(2)
assert.Equal(t, expectedSubCategory, actualSubCategory)
actualAccountName := dataRow.GetData(3)
assert.Equal(t, expectedAccountName, actualAccountName)
actualAmount := dataRow.GetData(4)
assert.Equal(t, expectedAmount, actualAmount)
}
func TestWritableDataTableAdd_NotExistsColumn(t *testing.T) {
columns := make([]DataTableColumn, 1)
columns[0] = DATA_TABLE_TRANSACTION_TIME
writableDataTable := CreateNewWritableDataTable(columns)
expectedTransactionUnixTime := time.Now().Unix()
expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local)
expectedTransactionType := "Expense"
writableDataTable.Add(map[DataTableColumn]string{
DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime,
DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType,
})
assert.Equal(t, 1, writableDataTable.DataRowCount())
dataRow := writableDataTable.Get(0)
assert.Equal(t, 1, dataRow.ColumnCount())
}
func TestWritableDataTableGet_NotExistsRow(t *testing.T) {
columns := make([]DataTableColumn, 1)
columns[0] = DATA_TABLE_TRANSACTION_TIME
writableDataTable := CreateNewWritableDataTable(columns)
assert.Equal(t, 0, writableDataTable.DataRowCount())
dataRow := writableDataTable.Get(0)
assert.Nil(t, dataRow)
}
func TestWritableDataRowGetData_NotExistsColumn(t *testing.T) {
columns := make([]DataTableColumn, 1)
columns[0] = DATA_TABLE_TRANSACTION_TIME
writableDataTable := CreateNewWritableDataTable(columns)
expectedTransactionUnixTime := time.Now().Unix()
expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local)
writableDataTable.Add(map[DataTableColumn]string{
DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime,
})
assert.Equal(t, 1, writableDataTable.DataRowCount())
dataRow := writableDataTable.Get(0)
assert.Equal(t, 1, dataRow.ColumnCount())
assert.Equal(t, "", dataRow.GetData(1))
}
func TestWritableDataTableDataRowIterator(t *testing.T) {
columns := make([]DataTableColumn, 5)
columns[0] = DATA_TABLE_TRANSACTION_TIME
columns[1] = DATA_TABLE_TRANSACTION_TYPE
columns[2] = DATA_TABLE_SUB_CATEGORY
columns[3] = DATA_TABLE_ACCOUNT_NAME
columns[4] = DATA_TABLE_AMOUNT
writableDataTable := CreateNewWritableDataTable(columns)
assert.Equal(t, 0, writableDataTable.DataRowCount())
expectedTransactionUnixTimes := make([]int64, 3)
expectedTextualTransactionTimes := make([]string, 3)
expectedTransactionTypes := make([]string, 3)
expectedSubCategories := make([]string, 3)
expectedAccountNames := make([]string, 3)
expectedAmounts := make([]string, 3)
expectedTransactionUnixTimes[0] = time.Now().Add(-5 * time.Hour).Unix()
expectedTextualTransactionTimes[0] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[0], time.Local)
expectedTransactionTypes[0] = "Balance Modification"
expectedSubCategories[0] = ""
expectedAccountNames[0] = "Test Account"
expectedAmounts[0] = "123.45"
writableDataTable.Add(map[DataTableColumn]string{
DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTimes[0],
DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[0],
DATA_TABLE_SUB_CATEGORY: expectedSubCategories[0],
DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[0],
DATA_TABLE_AMOUNT: expectedAmounts[0],
})
expectedTransactionUnixTimes[1] = time.Now().Add(-45 * time.Minute).Unix()
expectedTextualTransactionTimes[1] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[1], time.Local)
expectedTransactionTypes[1] = "Expense"
expectedSubCategories[1] = "Test Category2"
expectedAccountNames[1] = "Test Account"
expectedAmounts[1] = "-23.4"
writableDataTable.Add(map[DataTableColumn]string{
DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTimes[1],
DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[1],
DATA_TABLE_SUB_CATEGORY: expectedSubCategories[1],
DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[1],
DATA_TABLE_AMOUNT: expectedAmounts[1],
})
expectedTransactionUnixTimes[2] = time.Now().Unix()
expectedTextualTransactionTimes[2] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[2], time.Local)
expectedTransactionTypes[2] = "Income"
expectedSubCategories[2] = "Test Category3"
expectedAccountNames[2] = "Test Account2"
expectedAmounts[2] = "123"
writableDataTable.Add(map[DataTableColumn]string{
DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTimes[2],
DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[2],
DATA_TABLE_SUB_CATEGORY: expectedSubCategories[2],
DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[2],
DATA_TABLE_AMOUNT: expectedAmounts[2],
})
assert.Equal(t, 3, writableDataTable.DataRowCount())
index := 0
iterator := writableDataTable.DataRowIterator()
for iterator.HasNext() {
dataRow := iterator.Next(core.NewNullContext(), &models.User{})
actualTransactionTime, err := dataRow.GetTime(0, utils.GetTimezoneOffsetMinutes(time.Local))
assert.Nil(t, err)
actualTransactionUnixTime := actualTransactionTime.Unix()
assert.Equal(t, expectedTransactionUnixTimes[index], actualTransactionUnixTime)
actualTextualTransactionTime := dataRow.GetData(0)
assert.Equal(t, expectedTextualTransactionTimes[index], actualTextualTransactionTime)
actualTransactionType := dataRow.GetData(1)
assert.Equal(t, expectedTransactionTypes[index], actualTransactionType)
actualSubCategory := dataRow.GetData(2)
assert.Equal(t, expectedSubCategories[index], actualSubCategory)
actualAccountName := dataRow.GetData(3)
assert.Equal(t, expectedAccountNames[index], actualAccountName)
actualAmount := dataRow.GetData(4)
assert.Equal(t, expectedAmounts[index], actualAmount)
index++
}
assert.Equal(t, 3, index)
}
pkg/converters/datatable/writable_transaction_data_table.go
+169
View File
@@ -0,0 +1,169 @@
package datatable
import (
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// WritableTransactionDataTable defines the structure of writable transaction data table
type WritableTransactionDataTable struct {
allData []map[TransactionDataTableColumn]string
supportedColumns map[TransactionDataTableColumn]any
rowParser TransactionDataRowParser
addedColumns map[TransactionDataTableColumn]any
}
// WritableTransactionDataRow defines the structure of transaction data row of writable data table
type WritableTransactionDataRow struct {
dataTable *WritableTransactionDataTable
rowData map[TransactionDataTableColumn]string
rowDataValid bool
}
// WritableTransactionDataRowIterator defines the structure of transaction data row iterator of writable data table
type WritableTransactionDataRowIterator struct {
dataTable *WritableTransactionDataTable
nextIndex int
}
// Add appends a new record to data table
func (t *WritableTransactionDataTable) Add(data map[TransactionDataTableColumn]string) {
finalData := make(map[TransactionDataTableColumn]string, len(data))
for column, value := range data {
_, exists := t.supportedColumns[column]
if exists {
finalData[column] = value
}
}
t.allData = append(t.allData, finalData)
}
// Get returns the record in the specified index
func (t *WritableTransactionDataTable) Get(index int) *WritableTransactionDataRow {
if index >= len(t.allData) {
return nil
}
rowData := t.allData[index]
return &WritableTransactionDataRow{
dataTable: t,
rowData: rowData,
}
}
// HasColumn returns whether the data table has specified column
func (t *WritableTransactionDataTable) HasColumn(column TransactionDataTableColumn) bool {
_, exists := t.supportedColumns[column]
if exists {
return exists
}
if t.addedColumns != nil {
_, exists = t.addedColumns[column]
if exists {
return exists
}
}
return false
}
// TransactionRowCount returns the total count of transaction data row
func (t *WritableTransactionDataTable) TransactionRowCount() int {
return len(t.allData)
}
// TransactionRowIterator returns the iterator of transaction data row
func (t *WritableTransactionDataTable) TransactionRowIterator() TransactionDataRowIterator {
return &WritableTransactionDataRowIterator{
dataTable: t,
nextIndex: 0,
}
}
// ColumnCount returns the total count of column in this data row
func (r *WritableTransactionDataRow) ColumnCount() int {
return len(r.rowData)
}
// IsValid returns whether this row is valid data for importing
func (r *WritableTransactionDataRow) IsValid() bool {
return r.rowDataValid
}
// GetData returns the data in the specified column type
func (r *WritableTransactionDataRow) GetData(column TransactionDataTableColumn) string {
return r.rowData[column]
}
// HasNext returns whether the iterator does not reach the end
func (t *WritableTransactionDataRowIterator) HasNext() bool {
return t.nextIndex < len(t.dataTable.allData)
}
// Next returns the next transaction data row
func (t *WritableTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) {
if t.nextIndex >= len(t.dataTable.allData) {
return nil, nil
}
rowData := t.dataTable.allData[t.nextIndex]
rowDataValid := true
if t.dataTable.rowParser != nil {
rowData, rowDataValid, err = t.dataTable.rowParser.Parse(rowData)
if err != nil {
log.Errorf(ctx, "[writable_transaction_data_table.Next] cannot parse data row, because %s", err.Error())
return nil, err
}
}
t.nextIndex++
return &WritableTransactionDataRow{
dataTable: t.dataTable,
rowData: rowData,
rowDataValid: rowDataValid,
}, nil
}
// CreateNewWritableTransactionDataTable returns a new writable transaction data table according to the specified columns
func CreateNewWritableTransactionDataTable(columns []TransactionDataTableColumn) *WritableTransactionDataTable {
return CreateNewWritableTransactionDataTableWithRowParser(columns, nil)
}
// CreateNewWritableTransactionDataTableWithRowParser returns a new writable transaction data table according to the specified columns
func CreateNewWritableTransactionDataTableWithRowParser(columns []TransactionDataTableColumn, rowParser TransactionDataRowParser) *WritableTransactionDataTable {
supportedColumns := make(map[TransactionDataTableColumn]any, len(columns))
for i := 0; i < len(columns); i++ {
column := columns[i]
supportedColumns[column] = true
}
var addedColumns map[TransactionDataTableColumn]any
if rowParser != nil {
addedColumnsByParser := rowParser.GetAddedColumns()
addedColumns = make(map[TransactionDataTableColumn]any, len(addedColumnsByParser))
for i := 0; i < len(addedColumnsByParser); i++ {
addedColumns[addedColumnsByParser[i]] = true
}
}
return &WritableTransactionDataTable{
allData: make([]map[TransactionDataTableColumn]string, 0),
supportedColumns: supportedColumns,
rowParser: rowParser,
addedColumns: addedColumns,
}
}
pkg/converters/datatable/writable_transaction_data_table_test.go
+196
View File
@@ -0,0 +1,196 @@
package datatable
import (
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
func TestWritableDataTableAdd(t *testing.T) {
columns := make([]TransactionDataTableColumn, 5)
columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME
columns[1] = TRANSACTION_DATA_TABLE_TRANSACTION_TYPE
columns[2] = TRANSACTION_DATA_TABLE_SUB_CATEGORY
columns[3] = TRANSACTION_DATA_TABLE_ACCOUNT_NAME
columns[4] = TRANSACTION_DATA_TABLE_AMOUNT
writableDataTable := CreateNewWritableTransactionDataTable(columns)
assert.Equal(t, 0, writableDataTable.TransactionRowCount())
expectedTransactionTime := "2024-09-01 01:23:45"
expectedTransactionType := "Expense"
expectedSubCategory := "Test Category"
expectedAccountName := "Test Account"
expectedAmount := "123.45"
writableDataTable.Add(map[TransactionDataTableColumn]string{
TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTime,
TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType,
TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategory,
TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountName,
TRANSACTION_DATA_TABLE_AMOUNT: expectedAmount,
})
assert.Equal(t, 1, writableDataTable.TransactionRowCount())
dataRow := writableDataTable.Get(0)
actualTransactionTime := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME)
assert.Equal(t, expectedTransactionTime, actualTransactionTime)
actualTransactionType := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)
assert.Equal(t, expectedTransactionType, actualTransactionType)
actualSubCategory := dataRow.GetData(TRANSACTION_DATA_TABLE_SUB_CATEGORY)
assert.Equal(t, expectedSubCategory, actualSubCategory)
actualAccountName := dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
assert.Equal(t, expectedAccountName, actualAccountName)
actualAmount := dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT)
assert.Equal(t, expectedAmount, actualAmount)
}
func TestWritableDataTableAdd_NotExistsColumn(t *testing.T) {
columns := make([]TransactionDataTableColumn, 1)
columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME
writableDataTable := CreateNewWritableTransactionDataTable(columns)
expectedTransactionUnixTime := time.Now().Unix()
expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local)
expectedTransactionType := "Expense"
writableDataTable.Add(map[TransactionDataTableColumn]string{
TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime,
TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionType,
})
assert.Equal(t, 1, writableDataTable.TransactionRowCount())
dataRow := writableDataTable.Get(0)
assert.Equal(t, 1, dataRow.ColumnCount())
}
func TestWritableDataTableGet_NotExistsRow(t *testing.T) {
columns := make([]TransactionDataTableColumn, 1)
columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME
writableDataTable := CreateNewWritableTransactionDataTable(columns)
assert.Equal(t, 0, writableDataTable.TransactionRowCount())
dataRow := writableDataTable.Get(0)
assert.Nil(t, dataRow)
}
func TestWritableDataRowGetData_NotExistsColumn(t *testing.T) {
columns := make([]TransactionDataTableColumn, 1)
columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME
writableDataTable := CreateNewWritableTransactionDataTable(columns)
expectedTransactionUnixTime := time.Now().Unix()
expectedTextualTransactionTime := utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTime, time.Local)
writableDataTable.Add(map[TransactionDataTableColumn]string{
TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTextualTransactionTime,
})
assert.Equal(t, 1, writableDataTable.TransactionRowCount())
dataRow := writableDataTable.Get(0)
assert.Equal(t, 1, dataRow.ColumnCount())
assert.Equal(t, "", dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE))
}
func TestWritableDataTableDataRowIterator(t *testing.T) {
columns := make([]TransactionDataTableColumn, 5)
columns[0] = TRANSACTION_DATA_TABLE_TRANSACTION_TIME
columns[1] = TRANSACTION_DATA_TABLE_TRANSACTION_TYPE
columns[2] = TRANSACTION_DATA_TABLE_SUB_CATEGORY
columns[3] = TRANSACTION_DATA_TABLE_ACCOUNT_NAME
columns[4] = TRANSACTION_DATA_TABLE_AMOUNT
writableDataTable := CreateNewWritableTransactionDataTable(columns)
assert.Equal(t, 0, writableDataTable.TransactionRowCount())
expectedTransactionUnixTimes := make([]int64, 3)
expectedTransactionTimes := make([]string, 3)
expectedTransactionTypes := make([]string, 3)
expectedSubCategories := make([]string, 3)
expectedAccountNames := make([]string, 3)
expectedAmounts := make([]string, 3)
expectedTransactionUnixTimes[0] = time.Now().Add(-5 * time.Hour).Unix()
expectedTransactionTimes[0] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[0], time.Local)
expectedTransactionTypes[0] = "Balance Modification"
expectedSubCategories[0] = ""
expectedAccountNames[0] = "Test Account"
expectedAmounts[0] = "123.45"
writableDataTable.Add(map[TransactionDataTableColumn]string{
TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTimes[0],
TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[0],
TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategories[0],
TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[0],
TRANSACTION_DATA_TABLE_AMOUNT: expectedAmounts[0],
})
expectedTransactionUnixTimes[1] = time.Now().Add(-45 * time.Minute).Unix()
expectedTransactionTimes[1] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[1], time.Local)
expectedTransactionTypes[1] = "Expense"
expectedSubCategories[1] = "Test Category2"
expectedAccountNames[1] = "Test Account"
expectedAmounts[1] = "-23.4"
writableDataTable.Add(map[TransactionDataTableColumn]string{
TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTimes[1],
TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[1],
TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategories[1],
TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[1],
TRANSACTION_DATA_TABLE_AMOUNT: expectedAmounts[1],
})
expectedTransactionUnixTimes[2] = time.Now().Unix()
expectedTransactionTimes[2] = utils.FormatUnixTimeToLongDateTime(expectedTransactionUnixTimes[2], time.Local)
expectedTransactionTypes[2] = "Income"
expectedSubCategories[2] = "Test Category3"
expectedAccountNames[2] = "Test Account2"
expectedAmounts[2] = "123"
writableDataTable.Add(map[TransactionDataTableColumn]string{
TRANSACTION_DATA_TABLE_TRANSACTION_TIME: expectedTransactionTimes[2],
TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: expectedTransactionTypes[2],
TRANSACTION_DATA_TABLE_SUB_CATEGORY: expectedSubCategories[2],
TRANSACTION_DATA_TABLE_ACCOUNT_NAME: expectedAccountNames[2],
TRANSACTION_DATA_TABLE_AMOUNT: expectedAmounts[2],
})
assert.Equal(t, 3, writableDataTable.TransactionRowCount())
index := 0
iterator := writableDataTable.TransactionRowIterator()
for iterator.HasNext() {
dataRow, err := iterator.Next(core.NewNullContext(), &models.User{})
assert.Nil(t, err)
actualTransactionTime := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TIME)
assert.Equal(t, expectedTransactionTimes[index], actualTransactionTime)
actualTransactionType := dataRow.GetData(TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)
assert.Equal(t, expectedTransactionTypes[index], actualTransactionType)
actualSubCategory := dataRow.GetData(TRANSACTION_DATA_TABLE_SUB_CATEGORY)
assert.Equal(t, expectedSubCategories[index], actualSubCategory)
actualAccountName := dataRow.GetData(TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
assert.Equal(t, expectedAccountNames[index], actualAccountName)
actualAmount := dataRow.GetData(TRANSACTION_DATA_TABLE_AMOUNT)
assert.Equal(t, expectedAmounts[index], actualAmount)
index++
}
assert.Equal(t, 3, index)
}
pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter.go
+34 -34
View File
@@ -15,21 +15,21 @@ const ezbookkeepingLineSeparator = "\n"
const ezbookkeepingGeoLocationSeparator = " "
const ezbookkeepingTagSeparator = ";"
var ezbookkeepingDataColumnNameMapping = map[datatable.DataTableColumn]string{
datatable.DATA_TABLE_TRANSACTION_TIME: "Time",
datatable.DATA_TABLE_TRANSACTION_TIMEZONE: "Timezone",
datatable.DATA_TABLE_TRANSACTION_TYPE: "Type",
datatable.DATA_TABLE_CATEGORY: "Category",
datatable.DATA_TABLE_SUB_CATEGORY: "Sub Category",
datatable.DATA_TABLE_ACCOUNT_NAME: "Account",
datatable.DATA_TABLE_ACCOUNT_CURRENCY: "Account Currency",
datatable.DATA_TABLE_AMOUNT: "Amount",
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: "Account2",
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "Account2 Currency",
datatable.DATA_TABLE_RELATED_AMOUNT: "Account2 Amount",
datatable.DATA_TABLE_GEOGRAPHIC_LOCATION: "Geographic Location",
datatable.DATA_TABLE_TAGS: "Tags",
datatable.DATA_TABLE_DESCRIPTION: "Description",
var ezbookkeepingDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "Time",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: "Timezone",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "Type",
datatable.TRANSACTION_DATA_TABLE_CATEGORY: "Category",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "Sub Category",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "Account",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "Account Currency",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "Amount",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "Account2",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "Account2 Currency",
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "Account2 Amount",
datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION: "Geographic Location",
datatable.TRANSACTION_DATA_TABLE_TAGS: "Tags",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "Description",
}
var ezbookkeepingTransactionTypeNameMapping = map[models.TransactionType]string{
@@ -39,21 +39,21 @@ var ezbookkeepingTransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_TRANSFER: "Transfer",
}
var ezbookkeepingDataColumns = []datatable.DataTableColumn{
datatable.DATA_TABLE_TRANSACTION_TIME,
datatable.DATA_TABLE_TRANSACTION_TIMEZONE,
datatable.DATA_TABLE_TRANSACTION_TYPE,
datatable.DATA_TABLE_CATEGORY,
datatable.DATA_TABLE_SUB_CATEGORY,
datatable.DATA_TABLE_ACCOUNT_NAME,
datatable.DATA_TABLE_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_AMOUNT,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_RELATED_AMOUNT,
datatable.DATA_TABLE_GEOGRAPHIC_LOCATION,
datatable.DATA_TABLE_TAGS,
datatable.DATA_TABLE_DESCRIPTION,
var ezbookkeepingDataColumns = []datatable.TransactionDataTableColumn{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE,
datatable.TRANSACTION_DATA_TABLE_CATEGORY,
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY,
datatable.TRANSACTION_DATA_TABLE_AMOUNT,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME,
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY,
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT,
datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION,
datatable.TRANSACTION_DATA_TABLE_TAGS,
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION,
}
// ToExportedContent returns the exported transaction plain text data
@@ -67,7 +67,6 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ToExportedContent(ctx c
)
dataTableExporter := datatable.CreateNewExporter(
ezbookkeepingDataColumnNameMapping,
ezbookkeepingTransactionTypeNameMapping,
ezbookkeepingGeoLocationSeparator,
ezbookkeepingTagSeparator,
@@ -84,7 +83,7 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ToExportedContent(ctx c
// ParseImportedData returns the imported data by parsing the transaction plain text data
func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
dataTable, err := createNewezbookkeepingTransactionPlainTextDataTable(
dataTable, err := createNewezbookkeepingPlainTextDataTable(
string(data),
c.columnSeparator,
ezbookkeepingLineSeparator,
@@ -94,12 +93,13 @@ func (c *ezBookKeepingTransactionDataPlainTextConverter) ParseImportedData(ctx c
return nil, nil, nil, nil, nil, nil, err
}
transactionDataTable := datatable.CreateImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping)
dataTableImporter := datatable.CreateNewImporter(
ezbookkeepingDataColumnNameMapping,
ezbookkeepingTransactionTypeNameMapping,
ezbookkeepingGeoLocationSeparator,
ezbookkeepingTagSeparator,
)
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
pkg/converters/default/ezbookkeeping_transaction_data_plain_text_converter_test.go
+14 -2
View File
@@ -238,10 +238,22 @@ func TestEzBookKeepingPlainFileConverterParseImportedData_ParseValidTimezone(t *
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+
"2024-09-01 12:34:56,+08:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil)
"2024-09-01 12:34:56,-10:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725165296), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+
"2024-09-01 12:34:56,+00:00,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("Time,Timezone,Type,Sub Category,Account,Amount,Account2,Account2 Amount\n"+
"2024-09-01 12:34:56,+12:45,Expense,Test Category,Test Account,123.45,,"), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
}
func TestEzBookKeepingPlainFileConverterParseImportedData_ParseInvalidTimezone(t *testing.T) {
pkg/converters/default/ezbookkeeping_transaction_data_plain_text_data_table.go
+23 -42
View File
@@ -3,31 +3,27 @@ package _default
import (
"fmt"
"strings"
"time"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
// ezBookKeepingTransactionPlainTextDataTable defines the structure of ezbookkeeping transaction plain text data table
type ezBookKeepingTransactionPlainTextDataTable struct {
// ezBookKeepingPlainTextDataTable defines the structure of ezbookkeeping plain text data table
type ezBookKeepingPlainTextDataTable struct {
columnSeparator string
lineSeparator string
allLines []string
headerLineColumnNames []string
}
// ezBookKeepingTransactionPlainTextDataRow defines the structure of ezbookkeeping transaction plain text data row
type ezBookKeepingTransactionPlainTextDataRow struct {
// ezBookKeepingPlainTextDataRow defines the structure of ezbookkeeping plain text data row
type ezBookKeepingPlainTextDataRow struct {
allItems []string
}
// ezBookKeepingTransactionPlainTextDataRowIterator defines the structure of ezbookkeeping transaction plain text data row iterator
type ezBookKeepingTransactionPlainTextDataRowIterator struct {
dataTable *ezBookKeepingTransactionPlainTextDataTable
// ezBookKeepingPlainTextDataRowIterator defines the structure of ezbookkeeping plain text data row iterator
type ezBookKeepingPlainTextDataRowIterator struct {
dataTable *ezBookKeepingPlainTextDataTable
currentIndex int
}
@@ -35,14 +31,14 @@ type ezBookKeepingTransactionPlainTextDataRowIterator struct {
type ezBookKeepingTransactionPlainTextDataTableBuilder struct {
columnSeparator string
lineSeparator string
columns []datatable.DataTableColumn
dataColumnNameMapping map[datatable.DataTableColumn]string
columns []datatable.TransactionDataTableColumn
dataColumnNameMapping map[datatable.TransactionDataTableColumn]string
dataLineFormat string
builder *strings.Builder
}
// DataRowCount returns the total count of data row
func (t *ezBookKeepingTransactionPlainTextDataTable) DataRowCount() int {
func (t *ezBookKeepingPlainTextDataTable) DataRowCount() int {
if len(t.allLines) < 1 {
return 0
}
@@ -50,31 +46,26 @@ func (t *ezBookKeepingTransactionPlainTextDataTable) DataRowCount() int {
return len(t.allLines) - 1
}
// HeaderLineColumnNames returns the header column name list
func (t *ezBookKeepingTransactionPlainTextDataTable) HeaderLineColumnNames() []string {
// HeaderColumnNames returns the header column name list
func (t *ezBookKeepingPlainTextDataTable) HeaderColumnNames() []string {
return t.headerLineColumnNames
}
// DataRowIterator returns the iterator of data row
func (t *ezBookKeepingTransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &ezBookKeepingTransactionPlainTextDataRowIterator{
func (t *ezBookKeepingPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &ezBookKeepingPlainTextDataRowIterator{
dataTable: t,
currentIndex: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *ezBookKeepingTransactionPlainTextDataRow) IsValid() bool {
return true
}
// ColumnCount returns the total count of column in this data row
func (r *ezBookKeepingTransactionPlainTextDataRow) ColumnCount() int {
func (r *ezBookKeepingPlainTextDataRow) ColumnCount() int {
return len(r.allItems)
}
// GetData returns the data in the specified column index
func (r *ezBookKeepingTransactionPlainTextDataRow) GetData(columnIndex int) string {
func (r *ezBookKeepingPlainTextDataRow) GetData(columnIndex int) string {
if columnIndex >= len(r.allItems) {
return ""
}
@@ -82,23 +73,13 @@ func (r *ezBookKeepingTransactionPlainTextDataRow) GetData(columnIndex int) stri
return r.allItems[columnIndex]
}
// GetTime returns the time in the specified column index
func (r *ezBookKeepingTransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
return utils.ParseFromLongDateTime(r.GetData(columnIndex), timezoneOffset)
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *ezBookKeepingTransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return utils.ParseFromTimezoneOffset(r.GetData(columnIndex))
}
// HasNext returns whether the iterator does not reach the end
func (t *ezBookKeepingTransactionPlainTextDataRowIterator) HasNext() bool {
func (t *ezBookKeepingPlainTextDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allLines)
}
// Next returns the next imported data row
func (t *ezBookKeepingTransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
func (t *ezBookKeepingPlainTextDataRowIterator) Next() datatable.ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allLines) {
return nil
}
@@ -108,13 +89,13 @@ func (t *ezBookKeepingTransactionPlainTextDataRowIterator) Next(ctx core.Context
rowContent := t.dataTable.allLines[t.currentIndex]
rowItems := strings.Split(rowContent, t.dataTable.columnSeparator)
return &ezBookKeepingTransactionPlainTextDataRow{
return &ezBookKeepingPlainTextDataRow{
allItems: rowItems,
}
}
// AppendTransaction appends the specified transaction to data builder
func (b *ezBookKeepingTransactionPlainTextDataTableBuilder) AppendTransaction(data map[datatable.DataTableColumn]string) {
func (b *ezBookKeepingTransactionPlainTextDataTableBuilder) AppendTransaction(data map[datatable.TransactionDataTableColumn]string) {
dataRowParams := make([]any, len(b.columns))
for i := 0; i < len(b.columns); i++ {
@@ -175,7 +156,7 @@ func (b *ezBookKeepingTransactionPlainTextDataTableBuilder) generateDataLineForm
return ret.String()
}
func createNewezbookkeepingTransactionPlainTextDataTable(content string, columnSeparator string, lineSeparator string) (*ezBookKeepingTransactionPlainTextDataTable, error) {
func createNewezbookkeepingPlainTextDataTable(content string, columnSeparator string, lineSeparator string) (*ezBookKeepingPlainTextDataTable, error) {
allLines := strings.Split(content, lineSeparator)
if len(allLines) < 2 {
@@ -186,7 +167,7 @@ func createNewezbookkeepingTransactionPlainTextDataTable(content string, columnS
headerLine = strings.ReplaceAll(headerLine, "\r", "")
headerLineItems := strings.Split(headerLine, columnSeparator)
return &ezBookKeepingTransactionPlainTextDataTable{
return &ezBookKeepingPlainTextDataTable{
columnSeparator: columnSeparator,
lineSeparator: lineSeparator,
allLines: allLines,
@@ -194,7 +175,7 @@ func createNewezbookkeepingTransactionPlainTextDataTable(content string, columnS
}, nil
}
func createNewezbookkeepingTransactionPlainTextDataTableBuilder(transactionCount int, columns []datatable.DataTableColumn, dataColumnNameMapping map[datatable.DataTableColumn]string, columnSeparator string, lineSeparator string) *ezBookKeepingTransactionPlainTextDataTableBuilder {
func createNewezbookkeepingTransactionPlainTextDataTableBuilder(transactionCount int, columns []datatable.TransactionDataTableColumn, dataColumnNameMapping map[datatable.TransactionDataTableColumn]string, columnSeparator string, lineSeparator string) *ezBookKeepingTransactionPlainTextDataTableBuilder {
var builder strings.Builder
builder.Grow(transactionCount * 100)
pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer.go
+45 -48
View File
@@ -43,7 +43,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con
return nil, nil, nil, nil, nil, nil, err
}
if len(allLines) <= 1 {
if len(allLines) < 2 {
log.Errorf(ctx, "[feidee_mymoney_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile
}
@@ -71,36 +71,37 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con
return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow
}
newColumns := make([]datatable.DataTableColumn, 0, 11)
newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TYPE)
newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TIME)
newColumns := make([]datatable.TransactionDataTableColumn, 0, 11)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME)
if categoryColumnExists {
newColumns = append(newColumns, datatable.DATA_TABLE_CATEGORY)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_CATEGORY)
}
newColumns = append(newColumns, datatable.DATA_TABLE_SUB_CATEGORY)
newColumns = append(newColumns, datatable.DATA_TABLE_ACCOUNT_NAME)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
if accountCurrencyColumnExists {
newColumns = append(newColumns, datatable.DATA_TABLE_ACCOUNT_CURRENCY)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY)
}
newColumns = append(newColumns, datatable.DATA_TABLE_AMOUNT)
newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_ACCOUNT_NAME)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_AMOUNT)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME)
if accountCurrencyColumnExists {
newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY)
}
newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_AMOUNT)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT)
if descriptionColumnExists {
newColumns = append(newColumns, datatable.DATA_TABLE_DESCRIPTION)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_DESCRIPTION)
}
dataTable := datatable.CreateNewWritableDataTable(newColumns)
transferTransactionsMap := make(map[string]map[datatable.DataTableColumn]string, 0)
transactionRowParser := createFeideeMymoneyTransactionDataRowParser()
dataTable := datatable.CreateNewWritableTransactionDataTableWithRowParser(newColumns, transactionRowParser)
transferTransactionsMap := make(map[string]map[datatable.TransactionDataTableColumn]string, 0)
for i := 1; i < len(allLines); i++ {
items := allLines[i]
@@ -131,20 +132,20 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con
relatedIdColumnExists,
)
transactionType := data[datatable.DATA_TABLE_TRANSACTION_TYPE]
transactionType := data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]
if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText || transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText || transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText {
if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
} else if transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
} else if transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
}
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = ""
data[datatable.DATA_TABLE_RELATED_AMOUNT] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = ""
dataTable.Add(data)
} else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText || transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText {
if relatedId == "" {
@@ -159,18 +160,18 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con
continue
}
if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText && relatedData[datatable.DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferOutText {
relatedData[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
relatedData[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.DATA_TABLE_ACCOUNT_NAME]
relatedData[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY]
relatedData[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT]
if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferOutText {
relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = data[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
dataTable.Add(relatedData)
delete(transferTransactionsMap, relatedId)
} else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText && relatedData[datatable.DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferInText {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.DATA_TABLE_ACCOUNT_NAME]
data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.DATA_TABLE_ACCOUNT_CURRENCY]
data[datatable.DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.DATA_TABLE_AMOUNT]
} else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferInText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
dataTable.Add(data)
delete(transferTransactionsMap, relatedId)
} else {
@@ -188,11 +189,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) ParseImportedData(ctx core.Con
return nil, nil, nil, nil, nil, nil, errs.ErrFoundRecordNotHasRelatedRecord
}
dataTableImporter := datatable.CreateNewSimpleImporterFromWritableDataTableWithPostProcessFunc(
dataTable,
feideeMymoneyTransactionTypeNameMapping,
feideeMymoneyTransactionDataImporterPostProcess,
)
dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping)
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
@@ -253,40 +250,40 @@ func (c *feideeMymoneyTransactionDataCsvImporter) parseTransactionData(
descriptionColumnExists bool,
relatedIdColumnIdx int,
relatedIdColumnExists bool,
) (map[datatable.DataTableColumn]string, string) {
data := make(map[datatable.DataTableColumn]string, 11)
) (map[datatable.TransactionDataTableColumn]string, string) {
data := make(map[datatable.TransactionDataTableColumn]string, 11)
relatedId := ""
if timeColumnExists && timeColumnIdx < len(items) {
data[datatable.DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx]
}
if typeColumnExists && typeColumnIdx < len(items) {
data[datatable.DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx]
}
if categoryColumnExists && categoryColumnIdx < len(items) {
data[datatable.DATA_TABLE_CATEGORY] = items[categoryColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = items[categoryColumnIdx]
}
if subCategoryColumnExists && subCategoryColumnIdx < len(items) {
data[datatable.DATA_TABLE_SUB_CATEGORY] = items[subCategoryColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[subCategoryColumnIdx]
}
if accountColumnExists && accountColumnIdx < len(items) {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = items[accountColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = items[accountColumnIdx]
}
if accountCurrencyColumnExists && accountCurrencyColumnIdx < len(items) {
data[datatable.DATA_TABLE_ACCOUNT_CURRENCY] = items[accountCurrencyColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = items[accountCurrencyColumnIdx]
}
if amountColumnExists && amountColumnIdx < len(items) {
data[datatable.DATA_TABLE_AMOUNT] = items[amountColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[amountColumnIdx]
}
if descriptionColumnExists && descriptionColumnIdx < len(items) {
data[datatable.DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx]
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx]
}
if relatedIdColumnExists && relatedIdColumnIdx < len(items) {
@@ -296,7 +293,7 @@ func (c *feideeMymoneyTransactionDataCsvImporter) parseTransactionData(
return data, relatedId
}
func (c *feideeMymoneyTransactionDataCsvImporter) getRelatedIds(transferTransactionsMap map[string]map[datatable.DataTableColumn]string) string {
func (c *feideeMymoneyTransactionDataCsvImporter) getRelatedIds(transferTransactionsMap map[string]map[datatable.TransactionDataTableColumn]string) string {
builder := strings.Builder{}
for relatedId := range transferTransactionsMap {
pkg/converters/feidee/feidee_mymoney_transaction_data_csv_file_importer_test.go
+10 -4
View File
@@ -36,8 +36,8 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi
assert.Equal(t, 6, len(allNewTransactions))
assert.Equal(t, 2, len(allNewAccounts))
assert.Equal(t, 1, len(allNewSubExpenseCategories))
assert.Equal(t, 1, len(allNewSubIncomeCategories))
assert.Equal(t, 2, len(allNewSubExpenseCategories))
assert.Equal(t, 2, len(allNewSubIncomeCategories))
assert.Equal(t, 1, len(allNewSubTransferCategories))
assert.Equal(t, 0, len(allNewTags))
@@ -94,10 +94,16 @@ func TestFeideeMymoneyCsvFileImporterParseImportedData_MinimumValidData(t *testi
assert.Equal(t, "CNY", allNewAccounts[1].Currency)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid)
assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name)
assert.Equal(t, "", allNewSubExpenseCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[1].Uid)
assert.Equal(t, "Test Category2", allNewSubExpenseCategories[1].Name)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid)
assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name)
assert.Equal(t, "", allNewSubIncomeCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[1].Uid)
assert.Equal(t, "Test Category", allNewSubIncomeCategories[1].Name)
assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid)
assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name)
pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go
+9 -23
View File
@@ -2,19 +2,18 @@ package feidee
import (
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
var feideeMymoneyDataColumnNameMapping = map[datatable.DataTableColumn]string{
datatable.DATA_TABLE_TRANSACTION_TIME: "日期",
datatable.DATA_TABLE_TRANSACTION_TYPE: "交易类型",
datatable.DATA_TABLE_CATEGORY: "分类",
datatable.DATA_TABLE_SUB_CATEGORY: "子分类",
datatable.DATA_TABLE_ACCOUNT_NAME: "账户1",
datatable.DATA_TABLE_AMOUNT: "金额",
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2",
datatable.DATA_TABLE_DESCRIPTION: "备注",
var feideeMymoneyDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "日期",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "交易类型",
datatable.TRANSACTION_DATA_TABLE_CATEGORY: "分类",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "子分类",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "账户1",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "金额",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "备注",
}
var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{
@@ -23,16 +22,3 @@ var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_EXPENSE: "支出",
models.TRANSACTION_TYPE_TRANSFER: "转账",
}
func feideeMymoneyTransactionDataImporterPostProcess(ctx core.Context, transaction *models.ImportTransaction) error {
if transaction.Type == models.TRANSACTION_DB_TYPE_MODIFY_BALANCE {
if transaction.Amount >= 0 {
transaction.Type = models.TRANSACTION_DB_TYPE_INCOME
} else if transaction.Amount < 0 {
transaction.Amount = -transaction.Amount
transaction.Type = models.TRANSACTION_DB_TYPE_EXPENSE
}
}
return nil
}
pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go
+82
View File
@@ -0,0 +1,82 @@
package feidee
import (
"time"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
// feideeMymoneyTransactionDataRowParser defines the structure of feidee mymoney transaction data row parser
type feideeMymoneyTransactionDataRowParser struct {
}
// GetAddedColumns returns the added columns after converting the data row
func (p *feideeMymoneyTransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn {
return nil
}
// Parse returns the converted transaction data row
func (p *feideeMymoneyTransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) {
rowData = make(map[datatable.TransactionDataTableColumn]string, len(data))
for column, value := range data {
rowData[column] = value
}
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = p.getLongDateTime(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME])
}
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
if amount >= 0 {
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
} else {
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
}
return rowData, true, nil
}
// Parse returns the converted transaction data row
func (p *feideeMymoneyTransactionDataRowParser) getLongDateTime(str string) string {
if utils.IsValidLongDateTimeFormat(str) {
return str
}
utcTimezone := time.UTC
utcTimezoneOffsetMinutes := utils.GetTimezoneOffsetMinutes(utcTimezone)
if utils.IsValidLongDateTimeWithoutSecondFormat(str) {
dateTime, err := utils.ParseFromLongDateTimeWithoutSecond(str, utcTimezoneOffsetMinutes)
if err == nil {
return utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), utcTimezone)
}
}
if utils.IsValidLongDateFormat(str) {
dateTime, err := utils.ParseFromLongDateTimeWithoutSecond(str+" 00:00", utcTimezoneOffsetMinutes)
if err == nil {
return utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), utcTimezone)
}
}
return str
}
// createFeideeMymoneyTransactionDataRowParser returns feidee mymoney transaction data row parser
func createFeideeMymoneyTransactionDataRowParser() datatable.TransactionDataRowParser {
return &feideeMymoneyTransactionDataRowParser{}
}
pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer.go
+5 -7
View File
@@ -18,17 +18,15 @@ var (
// ParseImportedData returns the imported data by parsing the feidee mymoney transaction xls data
func (c *feideeMymoneyTransactionDataXlsImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
dataTable, err := createNewFeideeMymoneyTransactionExcelFileDataTable(data)
dataTable, err := datatable.CreateNewDefaultExcelFileImportedDataTable(data)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewSimpleImporterWithPostProcessFunc(
feideeMymoneyDataColumnNameMapping,
feideeMymoneyTransactionTypeNameMapping,
feideeMymoneyTransactionDataImporterPostProcess,
)
transactionRowParser := createFeideeMymoneyTransactionDataRowParser()
transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyDataColumnNameMapping, transactionRowParser)
dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping)
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
pkg/converters/feidee/feidee_mymoney_transaction_data_xls_file_importer_test.go
+12 -6
View File
@@ -29,8 +29,8 @@ func TestFeideeMymoneyTransactionDataXlsImporterParseImportedData_MinimumValidDa
assert.Equal(t, 7, len(allNewTransactions))
assert.Equal(t, 2, len(allNewAccounts))
assert.Equal(t, 2, len(allNewSubExpenseCategories))
assert.Equal(t, 2, len(allNewSubIncomeCategories))
assert.Equal(t, 3, len(allNewSubExpenseCategories))
assert.Equal(t, 3, len(allNewSubIncomeCategories))
assert.Equal(t, 1, len(allNewSubTransferCategories))
assert.Equal(t, 0, len(allNewTags))
@@ -95,16 +95,22 @@ func TestFeideeMymoneyTransactionDataXlsImporterParseImportedData_MinimumValidDa
assert.Equal(t, "CNY", allNewAccounts[1].Currency)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid)
assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name)
assert.Equal(t, "", allNewSubExpenseCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[1].Uid)
assert.Equal(t, "Test Category4", allNewSubExpenseCategories[1].Name)
assert.Equal(t, "Test Category2", allNewSubExpenseCategories[1].Name)
assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[2].Uid)
assert.Equal(t, "Test Category4", allNewSubExpenseCategories[2].Name)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid)
assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name)
assert.Equal(t, "", allNewSubIncomeCategories[0].Name)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[1].Uid)
assert.Equal(t, "Test Category5", allNewSubIncomeCategories[1].Name)
assert.Equal(t, "Test Category", allNewSubIncomeCategories[1].Name)
assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[2].Uid)
assert.Equal(t, "Test Category5", allNewSubIncomeCategories[2].Name)
assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid)
assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name)
pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go
+28 -14
View File
@@ -8,7 +8,28 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// fireflyiiiTransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data
var fireflyIIITransactionDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "date",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "type",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "category",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "source_name",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "currency_code",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "amount",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "destination_name",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "foreign_currency_code",
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "foreign_amount",
datatable.TRANSACTION_DATA_TABLE_TAGS: "tags",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "description",
}
var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance",
models.TRANSACTION_TYPE_INCOME: "Deposit",
models.TRANSACTION_TYPE_EXPENSE: "Withdrawal",
models.TRANSACTION_TYPE_TRANSFER: "Transfer",
}
// fireflyIIITransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data
type fireflyIIITransactionDataCsvImporter struct{}
// Initialize a firefly III transaction data csv file importer singleton instance
@@ -16,25 +37,18 @@ var (
FireflyIIITransactionDataCsvImporter = &fireflyIIITransactionDataCsvImporter{}
)
// ParseImportedData returns the imported data by parsing the firefly iii transaction csv data
// ParseImportedData returns the imported data by parsing the firefly III transaction csv data
func (c *fireflyIIITransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
reader := bytes.NewReader(data)
dataTable, err := createNewFireflyIIITransactionPlainTextDataTable(
ctx,
reader,
)
dataTable, err := datatable.CreateNewDefaultCsvDataTable(ctx, reader)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewImporter(
dataTable.GetDataColumnMapping(),
fireflyIIITransactionTypeNameMapping,
"",
",",
)
transactionRowParser := createFireflyIIITransactionDataRowParser()
transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser)
dataTableImporter := datatable.CreateNewImporter(fireflyIIITransactionTypeNameMapping, "", ",")
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer_test.go
+28
View File
@@ -114,6 +114,34 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidType(t *testing
assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidTimezone(t *testing.T) {
converter := FireflyIIITransactionDataCsvImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56-10:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+00:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+12:45,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidAccountCurrency(t *testing.T) {
converter := FireflyIIITransactionDataCsvImporter
context := core.NewNullContext()
pkg/converters/fireflyIII/fireflyiii_transaction_data_plain_text_data_table.go
-304
View File
@@ -1,304 +0,0 @@
package fireflyIII
import (
"encoding/csv"
"io"
"time"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
var fireflyIIITransactionSupportedColumns = []datatable.DataTableColumn{
datatable.DATA_TABLE_TRANSACTION_TIME,
datatable.DATA_TABLE_TRANSACTION_TYPE,
datatable.DATA_TABLE_SUB_CATEGORY,
datatable.DATA_TABLE_ACCOUNT_NAME,
datatable.DATA_TABLE_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_AMOUNT,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_RELATED_AMOUNT,
datatable.DATA_TABLE_TAGS,
datatable.DATA_TABLE_DESCRIPTION,
}
var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance",
models.TRANSACTION_TYPE_INCOME: "Deposit",
models.TRANSACTION_TYPE_EXPENSE: "Withdrawal",
models.TRANSACTION_TYPE_TRANSFER: "Transfer",
}
// fireflyIIITransactionPlainTextDataTable defines the structure of firefly III transaction plain text data table
type fireflyIIITransactionPlainTextDataTable struct {
allOriginalLines [][]string
originalHeaderLineColumnNames []string
originalColumnIndex map[datatable.DataTableColumn]int
}
// fireflyIIITransactionPlainTextDataRow defines the structure of firefly III transaction plain text data row
type fireflyIIITransactionPlainTextDataRow struct {
dataTable *fireflyIIITransactionPlainTextDataTable
originalItems []string
finalItems map[datatable.DataTableColumn]string
}
// fireflyIIITransactionPlainTextDataRowIterator defines the structure of firefly III transaction plain text data row iterator
type fireflyIIITransactionPlainTextDataRowIterator struct {
dataTable *fireflyIIITransactionPlainTextDataTable
currentIndex int
}
// DataRowCount returns the total count of data row
func (t *fireflyIIITransactionPlainTextDataTable) DataRowCount() int {
if len(t.allOriginalLines) < 1 {
return 0
}
return len(t.allOriginalLines) - 1
}
// GetDataColumnMapping returns data column map for data importer
func (t *fireflyIIITransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string {
dataColumnMapping := make(map[datatable.DataTableColumn]string, len(fireflyIIITransactionSupportedColumns))
for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
column := fireflyIIITransactionSupportedColumns[i]
dataColumnMapping[column] = utils.IntToString(int(column))
}
return dataColumnMapping
}
// HeaderLineColumnNames returns the header column name list
func (t *fireflyIIITransactionPlainTextDataTable) HeaderLineColumnNames() []string {
columnIndexes := make([]string, len(fireflyIIITransactionSupportedColumns))
for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
column := fireflyIIITransactionSupportedColumns[i]
if t.originalColumnIndex[column] >= 0 {
columnIndexes[i] = utils.IntToString(int(column))
} else {
columnIndexes[i] = "-1"
}
}
return columnIndexes
}
// DataRowIterator returns the iterator of data row
func (t *fireflyIIITransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &fireflyIIITransactionPlainTextDataRowIterator{
dataTable: t,
currentIndex: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *fireflyIIITransactionPlainTextDataRow) IsValid() bool {
return true
}
// ColumnCount returns the total count of column in this data row
func (r *fireflyIIITransactionPlainTextDataRow) ColumnCount() int {
return len(fireflyIIITransactionSupportedColumns)
}
// GetData returns the data in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetData(columnIndex int) string {
if columnIndex >= len(fireflyIIITransactionSupportedColumns) {
return ""
}
dataColumn := fireflyIIITransactionSupportedColumns[columnIndex]
return r.finalItems[dataColumn]
}
// GetTime returns the time in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
return utils.ParseFromLongDateTimeWithTimezone(r.GetData(columnIndex))
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return nil, errs.ErrNotSupported
}
// HasNext returns whether the iterator does not reach the end
func (t *fireflyIIITransactionPlainTextDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
}
// Next returns the next imported data row
func (t *fireflyIIITransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
return nil
}
t.currentIndex++
rowItems := t.dataTable.allOriginalLines[t.currentIndex]
finalItems := t.dataTable.parseTransactionData(rowItems)
return &fireflyIIITransactionPlainTextDataRow{
dataTable: t.dataTable,
originalItems: rowItems,
finalItems: finalItems,
}
}
func (t *fireflyIIITransactionPlainTextDataTable) parseTransactionData(items []string) map[datatable.DataTableColumn]string {
data := make(map[datatable.DataTableColumn]string, 12)
data[datatable.DATA_TABLE_SUB_CATEGORY] = ""
for column, index := range t.originalColumnIndex {
if index >= 0 && index < len(items) {
data[column] = items[index]
}
}
// trim trailing zero in decimal
if data[datatable.DATA_TABLE_AMOUNT] != "" {
data[datatable.DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
}
if data[datatable.DATA_TABLE_RELATED_AMOUNT] != "" {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_RELATED_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_RELATED_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount)
}
} else {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT]
}
// the related account currency field is foreign currency in firefly iii actually
if data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY]
}
// the destination account of modify balance transaction in firefly iii is the asset account
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// the destination account of income transaction in firefly iii is the asset account
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
}
return data
}
func createNewFireflyIIITransactionPlainTextDataTable(ctx core.Context, reader io.Reader) (*fireflyIIITransactionPlainTextDataTable, error) {
allOriginalLines, err := parseAllLinesFromFireflyIIITransactionPlainText(ctx, reader)
if err != nil {
return nil, err
}
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
originalHeaderItems := allOriginalLines[0]
originalHeaderItemMap := make(map[string]int)
for i := 0; i < len(originalHeaderItems); i++ {
originalHeaderItemMap[originalHeaderItems[i]] = i
}
typeColumnIdx, typeColumnExists := originalHeaderItemMap["type"]
amountColumnIdx, amountColumnExists := originalHeaderItemMap["amount"]
foreignAmountColumnIdx, foreignAmountColumnExists := originalHeaderItemMap["foreign_amount"]
currencyColumnIdx, currencyColumnExists := originalHeaderItemMap["currency_code"]
foreignCurrencyColumnIdx, foreignCurrencyColumnExists := originalHeaderItemMap["foreign_currency_code"]
descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["description"]
dateColumnIdx, dateColumnExists := originalHeaderItemMap["date"]
sourceNameColumnIdx, sourceNameColumnExists := originalHeaderItemMap["source_name"]
destinationNameColumnIdx, destinationNameColumnExists := originalHeaderItemMap["destination_name"]
categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["category"]
tagsColumnIdx, tagsColumnExists := originalHeaderItemMap["tags"]
if !typeColumnExists || !amountColumnExists || !dateColumnExists || !sourceNameColumnExists || !destinationNameColumnExists || !categoryColumnExists {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse firefly III csv data, because missing essential columns in header row")
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
if !foreignAmountColumnExists {
foreignAmountColumnIdx = -1
}
if !currencyColumnExists {
currencyColumnIdx = -1
}
if !foreignCurrencyColumnExists {
foreignCurrencyColumnIdx = -1
}
if !descriptionColumnExists {
descriptionColumnIdx = -1
}
if !tagsColumnExists {
tagsColumnIdx = -1
}
return &fireflyIIITransactionPlainTextDataTable{
allOriginalLines: allOriginalLines,
originalHeaderLineColumnNames: originalHeaderItems,
originalColumnIndex: map[datatable.DataTableColumn]int{
datatable.DATA_TABLE_TRANSACTION_TIME: dateColumnIdx,
datatable.DATA_TABLE_TRANSACTION_TYPE: typeColumnIdx,
datatable.DATA_TABLE_SUB_CATEGORY: categoryColumnIdx,
datatable.DATA_TABLE_ACCOUNT_NAME: sourceNameColumnIdx,
datatable.DATA_TABLE_ACCOUNT_CURRENCY: currencyColumnIdx,
datatable.DATA_TABLE_AMOUNT: amountColumnIdx,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: destinationNameColumnIdx,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: foreignCurrencyColumnIdx,
datatable.DATA_TABLE_RELATED_AMOUNT: foreignAmountColumnIdx,
datatable.DATA_TABLE_TAGS: tagsColumnIdx,
datatable.DATA_TABLE_DESCRIPTION: descriptionColumnIdx,
},
}, nil
}
func parseAllLinesFromFireflyIIITransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) {
csvReader := csv.NewReader(reader)
csvReader.FieldsPerRecord = -1
allOriginalLines := make([][]string, 0)
for {
items, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.parseAllLinesFromFireflyIIITransactionPlainText] cannot parse firefly III csv data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
allOriginalLines = append(allOriginalLines, items)
}
return allOriginalLines, nil
}
pkg/converters/fireflyIII/fireflyiii_transaction_data_row_parser.go
+89
View File
@@ -0,0 +1,89 @@
package fireflyIII
import (
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
// fireflyIIITransactionDataRowParser defines the structure of firefly III transaction data row parser
type fireflyIIITransactionDataRowParser struct {
}
// GetAddedColumns returns the added columns after converting the data row
func (p *fireflyIIITransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn {
return []datatable.TransactionDataTableColumn{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE,
}
}
// Parse returns the converted transaction data row
func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) {
rowData = make(map[datatable.TransactionDataTableColumn]string, len(data))
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
for column, value := range data {
rowData[column] = value
}
// parse long date time and timezone
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" {
dateTime, err := utils.ParseFromLongDateTimeWithTimezone(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME])
if err != nil {
return nil, false, errs.ErrTransactionTimeInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location())
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location())
}
// trim trailing zero in decimal
if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount)
} else {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
}
// the related account currency field is foreign currency in firefly III actually
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
}
// the destination account of modify balance transaction in firefly III is the asset account
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// the destination account of income transaction in firefly III is the asset account
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]
}
return rowData, true, nil
}
// createFireflyIIITransactionDataRowParser returns firefly III transaction data row parser
func createFireflyIIITransactionDataRowParser() datatable.TransactionDataRowParser {
return &fireflyIIITransactionDataRowParser{}
}