diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go index 92ecfe9e..1ab5d1e7 100644 --- a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go @@ -61,13 +61,13 @@ func (c *alipayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Contex enc := simplifiedchinese.GB18030 reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder()) - dataTable, err := c.createNewAlipayImportedDataTable(ctx, reader, c.fileHeaderLine, c.dataHeaderStartContent, c.dataBottomEndLineRune) + dataTable, err := c.createNewAlipayBasicDataTable(ctx, reader, c.fileHeaderLine, c.dataHeaderStartContent, c.dataBottomEndLineRune) if err != nil { return nil, nil, nil, nil, nil, nil, err } - commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable) + commonDataTable := datatable.CreateNewCommonDataTableFromBasicDataTable(dataTable) if !commonDataTable.HasColumn(c.originalColumnNames.timeColumnName) || !commonDataTable.HasColumn(c.originalColumnNames.amountColumnName) || @@ -77,14 +77,14 @@ func (c *alipayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Contex return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } - transactionRowParser := createAlipayTransactionDataRowParser(c.originalColumnNames) - transactionDataTable := datatable.CreateNewCommonTransactionDataTable(commonDataTable, alipayTransactionSupportedColumns, transactionRowParser) + transactionRowParser := createAlipayTransactionDataRowParser(c.originalColumnNames, dataTable.HeaderColumnNames()) + transactionDataTable := datatable.CreateNewTransactionDataTableFromCommonDataTable(commonDataTable, alipayTransactionSupportedColumns, transactionRowParser) dataTableImporter := converter.CreateNewSimpleImporterWithTypeNameMapping(alipayTransactionTypeNameMapping) return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } -func (c *alipayTransactionDataCsvFileImporter) createNewAlipayImportedDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) (datatable.ImportedDataTable, error) { +func (c *alipayTransactionDataCsvFileImporter) createNewAlipayBasicDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) (datatable.BasicDataTable, error) { csvReader := csv.NewReader(reader) csvReader.FieldsPerRecord = -1 @@ -100,7 +100,7 @@ func (c *alipayTransactionDataCsvFileImporter) createNewAlipayImportedDataTable( } if err != nil { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse alipay csv data, because %s", err.Error()) + log.Errorf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] cannot parse alipay csv data, because %s", err.Error()) return nil, errs.ErrInvalidCSVFile } @@ -111,7 +111,7 @@ func (c *alipayTransactionDataCsvFileImporter) createNewAlipayImportedDataTable( hasFileHeader = true continue } else { - log.Warnf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) + log.Warnf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) continue } } @@ -139,7 +139,7 @@ func (c *alipayTransactionDataCsvFileImporter) createNewAlipayImportedDataTable( } if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) + log.Errorf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } @@ -152,11 +152,11 @@ func (c *alipayTransactionDataCsvFileImporter) createNewAlipayImportedDataTable( } if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse import data, because data table row count is less 1") + log.Errorf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] cannot parse import data, because data table row count is less 1") return nil, errs.ErrNotFoundTransactionDataInFile } - dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines) + dataTable := csvdatatable.CreateNewCustomCsvBasicDataTable(allOriginalLines) return dataTable, nil } diff --git a/pkg/converters/alipay/alipay_transaction_data_row_parser.go b/pkg/converters/alipay/alipay_transaction_data_row_parser.go index 8c36362c..47757b67 100644 --- a/pkg/converters/alipay/alipay_transaction_data_row_parser.go +++ b/pkg/converters/alipay/alipay_transaction_data_row_parser.go @@ -26,11 +26,12 @@ const alipayTransactionDataProductNameRepaymentText = "还款" // alipayTransactionDataRowParser defines the structure of alipay transaction data row parser type alipayTransactionDataRowParser struct { - columns alipayTransactionColumnNames + columns alipayTransactionColumnNames + existedOriginalDataColumns map[string]bool } // Parse returns the converted transaction data row -func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.User, dataTable *datatable.CommonTransactionDataTable, dataRow datatable.CommonDataRow, rowId string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { +func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.User, dataRow datatable.CommonDataTableRow, rowId string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { if dataRow.GetData(p.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && dataRow.GetData(p.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && dataRow.GetData(p.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { @@ -50,23 +51,23 @@ func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.Us data := make(map[datatable.TransactionDataTableColumn]string, len(alipayTransactionSupportedColumns)) - if dataTable.HasOriginalColumn(p.columns.timeColumnName) { + if p.hasOriginalColumn(p.columns.timeColumnName) { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = dataRow.GetData(p.columns.timeColumnName) } - if dataTable.HasOriginalColumn(p.columns.categoryColumnName) { + if p.hasOriginalColumn(p.columns.categoryColumnName) { data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(p.columns.categoryColumnName) } else { data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" } - if dataTable.HasOriginalColumn(p.columns.amountColumnName) { + if p.hasOriginalColumn(p.columns.amountColumnName) { data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = dataRow.GetData(p.columns.amountColumnName) } - if dataTable.HasOriginalColumn(p.columns.descriptionColumnName) && dataRow.GetData(p.columns.descriptionColumnName) != "" { + if p.hasOriginalColumn(p.columns.descriptionColumnName) && dataRow.GetData(p.columns.descriptionColumnName) != "" { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(p.columns.descriptionColumnName) - } else if dataTable.HasOriginalColumn(p.columns.productNameColumnName) && dataRow.GetData(p.columns.productNameColumnName) != "" { + } else if p.hasOriginalColumn(p.columns.productNameColumnName) && dataRow.GetData(p.columns.productNameColumnName) != "" { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(p.columns.productNameColumnName) } else { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = "" @@ -74,13 +75,13 @@ func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.Us relatedAccountName := "" - if dataTable.HasOriginalColumn(p.columns.relatedAccountColumnName) { + if p.hasOriginalColumn(p.columns.relatedAccountColumnName) { relatedAccountName = dataRow.GetData(p.columns.relatedAccountColumnName) } statusName := "" - if dataTable.HasOriginalColumn(p.columns.statusColumnName) { + if p.hasOriginalColumn(p.columns.statusColumnName) { statusName = dataRow.GetData(p.columns.statusColumnName) } @@ -92,7 +93,7 @@ func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.Us localeTextItems := locales.GetLocaleTextItems(locale) - if dataTable.HasOriginalColumn(p.columns.typeColumnName) { + if p.hasOriginalColumn(p.columns.typeColumnName) { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(p.columns.typeColumnName) if dataRow.GetData(p.columns.typeColumnName) == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { @@ -117,11 +118,11 @@ func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.Us targetName := "" productName := "" - if dataTable.HasOriginalColumn(p.columns.targetNameColumnName) { + if p.hasOriginalColumn(p.columns.targetNameColumnName) { targetName = dataRow.GetData(p.columns.targetNameColumnName) } - if dataTable.HasOriginalColumn(p.columns.productNameColumnName) { + if p.hasOriginalColumn(p.columns.productNameColumnName) { productName = dataRow.GetData(p.columns.productNameColumnName) } @@ -170,9 +171,21 @@ func (p *alipayTransactionDataRowParser) Parse(ctx core.Context, user *models.Us return data, true, nil } +func (p *alipayTransactionDataRowParser) hasOriginalColumn(columnName string) bool { + _, exists := p.existedOriginalDataColumns[columnName] + return exists +} + // createAlipayTransactionDataRowParser returns alipay transaction data row parser -func createAlipayTransactionDataRowParser(originalColumnNames alipayTransactionColumnNames) datatable.CommonTransactionDataRowParser { +func createAlipayTransactionDataRowParser(originalColumnNames alipayTransactionColumnNames, headerColumnNames []string) datatable.CommonTransactionDataRowParser { + existedOriginalDataColumns := make(map[string]bool, len(headerColumnNames)) + + for i := 0; i < len(headerColumnNames); i++ { + existedOriginalDataColumns[headerColumnNames[i]] = true + } + return &alipayTransactionDataRowParser{ - columns: originalColumnNames, + columns: originalColumnNames, + existedOriginalDataColumns: existedOriginalDataColumns, } } diff --git a/pkg/converters/beancount/beancount_transaction_data_table.go b/pkg/converters/beancount/beancount_transaction_data_table.go index 4b2a9a6a..4ff5c6f4 100644 --- a/pkg/converters/beancount/beancount_transaction_data_table.go +++ b/pkg/converters/beancount/beancount_transaction_data_table.go @@ -85,7 +85,7 @@ func (t *beancountTransactionDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allData) } -// Next returns the next imported data row +// Next returns the next transaction data row func (t *beancountTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { if t.currentIndex+1 >= len(t.dataTable.allData) { return nil, nil diff --git a/pkg/converters/camt/camt_statement_transaction_data_table.go b/pkg/converters/camt/camt_statement_transaction_data_table.go index c4de826f..5bcd2b82 100644 --- a/pkg/converters/camt/camt_statement_transaction_data_table.go +++ b/pkg/converters/camt/camt_statement_transaction_data_table.go @@ -138,7 +138,7 @@ func (t *camtStatementTransactionDataRowIterator) HasNext() bool { return false } -// Next returns the next imported data row +// Next returns the next transaction data row func (t *camtStatementTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { allStatements := t.dataTable.allStatements diff --git a/pkg/converters/csv/csv_file_basic_data_table.go b/pkg/converters/csv/csv_file_basic_data_table.go new file mode 100644 index 00000000..628333c2 --- /dev/null +++ b/pkg/converters/csv/csv_file_basic_data_table.go @@ -0,0 +1,138 @@ +package csv + +import ( + "encoding/csv" + "fmt" + "io" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +// CsvFileBasicDataTable defines the structure of csv data table +type CsvFileBasicDataTable struct { + allLines [][]string +} + +// CsvFileBasicDataTableRow defines the structure of csv data table row +type CsvFileBasicDataTableRow struct { + dataTable *CsvFileBasicDataTable + allItems []string +} + +// CsvFileBasicDataTableRowIterator defines the structure of csv data table row iterator +type CsvFileBasicDataTableRowIterator struct { + dataTable *CsvFileBasicDataTable + currentIndex int +} + +// DataRowCount returns the total count of data row +func (t *CsvFileBasicDataTable) DataRowCount() int { + if len(t.allLines) < 1 { + return 0 + } + + return len(t.allLines) - 1 +} + +// HeaderColumnNames returns the header column name list +func (t *CsvFileBasicDataTable) HeaderColumnNames() []string { + if len(t.allLines) < 1 { + return nil + } + + return t.allLines[0] +} + +// DataRowIterator returns the iterator of data row +func (t *CsvFileBasicDataTable) DataRowIterator() datatable.BasicDataTableRowIterator { + return &CsvFileBasicDataTableRowIterator{ + dataTable: t, + currentIndex: 0, + } +} + +// ColumnCount returns the total count of column in this data row +func (r *CsvFileBasicDataTableRow) ColumnCount() int { + return len(r.allItems) +} + +// GetData returns the data in the specified column index +func (r *CsvFileBasicDataTableRow) GetData(columnIndex int) string { + if columnIndex >= len(r.allItems) { + return "" + } + + return r.allItems[columnIndex] +} + +// HasNext returns whether the iterator does not reach the end +func (t *CsvFileBasicDataTableRowIterator) HasNext() bool { + return t.currentIndex+1 < len(t.dataTable.allLines) +} + +// CurrentRowId returns current index +func (t *CsvFileBasicDataTableRowIterator) CurrentRowId() string { + return fmt.Sprintf("line#%d", t.currentIndex) +} + +// Next returns the next basic data row +func (t *CsvFileBasicDataTableRowIterator) Next() datatable.BasicDataTableRow { + if t.currentIndex+1 >= len(t.dataTable.allLines) { + return nil + } + + t.currentIndex++ + + rowItems := t.dataTable.allLines[t.currentIndex] + + return &CsvFileBasicDataTableRow{ + dataTable: t.dataTable, + allItems: rowItems, + } +} + +// CreateNewCsvBasicDataTable returns comma separated values data table by io readers +func CreateNewCsvBasicDataTable(ctx core.Context, reader io.Reader) (datatable.BasicDataTable, error) { + return createNewCsvFileBasicDataTable(ctx, reader, ',') +} + +// CreateNewCustomCsvBasicDataTable returns character separated values data table by io readers +func CreateNewCustomCsvBasicDataTable(allLines [][]string) datatable.BasicDataTable { + return &CsvFileBasicDataTable{ + allLines: allLines, + } +} + +func createNewCsvFileBasicDataTable(ctx core.Context, reader io.Reader, separator rune) (*CsvFileBasicDataTable, error) { + csvReader := csv.NewReader(reader) + csvReader.Comma = separator + csvReader.FieldsPerRecord = -1 + + allLines := make([][]string, 0) + + for { + items, err := csvReader.Read() + + if err == io.EOF { + break + } + + if err != nil { + log.Errorf(ctx, "[csv_file_basic_data_table.createNewCsvFileDataTable] cannot parse csv data, because %s", err.Error()) + return nil, errs.ErrInvalidCSVFile + } + + if len(items) == 1 && items[0] == "" { + continue + } + + allLines = append(allLines, items) + } + + return &CsvFileBasicDataTable{ + allLines: allLines, + }, nil +} diff --git a/pkg/converters/csv/csv_file_imported_data_table_test.go b/pkg/converters/csv/csv_file_basic_data_table_test.go similarity index 70% rename from pkg/converters/csv/csv_file_imported_data_table_test.go rename to pkg/converters/csv/csv_file_basic_data_table_test.go index 29d7348f..086d06ff 100644 --- a/pkg/converters/csv/csv_file_imported_data_table_test.go +++ b/pkg/converters/csv/csv_file_basic_data_table_test.go @@ -9,8 +9,8 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/core" ) -func TestCsvFileImportedDataTableDataRowCount(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableDataRowCount(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, {"A2", "B2", "C2"}, {"A3", "B3", "C3"}, @@ -19,22 +19,22 @@ func TestCsvFileImportedDataTableDataRowCount(t *testing.T) { assert.Equal(t, 2, datatable.DataRowCount()) } -func TestCsvFileImportedDataTableDataRowCount_OnlyHeaderLine(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableDataRowCount_OnlyHeaderLine(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, }) assert.Equal(t, 0, datatable.DataRowCount()) } -func TestCsvFileImportedDataTableDataRowCount_EmptyContent(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{}) +func TestCsvFileBasicDataTableDataRowCount_EmptyContent(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{}) assert.Equal(t, 0, datatable.DataRowCount()) } -func TestCsvFileImportedDataTableHeaderColumnNames(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableHeaderColumnNames(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, {"A2", "B2", "C2"}, {"A3", "B3", "C3"}, @@ -43,14 +43,14 @@ func TestCsvFileImportedDataTableHeaderColumnNames(t *testing.T) { assert.EqualValues(t, []string{"A1", "B1", "C1"}, datatable.HeaderColumnNames()) } -func TestCsvFileImportedDataTableHeaderColumnNames_EmptyContent(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{}) +func TestCsvFileBasicDataTableHeaderColumnNames_EmptyContent(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{}) assert.Nil(t, datatable.HeaderColumnNames()) } -func TestCsvFileImportedDataRowIterator(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableRowIterator(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, {"A2", "B2", "C2"}, {"A3", "B3", "C3"}, @@ -76,8 +76,8 @@ func TestCsvFileImportedDataRowIterator(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestCsvFileImportedDataRowColumnCount(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableRowColumnCount(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, {"A2", "B2", "C2"}, {"A3", "B3", "C3"}, @@ -92,8 +92,8 @@ func TestCsvFileImportedDataRowColumnCount(t *testing.T) { assert.EqualValues(t, 3, row2.ColumnCount()) } -func TestCsvFileImportedDataRowGetData(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableRowGetData(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, {"A2", "B2", "C2"}, {"A3", "B3", "C3"}, @@ -112,8 +112,8 @@ func TestCsvFileImportedDataRowGetData(t *testing.T) { assert.Equal(t, "C3", row2.GetData(2)) } -func TestCsvFileImportedDataRowGetData_GetNotExistedColumnData(t *testing.T) { - datatable := CreateNewCustomCsvImportedDataTable([][]string{ +func TestCsvFileBasicDataTableRowGetData_GetNotExistedColumnData(t *testing.T) { + datatable := CreateNewCustomCsvBasicDataTable([][]string{ {"A1", "B1", "C1"}, {"A2", "B2", "C2"}, {"A3", "B3", "C3"}, @@ -125,12 +125,12 @@ func TestCsvFileImportedDataRowGetData_GetNotExistedColumnData(t *testing.T) { assert.Equal(t, "", row1.GetData(3)) } -func TestCreateNewCsvImportedDataTable(t *testing.T) { +func TestCreateNewCsvBasicDataTable(t *testing.T) { context := core.NewNullContext() reader := bytes.NewReader([]byte("A1,B1,C1\n" + "A2,B2,C2\n" + "A3,B3,C3\n")) - datatable, err := CreateNewCsvImportedDataTable(context, reader) + datatable, err := CreateNewCsvBasicDataTable(context, reader) assert.Nil(t, err) assert.Equal(t, 2, datatable.DataRowCount()) @@ -153,14 +153,14 @@ func TestCreateNewCsvImportedDataTable(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestCreateNewCsvImportedDataTable_SkipBlankLine(t *testing.T) { +func TestCreateNewCsvBasicDataTable_SkipBlankLine(t *testing.T) { context := core.NewNullContext() reader := bytes.NewReader([]byte("\n" + "A1,B1,C1\n" + "A2,B2,C2\n" + "\n" + "A3,B3,C3\n")) - datatable, err := CreateNewCsvImportedDataTable(context, reader) + datatable, err := CreateNewCsvBasicDataTable(context, reader) assert.Nil(t, err) assert.Equal(t, 2, datatable.DataRowCount()) diff --git a/pkg/converters/csv/csv_file_imported_data_table.go b/pkg/converters/csv/csv_file_imported_data_table.go deleted file mode 100644 index a26ded14..00000000 --- a/pkg/converters/csv/csv_file_imported_data_table.go +++ /dev/null @@ -1,138 +0,0 @@ -package csv - -import ( - "encoding/csv" - "fmt" - "io" - - "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/errs" - "github.com/mayswind/ezbookkeeping/pkg/log" -) - -// CsvFileImportedDataTable defines the structure of csv data table -type CsvFileImportedDataTable struct { - allLines [][]string -} - -// CsvFileImportedDataRow defines the structure of csv data table row -type CsvFileImportedDataRow struct { - dataTable *CsvFileImportedDataTable - allItems []string -} - -// CsvFileImportedDataRowIterator defines the structure of csv data table row iterator -type CsvFileImportedDataRowIterator struct { - dataTable *CsvFileImportedDataTable - currentIndex int -} - -// DataRowCount returns the total count of data row -func (t *CsvFileImportedDataTable) DataRowCount() int { - if len(t.allLines) < 1 { - return 0 - } - - return len(t.allLines) - 1 -} - -// HeaderColumnNames returns the header column name list -func (t *CsvFileImportedDataTable) HeaderColumnNames() []string { - if len(t.allLines) < 1 { - return nil - } - - return t.allLines[0] -} - -// DataRowIterator returns the iterator of data row -func (t *CsvFileImportedDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &CsvFileImportedDataRowIterator{ - dataTable: t, - currentIndex: 0, - } -} - -// ColumnCount returns the total count of column in this data row -func (r *CsvFileImportedDataRow) ColumnCount() int { - return len(r.allItems) -} - -// GetData returns the data in the specified column index -func (r *CsvFileImportedDataRow) GetData(columnIndex int) string { - if columnIndex >= len(r.allItems) { - return "" - } - - return r.allItems[columnIndex] -} - -// HasNext returns whether the iterator does not reach the end -func (t *CsvFileImportedDataRowIterator) HasNext() bool { - return t.currentIndex+1 < len(t.dataTable.allLines) -} - -// CurrentRowId returns current index -func (t *CsvFileImportedDataRowIterator) CurrentRowId() string { - return fmt.Sprintf("line#%d", t.currentIndex) -} - -// Next returns the next imported data row -func (t *CsvFileImportedDataRowIterator) Next() datatable.ImportedDataRow { - if t.currentIndex+1 >= len(t.dataTable.allLines) { - return nil - } - - t.currentIndex++ - - rowItems := t.dataTable.allLines[t.currentIndex] - - return &CsvFileImportedDataRow{ - dataTable: t.dataTable, - allItems: rowItems, - } -} - -// CreateNewCsvImportedDataTable returns comma separated values data table by io readers -func CreateNewCsvImportedDataTable(ctx core.Context, reader io.Reader) (*CsvFileImportedDataTable, error) { - return createNewCsvFileDataTable(ctx, reader, ',') -} - -// CreateNewCustomCsvImportedDataTable returns character separated values data table by io readers -func CreateNewCustomCsvImportedDataTable(allLines [][]string) *CsvFileImportedDataTable { - return &CsvFileImportedDataTable{ - allLines: allLines, - } -} - -func createNewCsvFileDataTable(ctx core.Context, reader io.Reader, separator rune) (*CsvFileImportedDataTable, error) { - csvReader := csv.NewReader(reader) - csvReader.Comma = separator - csvReader.FieldsPerRecord = -1 - - allLines := make([][]string, 0) - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[csv_file_imported_data_table.createNewCsvFileDataTable] cannot parse csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - if len(items) == 1 && items[0] == "" { - continue - } - - allLines = append(allLines, items) - } - - return &CsvFileImportedDataTable{ - allLines: allLines, - }, nil -} diff --git a/pkg/converters/datatable/imported_data_table.go b/pkg/converters/datatable/basic_data_table.go similarity index 57% rename from pkg/converters/datatable/imported_data_table.go rename to pkg/converters/datatable/basic_data_table.go index 9e6992b7..fdf6f609 100644 --- a/pkg/converters/datatable/imported_data_table.go +++ b/pkg/converters/datatable/basic_data_table.go @@ -1,7 +1,7 @@ package datatable -// ImportedDataTable defines the structure of imported data table -type ImportedDataTable interface { +// BasicDataTable defines the structure of basic data table +type BasicDataTable interface { // DataRowCount returns the total count of data row DataRowCount() int @@ -9,11 +9,11 @@ type ImportedDataTable interface { HeaderColumnNames() []string // DataRowIterator returns the iterator of data row - DataRowIterator() ImportedDataRowIterator + DataRowIterator() BasicDataTableRowIterator } -// ImportedDataRow defines the structure of imported data row -type ImportedDataRow interface { +// BasicDataTableRow defines the structure of basic data row +type BasicDataTableRow interface { // ColumnCount returns the total count of column in this data row ColumnCount() int @@ -21,14 +21,14 @@ type ImportedDataRow interface { GetData(columnIndex int) string } -// ImportedDataRowIterator defines the structure of imported data row iterator -type ImportedDataRowIterator interface { +// BasicDataTableRowIterator defines the structure of basic data row iterator +type BasicDataTableRowIterator interface { // HasNext returns whether the iterator does not reach the end HasNext() bool // CurrentRowId returns current row id CurrentRowId() string - // Next returns the next imported data row - Next() ImportedDataRow + // Next returns the next basic data row + Next() BasicDataTableRow } diff --git a/pkg/converters/datatable/basic_data_table_to_common_data_table_wrapper.go b/pkg/converters/datatable/basic_data_table_to_common_data_table_wrapper.go new file mode 100644 index 00000000..a15755c3 --- /dev/null +++ b/pkg/converters/datatable/basic_data_table_to_common_data_table_wrapper.go @@ -0,0 +1,107 @@ +package datatable + +// basicDataTableToCommonDataTableWrapper defines the structure of basic data table to common data table wrapper +type basicDataTableToCommonDataTableWrapper struct { + innerDataTable BasicDataTable + dataColumnIndexes map[string]int +} + +// basicDataTableToCommonDataTableWrapperRow defines the data row structure of basic data table to common data table wrapper +type basicDataTableToCommonDataTableWrapperRow struct { + rowData map[string]string +} + +// basicDataTableToCommonDataTableWrapperRowIterator defines the data row iterator structure of basic data table to common data table wrapper +type basicDataTableToCommonDataTableWrapperRowIterator struct { + commonDataTable *basicDataTableToCommonDataTableWrapper + innerIterator BasicDataTableRowIterator +} + +// HeaderColumnCount returns the total count of column in header row +func (t *basicDataTableToCommonDataTableWrapper) HeaderColumnCount() int { + return len(t.innerDataTable.HeaderColumnNames()) +} + +// HasColumn returns whether the data table has specified column name +func (t *basicDataTableToCommonDataTableWrapper) HasColumn(columnName string) bool { + index, exists := t.dataColumnIndexes[columnName] + return exists && index >= 0 +} + +// DataRowCount returns the total count of common data row +func (t *basicDataTableToCommonDataTableWrapper) DataRowCount() int { + return t.innerDataTable.DataRowCount() +} + +// DataRowIterator returns the iterator of common data row +func (t *basicDataTableToCommonDataTableWrapper) DataRowIterator() CommonDataTableRowIterator { + return &basicDataTableToCommonDataTableWrapperRowIterator{ + commonDataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), + } +} + +// HasData returns whether the common data row has specified column data +func (r *basicDataTableToCommonDataTableWrapperRow) HasData(columnName string) bool { + _, exists := r.rowData[columnName] + return exists +} + +// ColumnCount returns the total count of column in this data row +func (r *basicDataTableToCommonDataTableWrapperRow) ColumnCount() int { + return len(r.rowData) +} + +// GetData returns the data in the specified column name +func (r *basicDataTableToCommonDataTableWrapperRow) GetData(columnName string) string { + return r.rowData[columnName] +} + +// HasNext returns whether the iterator does not reach the end +func (t *basicDataTableToCommonDataTableWrapperRowIterator) HasNext() bool { + return t.innerIterator.HasNext() +} + +// CurrentRowId returns current row id +func (t *basicDataTableToCommonDataTableWrapperRowIterator) CurrentRowId() string { + return t.innerIterator.CurrentRowId() +} + +// Next returns the next common data row +func (t *basicDataTableToCommonDataTableWrapperRowIterator) Next() CommonDataTableRow { + basicDataRow := t.innerIterator.Next() + + if basicDataRow == nil { + return nil + } + + rowData := make(map[string]string, len(t.commonDataTable.dataColumnIndexes)) + + for column, columnIndex := range t.commonDataTable.dataColumnIndexes { + if columnIndex < 0 || columnIndex >= basicDataRow.ColumnCount() { + continue + } + + value := basicDataRow.GetData(columnIndex) + rowData[column] = value + } + + return &basicDataTableToCommonDataTableWrapperRow{ + rowData: rowData, + } +} + +// CreateNewCommonDataTableFromBasicDataTable returns common data table from basic data table +func CreateNewCommonDataTableFromBasicDataTable(dataTable BasicDataTable) CommonDataTable { + headerLineItems := dataTable.HeaderColumnNames() + dataColumnIndexes := make(map[string]int, len(headerLineItems)) + + for i := 0; i < len(headerLineItems); i++ { + dataColumnIndexes[headerLineItems[i]] = i + } + + return &basicDataTableToCommonDataTableWrapper{ + innerDataTable: dataTable, + dataColumnIndexes: dataColumnIndexes, + } +} diff --git a/pkg/converters/datatable/imported_transaction_data_table.go b/pkg/converters/datatable/basic_data_table_to_transaction_data_table_wrapper.go similarity index 51% rename from pkg/converters/datatable/imported_transaction_data_table.go rename to pkg/converters/datatable/basic_data_table_to_transaction_data_table_wrapper.go index 4b393c5a..9802f41c 100644 --- a/pkg/converters/datatable/imported_transaction_data_table.go +++ b/pkg/converters/datatable/basic_data_table_to_transaction_data_table_wrapper.go @@ -7,30 +7,30 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/models" ) -// ImportedTransactionDataTable defines the structure of imported transaction data table -type ImportedTransactionDataTable struct { - innerDataTable ImportedDataTable +// basicDataTableToTransactionDataTableWrapper defines the structure of basic data table to transaction data table wrapper +type basicDataTableToTransactionDataTableWrapper struct { + innerDataTable BasicDataTable dataColumnMapping map[TransactionDataTableColumn]string dataColumnIndexes map[TransactionDataTableColumn]int rowParser TransactionDataRowParser addedColumns map[TransactionDataTableColumn]bool } -// ImportedTransactionDataRow defines the structure of imported transaction data row -type ImportedTransactionDataRow struct { - transactionDataTable *ImportedTransactionDataTable +// basicDataTableToTransactionDataTableWrapperRow defines the data row structure of basic data table to transaction data table wrapper +type basicDataTableToTransactionDataTableWrapperRow struct { + transactionDataTable *basicDataTableToTransactionDataTableWrapper rowData map[TransactionDataTableColumn]string rowDataValid bool } -// ImportedTransactionDataRowIterator defines the structure of imported transaction data row iterator -type ImportedTransactionDataRowIterator struct { - transactionDataTable *ImportedTransactionDataTable - innerIterator ImportedDataRowIterator +// basicDataTableToTransactionDataTableWrapperRowIterator defines the data row iterator structure of basic data table to transaction data table wrapper +type basicDataTableToTransactionDataTableWrapperRowIterator struct { + transactionDataTable *basicDataTableToTransactionDataTableWrapper + innerIterator BasicDataTableRowIterator } // HasColumn returns whether the data table has specified column -func (t *ImportedTransactionDataTable) HasColumn(column TransactionDataTableColumn) bool { +func (t *basicDataTableToTransactionDataTableWrapper) HasColumn(column TransactionDataTableColumn) bool { index, exists := t.dataColumnIndexes[column] if exists && index >= 0 { @@ -49,25 +49,25 @@ func (t *ImportedTransactionDataTable) HasColumn(column TransactionDataTableColu } // TransactionRowCount returns the total count of transaction data row -func (t *ImportedTransactionDataTable) TransactionRowCount() int { +func (t *basicDataTableToTransactionDataTableWrapper) TransactionRowCount() int { return t.innerDataTable.DataRowCount() } // TransactionRowIterator returns the iterator of transaction data row -func (t *ImportedTransactionDataTable) TransactionRowIterator() TransactionDataRowIterator { - return &ImportedTransactionDataRowIterator{ +func (t *basicDataTableToTransactionDataTableWrapper) TransactionRowIterator() TransactionDataRowIterator { + return &basicDataTableToTransactionDataTableWrapperRowIterator{ transactionDataTable: t, innerIterator: t.innerDataTable.DataRowIterator(), } } // IsValid returns whether this row is valid data for importing -func (r *ImportedTransactionDataRow) IsValid() bool { +func (r *basicDataTableToTransactionDataTableWrapperRow) IsValid() bool { return r.rowDataValid } // GetData returns the data in the specified column type -func (r *ImportedTransactionDataRow) GetData(column TransactionDataTableColumn) string { +func (r *basicDataTableToTransactionDataTableWrapperRow) GetData(column TransactionDataTableColumn) string { if !r.rowDataValid { return "" } @@ -90,28 +90,28 @@ func (r *ImportedTransactionDataRow) GetData(column TransactionDataTableColumn) } // HasNext returns whether the iterator does not reach the end -func (t *ImportedTransactionDataRowIterator) HasNext() bool { +func (t *basicDataTableToTransactionDataTableWrapperRowIterator) HasNext() bool { return t.innerIterator.HasNext() } // Next returns the next transaction data row -func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) { - importedRow := t.innerIterator.Next() +func (t *basicDataTableToTransactionDataTableWrapperRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) { + basicDataRow := t.innerIterator.Next() - if importedRow == nil { + if basicDataRow == nil { return nil, nil } - if importedRow.ColumnCount() == 1 && importedRow.GetData(0) == "" { - return &ImportedTransactionDataRow{ + if basicDataRow.ColumnCount() == 1 && basicDataRow.GetData(0) == "" { + return &basicDataTableToTransactionDataTableWrapperRow{ transactionDataTable: t.transactionDataTable, rowData: nil, rowDataValid: false, }, nil } - if importedRow.ColumnCount() < len(t.transactionDataTable.dataColumnIndexes) { - log.Errorf(ctx, "[imported_transaction_data_table.Next] cannot parse data row, because may missing some columns (column count %d in data row is less than header column count %d)", importedRow.ColumnCount(), len(t.transactionDataTable.dataColumnIndexes)) + if basicDataRow.ColumnCount() < len(t.transactionDataTable.dataColumnIndexes) { + log.Errorf(ctx, "[basic_data_table_to_transaction_data_table_wrapper.Next] cannot parse data row, because may missing some columns (column count %d in data row is less than header column count %d)", basicDataRow.ColumnCount(), len(t.transactionDataTable.dataColumnIndexes)) return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } @@ -119,11 +119,11 @@ func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models rowDataValid := true for column, columnIndex := range t.transactionDataTable.dataColumnIndexes { - if columnIndex < 0 || columnIndex >= importedRow.ColumnCount() { + if columnIndex < 0 || columnIndex >= basicDataRow.ColumnCount() { continue } - value := importedRow.GetData(columnIndex) + value := basicDataRow.GetData(columnIndex) rowData[column] = value } @@ -131,25 +131,25 @@ func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models rowData, rowDataValid, err = t.transactionDataTable.rowParser.Parse(rowData) if err != nil { - log.Errorf(ctx, "[imported_transaction_data_table.Next] cannot parse data row, because %s", err.Error()) + log.Errorf(ctx, "[basic_data_table_to_transaction_data_table_wrapper.Next] cannot parse data row, because %s", err.Error()) return nil, err } } - return &ImportedTransactionDataRow{ + return &basicDataTableToTransactionDataTableWrapperRow{ transactionDataTable: t.transactionDataTable, rowData: rowData, rowDataValid: rowDataValid, }, nil } -// CreateNewImportedTransactionDataTable returns transaction data table from imported data table -func CreateNewImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable { - return CreateNewImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil) +// CreateNewTransactionDataTableFromBasicDataTable returns transaction data table from basic data table +func CreateNewTransactionDataTableFromBasicDataTable(dataTable BasicDataTable, dataColumnMapping map[TransactionDataTableColumn]string) TransactionDataTable { + return CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable, dataColumnMapping, nil) } -// CreateNewImportedTransactionDataTableWithRowParser returns transaction data table from imported data table -func CreateNewImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable { +// CreateNewTransactionDataTableFromBasicDataTableWithRowParser returns transaction data table from basic data table +func CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable BasicDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) TransactionDataTable { headerLineItems := dataTable.HeaderColumnNames() headerItemMap := make(map[string]int, len(headerLineItems)) @@ -178,7 +178,7 @@ func CreateNewImportedTransactionDataTableWithRowParser(dataTable ImportedDataTa } } - return &ImportedTransactionDataTable{ + return &basicDataTableToTransactionDataTableWrapper{ innerDataTable: dataTable, dataColumnMapping: dataColumnMapping, dataColumnIndexes: dataColumnIndexes, diff --git a/pkg/converters/datatable/common_data_table.go b/pkg/converters/datatable/common_data_table.go index 14bf9ef4..9cc9996b 100644 --- a/pkg/converters/datatable/common_data_table.go +++ b/pkg/converters/datatable/common_data_table.go @@ -12,11 +12,11 @@ type CommonDataTable interface { DataRowCount() int // DataRowIterator returns the iterator of common data row - DataRowIterator() CommonDataRowIterator + DataRowIterator() CommonDataTableRowIterator } -// CommonDataRow defines the structure of common data row -type CommonDataRow interface { +// CommonDataTableRow defines the structure of common data row +type CommonDataTableRow interface { // ColumnCount returns the total count of column in this data row ColumnCount() int @@ -27,8 +27,8 @@ type CommonDataRow interface { GetData(columnName string) string } -// CommonDataRowIterator defines the structure of common data row iterator -type CommonDataRowIterator interface { +// CommonDataTableRowIterator defines the structure of common data row iterator +type CommonDataTableRowIterator interface { // HasNext returns whether the iterator does not reach the end HasNext() bool @@ -36,5 +36,5 @@ type CommonDataRowIterator interface { CurrentRowId() string // Next returns the next common data row - Next() CommonDataRow + Next() CommonDataTableRow } diff --git a/pkg/converters/datatable/common_data_table_to_transaction_data_table_wrapper.go b/pkg/converters/datatable/common_data_table_to_transaction_data_table_wrapper.go new file mode 100644 index 00000000..dba889b7 --- /dev/null +++ b/pkg/converters/datatable/common_data_table_to_transaction_data_table_wrapper.go @@ -0,0 +1,109 @@ +package datatable + +import ( + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" +) + +// CommonTransactionDataRowParser defines the structure of common transaction data row parser +type CommonTransactionDataRowParser interface { + // Parse returns the converted transaction data row + Parse(ctx core.Context, user *models.User, dataRow CommonDataTableRow, rowId string) (rowData map[TransactionDataTableColumn]string, rowDataValid bool, err error) +} + +// commonDataTableToTransactionDataTableWrapper defines the structure of common data table to transaction data table wrapper +type commonDataTableToTransactionDataTableWrapper struct { + innerDataTable CommonDataTable + supportedDataColumns map[TransactionDataTableColumn]bool + rowParser CommonTransactionDataRowParser +} + +// commonDataTableToTransactionDataTableWrapperRow defines the data row structure of common data table to transaction data table wrapper +type commonDataTableToTransactionDataTableWrapperRow struct { + transactionDataTable *commonDataTableToTransactionDataTableWrapper + rowData map[TransactionDataTableColumn]string + rowDataValid bool +} + +// commonDataTableToTransactionDataTableWrapperRowIterator defines the data row iterator structure of common data table to transaction data table wrapper +type commonDataTableToTransactionDataTableWrapperRowIterator struct { + transactionDataTable *commonDataTableToTransactionDataTableWrapper + innerIterator CommonDataTableRowIterator +} + +// HasColumn returns whether the data table has specified column +func (t *commonDataTableToTransactionDataTableWrapper) HasColumn(column TransactionDataTableColumn) bool { + _, exists := t.supportedDataColumns[column] + return exists +} + +// TransactionRowCount returns the total count of transaction data row +func (t *commonDataTableToTransactionDataTableWrapper) TransactionRowCount() int { + return t.innerDataTable.DataRowCount() +} + +// TransactionRowIterator returns the iterator of transaction data row +func (t *commonDataTableToTransactionDataTableWrapper) TransactionRowIterator() TransactionDataRowIterator { + return &commonDataTableToTransactionDataTableWrapperRowIterator{ + transactionDataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), + } +} + +// IsValid returns whether this row is valid data for importing +func (r *commonDataTableToTransactionDataTableWrapperRow) IsValid() bool { + return r.rowDataValid +} + +// GetData returns the data in the specified column type +func (r *commonDataTableToTransactionDataTableWrapperRow) GetData(column TransactionDataTableColumn) string { + if !r.rowDataValid { + return "" + } + + _, exists := r.transactionDataTable.supportedDataColumns[column] + + if !exists { + return "" + } + + return r.rowData[column] +} + +// HasNext returns whether the iterator does not reach the end +func (t *commonDataTableToTransactionDataTableWrapperRowIterator) HasNext() bool { + return t.innerIterator.HasNext() +} + +// Next returns the next transaction data row +func (t *commonDataTableToTransactionDataTableWrapperRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) { + commonDataRow := t.innerIterator.Next() + + if commonDataRow == nil { + return nil, nil + } + + rowId := t.innerIterator.CurrentRowId() + rowData, rowDataValid, err := t.transactionDataTable.rowParser.Parse(ctx, user, commonDataRow, rowId) + + if err != nil { + log.Errorf(ctx, "[common_data_table_to_transaction_data_table_wrapper.Next] cannot parse data row, because %s", err.Error()) + return nil, err + } + + return &commonDataTableToTransactionDataTableWrapperRow{ + transactionDataTable: t.transactionDataTable, + rowData: rowData, + rowDataValid: rowDataValid, + }, nil +} + +// CreateNewTransactionDataTableFromCommonDataTable returns transaction data table from Common data table +func CreateNewTransactionDataTableFromCommonDataTable(dataTable CommonDataTable, supportedDataColumns map[TransactionDataTableColumn]bool, rowParser CommonTransactionDataRowParser) TransactionDataTable { + return &commonDataTableToTransactionDataTableWrapper{ + innerDataTable: dataTable, + supportedDataColumns: supportedDataColumns, + rowParser: rowParser, + } +} diff --git a/pkg/converters/datatable/common_transaction_data_table.go b/pkg/converters/datatable/common_transaction_data_table.go deleted file mode 100644 index 57399de3..00000000 --- a/pkg/converters/datatable/common_transaction_data_table.go +++ /dev/null @@ -1,114 +0,0 @@ -package datatable - -import ( - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/log" - "github.com/mayswind/ezbookkeeping/pkg/models" -) - -// CommonTransactionDataTable defines the structure of common transaction data table -type CommonTransactionDataTable struct { - innerDataTable CommonDataTable - supportedDataColumns map[TransactionDataTableColumn]bool - rowParser CommonTransactionDataRowParser -} - -// CommonTransactionDataRow defines the structure of common transaction data row -type CommonTransactionDataRow struct { - transactionDataTable *CommonTransactionDataTable - rowData map[TransactionDataTableColumn]string - rowDataValid bool -} - -// CommonTransactionDataRowIterator defines the structure of common transaction data row iterator -type CommonTransactionDataRowIterator struct { - transactionDataTable *CommonTransactionDataTable - innerIterator CommonDataRowIterator -} - -// CommonTransactionDataRowParser defines the structure of common transaction data row parser -type CommonTransactionDataRowParser interface { - // Parse returns the converted transaction data row - Parse(ctx core.Context, user *models.User, dataTable *CommonTransactionDataTable, dataRow CommonDataRow, rowId string) (rowData map[TransactionDataTableColumn]string, rowDataValid bool, err error) -} - -// HasColumn returns whether the data table has specified column -func (t *CommonTransactionDataTable) HasColumn(column TransactionDataTableColumn) bool { - _, exists := t.supportedDataColumns[column] - return exists -} - -// HasOriginalColumn returns whether the original data table has specified column name -func (t *CommonTransactionDataTable) HasOriginalColumn(columnName string) bool { - return columnName != "" && t.innerDataTable.HasColumn(columnName) -} - -// TransactionRowCount returns the total count of transaction data row -func (t *CommonTransactionDataTable) TransactionRowCount() int { - return t.innerDataTable.DataRowCount() -} - -// TransactionRowIterator returns the iterator of transaction data row -func (t *CommonTransactionDataTable) TransactionRowIterator() TransactionDataRowIterator { - return &CommonTransactionDataRowIterator{ - transactionDataTable: t, - innerIterator: t.innerDataTable.DataRowIterator(), - } -} - -// IsValid returns whether this row is valid data for importing -func (r *CommonTransactionDataRow) IsValid() bool { - return r.rowDataValid -} - -// GetData returns the data in the specified column type -func (r *CommonTransactionDataRow) GetData(column TransactionDataTableColumn) string { - if !r.rowDataValid { - return "" - } - - _, exists := r.transactionDataTable.supportedDataColumns[column] - - if !exists { - return "" - } - - return r.rowData[column] -} - -// HasNext returns whether the iterator does not reach the end -func (t *CommonTransactionDataRowIterator) HasNext() bool { - return t.innerIterator.HasNext() -} - -// Next returns the next transaction data row -func (t *CommonTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow TransactionDataRow, err error) { - commonRow := t.innerIterator.Next() - - if commonRow == nil { - return nil, nil - } - - rowId := t.innerIterator.CurrentRowId() - rowData, rowDataValid, err := t.transactionDataTable.rowParser.Parse(ctx, user, t.transactionDataTable, commonRow, rowId) - - if err != nil { - log.Errorf(ctx, "[common_transaction_data_table.Next] cannot parse data row, because %s", err.Error()) - return nil, err - } - - return &CommonTransactionDataRow{ - transactionDataTable: t.transactionDataTable, - rowData: rowData, - rowDataValid: rowDataValid, - }, nil -} - -// CreateNewCommonTransactionDataTable returns transaction data table from Common data table -func CreateNewCommonTransactionDataTable(dataTable CommonDataTable, supportedDataColumns map[TransactionDataTableColumn]bool, rowParser CommonTransactionDataRowParser) *CommonTransactionDataTable { - return &CommonTransactionDataTable{ - innerDataTable: dataTable, - supportedDataColumns: supportedDataColumns, - rowParser: rowParser, - } -} diff --git a/pkg/converters/datatable/imported_common_data_table.go b/pkg/converters/datatable/imported_common_data_table.go deleted file mode 100644 index 2a827103..00000000 --- a/pkg/converters/datatable/imported_common_data_table.go +++ /dev/null @@ -1,107 +0,0 @@ -package datatable - -// ImportedCommonDataTable defines the structure of imported common data table -type ImportedCommonDataTable struct { - innerDataTable ImportedDataTable - dataColumnIndexes map[string]int -} - -// ImportedCommonDataRow defines the structure of imported common data row -type ImportedCommonDataRow struct { - rowData map[string]string -} - -// ImportedCommonDataRowIterator defines the structure of imported common data row iterator -type ImportedCommonDataRowIterator struct { - commonDataTable *ImportedCommonDataTable - innerIterator ImportedDataRowIterator -} - -// HeaderColumnCount returns the total count of column in header row -func (t *ImportedCommonDataTable) HeaderColumnCount() int { - return len(t.innerDataTable.HeaderColumnNames()) -} - -// HasColumn returns whether the data table has specified column name -func (t *ImportedCommonDataTable) HasColumn(columnName string) bool { - index, exists := t.dataColumnIndexes[columnName] - return exists && index >= 0 -} - -// DataRowCount returns the total count of common data row -func (t *ImportedCommonDataTable) DataRowCount() int { - return t.innerDataTable.DataRowCount() -} - -// DataRowIterator returns the iterator of common data row -func (t *ImportedCommonDataTable) DataRowIterator() CommonDataRowIterator { - return &ImportedCommonDataRowIterator{ - commonDataTable: t, - innerIterator: t.innerDataTable.DataRowIterator(), - } -} - -// HasData returns whether the common data row has specified column data -func (r *ImportedCommonDataRow) HasData(columnName string) bool { - _, exists := r.rowData[columnName] - return exists -} - -// ColumnCount returns the total count of column in this data row -func (r *ImportedCommonDataRow) ColumnCount() int { - return len(r.rowData) -} - -// GetData returns the data in the specified column name -func (r *ImportedCommonDataRow) GetData(columnName string) string { - return r.rowData[columnName] -} - -// HasNext returns whether the iterator does not reach the end -func (t *ImportedCommonDataRowIterator) HasNext() bool { - return t.innerIterator.HasNext() -} - -// CurrentRowId returns current row id -func (t *ImportedCommonDataRowIterator) CurrentRowId() string { - return t.innerIterator.CurrentRowId() -} - -// Next returns the next common data row -func (t *ImportedCommonDataRowIterator) Next() CommonDataRow { - importedRow := t.innerIterator.Next() - - if importedRow == nil { - return nil - } - - rowData := make(map[string]string, len(t.commonDataTable.dataColumnIndexes)) - - for column, columnIndex := range t.commonDataTable.dataColumnIndexes { - if columnIndex < 0 || columnIndex >= importedRow.ColumnCount() { - continue - } - - value := importedRow.GetData(columnIndex) - rowData[column] = value - } - - return &ImportedCommonDataRow{ - rowData: rowData, - } -} - -// CreateNewImportedCommonDataTable returns common data table from imported data table -func CreateNewImportedCommonDataTable(dataTable ImportedDataTable) *ImportedCommonDataTable { - headerLineItems := dataTable.HeaderColumnNames() - dataColumnIndexes := make(map[string]int, len(headerLineItems)) - - for i := 0; i < len(headerLineItems); i++ { - dataColumnIndexes[headerLineItems[i]] = i - } - - return &ImportedCommonDataTable{ - innerDataTable: dataTable, - dataColumnIndexes: dataColumnIndexes, - } -} diff --git a/pkg/converters/default/default_transaction_data_plain_text_converter.go b/pkg/converters/default/default_transaction_data_plain_text_converter.go index 7e61abc8..698f0281 100644 --- a/pkg/converters/default/default_transaction_data_plain_text_converter.go +++ b/pkg/converters/default/default_transaction_data_plain_text_converter.go @@ -95,7 +95,7 @@ func (c *defaultTransactionDataPlainTextConverter) ParseImportedData(ctx core.Co return nil, nil, nil, nil, nil, nil, err } - transactionDataTable := datatable.CreateNewImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping) + transactionDataTable := datatable.CreateNewTransactionDataTableFromBasicDataTable(dataTable, ezbookkeepingDataColumnNameMapping) dataTableImporter := converter.CreateNewImporterWithTypeNameMapping( ezbookkeepingTransactionTypeNameMapping, diff --git a/pkg/converters/default/default_transaction_plain_text_data_table.go b/pkg/converters/default/default_transaction_plain_text_data_table.go index c0d88d8c..33f68371 100644 --- a/pkg/converters/default/default_transaction_plain_text_data_table.go +++ b/pkg/converters/default/default_transaction_plain_text_data_table.go @@ -52,7 +52,7 @@ func (t *defaultPlainTextDataTable) HeaderColumnNames() []string { } // DataRowIterator returns the iterator of data row -func (t *defaultPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator { +func (t *defaultPlainTextDataTable) DataRowIterator() datatable.BasicDataTableRowIterator { return &defaultPlainTextDataRowIterator{ dataTable: t, currentIndex: 0, @@ -83,8 +83,8 @@ func (t *defaultPlainTextDataRowIterator) CurrentRowId() string { return fmt.Sprintf("line#%d", t.currentIndex) } -// Next returns the next imported data row -func (t *defaultPlainTextDataRowIterator) Next() datatable.ImportedDataRow { +// Next returns the next basic data row +func (t *defaultPlainTextDataRowIterator) Next() datatable.BasicDataTableRow { if t.currentIndex+1 >= len(t.dataTable.allLines) { return nil } diff --git a/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go index 4164add4..d7713d57 100644 --- a/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go +++ b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go @@ -157,7 +157,7 @@ func (c *customTransactionDataDsvFileImporter) ParseImportedData(ctx core.Contex allLines = append([][]string{{}}, allLines...) } - dataTable := csvconverter.CreateNewCustomCsvImportedDataTable(allLines) + dataTable := csvconverter.CreateNewCustomCsvBasicDataTable(allLines) transactionDataTable := CreateNewCustomPlainTextDataTable(dataTable, c.columnIndexMapping, c.transactionTypeNameMapping, c.timeFormat, c.timezoneFormat, c.amountDecimalSeparator, c.amountDigitGroupingSymbol) dataTableImporter := converter.CreateNewImporterWithTypeNameMapping(customTransactionTypeNameMapping, c.geoLocationSeparator, c.geoLocationOrder, c.transactionTagSeparator) diff --git a/pkg/converters/dsv/custom_transaction_plain_text_data_table.go b/pkg/converters/dsv/custom_transaction_plain_text_data_table.go index f92ab80f..c5528f51 100644 --- a/pkg/converters/dsv/custom_transaction_plain_text_data_table.go +++ b/pkg/converters/dsv/custom_transaction_plain_text_data_table.go @@ -14,7 +14,7 @@ import ( // customPlainTextDataTable defines the structure of custom plain text transaction data table type customPlainTextDataTable struct { - innerDataTable datatable.ImportedDataTable + innerDataTable datatable.BasicDataTable columnIndexMapping map[datatable.TransactionDataTableColumn]int transactionTypeNameMapping map[string]models.TransactionType timeFormat string @@ -34,7 +34,7 @@ type customPlainTextDataRow struct { // customPlainTextDataRowIterator defines the structure of custom plain text transaction data row iterator type customPlainTextDataRowIterator struct { transactionDataTable *customPlainTextDataTable - innerIterator datatable.ImportedDataRowIterator + innerIterator datatable.BasicDataTableRowIterator } // HasColumn returns whether the data table has specified column @@ -105,7 +105,7 @@ func (t *customPlainTextDataRowIterator) Next(ctx core.Context, user *models.Use }, nil } -func (t *customPlainTextDataRowIterator) parseTransaction(ctx core.Context, user *models.User, row datatable.ImportedDataRow) (map[datatable.TransactionDataTableColumn]string, bool, error) { +func (t *customPlainTextDataRowIterator) parseTransaction(ctx core.Context, user *models.User, row datatable.BasicDataTableRow) (map[datatable.TransactionDataTableColumn]string, bool, error) { rowData := make(map[datatable.TransactionDataTableColumn]string, len(t.transactionDataTable.columnIndexMapping)) for column, columnIndex := range t.transactionDataTable.columnIndexMapping { @@ -236,7 +236,7 @@ func (t *customPlainTextDataRowIterator) parseAmount(ctx core.Context, amountVal } // CreateNewCustomPlainTextDataTable returns transaction data table from imported data table -func CreateNewCustomPlainTextDataTable(dataTable datatable.ImportedDataTable, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, timeFormat string, timezoneFormat string, amountDecimalSeparator string, amountDigitGroupingSymbol string) *customPlainTextDataTable { +func CreateNewCustomPlainTextDataTable(dataTable datatable.BasicDataTable, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, timeFormat string, timezoneFormat string, amountDecimalSeparator string, amountDigitGroupingSymbol string) *customPlainTextDataTable { timeFormatIncludeTimezone := strings.Contains(timeFormat, "z") || strings.Contains(timeFormat, "Z") return &customPlainTextDataTable{ diff --git a/pkg/converters/excel/excel_mscfb_file_imported_data_table.go b/pkg/converters/excel/excel_mscfb_file_basic_data_table.go similarity index 66% rename from pkg/converters/excel/excel_mscfb_file_imported_data_table.go rename to pkg/converters/excel/excel_mscfb_file_basic_data_table.go index 537194f9..243f13c6 100644 --- a/pkg/converters/excel/excel_mscfb_file_imported_data_table.go +++ b/pkg/converters/excel/excel_mscfb_file_basic_data_table.go @@ -10,27 +10,27 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/errs" ) -// ExcelMSCFBFileImportedDataTable defines the structure of excel (microsoft compound file binary) file data table -type ExcelMSCFBFileImportedDataTable struct { +// ExcelMSCFBFileBasicDataTable defines the structure of excel (microsoft compound file binary) file data table +type ExcelMSCFBFileBasicDataTable struct { workbook *xls.WorkBook headerLineColumnNames []string } -// ExcelMSCFBFileDataRow defines the structure of excel (microsoft compound file binary) file data table row -type ExcelMSCFBFileDataRow struct { +// ExcelMSCFBFileBasicDataTableRow defines the structure of excel (microsoft compound file binary) file data table row +type ExcelMSCFBFileBasicDataTableRow struct { sheet *xls.WorkSheet rowIndex int } -// ExcelMSCFBFileDataRowIterator defines the structure of excel (microsoft compound file binary) file data table row iterator -type ExcelMSCFBFileDataRowIterator struct { - dataTable *ExcelMSCFBFileImportedDataTable +// ExcelMSCFBFileBasicDataTableRowIterator defines the structure of excel (microsoft compound file binary) file data table row iterator +type ExcelMSCFBFileBasicDataTableRowIterator struct { + dataTable *ExcelMSCFBFileBasicDataTable currentSheetIndex int currentRowIndexInSheet uint16 } // DataRowCount returns the total count of data row -func (t *ExcelMSCFBFileImportedDataTable) DataRowCount() int { +func (t *ExcelMSCFBFileBasicDataTable) DataRowCount() int { totalDataRowCount := 0 for i := 0; i < t.workbook.NumSheets(); i++ { @@ -47,13 +47,13 @@ func (t *ExcelMSCFBFileImportedDataTable) DataRowCount() int { } // HeaderColumnNames returns the header column name list -func (t *ExcelMSCFBFileImportedDataTable) HeaderColumnNames() []string { +func (t *ExcelMSCFBFileBasicDataTable) HeaderColumnNames() []string { return t.headerLineColumnNames } // DataRowIterator returns the iterator of data row -func (t *ExcelMSCFBFileImportedDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &ExcelMSCFBFileDataRowIterator{ +func (t *ExcelMSCFBFileBasicDataTable) DataRowIterator() datatable.BasicDataTableRowIterator { + return &ExcelMSCFBFileBasicDataTableRowIterator{ dataTable: t, currentSheetIndex: 0, currentRowIndexInSheet: 0, @@ -61,19 +61,19 @@ func (t *ExcelMSCFBFileImportedDataTable) DataRowIterator() datatable.ImportedDa } // ColumnCount returns the total count of column in this data row -func (r *ExcelMSCFBFileDataRow) ColumnCount() int { +func (r *ExcelMSCFBFileBasicDataTableRow) ColumnCount() int { row := r.sheet.Row(r.rowIndex) return row.LastCol() + 1 } // GetData returns the data in the specified column index -func (r *ExcelMSCFBFileDataRow) GetData(columnIndex int) string { +func (r *ExcelMSCFBFileBasicDataTableRow) GetData(columnIndex int) string { row := r.sheet.Row(r.rowIndex) return row.Col(columnIndex) } // HasNext returns whether the iterator does not reach the end -func (t *ExcelMSCFBFileDataRowIterator) HasNext() bool { +func (t *ExcelMSCFBFileBasicDataTableRowIterator) HasNext() bool { workbook := t.dataTable.workbook if t.currentSheetIndex >= workbook.NumSheets() { @@ -100,12 +100,12 @@ func (t *ExcelMSCFBFileDataRowIterator) HasNext() bool { } // CurrentRowId returns current index -func (t *ExcelMSCFBFileDataRowIterator) CurrentRowId() string { - return fmt.Sprintf("table#%d-row#%d", t.currentSheetIndex, t.currentRowIndexInSheet) +func (t *ExcelMSCFBFileBasicDataTableRowIterator) CurrentRowId() string { + return fmt.Sprintf("sheet#%d-row#%d", t.currentSheetIndex, t.currentRowIndexInSheet) } -// Next returns the next imported data row -func (t *ExcelMSCFBFileDataRowIterator) Next() datatable.ImportedDataRow { +// Next returns the next basic data row +func (t *ExcelMSCFBFileBasicDataTableRowIterator) Next() datatable.BasicDataTableRow { workbook := t.dataTable.workbook currentRowIndexInTable := t.currentRowIndexInSheet @@ -133,14 +133,14 @@ func (t *ExcelMSCFBFileDataRowIterator) Next() datatable.ImportedDataRow { return nil } - return &ExcelMSCFBFileDataRow{ + return &ExcelMSCFBFileBasicDataTableRow{ sheet: currentSheet, rowIndex: int(t.currentRowIndexInSheet), } } -// CreateNewExcelMSCFBFileImportedDataTable returns excel (microsoft compound file binary) data table by file binary data -func CreateNewExcelMSCFBFileImportedDataTable(data []byte) (*ExcelMSCFBFileImportedDataTable, error) { +// CreateNewExcelMSCFBFileBasicDataTable returns excel (microsoft compound file binary) data table by file binary data +func CreateNewExcelMSCFBFileBasicDataTable(data []byte) (datatable.BasicDataTable, error) { reader := bytes.NewReader(data) workbook, err := xls.OpenReader(reader, "") @@ -184,7 +184,7 @@ func CreateNewExcelMSCFBFileImportedDataTable(data []byte) (*ExcelMSCFBFileImpor } } - return &ExcelMSCFBFileImportedDataTable{ + return &ExcelMSCFBFileBasicDataTable{ workbook: workbook, headerLineColumnNames: headerRowItems, }, nil diff --git a/pkg/converters/excel/excel_mscfb_file_imported_data_table_test.go b/pkg/converters/excel/excel_mscfb_file_basic_data_table_test.go similarity index 71% rename from pkg/converters/excel/excel_mscfb_file_imported_data_table_test.go rename to pkg/converters/excel/excel_mscfb_file_basic_data_table_test.go index 31ba78b2..aa4a82eb 100644 --- a/pkg/converters/excel/excel_mscfb_file_imported_data_table_test.go +++ b/pkg/converters/excel/excel_mscfb_file_basic_data_table_test.go @@ -9,63 +9,63 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/errs" ) -func TestExcelMSCFBFileImportedDataTableDataRowCount(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableDataRowCount(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 2, datatable.DataRowCount()) } -func TestExcelMSCFBFileImportedDataTableDataRowCount_MultipleSheets(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableDataRowCount_MultipleSheets(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 5, datatable.DataRowCount()) } -func TestExcelMSCFBFileImportedDataTableDataRowCount_OnlyHeaderLine(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableDataRowCount_OnlyHeaderLine(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/only_one_row_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 0, datatable.DataRowCount()) } -func TestExcelMSCFBFileImportedDataTableDataRowCount_EmptyContent(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableDataRowCount_EmptyContent(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 0, datatable.DataRowCount()) } -func TestExcelMSCFBFileImportedDataTableHeaderColumnNames(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableHeaderColumnNames(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.EqualValues(t, []string{"A1", "B1", "C1"}, datatable.HeaderColumnNames()) } -func TestExcelMSCFBFileImportedDataTableHeaderColumnNames_EmptyContent(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableHeaderColumnNames_EmptyContent(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.Nil(t, datatable.HeaderColumnNames()) } -func TestExcelMSCFBFileDataRowIterator(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowIterator(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.True(t, iterator.HasNext()) @@ -86,11 +86,11 @@ func TestExcelMSCFBFileDataRowIterator(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelMSCFBFileDataRowIterator_MultipleSheets(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowIterator_MultipleSheets(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.True(t, iterator.HasNext()) @@ -123,11 +123,11 @@ func TestExcelMSCFBFileDataRowIterator_MultipleSheets(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelMSCFBFileDataRowIterator_OnlyHeaderLine(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowIterator_OnlyHeaderLine(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/only_one_row_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.False(t, iterator.HasNext()) @@ -140,11 +140,11 @@ func TestExcelMSCFBFileDataRowIterator_OnlyHeaderLine(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelMSCFBFileDataRowIterator_EmptyContent(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowIterator_EmptyContent(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.False(t, iterator.HasNext()) @@ -157,11 +157,11 @@ func TestExcelMSCFBFileDataRowIterator_EmptyContent(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelMSCFBFileDataRowColumnCount(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowColumnCount(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() row1 := iterator.Next() @@ -171,11 +171,11 @@ func TestExcelMSCFBFileDataRowColumnCount(t *testing.T) { assert.EqualValues(t, 4, row2.ColumnCount()) } -func TestExcelMSCFBFileDataRowGetData(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowGetData(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() row1 := iterator.Next() @@ -189,22 +189,22 @@ func TestExcelMSCFBFileDataRowGetData(t *testing.T) { assert.Equal(t, "C3", row2.GetData(2)) } -func TestExcelMSCFBFileDataRowGetData_GetNotExistedColumnData(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowGetData_GetNotExistedColumnData(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() row1 := iterator.Next() assert.Equal(t, "", row1.GetData(3)) } -func TestExcelMSCFBFileDataRowGetData_MultipleSheets(t *testing.T) { +func TestExcelMSCFBFileBasicDataTableRowGetData_MultipleSheets(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xls") assert.Nil(t, err) - datatable, err := CreateNewExcelMSCFBFileImportedDataTable(testdata) + datatable, err := CreateNewExcelMSCFBFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() sheet1Row1 := iterator.Next() @@ -237,10 +237,10 @@ func TestExcelMSCFBFileDataRowGetData_MultipleSheets(t *testing.T) { assert.Equal(t, "5-C3", sheet5Row2.GetData(2)) } -func TestCreateNewExcelMSCFBFileImportedDataTable_MultipleSheetsWithDifferentHeaders(t *testing.T) { +func TestCreateNewExcelMSCFBFileBasicDataTable_MultipleSheetsWithDifferentHeaders(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_with_different_header_row_excel_file.xls") assert.Nil(t, err) - _, err = CreateNewExcelMSCFBFileImportedDataTable(testdata) + _, err = CreateNewExcelMSCFBFileBasicDataTable(testdata) assert.EqualError(t, err, errs.ErrFieldsInMultiTableAreDifferent.Message) } diff --git a/pkg/converters/excel/excel_ooxml_file_imported_data_table.go b/pkg/converters/excel/excel_ooxml_file_basic_data_table.go similarity index 69% rename from pkg/converters/excel/excel_ooxml_file_imported_data_table.go rename to pkg/converters/excel/excel_ooxml_file_basic_data_table.go index 4dc01a62..c3b4e62f 100644 --- a/pkg/converters/excel/excel_ooxml_file_imported_data_table.go +++ b/pkg/converters/excel/excel_ooxml_file_basic_data_table.go @@ -16,28 +16,28 @@ type excelOOXMLSheet struct { allData [][]string } -// ExcelOOXMLFileImportedDataTable defines the structure of excel (Office Open XML) file data table -type ExcelOOXMLFileImportedDataTable struct { +// ExcelOOXMLFileBasicDataTable defines the structure of excel (Office Open XML) file data table +type ExcelOOXMLFileBasicDataTable struct { sheets []*excelOOXMLSheet headerLineColumnNames []string } -// ExcelOOXMLFileDataRow defines the structure of excel (Office Open XML) file data table row -type ExcelOOXMLFileDataRow struct { +// ExcelOOXMLFileBasicDataTableRow defines the structure of excel (Office Open XML) file data table row +type ExcelOOXMLFileBasicDataTableRow struct { sheet *excelOOXMLSheet rowData []string rowIndex int } -// ExcelOOXMLFileDataRowIterator defines the structure of excel (Office Open XML) file data table row iterator -type ExcelOOXMLFileDataRowIterator struct { - dataTable *ExcelOOXMLFileImportedDataTable +// ExcelOOXMLFileBasicDataTableRowIterator defines the structure of excel (Office Open XML) file data table row iterator +type ExcelOOXMLFileBasicDataTableRowIterator struct { + dataTable *ExcelOOXMLFileBasicDataTable currentSheetIndex int currentRowIndexInSheet int } // DataRowCount returns the total count of data row -func (t *ExcelOOXMLFileImportedDataTable) DataRowCount() int { +func (t *ExcelOOXMLFileBasicDataTable) DataRowCount() int { totalDataRowCount := 0 for i := 0; i < len(t.sheets); i++ { @@ -54,13 +54,13 @@ func (t *ExcelOOXMLFileImportedDataTable) DataRowCount() int { } // HeaderColumnNames returns the header column name list -func (t *ExcelOOXMLFileImportedDataTable) HeaderColumnNames() []string { +func (t *ExcelOOXMLFileBasicDataTable) HeaderColumnNames() []string { return t.headerLineColumnNames } // DataRowIterator returns the iterator of data row -func (t *ExcelOOXMLFileImportedDataTable) DataRowIterator() datatable.ImportedDataRowIterator { - return &ExcelOOXMLFileDataRowIterator{ +func (t *ExcelOOXMLFileBasicDataTable) DataRowIterator() datatable.BasicDataTableRowIterator { + return &ExcelOOXMLFileBasicDataTableRowIterator{ dataTable: t, currentSheetIndex: 0, currentRowIndexInSheet: 0, @@ -68,12 +68,12 @@ func (t *ExcelOOXMLFileImportedDataTable) DataRowIterator() datatable.ImportedDa } // ColumnCount returns the total count of column in this data row -func (r *ExcelOOXMLFileDataRow) ColumnCount() int { +func (r *ExcelOOXMLFileBasicDataTableRow) ColumnCount() int { return len(r.rowData) } // GetData returns the data in the specified column index -func (r *ExcelOOXMLFileDataRow) GetData(columnIndex int) string { +func (r *ExcelOOXMLFileBasicDataTableRow) GetData(columnIndex int) string { if columnIndex < 0 || columnIndex >= len(r.rowData) { return "" } @@ -82,7 +82,7 @@ func (r *ExcelOOXMLFileDataRow) GetData(columnIndex int) string { } // HasNext returns whether the iterator does not reach the end -func (t *ExcelOOXMLFileDataRowIterator) HasNext() bool { +func (t *ExcelOOXMLFileBasicDataTableRowIterator) HasNext() bool { sheets := t.dataTable.sheets if t.currentSheetIndex >= len(sheets) { @@ -109,12 +109,12 @@ func (t *ExcelOOXMLFileDataRowIterator) HasNext() bool { } // CurrentRowId returns current index -func (t *ExcelOOXMLFileDataRowIterator) CurrentRowId() string { - return fmt.Sprintf("table#%d-row#%d", t.currentSheetIndex, t.currentRowIndexInSheet) +func (t *ExcelOOXMLFileBasicDataTableRowIterator) CurrentRowId() string { + return fmt.Sprintf("sheet#%d-row#%d", t.currentSheetIndex, t.currentRowIndexInSheet) } -// Next returns the next imported data row -func (t *ExcelOOXMLFileDataRowIterator) Next() datatable.ImportedDataRow { +// Next returns the next basic data row +func (t *ExcelOOXMLFileBasicDataTableRowIterator) Next() datatable.BasicDataTableRow { sheets := t.dataTable.sheets currentRowIndexInTable := t.currentRowIndexInSheet @@ -142,15 +142,15 @@ func (t *ExcelOOXMLFileDataRowIterator) Next() datatable.ImportedDataRow { return nil } - return &ExcelOOXMLFileDataRow{ + return &ExcelOOXMLFileBasicDataTableRow{ sheet: currentSheet, rowData: currentSheet.allData[t.currentRowIndexInSheet], rowIndex: t.currentRowIndexInSheet, } } -// CreateNewExcelOOXMLFileImportedDataTable returns excel (Office Open XML) data table by file binary data -func CreateNewExcelOOXMLFileImportedDataTable(data []byte) (*ExcelOOXMLFileImportedDataTable, error) { +// CreateNewExcelOOXMLFileBasicDataTable returns excel (Office Open XML) data table by file binary data +func CreateNewExcelOOXMLFileBasicDataTable(data []byte) (datatable.BasicDataTable, error) { reader := bytes.NewReader(data) file, err := excelize.OpenReader(reader) @@ -204,7 +204,7 @@ func CreateNewExcelOOXMLFileImportedDataTable(data []byte) (*ExcelOOXMLFileImpor }) } - return &ExcelOOXMLFileImportedDataTable{ + return &ExcelOOXMLFileBasicDataTable{ sheets: sheets, headerLineColumnNames: headerRowItems, }, nil diff --git a/pkg/converters/excel/excel_ooxml_file_imported_data_table_test.go b/pkg/converters/excel/excel_ooxml_file_basic_data_table_test.go similarity index 71% rename from pkg/converters/excel/excel_ooxml_file_imported_data_table_test.go rename to pkg/converters/excel/excel_ooxml_file_basic_data_table_test.go index d70ae52f..08cb0b83 100644 --- a/pkg/converters/excel/excel_ooxml_file_imported_data_table_test.go +++ b/pkg/converters/excel/excel_ooxml_file_basic_data_table_test.go @@ -9,63 +9,63 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/errs" ) -func TestExcelOOXMLFileImportedDataTableDataRowCount(t *testing.T) { +func TestExcelOOXMLFileBasicDataTableDataRowCount(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 2, datatable.DataRowCount()) } -func TestExcelOOXMLFileImportedDataTableDataRowCount_MultipleSheets(t *testing.T) { +func TestExcelOOXMLFileBasicDataTableDataRowCount_MultipleSheets(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 5, datatable.DataRowCount()) } -func TestExcelOOXMLFileImportedDataTableDataRowCount_OnlyHeaderLine(t *testing.T) { +func TestExcelOOXMLFileBasicDataTableDataRowCount_OnlyHeaderLine(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/only_one_row_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 0, datatable.DataRowCount()) } -func TestExcelOOXMLFileImportedDataTableDataRowCount_EmptyContent(t *testing.T) { +func TestExcelOOXMLFileBasicDataTableDataRowCount_EmptyContent(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.Nil(t, err) assert.Equal(t, 0, datatable.DataRowCount()) } -func TestExcelOOXMLFileImportedDataTableHeaderColumnNames(t *testing.T) { +func TestExcelOOXMLFileBasicDataTableHeaderColumnNames(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.EqualValues(t, []string{"A1", "B1", "C1"}, datatable.HeaderColumnNames()) } -func TestExcelOOXMLFileImportedDataTableHeaderColumnNames_EmptyContent(t *testing.T) { +func TestExcelOOXMLFileBasicDataTableHeaderColumnNames_EmptyContent(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.Nil(t, datatable.HeaderColumnNames()) } -func TestExcelOOXMLFileDataRowIterator(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowIterator(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.True(t, iterator.HasNext()) @@ -86,11 +86,11 @@ func TestExcelOOXMLFileDataRowIterator(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelOOXMLFileDataRowIterator_MultipleSheets(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowIterator_MultipleSheets(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.True(t, iterator.HasNext()) @@ -123,11 +123,11 @@ func TestExcelOOXMLFileDataRowIterator_MultipleSheets(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelOOXMLFileDataRowIterator_OnlyHeaderLine(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowIterator_OnlyHeaderLine(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/only_one_row_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.False(t, iterator.HasNext()) @@ -140,11 +140,11 @@ func TestExcelOOXMLFileDataRowIterator_OnlyHeaderLine(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelOOXMLFileDataRowIterator_EmptyContent(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowIterator_EmptyContent(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() assert.False(t, iterator.HasNext()) @@ -157,11 +157,11 @@ func TestExcelOOXMLFileDataRowIterator_EmptyContent(t *testing.T) { assert.False(t, iterator.HasNext()) } -func TestExcelOOXMLFileDataRowColumnCount(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowColumnCount(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() row1 := iterator.Next() @@ -171,11 +171,11 @@ func TestExcelOOXMLFileDataRowColumnCount(t *testing.T) { assert.EqualValues(t, 3, row2.ColumnCount()) } -func TestExcelOOXMLFileDataRowGetData(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowGetData(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() row1 := iterator.Next() @@ -189,22 +189,22 @@ func TestExcelOOXMLFileDataRowGetData(t *testing.T) { assert.Equal(t, "C3", row2.GetData(2)) } -func TestExcelOOXMLFileDataRowGetData_GetNotExistedColumnData(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowGetData_GetNotExistedColumnData(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() row1 := iterator.Next() assert.Equal(t, "", row1.GetData(3)) } -func TestExcelOOXMLFileDataRowGetData_MultipleSheets(t *testing.T) { +func TestExcelOOXMLFileBasicDataRowGetData_MultipleSheets(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xlsx") assert.Nil(t, err) - datatable, err := CreateNewExcelOOXMLFileImportedDataTable(testdata) + datatable, err := CreateNewExcelOOXMLFileBasicDataTable(testdata) iterator := datatable.DataRowIterator() sheet1Row1 := iterator.Next() @@ -237,10 +237,10 @@ func TestExcelOOXMLFileDataRowGetData_MultipleSheets(t *testing.T) { assert.Equal(t, "5-C3", sheet5Row2.GetData(2)) } -func TestCreateNewExcelOOXMLFileImportedDataTable_MultipleSheetsWithDifferentHeaders(t *testing.T) { +func TestCreateNewExcelOOXMLFileBasicDataTable_MultipleSheetsWithDifferentHeaders(t *testing.T) { testdata, err := os.ReadFile("../../../testdata/multiple_sheets_with_different_header_row_excel_file.xlsx") assert.Nil(t, err) - _, err = CreateNewExcelOOXMLFileImportedDataTable(testdata) + _, err = CreateNewExcelOOXMLFileBasicDataTable(testdata) assert.EqualError(t, err, errs.ErrFieldsInMultiTableAreDifferent.Message) } diff --git a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go index 7caf5687..01481c6a 100644 --- a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go @@ -60,13 +60,13 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c fallback := unicode.UTF8.NewDecoder() reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) - dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, reader) + dataTable, err := c.createNewFeideeMymoneyAppBasicDataTable(ctx, reader) if err != nil { return nil, nil, nil, nil, nil, nil, err } - commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable) + commonDataTable := datatable.CreateNewCommonDataTableFromBasicDataTable(dataTable) if !commonDataTable.HasColumn(feideeMymoneyAppTransactionTimeColumnName) || !commonDataTable.HasColumn(feideeMymoneyAppTransactionTypeColumnName) || @@ -89,7 +89,7 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } -func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, reader io.Reader) (datatable.ImportedDataTable, error) { +func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppBasicDataTable(ctx core.Context, reader io.Reader) (datatable.BasicDataTable, error) { csvReader := csv.NewReader(reader) csvReader.FieldsPerRecord = -1 @@ -132,7 +132,7 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyA return nil, errs.ErrNotFoundTransactionDataInFile } - dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines) + dataTable := csvdatatable.CreateNewCustomCsvBasicDataTable(allOriginalLines) return dataTable, nil } diff --git a/pkg/converters/feidee/feidee_mymoney_elecloud_transaction_data_xlsx_file_importer.go b/pkg/converters/feidee/feidee_mymoney_elecloud_transaction_data_xlsx_file_importer.go index b1af8255..50e990c7 100644 --- a/pkg/converters/feidee/feidee_mymoney_elecloud_transaction_data_xlsx_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_elecloud_transaction_data_xlsx_file_importer.go @@ -32,14 +32,14 @@ var ( // ParseImportedData returns the imported data by parsing the feidee mymoney (elecloud) transaction xlsx data func (c *feideeMymoneyElecloudTransactionDataXlsxFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]map[string]*models.TransactionCategory, incomeCategoryMap map[string]map[string]*models.TransactionCategory, transferCategoryMap map[string]map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - dataTable, err := excel.CreateNewExcelOOXMLFileImportedDataTable(data) + dataTable, err := excel.CreateNewExcelOOXMLFileBasicDataTable(data) if err != nil { return nil, nil, nil, nil, nil, nil, err } transactionRowParser := createFeideeMymoneyElecloudTransactionDataRowParser() - transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyElecloudDataColumnNameMapping, transactionRowParser) + transactionDataTable := datatable.CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable, feideeMymoneyElecloudDataColumnNameMapping, transactionRowParser) dataTableImporter := converter.CreateNewSimpleImporter(feideeMymoneyElecloudTransactionTypeNameMapping) return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) diff --git a/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go b/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go index 0833ba5b..feb993b3 100644 --- a/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go @@ -31,14 +31,14 @@ var ( // ParseImportedData returns the imported data by parsing the feidee mymoney (web) transaction xls data func (c *feideeMymoneyWebTransactionDataXlsFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]map[string]*models.TransactionCategory, incomeCategoryMap map[string]map[string]*models.TransactionCategory, transferCategoryMap map[string]map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - dataTable, err := excel.CreateNewExcelMSCFBFileImportedDataTable(data) + dataTable, err := excel.CreateNewExcelMSCFBFileBasicDataTable(data) if err != nil { return nil, nil, nil, nil, nil, nil, err } transactionRowParser := createFeideeMymoneyTransactionDataRowParser() - transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyWebDataColumnNameMapping, transactionRowParser) + transactionDataTable := datatable.CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable, feideeMymoneyWebDataColumnNameMapping, transactionRowParser) dataTableImporter := converter.CreateNewSimpleImporterWithTypeNameMapping(feideeMymoneyTransactionTypeNameMapping) return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) diff --git a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go index 706050b4..1bd7db51 100644 --- a/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go +++ b/pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go @@ -42,14 +42,14 @@ var ( // ParseImportedData returns the imported data by parsing the firefly III transaction csv data func (c *fireflyIIITransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]map[string]*models.TransactionCategory, incomeCategoryMap map[string]map[string]*models.TransactionCategory, transferCategoryMap map[string]map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { reader := bytes.NewReader(data) - dataTable, err := csv.CreateNewCsvImportedDataTable(ctx, reader) + dataTable, err := csv.CreateNewCsvBasicDataTable(ctx, reader) if err != nil { return nil, nil, nil, nil, nil, nil, err } transactionRowParser := createFireflyIIITransactionDataRowParser() - transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser) + transactionDataTable := datatable.CreateNewTransactionDataTableFromBasicDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser) dataTableImporter := converter.CreateNewImporterWithTypeNameMapping(fireflyIIITransactionTypeNameMapping, "", "", ",") return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) diff --git a/pkg/converters/gnucash/gnucash_transaction_data_table.go b/pkg/converters/gnucash/gnucash_transaction_data_table.go index 76cb3870..40217412 100644 --- a/pkg/converters/gnucash/gnucash_transaction_data_table.go +++ b/pkg/converters/gnucash/gnucash_transaction_data_table.go @@ -86,7 +86,7 @@ func (t *gnucashTransactionDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allData) } -// Next returns the next imported data row +// Next returns the next transaction data row func (t *gnucashTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { if t.currentIndex+1 >= len(t.dataTable.allData) { return nil, nil diff --git a/pkg/converters/iif/iif_transaction_data_table.go b/pkg/converters/iif/iif_transaction_data_table.go index 4f93f59b..f2256a26 100644 --- a/pkg/converters/iif/iif_transaction_data_table.go +++ b/pkg/converters/iif/iif_transaction_data_table.go @@ -144,7 +144,7 @@ func (t *iifTransactionDataRowIterator) HasNext() bool { return false } -// Next returns the next imported data row +// Next returns the next transaction data row func (t *iifTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { allDatasets := t.dataTable.transactionDatasets diff --git a/pkg/converters/ofx/ofx_transaction_data_table.go b/pkg/converters/ofx/ofx_transaction_data_table.go index 05e0794e..7aad9c2a 100644 --- a/pkg/converters/ofx/ofx_transaction_data_table.go +++ b/pkg/converters/ofx/ofx_transaction_data_table.go @@ -93,7 +93,7 @@ func (t *ofxTransactionDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allData) } -// Next returns the next imported data row +// Next returns the next transaction data row func (t *ofxTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { if t.currentIndex+1 >= len(t.dataTable.allData) { return nil, nil diff --git a/pkg/converters/qif/qif_transaction_data_table.go b/pkg/converters/qif/qif_transaction_data_table.go index a4708289..5f5667c3 100644 --- a/pkg/converters/qif/qif_transaction_data_table.go +++ b/pkg/converters/qif/qif_transaction_data_table.go @@ -93,7 +93,7 @@ func (t *qifTransactionDataRowIterator) HasNext() bool { return t.currentIndex+1 < len(t.dataTable.allData) } -// Next returns the next imported data row +// Next returns the next transaction data row func (t *qifTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { if t.currentIndex+1 >= len(t.dataTable.allData) { return nil, nil diff --git a/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go b/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go index 6757c68d..7f6b2afc 100644 --- a/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go +++ b/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go @@ -49,13 +49,13 @@ func (c *wechatPayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Con fallback := unicode.UTF8.NewDecoder() reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) - dataTable, err := c.createNewWeChatPayImportedDataTable(ctx, reader) + dataTable, err := c.createNewWeChatPayBasicDataTable(ctx, reader) if err != nil { return nil, nil, nil, nil, nil, nil, err } - commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable) + commonDataTable := datatable.CreateNewCommonDataTableFromBasicDataTable(dataTable) if !commonDataTable.HasColumn(wechatPayTransactionTimeColumnName) || !commonDataTable.HasColumn(wechatPayTransactionCategoryColumnName) || @@ -66,14 +66,14 @@ func (c *wechatPayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Con return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow } - transactionRowParser := createWeChatPayTransactionDataRowParser() - transactionDataTable := datatable.CreateNewCommonTransactionDataTable(commonDataTable, wechatPayTransactionSupportedColumns, transactionRowParser) + transactionRowParser := createWeChatPayTransactionDataRowParser(dataTable.HeaderColumnNames()) + transactionDataTable := datatable.CreateNewTransactionDataTableFromCommonDataTable(commonDataTable, wechatPayTransactionSupportedColumns, transactionRowParser) dataTableImporter := converter.CreateNewSimpleImporterWithTypeNameMapping(wechatPayTransactionTypeNameMapping) return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } -func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedDataTable(ctx core.Context, reader io.Reader) (datatable.ImportedDataTable, error) { +func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayBasicDataTable(ctx core.Context, reader io.Reader) (datatable.BasicDataTable, error) { csvReader := csv.NewReader(reader) csvReader.FieldsPerRecord = -1 @@ -89,7 +89,7 @@ func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedData } if err != nil { - log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayImportedDataTable] cannot parse wechat pay csv data, because %s", err.Error()) + log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] cannot parse wechat pay csv data, because %s", err.Error()) return nil, errs.ErrInvalidCSVFile } @@ -100,7 +100,7 @@ func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedData hasFileHeader = true continue } else { - log.Warnf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayImportedDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) + log.Warnf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) continue } } @@ -126,7 +126,7 @@ func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedData } if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { - log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayImportedDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) + log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow } @@ -139,11 +139,11 @@ func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedData } if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayImportedDataTable] cannot parse import data, because data table row count is less 1") + log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] cannot parse import data, because data table row count is less 1") return nil, errs.ErrNotFoundTransactionDataInFile } - dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines) + dataTable := csvdatatable.CreateNewCustomCsvBasicDataTable(allOriginalLines) return dataTable, nil } diff --git a/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go b/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go index df4302ce..75b8934e 100644 --- a/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go +++ b/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go @@ -31,10 +31,11 @@ const wechatPayTransactionDataStatusRefundName = "退款" // weChatPayTransactionDataRowParser defines the structure of wechat pay transaction data row parser type weChatPayTransactionDataRowParser struct { + existedOriginalDataColumns map[string]bool } // Parse returns the converted transaction data row -func (t *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models.User, dataTable *datatable.CommonTransactionDataTable, dataRow datatable.CommonDataRow, rowId string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { +func (p *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models.User, dataRow datatable.CommonDataTableRow, rowId string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) { if dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { @@ -44,15 +45,15 @@ func (t *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models data := make(map[datatable.TransactionDataTableColumn]string, len(wechatPayTransactionSupportedColumns)) - if dataTable.HasOriginalColumn(wechatPayTransactionTimeColumnName) { + if p.hasOriginalColumn(wechatPayTransactionTimeColumnName) { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = dataRow.GetData(wechatPayTransactionTimeColumnName) } - if dataTable.HasOriginalColumn(wechatPayTransactionCategoryColumnName) { + if p.hasOriginalColumn(wechatPayTransactionCategoryColumnName) { data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(wechatPayTransactionCategoryColumnName) } - if dataTable.HasOriginalColumn(wechatPayTransactionAmountColumnName) { + if p.hasOriginalColumn(wechatPayTransactionAmountColumnName) { amount, success := utils.ParseFirstConsecutiveNumber(dataRow.GetData(wechatPayTransactionAmountColumnName)) if !success { @@ -63,9 +64,9 @@ func (t *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = amount } - if dataTable.HasOriginalColumn(wechatPayTransactionDescriptionColumnName) && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "" && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "/" { + if p.hasOriginalColumn(wechatPayTransactionDescriptionColumnName) && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "" && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "/" { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(wechatPayTransactionDescriptionColumnName) - } else if dataTable.HasOriginalColumn(wechatPayTransactionProductNameColumnName) && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "" && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "/" { + } else if p.hasOriginalColumn(wechatPayTransactionProductNameColumnName) && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "" && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "/" { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(wechatPayTransactionProductNameColumnName) } else { data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = "" @@ -73,13 +74,13 @@ func (t *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models relatedAccountName := "" - if dataTable.HasOriginalColumn(wechatPayTransactionRelatedAccountColumnName) { + if p.hasOriginalColumn(wechatPayTransactionRelatedAccountColumnName) { relatedAccountName = dataRow.GetData(wechatPayTransactionRelatedAccountColumnName) } statusName := "" - if dataTable.HasOriginalColumn(wechatPayTransactionStatusColumnName) { + if p.hasOriginalColumn(wechatPayTransactionStatusColumnName) { statusName = dataRow.GetData(wechatPayTransactionStatusColumnName) } @@ -91,7 +92,7 @@ func (t *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models localeTextItems := locales.GetLocaleTextItems(locale) - if dataTable.HasOriginalColumn(wechatPayTransactionTypeColumnName) { + if p.hasOriginalColumn(wechatPayTransactionTypeColumnName) { data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(wechatPayTransactionTypeColumnName) if dataRow.GetData(wechatPayTransactionTypeColumnName) == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { @@ -132,7 +133,20 @@ func (t *weChatPayTransactionDataRowParser) Parse(ctx core.Context, user *models return data, true, nil } -// createWeChatPayTransactionDataRowParser returns wechat pay transaction data row parser -func createWeChatPayTransactionDataRowParser() datatable.CommonTransactionDataRowParser { - return &weChatPayTransactionDataRowParser{} +func (p *weChatPayTransactionDataRowParser) hasOriginalColumn(columnName string) bool { + _, exists := p.existedOriginalDataColumns[columnName] + return exists +} + +// createWeChatPayTransactionDataRowParser returns wechat pay transaction data row parser +func createWeChatPayTransactionDataRowParser(headerColumnNames []string) datatable.CommonTransactionDataRowParser { + existedOriginalDataColumns := make(map[string]bool, len(headerColumnNames)) + + for i := 0; i < len(headerColumnNames); i++ { + existedOriginalDataColumns[headerColumnNames[i]] = true + } + + return &weChatPayTransactionDataRowParser{ + existedOriginalDataColumns: existedOriginalDataColumns, + } }