diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go index 44c41d7c..4f83fe1a 100644 --- a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go @@ -2,21 +2,17 @@ package alipay import ( "bytes" - "encoding/csv" - "io" - "strings" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" "github.com/mayswind/ezbookkeeping/pkg/converters/converter" - csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/log" "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" ) var alipayTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{ @@ -61,7 +57,13 @@ func (c *alipayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Contex enc := simplifiedchinese.GB18030 reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder()) - dataTable, err := c.createNewAlipayBasicDataTable(ctx, reader, c.fileHeaderLine, c.dataHeaderStartContent, c.dataBottomEndLineRune) + csvDataTable, err := csv.CreateNewCsvBasicDataTable(ctx, reader, false) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + dataTable, err := createNewAlipayTransactionBasicDataTable(ctx, csvDataTable, c.fileHeaderLine, c.dataHeaderStartContent, c.dataBottomEndLineRune) if err != nil { return nil, nil, nil, nil, nil, nil, err @@ -83,80 +85,3 @@ func (c *alipayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Contex return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } - -func (c *alipayTransactionDataCsvFileImporter) createNewAlipayBasicDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) (datatable.BasicDataTable, error) { - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 - - allOriginalLines := make([][]string, 0) - hasFileHeader := false - foundContentBeforeDataHeaderLine := false - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] cannot parse alipay csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - if !hasFileHeader { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], fileHeaderLine) == 0 { - hasFileHeader = true - continue - } else { - log.Warnf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) - continue - } - } - - if !foundContentBeforeDataHeaderLine { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], dataHeaderStartContent) >= 0 { - foundContentBeforeDataHeaderLine = true - continue - } else { - continue - } - } - - if foundContentBeforeDataHeaderLine { - if len(items) <= 0 { - continue - } else if len(items) == 1 && dataBottomEndLineRune > 0 && utils.ContainsOnlyOneRune(items[0], dataBottomEndLineRune) { - break - } - - for i := 0; i < len(items); i++ { - items[i] = strings.Trim(items[i], " ") - } - - if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { - log.Errorf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) - return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow - } - - allOriginalLines = append(allOriginalLines, items) - } - } - - if !hasFileHeader || !foundContentBeforeDataHeaderLine { - return nil, errs.ErrInvalidFileHeader - } - - if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[alipay_transaction_data_csv_file_importer.createNewAlipayBasicDataTable] cannot parse import data, because data table row count is less 1") - return nil, errs.ErrNotFoundTransactionDataInFile - } - - dataTable := csvdatatable.CreateNewCustomCsvBasicDataTable(allOriginalLines, true) - - return dataTable, nil -} diff --git a/pkg/converters/alipay/alipay_transaction_data_extrator.go b/pkg/converters/alipay/alipay_transaction_data_extrator.go new file mode 100644 index 00000000..0b228dc2 --- /dev/null +++ b/pkg/converters/alipay/alipay_transaction_data_extrator.go @@ -0,0 +1,78 @@ +package alipay + +import ( + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +func createNewAlipayTransactionBasicDataTable(ctx core.Context, originalDataTable datatable.BasicDataTable, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) (datatable.BasicDataTable, error) { + iterator := originalDataTable.DataRowIterator() + allOriginalLines := make([][]string, 0) + hasFileHeader := false + foundContentBeforeDataHeaderLine := false + + for iterator.HasNext() { + row := iterator.Next() + + if !hasFileHeader { + if row.ColumnCount() <= 0 { + continue + } else if strings.Index(row.GetData(0), fileHeaderLine) == 0 { + hasFileHeader = true + continue + } else { + log.Warnf(ctx, "[alipay_transaction_data_extrator.createNewAlipayTransactionBasicDataTable] read unexpected line in row \"%s\" before read file header", iterator.CurrentRowId()) + continue + } + } + + if !foundContentBeforeDataHeaderLine { + if row.ColumnCount() <= 0 { + continue + } else if strings.Index(row.GetData(0), dataHeaderStartContent) >= 0 { + foundContentBeforeDataHeaderLine = true + continue + } else { + continue + } + } + + if foundContentBeforeDataHeaderLine { + if row.ColumnCount() <= 0 { + continue + } else if row.ColumnCount() == 1 && dataBottomEndLineRune > 0 && utils.ContainsOnlyOneRune(row.GetData(0), dataBottomEndLineRune) { + break + } + + items := make([]string, row.ColumnCount()) + + for i := 0; i < row.ColumnCount(); i++ { + items[i] = strings.Trim(row.GetData(i), " ") + } + + if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { + log.Errorf(ctx, "[alipay_transaction_data_extrator.createNewAlipayTransactionBasicDataTable] cannot parse row \"%s\", because may missing some columns (column count %d in data row is less than header column count %d)", iterator.CurrentRowId(), len(items), len(allOriginalLines[0])) + return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + } + + allOriginalLines = append(allOriginalLines, items) + } + } + + if !hasFileHeader || !foundContentBeforeDataHeaderLine { + return nil, errs.ErrInvalidFileHeader + } + + if len(allOriginalLines) < 2 { + log.Errorf(ctx, "[alipay_transaction_data_extrator.createNewAlipayTransactionBasicDataTable] cannot parse import data, because data table row count is less 1") + return nil, errs.ErrNotFoundTransactionDataInFile + } + + return csv.CreateNewCustomCsvBasicDataTable(allOriginalLines, true), nil +} diff --git a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go index 7cb93248..6890a7fc 100644 --- a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go @@ -2,14 +2,13 @@ package feidee import ( "bytes" - "encoding/csv" - "golang.org/x/text/encoding/unicode" - "golang.org/x/text/transform" - "io" "strings" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" + "github.com/mayswind/ezbookkeeping/pkg/converters/converter" - csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -60,7 +59,13 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c fallback := unicode.UTF8.NewDecoder() reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) - dataTable, err := c.createNewFeideeMymoneyAppBasicDataTable(ctx, reader) + csvDataTable, err := csv.CreateNewCsvBasicDataTable(ctx, reader, false) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + dataTable, err := createNewFeideeMymoneyAppTransactionBasicDataTable(ctx, csvDataTable) if err != nil { return nil, nil, nil, nil, nil, nil, err @@ -89,54 +94,6 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } -func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppBasicDataTable(ctx core.Context, reader io.Reader) (datatable.BasicDataTable, error) { - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 - - allOriginalLines := make([][]string, 0) - hasFileHeader := false - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse feidee mymoney csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - if !hasFileHeader { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 { - hasFileHeader = true - continue - } else { - log.Warnf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) - continue - } - } - - allOriginalLines = append(allOriginalLines, items) - } - - if !hasFileHeader { - return nil, errs.ErrInvalidFileHeader - } - - if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse import data, because data table row count is less 1") - return nil, errs.ErrNotFoundTransactionDataInFile - } - - dataTable := csvdatatable.CreateNewCustomCsvBasicDataTable(allOriginalLines, true) - - return dataTable, nil -} - func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppTransactionDataTable(ctx core.Context, commonDataTable datatable.CommonDataTable) (datatable.TransactionDataTable, error) { newColumns := make([]datatable.TransactionDataTableColumn, 0, 11) newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE) diff --git a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_extrator.go b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_extrator.go new file mode 100644 index 00000000..9e0136be --- /dev/null +++ b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_extrator.go @@ -0,0 +1,52 @@ +package feidee + +import ( + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +func createNewFeideeMymoneyAppTransactionBasicDataTable(ctx core.Context, originalDataTable datatable.BasicDataTable) (datatable.BasicDataTable, error) { + iterator := originalDataTable.DataRowIterator() + allOriginalLines := make([][]string, 0) + hasFileHeader := false + + for iterator.HasNext() { + row := iterator.Next() + + if !hasFileHeader { + if row.ColumnCount() <= 0 { + continue + } else if strings.Index(row.GetData(0), feideeMymoneyAppTransactionDataCsvFileHeader) == 0 { + hasFileHeader = true + continue + } else { + log.Warnf(ctx, "[feidee_mymoney_app_transaction_data_extrator.createNewFeideeMymoneyAppTransactionBasicDataTable] read unexpected line in row \"%s\" before read file header", iterator.CurrentRowId()) + continue + } + } + + items := make([]string, row.ColumnCount()) + + for i := 0; i < row.ColumnCount(); i++ { + items[i] = strings.Trim(row.GetData(i), " ") + } + + allOriginalLines = append(allOriginalLines, items) + } + + if !hasFileHeader { + return nil, errs.ErrInvalidFileHeader + } + + if len(allOriginalLines) < 2 { + log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_extrator.createNewFeideeMymoneyAppTransactionBasicDataTable] cannot parse import data, because data table row count is less 1") + return nil, errs.ErrNotFoundTransactionDataInFile + } + + return csv.CreateNewCustomCsvBasicDataTable(allOriginalLines, true), nil +} diff --git a/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go b/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go index d9357f17..77eb2f79 100644 --- a/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go +++ b/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go @@ -2,14 +2,12 @@ package wechat import ( "bytes" - "encoding/csv" + "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" - "io" - "strings" "github.com/mayswind/ezbookkeeping/pkg/converters/converter" - csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/csv" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -49,7 +47,13 @@ func (c *wechatPayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Con fallback := unicode.UTF8.NewDecoder() reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) - dataTable, err := c.createNewWeChatPayBasicDataTable(ctx, reader) + csvDataTable, err := csv.CreateNewCsvBasicDataTable(ctx, reader, false) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + dataTable, err := createNewWeChatPayTransactionBasicDataTable(ctx, csvDataTable) if err != nil { return nil, nil, nil, nil, nil, nil, err @@ -72,78 +76,3 @@ func (c *wechatPayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Con return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } - -func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayBasicDataTable(ctx core.Context, reader io.Reader) (datatable.BasicDataTable, error) { - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 - - allOriginalLines := make([][]string, 0) - hasFileHeader := false - foundContentBeforeDataHeaderLine := false - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] cannot parse wechat pay csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - if !hasFileHeader { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], wechatPayTransactionDataCsvFileHeader) == 0 { - hasFileHeader = true - continue - } else { - log.Warnf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) - continue - } - } - - if !foundContentBeforeDataHeaderLine { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], wechatPayTransactionDataHeaderStartContentBeginning) == 0 { - foundContentBeforeDataHeaderLine = true - continue - } else { - continue - } - } - - if foundContentBeforeDataHeaderLine { - if len(items) <= 0 { - continue - } - - for i := 0; i < len(items); i++ { - items[i] = strings.Trim(items[i], " ") - } - - if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { - log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0])) - return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow - } - - allOriginalLines = append(allOriginalLines, items) - } - } - - if !hasFileHeader || !foundContentBeforeDataHeaderLine { - return nil, errs.ErrInvalidFileHeader - } - - if len(allOriginalLines) < 2 { - log.Errorf(ctx, "[wechat_pay_transaction_data_csv_file_importer.createNewWeChatPayBasicDataTable] cannot parse import data, because data table row count is less 1") - return nil, errs.ErrNotFoundTransactionDataInFile - } - - dataTable := csvdatatable.CreateNewCustomCsvBasicDataTable(allOriginalLines, true) - - return dataTable, nil -} diff --git a/pkg/converters/wechat/wechat_pay_transaction_data_extrator.go b/pkg/converters/wechat/wechat_pay_transaction_data_extrator.go new file mode 100644 index 00000000..96de676d --- /dev/null +++ b/pkg/converters/wechat/wechat_pay_transaction_data_extrator.go @@ -0,0 +1,75 @@ +package wechat + +import ( + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +func createNewWeChatPayTransactionBasicDataTable(ctx core.Context, originalDataTable datatable.BasicDataTable) (datatable.BasicDataTable, error) { + iterator := originalDataTable.DataRowIterator() + allOriginalLines := make([][]string, 0) + hasFileHeader := false + foundContentBeforeDataHeaderLine := false + + for iterator.HasNext() { + row := iterator.Next() + + if !hasFileHeader { + if row.ColumnCount() <= 0 { + continue + } else if strings.Index(row.GetData(0), wechatPayTransactionDataCsvFileHeader) == 0 { + hasFileHeader = true + continue + } else { + log.Warnf(ctx, "[wechat_pay_transaction_data_extrator.createNewWeChatPayTransactionBasicDataTable] read unexpected line in row \"%s\" before read file header", iterator.CurrentRowId()) + continue + } + } + + if !foundContentBeforeDataHeaderLine { + if row.ColumnCount() <= 0 { + continue + } else if strings.Index(row.GetData(0), wechatPayTransactionDataHeaderStartContentBeginning) == 0 { + foundContentBeforeDataHeaderLine = true + continue + } else { + continue + } + } + + if foundContentBeforeDataHeaderLine { + if row.ColumnCount() <= 0 { + continue + } + + items := make([]string, row.ColumnCount()) + + for i := 0; i < row.ColumnCount(); i++ { + items[i] = strings.Trim(row.GetData(i), " ") + } + + if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) { + log.Errorf(ctx, "[wechat_pay_transaction_data_extrator.createNewWeChatPayTransactionBasicDataTable] cannot parse row \"%s\", because may missing some columns (column count %d in data row is less than header column count %d)", iterator.CurrentRowId(), len(items), len(allOriginalLines[0])) + return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + } + + allOriginalLines = append(allOriginalLines, items) + } + } + + if !hasFileHeader || !foundContentBeforeDataHeaderLine { + return nil, errs.ErrInvalidFileHeader + } + + if len(allOriginalLines) < 2 { + log.Errorf(ctx, "[wechat_pay_transaction_data_extrator.createNewWeChatPayTransactionBasicDataTable] cannot parse import data, because data table row count is less 1") + return nil, errs.ErrNotFoundTransactionDataInFile + } + + return csv.CreateNewCustomCsvBasicDataTable(allOriginalLines, true), nil +}