diff --git a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go index a8db7f65..46518344 100644 --- a/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go +++ b/pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go @@ -1,7 +1,10 @@ package feidee import ( + "bytes" "encoding/csv" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" "io" "strings" @@ -14,7 +17,6 @@ import ( ) const feideeMymoneyAppTransactionDataCsvFileHeader = "随手记导出文件(headers:v5;" -const feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + feideeMymoneyAppTransactionDataCsvFileHeader const feideeMymoneyAppTransactionTimeColumnName = "日期" const feideeMymoneyAppTransactionTypeColumnName = "交易类型" @@ -53,8 +55,10 @@ var ( // ParseImportedData returns the imported data by parsing the feidee mymoney app transaction csv data func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - content := string(data) - dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, content) + fallback := unicode.UTF8.NewDecoder() + reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) + + dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, reader) if err != nil { return nil, nil, nil, nil, nil, nil, err @@ -83,12 +87,8 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) } -func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, content string) (datatable.ImportedDataTable, error) { - if strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) != 0 { - return nil, errs.ErrInvalidFileHeader - } - - csvReader := csv.NewReader(strings.NewReader(content)) +func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, reader io.Reader) (datatable.ImportedDataTable, error) { + csvReader := csv.NewReader(reader) csvReader.FieldsPerRecord = -1 allOriginalLines := make([][]string, 0) @@ -109,7 +109,7 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyA if !hasFileHeader { if len(items) <= 0 { continue - } else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) == 0 { + } else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 { hasFileHeader = true continue } else { diff --git a/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go b/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go index 1aa24b91..eb492417 100644 --- a/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go +++ b/pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go @@ -3,6 +3,8 @@ package wechat import ( "bytes" "encoding/csv" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" "io" "strings" @@ -43,7 +45,8 @@ var ( // ParseImportedData returns the imported data by parsing the wechat pay transaction csv data func (c *wechatPayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - reader := bytes.NewReader(data) + fallback := unicode.UTF8.NewDecoder() + reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) dataTable, err := c.createNewWeChatPayImportedDataTable(ctx, reader) @@ -92,7 +95,7 @@ func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedData if !hasFileHeader { if len(items) <= 0 { continue - } else if strings.Index(items[0], wechatPayTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], wechatPayTransactionDataCsvFileHeaderWithUtf8Bom) == 0 { + } else if strings.Index(items[0], wechatPayTransactionDataCsvFileHeader) == 0 { hasFileHeader = true continue } else { diff --git a/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go b/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go index 000f7398..5baa45c1 100644 --- a/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go +++ b/pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go @@ -13,7 +13,6 @@ import ( ) const wechatPayTransactionDataCsvFileHeader = "微信支付账单明细" -const wechatPayTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + wechatPayTransactionDataCsvFileHeader const wechatPayTransactionDataHeaderStartContentBeginning = "----------------------微信支付账单明细列表--------------------" const wechatPayTransactionTimeColumnName = "交易时间"