code refactor

pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go

@@ -1,7 +1,10 @@
 package feidee
 
 import (
+	"bytes"
 	"encoding/csv"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	"io"
 	"strings"
 
@@ -14,7 +17,6 @@ import (
 )
 
 const feideeMymoneyAppTransactionDataCsvFileHeader = "随手记导出文件(headers:v5;"
-const feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + feideeMymoneyAppTransactionDataCsvFileHeader
 
 const feideeMymoneyAppTransactionTimeColumnName = "日期"
 const feideeMymoneyAppTransactionTypeColumnName = "交易类型"
@@ -53,8 +55,10 @@ var (
 
 // ParseImportedData returns the imported data by parsing the feidee mymoney app transaction csv data
 func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
-	content := string(data)
+	fallback := unicode.UTF8.NewDecoder()
-	dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, content)
+	reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback))
+
+	dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, reader)
 
 	if err != nil {
 		return nil, nil, nil, nil, nil, nil, err
@@ -83,12 +87,8 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx c
 	return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
 }
 
-func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, content string) (datatable.ImportedDataTable, error) {
+func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, reader io.Reader) (datatable.ImportedDataTable, error) {
-	if strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) != 0 {
+	csvReader := csv.NewReader(reader)
-		return nil, errs.ErrInvalidFileHeader
-	}
-
-	csvReader := csv.NewReader(strings.NewReader(content))
 	csvReader.FieldsPerRecord = -1
 
 	allOriginalLines := make([][]string, 0)
@@ -109,7 +109,7 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyA
 		if !hasFileHeader {
 			if len(items) <= 0 {
 				continue
-			} else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) == 0 {
+			} else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 {
 				hasFileHeader = true
 				continue
 			} else {

pkg/converters/wechat/wechat_pay_transaction_data_csv_file_importer.go

@@ -3,6 +3,8 @@ package wechat
 import (
 	"bytes"
 	"encoding/csv"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	"io"
 	"strings"
 
@@ -43,7 +45,8 @@ var (
 
 // ParseImportedData returns the imported data by parsing the wechat pay transaction csv data
 func (c *wechatPayTransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
-	reader := bytes.NewReader(data)
+	fallback := unicode.UTF8.NewDecoder()
+	reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback))
 
 	dataTable, err := c.createNewWeChatPayImportedDataTable(ctx, reader)
 
@@ -92,7 +95,7 @@ func (c *wechatPayTransactionDataCsvFileImporter) createNewWeChatPayImportedData
 		if !hasFileHeader {
 			if len(items) <= 0 {
 				continue
-			} else if strings.Index(items[0], wechatPayTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], wechatPayTransactionDataCsvFileHeaderWithUtf8Bom) == 0 {
+			} else if strings.Index(items[0], wechatPayTransactionDataCsvFileHeader) == 0 {
 				hasFileHeader = true
 				continue
 			} else {

pkg/converters/wechat/wechat_pay_transaction_data_row_parser.go

@@ -13,7 +13,6 @@ import (
 )
 
 const wechatPayTransactionDataCsvFileHeader = "微信支付账单明细"
-const wechatPayTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + wechatPayTransactionDataCsvFileHeader
 const wechatPayTransactionDataHeaderStartContentBeginning = "----------------------微信支付账单明细列表--------------------"
 
 const wechatPayTransactionTimeColumnName = "交易时间"