From 45d348c0ef724d836347eb1c6be8610aff66a57b Mon Sep 17 00:00:00 2001 From: MaysWind Date: Tue, 8 Oct 2024 00:23:22 +0800 Subject: [PATCH] support importing transaction data from alipay app --- ..._app_transaction_data_csv_file_importer.go | 25 ++ ...ipay_transaction_data_csv_file_importer.go | 373 ++++++++++++++++++ ..._web_transaction_data_csv_file_importer.go | 361 +---------------- pkg/converters/transaction_data_converters.go | 2 + src/consts/file.js | 9 + src/locales/en.json | 3 +- src/locales/zh_Hans.json | 3 +- 7 files changed, 432 insertions(+), 344 deletions(-) create mode 100644 pkg/converters/alipay/alipay_app_transaction_data_csv_file_importer.go create mode 100644 pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go diff --git a/pkg/converters/alipay/alipay_app_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_app_transaction_data_csv_file_importer.go new file mode 100644 index 00000000..aa80e134 --- /dev/null +++ b/pkg/converters/alipay/alipay_app_transaction_data_csv_file_importer.go @@ -0,0 +1,25 @@ +package alipay + +// alipayAppTransactionDataCsvImporter defines the structure of alipay app csv importer for transaction data +type alipayAppTransactionDataCsvImporter struct { + alipayTransactionDataCsvImporter +} + +// Initialize a alipay app transaction data csv file importer singleton instance +var ( + AlipayAppTransactionDataCsvImporter = &alipayAppTransactionDataCsvImporter{ + alipayTransactionDataCsvImporter{ + fileHeaderLine: "------------------------------------------------------------------------------------", + dataHeaderStartContent: "支付宝（中国）网络技术有限公司 电子客户回单", + timeColumnName: "交易时间", + categoryColumnName: "交易分类", + targetNameColumnName: "交易对方", + productNameColumnName: "商品说明", + amountColumnName: "金额", + typeColumnName: "收/支", + relatedAccountColumnName: "收/付款方式", + statusColumnName: "交易状态", + descriptionColumnName: "备注", + }, + } +) diff --git a/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go new file mode 100644 index 00000000..bf82029e --- /dev/null +++ b/pkg/converters/alipay/alipay_transaction_data_csv_file_importer.go @@ -0,0 +1,373 @@ +package alipay + +import ( + "bytes" + "encoding/csv" + "fmt" + "io" + "strings" + + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/locales" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +const alipayTransactionDataStatusSuccessName = "交易成功" +const alipayTransactionDataStatusPaymentSuccessName = "支付成功" +const alipayTransactionDataStatusRepaymentSuccessName = "还款成功" +const alipayTransactionDataStatusClosedName = "交易关闭" +const alipayTransactionDataStatusRefundSuccessName = "退款成功" +const alipayTransactionDataStatusTaxRefundSuccessName = "退税成功" + +const alipayTransactionDataProductNameRechargePrefix = "充值-" +const alipayTransactionDataProductNameCashWithdrawalPrefix = "提现-" +const alipayTransactionDataProductNameTransferInText = "转入" +const alipayTransactionDataProductNameTransferOutText = "转出" +const alipayTransactionDataProductNameRepaymentText = "还款" + +var alipayTransactionTypeNameMapping = map[models.TransactionType]string{ + models.TRANSACTION_TYPE_INCOME: "收入", + models.TRANSACTION_TYPE_EXPENSE: "支出", + models.TRANSACTION_TYPE_TRANSFER: "不计收支", +} + +// alipayTransactionDataCsvImporter defines the structure of alipay csv importer for transaction data +type alipayTransactionDataCsvImporter struct { + fileHeaderLine string + dataHeaderStartContent string + dataBottomEndLineRune rune + timeColumnName string + categoryColumnName string + targetNameColumnName string + productNameColumnName string + amountColumnName string + typeColumnName string + relatedAccountColumnName string + statusColumnName string + descriptionColumnName string +} + +// ParseImportedData returns the imported data by parsing the alipay transaction csv data +func (c *alipayTransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { + enc := simplifiedchinese.GB18030 + reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder()) + allLines, err := c.parseAllLinesFromCsvData(ctx, reader) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + if len(allLines) <= 1 { + log.Errorf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) + return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile + } + + headerLineItems := allLines[0] + headerItemMap := make(map[string]int) + + for i := 0; i < len(headerLineItems); i++ { + headerItemMap[headerLineItems[i]] = i + } + + timeColumnIdx, timeColumnExists := headerItemMap[c.timeColumnName] + categoryColumnIdx, categoryColumnExists := headerItemMap[c.categoryColumnName] + targetNameColumnIdx, targetNameColumnExists := headerItemMap[c.targetNameColumnName] + productNameColumnIdx, productNameColumnExists := headerItemMap[c.productNameColumnName] + amountColumnIdx, amountColumnExists := headerItemMap[c.amountColumnName] + typeColumnIdx, typeColumnExists := headerItemMap[c.typeColumnName] + relatedAccountColumnIdx, relatedAccountColumnExists := headerItemMap[c.relatedAccountColumnName] + statusColumnIdx, statusColumnExists := headerItemMap[c.statusColumnName] + descriptionColumnIdx, descriptionColumnExists := headerItemMap[c.descriptionColumnName] + + if !timeColumnExists || !amountColumnExists || !typeColumnExists || !statusColumnExists { + log.Errorf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) + return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow + } + + newColumns := make([]datatable.DataTableColumn, 0, 7) + newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TYPE) + newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TIME) + newColumns = append(newColumns, datatable.DATA_TABLE_SUB_CATEGORY) + newColumns = append(newColumns, datatable.DATA_TABLE_ACCOUNT_NAME) + newColumns = append(newColumns, datatable.DATA_TABLE_AMOUNT) + newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_ACCOUNT_NAME) + newColumns = append(newColumns, datatable.DATA_TABLE_DESCRIPTION) + + dataTable := datatable.CreateNewWritableDataTable(newColumns) + + for i := 1; i < len(allLines); i++ { + items := allLines[i] + + if len(items) < len(headerLineItems) { + log.Errorf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] cannot parse row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", i, user.Uid, len(items), len(headerLineItems)) + return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow + } + + if items[typeColumnIdx] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && + items[typeColumnIdx] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && + items[typeColumnIdx] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + log.Warnf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] skip parsing transaction in row \"index:%d\" for user \"uid:%d\", because type is \"%s\"", i, user.Uid, items[typeColumnIdx]) + continue + } + + if items[statusColumnIdx] != alipayTransactionDataStatusSuccessName && + items[statusColumnIdx] != alipayTransactionDataStatusPaymentSuccessName && + items[statusColumnIdx] != alipayTransactionDataStatusRepaymentSuccessName && + items[statusColumnIdx] != alipayTransactionDataStatusClosedName && + items[statusColumnIdx] != alipayTransactionDataStatusRefundSuccessName && + items[statusColumnIdx] != alipayTransactionDataStatusTaxRefundSuccessName { + log.Warnf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] skip parsing transaction in row \"index:%d\" for user \"uid:%d\", because status is \"%s\"", i, user.Uid, items[statusColumnIdx]) + continue + } + + data, errMsg := c.parseTransactionData(ctx, + user, + items, + timeColumnIdx, + timeColumnExists, + categoryColumnIdx, + categoryColumnExists, + targetNameColumnIdx, + targetNameColumnExists, + productNameColumnIdx, + productNameColumnExists, + amountColumnIdx, + amountColumnExists, + typeColumnIdx, + typeColumnExists, + relatedAccountColumnIdx, + relatedAccountColumnExists, + statusColumnIdx, + statusColumnExists, + descriptionColumnIdx, + descriptionColumnExists, + ) + + if data == nil { + log.Warnf(ctx, "[alipayTransactionDataCsvImporter.ParseImportedData] skip parsing transaction in row \"index:%d\" for user \"uid:%d\", because %s", i, user.Uid, errMsg) + continue + } + + dataTable.Add(data) + } + + dataTableImporter := datatable.CreateNewSimpleImporterFromWritableDataTable( + dataTable, + alipayTransactionTypeNameMapping, + ) + + return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) +} + +func (c *alipayTransactionDataCsvImporter) parseAllLinesFromCsvData(ctx core.Context, reader io.Reader) ([][]string, error) { + csvReader := csv.NewReader(reader) + csvReader.FieldsPerRecord = -1 + + allLines := make([][]string, 0) + hasFileHeader := false + foundContentBeforeDataHeaderLine := false + + for { + items, err := csvReader.Read() + + if err == io.EOF { + break + } + + if err != nil { + log.Errorf(ctx, "[alipayTransactionDataCsvImporter.parseAllLinesFromCsvData] cannot parse alipay csv data, because %s", err.Error()) + return nil, errs.ErrInvalidCSVFile + } + + if !hasFileHeader { + if len(items) <= 0 { + continue + } else if strings.Index(items[0], c.fileHeaderLine) == 0 { + hasFileHeader = true + continue + } else { + log.Warnf(ctx, "[alipayTransactionDataCsvImporter.parseAllLinesFromCsvData] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) + } + } + + if !foundContentBeforeDataHeaderLine { + if len(items) <= 0 { + continue + } else if strings.Index(items[0], c.dataHeaderStartContent) >= 0 { + foundContentBeforeDataHeaderLine = true + continue + } else { + continue + } + } + + if foundContentBeforeDataHeaderLine { + if len(items) <= 0 { + continue + } else if len(items) == 1 && c.dataBottomEndLineRune > 0 && utils.ContainsOnlyOneRune(items[0], c.dataBottomEndLineRune) { + break + } + + for i := 0; i < len(items); i++ { + items[i] = strings.Trim(items[i], " ") + } + + allLines = append(allLines, items) + } + } + + if !hasFileHeader || !foundContentBeforeDataHeaderLine { + return nil, errs.ErrInvalidFileHeader + } + + return allLines, nil +} + +func (c *alipayTransactionDataCsvImporter) parseTransactionData( + ctx core.Context, + user *models.User, + items []string, + timeColumnIdx int, + timeColumnExists bool, + categoryColumnIdx int, + categoryColumnExists bool, + targetNameColumnIdx int, + targetNameColumnExists bool, + productNameColumnIdx int, + productNameColumnExists bool, + amountColumnIdx int, + amountColumnExists bool, + typeColumnIdx int, + typeColumnExists bool, + relatedAccountColumnIdx int, + relatedAccountColumnExists bool, + statusColumnIdx int, + statusColumnExists bool, + descriptionColumnIdx int, + descriptionColumnExists bool, +) (map[datatable.DataTableColumn]string, string) { + data := make(map[datatable.DataTableColumn]string, 11) + + if timeColumnExists && timeColumnIdx < len(items) { + data[datatable.DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx] + } + + if categoryColumnExists && categoryColumnIdx < len(items) { + data[datatable.DATA_TABLE_SUB_CATEGORY] = items[categoryColumnIdx] + } else { + data[datatable.DATA_TABLE_SUB_CATEGORY] = "" + } + + if amountColumnExists && amountColumnIdx < len(items) { + data[datatable.DATA_TABLE_AMOUNT] = items[amountColumnIdx] + } + + if descriptionColumnExists && descriptionColumnIdx < len(items) && items[descriptionColumnIdx] != "" { + data[datatable.DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx] + } else if productNameColumnExists && productNameColumnIdx < len(items) && items[productNameColumnIdx] != "" { + data[datatable.DATA_TABLE_DESCRIPTION] = items[productNameColumnIdx] + } else { + data[datatable.DATA_TABLE_DESCRIPTION] = "" + } + + relatedAccountName := "" + + if relatedAccountColumnExists && relatedAccountColumnIdx < len(items) { + relatedAccountName = items[relatedAccountColumnIdx] + } + + statusName := "" + + if statusColumnExists && statusColumnIdx < len(items) { + statusName = items[statusColumnIdx] + } + + locale := user.Language + + if locale == "" { + locale = ctx.GetClientLocale() + } + + localeTextItems := locales.GetLocaleTextItems(locale) + + if typeColumnExists && typeColumnIdx < len(items) { + data[datatable.DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx] + + if items[typeColumnIdx] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { + if statusName == alipayTransactionDataStatusClosedName { + return nil, fmt.Sprintf("income transaction is closed") + } + + if statusName == alipayTransactionDataStatusSuccessName { + data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + } else { + data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + } + } else if items[typeColumnIdx] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { + if statusName == alipayTransactionDataStatusClosedName { + return nil, fmt.Sprintf("non-income/expense transaction is closed") + } + + targetName := "" + productName := "" + + if targetNameColumnExists && targetNameColumnIdx < len(items) { + targetName = items[targetNameColumnIdx] + } + + if productNameColumnExists && productNameColumnIdx < len(items) { + productName = items[productNameColumnIdx] + } + + if statusName == alipayTransactionDataStatusRefundSuccessName { + data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] + data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + } else { + if strings.Index(productName, alipayTransactionDataProductNameRechargePrefix) == 0 { // transfer to alipay wallet + data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay + } else if strings.Index(productName, alipayTransactionDataProductNameCashWithdrawalPrefix) == 0 { // transfer from alipay wallet + data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + } else if strings.Index(productName, alipayTransactionDataProductNameTransferInText) >= 0 { // transfer in + data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + } else if strings.Index(productName, alipayTransactionDataProductNameTransferOutText) >= 0 { // transfer out + data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + } else if strings.Index(productName, alipayTransactionDataProductNameRepaymentText) >= 0 { // repayment + data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName + } else { + return nil, fmt.Sprintf("product name (\"%s\") is unknown", productName) + } + } + } else { + data[datatable.DATA_TABLE_ACCOUNT_NAME] = relatedAccountName + data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + } + } + + if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && statusName != "" { + if statusName == alipayTransactionDataStatusRefundSuccessName || statusName == alipayTransactionDataStatusTaxRefundSuccessName { + amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT]) + + if err == nil { + data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] + data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) + } + } + } + + return data, "" +} diff --git a/pkg/converters/alipay/alipay_web_transaction_data_csv_file_importer.go b/pkg/converters/alipay/alipay_web_transaction_data_csv_file_importer.go index e015555b..27904ae5 100644 --- a/pkg/converters/alipay/alipay_web_transaction_data_csv_file_importer.go +++ b/pkg/converters/alipay/alipay_web_transaction_data_csv_file_importer.go @@ -1,349 +1,26 @@ package alipay -import ( - "bytes" - "encoding/csv" - "fmt" - "io" - "strings" - - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/transform" - - "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" - "github.com/mayswind/ezbookkeeping/pkg/core" - "github.com/mayswind/ezbookkeeping/pkg/errs" - "github.com/mayswind/ezbookkeeping/pkg/locales" - "github.com/mayswind/ezbookkeeping/pkg/log" - "github.com/mayswind/ezbookkeeping/pkg/models" - "github.com/mayswind/ezbookkeeping/pkg/utils" -) - -const alipayWebTransactionDataCsvFileHeader = "支付宝交易记录明细查询" -const alipayWebTransactionDataCsvDataHeaderLineStartContent = "交易记录明细列表" -const alipayWebTransactionDataCsvDataDataLineEndLineRune = '-' - -const alipayWebTransactionDataStatusSuccessName = "交易成功" -const alipayWebTransactionDataStatusPaymentSuccessName = "支付成功" -const alipayWebTransactionDataStatusRepaymentSuccessName = "还款成功" -const alipayWebTransactionDataStatusClosedName = "交易关闭" -const alipayWebTransactionDataStatusRefundSuccessName = "退款成功" -const alipayWebTransactionDataStatusTaxRefundSuccessName = "退税成功" - -const alipayWebTransactionDataProductNameRechargePrefix = "充值-" -const alipayWebTransactionDataProductNameCashWithdrawalPrefix = "提现-" -const alipayWebTransactionDataProductNameTransferInText = "转入" -const alipayWebTransactionDataProductNameTransferOutText = "转出" -const alipayWebTransactionDataProductNameRepaymentText = "还款" - -var alipayTransactionTypeNameMapping = map[models.TransactionType]string{ - models.TRANSACTION_TYPE_INCOME: "收入", - models.TRANSACTION_TYPE_EXPENSE: "支出", - models.TRANSACTION_TYPE_TRANSFER: "不计收支", -} - // alipayWebTransactionDataCsvImporter defines the structure of alipay (web) csv importer for transaction data -type alipayWebTransactionDataCsvImporter struct{} +type alipayWebTransactionDataCsvImporter struct { + alipayTransactionDataCsvImporter +} // Initialize a alipay (web) transaction data csv file importer singleton instance var ( - AlipayWebTransactionDataCsvImporter = &alipayWebTransactionDataCsvImporter{} + AlipayWebTransactionDataCsvImporter = &alipayWebTransactionDataCsvImporter{ + alipayTransactionDataCsvImporter{ + fileHeaderLine: "支付宝交易记录明细查询", + dataHeaderStartContent: "交易记录明细列表", + dataBottomEndLineRune: '-', + timeColumnName: "交易创建时间", + categoryColumnName: "", + targetNameColumnName: "交易对方", + productNameColumnName: "商品名称", + amountColumnName: "金额（元）", + typeColumnName: "收/支", + relatedAccountColumnName: "", + statusColumnName: "交易状态", + descriptionColumnName: "备注", + }, + } ) - -// ParseImportedData returns the imported data by parsing the alipay (web) transaction csv data -func (c *alipayWebTransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { - enc := simplifiedchinese.GB18030 - reader := transform.NewReader(bytes.NewReader(data), enc.NewDecoder()) - allLines, err := c.parseAllLinesFromCsvData(ctx, reader) - - if err != nil { - return nil, nil, nil, nil, nil, nil, err - } - - if len(allLines) <= 1 { - log.Errorf(ctx, "[alipayWebTransactionDataCsvImporter.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile - } - - headerLineItems := allLines[0] - headerItemMap := make(map[string]int) - - for i := 0; i < len(headerLineItems); i++ { - headerItemMap[headerLineItems[i]] = i - } - - timeColumnIdx, timeColumnExists := headerItemMap["交易创建时间"] - targetNameColumnIdx, targetNameColumnExists := headerItemMap["交易对方"] - productNameColumnIdx, productNameColumnExists := headerItemMap["商品名称"] - amountColumnIdx, amountColumnExists := headerItemMap["金额（元）"] - typeColumnIdx, typeColumnExists := headerItemMap["收/支"] - statusColumnIdx, statusColumnExists := headerItemMap["交易状态"] - descriptionColumnIdx, descriptionColumnExists := headerItemMap["备注"] - - if !timeColumnExists || !amountColumnExists || !typeColumnExists || !statusColumnExists { - log.Errorf(ctx, "[alipayWebTransactionDataCsvImporter.ParseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid) - return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow - } - - newColumns := make([]datatable.DataTableColumn, 0, 7) - newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TYPE) - newColumns = append(newColumns, datatable.DATA_TABLE_TRANSACTION_TIME) - newColumns = append(newColumns, datatable.DATA_TABLE_SUB_CATEGORY) - newColumns = append(newColumns, datatable.DATA_TABLE_ACCOUNT_NAME) - newColumns = append(newColumns, datatable.DATA_TABLE_AMOUNT) - newColumns = append(newColumns, datatable.DATA_TABLE_RELATED_ACCOUNT_NAME) - newColumns = append(newColumns, datatable.DATA_TABLE_DESCRIPTION) - - dataTable := datatable.CreateNewWritableDataTable(newColumns) - - for i := 1; i < len(allLines); i++ { - items := allLines[i] - - if len(items) < len(headerLineItems) { - log.Errorf(ctx, "[alipayWebTransactionDataCsvImporter.ParseImportedData] cannot parse row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", i, user.Uid, len(items), len(headerLineItems)) - return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow - } - - if items[typeColumnIdx] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && - items[typeColumnIdx] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] && - items[typeColumnIdx] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { - log.Warnf(ctx, "[alipayWebTransactionDataCsvImporter.ParseImportedData] skip parsing transaction in row \"index:%d\" for user \"uid:%d\", because type is \"%s\"", i, user.Uid, items[typeColumnIdx]) - continue - } - - if items[statusColumnIdx] != alipayWebTransactionDataStatusSuccessName && - items[statusColumnIdx] != alipayWebTransactionDataStatusPaymentSuccessName && - items[statusColumnIdx] != alipayWebTransactionDataStatusRepaymentSuccessName && - items[statusColumnIdx] != alipayWebTransactionDataStatusClosedName && - items[statusColumnIdx] != alipayWebTransactionDataStatusRefundSuccessName && - items[statusColumnIdx] != alipayWebTransactionDataStatusTaxRefundSuccessName { - log.Warnf(ctx, "[alipayWebTransactionDataCsvImporter.ParseImportedData] skip parsing transaction in row \"index:%d\" for user \"uid:%d\", because status is \"%s\"", i, user.Uid, items[statusColumnIdx]) - continue - } - - data, errMsg := c.parseTransactionData(ctx, - user, - items, - timeColumnIdx, - timeColumnExists, - targetNameColumnIdx, - targetNameColumnExists, - productNameColumnIdx, - productNameColumnExists, - amountColumnIdx, - amountColumnExists, - typeColumnIdx, - typeColumnExists, - statusColumnIdx, - statusColumnExists, - descriptionColumnIdx, - descriptionColumnExists, - ) - - if data == nil { - log.Warnf(ctx, "[alipayWebTransactionDataCsvImporter.ParseImportedData] skip parsing transaction in row \"index:%d\" for user \"uid:%d\", because %s", i, user.Uid, errMsg) - continue - } - - dataTable.Add(data) - } - - dataTableImporter := datatable.CreateNewSimpleImporterFromWritableDataTable( - dataTable, - alipayTransactionTypeNameMapping, - ) - - return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) -} - -func (c *alipayWebTransactionDataCsvImporter) parseAllLinesFromCsvData(ctx core.Context, reader io.Reader) ([][]string, error) { - csvReader := csv.NewReader(reader) - csvReader.FieldsPerRecord = -1 - - allLines := make([][]string, 0) - hasFileHeader := false - foundContentBeforeDataHeaderLine := false - - for { - items, err := csvReader.Read() - - if err == io.EOF { - break - } - - if err != nil { - log.Errorf(ctx, "[alipayWebTransactionDataCsvImporter.parseAllLinesFromCsvData] cannot parse alipay csv data, because %s", err.Error()) - return nil, errs.ErrInvalidCSVFile - } - - if !hasFileHeader { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], alipayWebTransactionDataCsvFileHeader) == 0 { - hasFileHeader = true - continue - } else { - log.Warnf(ctx, "[alipayWebTransactionDataCsvImporter.parseAllLinesFromCsvData] read unexpected line before read file header, line content is %s", strings.Join(items, ",")) - } - } - - if !foundContentBeforeDataHeaderLine { - if len(items) <= 0 { - continue - } else if strings.Index(items[0], alipayWebTransactionDataCsvDataHeaderLineStartContent) >= 0 { - foundContentBeforeDataHeaderLine = true - continue - } else { - continue - } - } - - if foundContentBeforeDataHeaderLine { - if len(items) <= 0 { - continue - } else if len(items) == 1 && utils.ContainsOnlyOneRune(items[0], alipayWebTransactionDataCsvDataDataLineEndLineRune) { - break - } - - for i := 0; i < len(items); i++ { - items[i] = strings.Trim(items[i], " ") - } - - allLines = append(allLines, items) - } - } - - if !hasFileHeader || !foundContentBeforeDataHeaderLine { - return nil, errs.ErrInvalidFileHeader - } - - return allLines, nil -} - -func (c *alipayWebTransactionDataCsvImporter) parseTransactionData( - ctx core.Context, - user *models.User, - items []string, - timeColumnIdx int, - timeColumnExists bool, - targetNameColumnIdx int, - targetNameColumnExists bool, - productNameColumnIdx int, - productNameColumnExists bool, - amountColumnIdx int, - amountColumnExists bool, - typeColumnIdx int, - typeColumnExists bool, - statusColumnIdx int, - statusColumnExists bool, - descriptionColumnIdx int, - descriptionColumnExists bool, -) (map[datatable.DataTableColumn]string, string) { - data := make(map[datatable.DataTableColumn]string, 11) - - if timeColumnExists && timeColumnIdx < len(items) { - data[datatable.DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx] - } - - if amountColumnExists && amountColumnIdx < len(items) { - data[datatable.DATA_TABLE_AMOUNT] = items[amountColumnIdx] - } - - data[datatable.DATA_TABLE_SUB_CATEGORY] = "" - - if descriptionColumnExists && descriptionColumnIdx < len(items) && items[descriptionColumnIdx] != "" { - data[datatable.DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx] - } else if productNameColumnExists && productNameColumnIdx < len(items) && items[productNameColumnIdx] != "" { - data[datatable.DATA_TABLE_DESCRIPTION] = items[productNameColumnIdx] - } else { - data[datatable.DATA_TABLE_DESCRIPTION] = "" - } - - statusName := "" - - if statusColumnExists && statusColumnIdx < len(items) { - statusName = items[statusColumnIdx] - } - - locale := user.Language - - if locale == "" { - locale = ctx.GetClientLocale() - } - - localeTextItems := locales.GetLocaleTextItems(locale) - - if typeColumnExists && typeColumnIdx < len(items) { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx] - - if items[typeColumnIdx] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] { - if statusName == alipayWebTransactionDataStatusClosedName { - return nil, fmt.Sprintf("income transaction is closed") - } - - if statusName == alipayWebTransactionDataStatusSuccessName { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" - } else { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" - } - } else if items[typeColumnIdx] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] { - if statusName == alipayWebTransactionDataStatusClosedName { - return nil, fmt.Sprintf("non-income/expense transaction is closed") - } - - targetName := "" - productName := "" - - if targetNameColumnExists && targetNameColumnIdx < len(items) { - targetName = items[targetNameColumnIdx] - } - - if productNameColumnExists && productNameColumnIdx < len(items) { - productName = items[productNameColumnIdx] - } - - if statusName == alipayWebTransactionDataStatusRefundSuccessName { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" - } else { - if strings.Index(productName, alipayWebTransactionDataProductNameRechargePrefix) == 0 { // transfer to alipay wallet - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay - } else if strings.Index(productName, alipayWebTransactionDataProductNameCashWithdrawalPrefix) == 0 { // transfer from alipay wallet - data[datatable.DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.Alipay - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName - } else if strings.Index(productName, alipayWebTransactionDataProductNameTransferInText) >= 0 { // transfer in - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName - } else if strings.Index(productName, alipayWebTransactionDataProductNameTransferOutText) >= 0 { // transfer out - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName - } else if strings.Index(productName, alipayWebTransactionDataProductNameRepaymentText) >= 0 { // repayment - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName - } else { - return nil, fmt.Sprintf("product name (\"%s\") is unknown", productName) - } - } - } else { - data[datatable.DATA_TABLE_ACCOUNT_NAME] = "" - data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME] = "" - } - } - - if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] && statusName != "" { - if statusName == alipayWebTransactionDataStatusRefundSuccessName || statusName == alipayWebTransactionDataStatusTaxRefundSuccessName { - amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT]) - - if err == nil { - data[datatable.DATA_TABLE_TRANSACTION_TYPE] = alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] - data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount) - } - } - } - - return data, "" -} diff --git a/pkg/converters/transaction_data_converters.go b/pkg/converters/transaction_data_converters.go index 4d429567..481ab964 100644 --- a/pkg/converters/transaction_data_converters.go +++ b/pkg/converters/transaction_data_converters.go @@ -29,6 +29,8 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter, return feidee.FeideeMymoneyTransactionDataCsvImporter, nil } else if fileType == "feidee_mymoney_xls" { return feidee.FeideeMymoneyTransactionDataXlsImporter, nil + } else if fileType == "alipay_app_csv" { + return alipay.AlipayAppTransactionDataCsvImporter, nil } else if fileType == "alipay_web_csv" { return alipay.AlipayWebTransactionDataCsvImporter, nil } else { diff --git a/src/consts/file.js b/src/consts/file.js index 291fa30c..9da2e502 100644 --- a/src/consts/file.js +++ b/src/consts/file.js @@ -37,6 +37,15 @@ const supportedImportFileTypes = [ anchor: '如何获取金蝶随手记web版数据导出文件' } }, + { + type: 'alipay_app_csv', + name: 'Alipay (App) Data Export File', + extensions: '.csv', + document: { + supportMultiLanguages: 'zh-Hans', + anchor: '如何获取支付宝app数据导出文件' + } + }, { type: 'alipay_web_csv', name: 'Alipay (Web) Data Export File', diff --git a/src/locales/en.json b/src/locales/en.json index c86f62bf..a7e5cbb4 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1514,8 +1514,9 @@ "How to export this file?": "How to export this file?", "ezbookkeeping Data Export File (CSV)": "ezbookkeeping Data Export File (CSV)", "ezbookkeeping Data Export File (TSV)": "ezbookkeeping Data Export File (TSV)", - "Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web) Data Export File", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App) Data Export File", + "Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web) Data Export File", + "Alipay (App) Data Export File": "Alipay (App) Data Export File", "Alipay (Web) Data Export File": "Alipay (Web) Data Export File", "Data File": "Data File", "No data to import": "No data to import", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index 9601d6a7..de1ed2b9 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -1514,8 +1514,9 @@ "How to export this file?": "如何导出该文件？", "ezbookkeeping Data Export File (CSV)": "ezbookkeeping 数据导出文件 (CSV)", "ezbookkeeping Data Export File (TSV)": "ezbookkeeping 数据导出文件 (TSV)", - "Feidee MyMoney (Web) Data Export File": "金蝶随手记 (Web版) 数据导出文件", "Feidee MyMoney (App) Data Export File": "金蝶随手记 (App) 数据导出文件", + "Feidee MyMoney (Web) Data Export File": "金蝶随手记 (Web版) 数据导出文件", + "Alipay (App) Data Export File": "支付宝 (App) 数据导出文件", "Alipay (Web) Data Export File": "支付宝 (网页版) 数据导出文件", "Data File": "数据文件", "No data to import": "没有可以导入的数据",