zhengchen.tao/ezbookkeeping
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

code refactor

Browse Source
This commit is contained in:
MaysWind
2024-10-17 01:13:35 +08:00
parent 6c285a0856
commit 34773537c2
16 changed files with 621 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/converters/alipay/alipay_transaction_csv_data_table.go
+92 -152
View File
@@ -2,10 +2,10 @@ package alipay
import (
"encoding/csv"
"fmt"
"io"
"strings"
csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
@@ -53,31 +53,20 @@ type alipayTransactionColumnNames struct {
// alipayTransactionDataTable defines the structure of alipay transaction plain text data table
type alipayTransactionDataTable struct {
allOriginalLines [][]string
originalHeaderLineColumnNames []string
originalTimeColumnIndex int
originalCategoryColumnIndex int
originalTargetNameColumnIndex int
originalProductNameColumnIndex int
originalAmountColumnIndex int
originalTypeColumnIndex int
originalRelatedAccountColumnIndex int
originalStatusColumnIndex int
originalDescriptionColumnIndex int
innerDataTable datatable.CommonDataTable
columns alipayTransactionColumnNames
}
// alipayTransactionDataRow defines the structure of alipay transaction plain text data row
type alipayTransactionDataRow struct {
dataTable *alipayTransactionDataTable
isValid bool
originalItems []string
finalItems map[datatable.TransactionDataTableColumn]string
isValid bool
finalItems map[datatable.TransactionDataTableColumn]string
}
// alipayTransactionDataRowIterator defines the structure of alipay transaction plain text data row iterator
type alipayTransactionDataRowIterator struct {
dataTable *alipayTransactionDataTable
currentIndex int
dataTable *alipayTransactionDataTable
innerIterator datatable.CommonDataRowIterator
}
// HasColumn returns whether the transaction data table has specified column
@@ -88,18 +77,14 @@ func (t *alipayTransactionDataTable) HasColumn(column datatable.TransactionDataT
// TransactionRowCount returns the total count of transaction data row
func (t *alipayTransactionDataTable) TransactionRowCount() int {
if len(t.allOriginalLines) < 1 {
return 0
}
return len(t.allOriginalLines) - 1
return t.innerDataTable.DataRowCount()
}
// TransactionRowIterator returns the iterator of transaction data row
func (t *alipayTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator {
return &alipayTransactionDataRowIterator{
dataTable: t,
currentIndex: 0,
dataTable: t,
innerIterator: t.innerDataTable.DataRowIterator(),
}
}
@@ -121,94 +106,85 @@ func (r *alipayTransactionDataRow) GetData(column datatable.TransactionDataTable
// HasNext returns whether the iterator does not reach the end
func (t *alipayTransactionDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
return t.innerIterator.HasNext()
}
// Next returns the next imported data row
func (t *alipayTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) {
if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
importedRow := t.innerIterator.Next()
if importedRow == nil {
return nil, nil
}
t.currentIndex++
finalItems, isValid, err := t.dataTable.parseTransactionData(ctx, user, importedRow, t.innerIterator.CurrentRowId())
rowItems := t.dataTable.allOriginalLines[t.currentIndex]
isValid := true
if t.dataTable.originalTypeColumnIndex >= 0 &&
rowItems[t.dataTable.originalTypeColumnIndex] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] &&
rowItems[t.dataTable.originalTypeColumnIndex] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] &&
rowItems[t.dataTable.originalTypeColumnIndex] != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
log.Warnf(ctx, "[alipay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because type is \"%s\"", t.currentIndex, rowItems[t.dataTable.originalTypeColumnIndex])
isValid = false
}
if t.dataTable.originalStatusColumnIndex >= 0 &&
rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusSuccessName &&
rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusPaymentSuccessName &&
rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusRepaymentSuccessName &&
rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusClosedName &&
rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusRefundSuccessName &&
rowItems[t.dataTable.originalStatusColumnIndex] != alipayTransactionDataStatusTaxRefundSuccessName {
log.Warnf(ctx, "[alipay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because status is \"%s\"", t.currentIndex, rowItems[t.dataTable.originalStatusColumnIndex])
isValid = false
}
var finalItems map[datatable.TransactionDataTableColumn]string
var errMsg string
if isValid {
finalItems, errMsg = t.dataTable.parseTransactionData(ctx, user, rowItems)
if finalItems == nil {
log.Warnf(ctx, "[alipay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because %s", t.currentIndex, errMsg)
isValid = false
}
if err != nil {
return nil, err
}
return &alipayTransactionDataRow{
dataTable: t.dataTable,
isValid: isValid,
originalItems: rowItems,
finalItems: finalItems,
isValid: isValid,
finalItems: finalItems,
}, nil
}
func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.TransactionDataTableColumn]string, string) {
data := make(map[datatable.TransactionDataTableColumn]string, len(alipayTransactionSupportedColumns))
func (t *alipayTransactionDataTable) hasOriginalColumn(columnName string) bool {
return columnName != "" && t.innerDataTable.HasColumn(columnName)
}
if t.originalTimeColumnIndex >= 0 && t.originalTimeColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex]
func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, dataRow datatable.CommonDataRow, rowId string) (map[datatable.TransactionDataTableColumn]string, bool, error) {
if dataRow.GetData(t.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] &&
dataRow.GetData(t.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] &&
dataRow.GetData(t.columns.typeColumnName) != alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
log.Warnf(ctx, "[alipay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because type is \"%s\"", rowId, dataRow.GetData(t.columns.typeColumnName))
return nil, false, nil
}
if t.originalCategoryColumnIndex >= 0 && t.originalCategoryColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex]
if dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusSuccessName &&
dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusPaymentSuccessName &&
dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusRepaymentSuccessName &&
dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusClosedName &&
dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusRefundSuccessName &&
dataRow.GetData(t.columns.statusColumnName) != alipayTransactionDataStatusTaxRefundSuccessName {
log.Warnf(ctx, "[alipay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because status is \"%s\"", rowId, dataRow.GetData(t.columns.statusColumnName))
return nil, false, nil
}
data := make(map[datatable.TransactionDataTableColumn]string, len(alipayTransactionSupportedColumns))
if t.hasOriginalColumn(t.columns.timeColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = dataRow.GetData(t.columns.timeColumnName)
}
if t.hasOriginalColumn(t.columns.categoryColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(t.columns.categoryColumnName)
} else {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
}
if t.originalAmountColumnIndex >= 0 && t.originalAmountColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex]
if t.hasOriginalColumn(t.columns.amountColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = dataRow.GetData(t.columns.amountColumnName)
}
if t.originalDescriptionColumnIndex >= 0 && t.originalDescriptionColumnIndex < len(items) && items[t.originalDescriptionColumnIndex] != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex]
} else if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) && items[t.originalProductNameColumnIndex] != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex]
if t.hasOriginalColumn(t.columns.descriptionColumnName) && dataRow.GetData(t.columns.descriptionColumnName) != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(t.columns.descriptionColumnName)
} else if t.hasOriginalColumn(t.columns.productNameColumnName) && dataRow.GetData(t.columns.productNameColumnName) != "" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(t.columns.productNameColumnName)
} else {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = ""
}
relatedAccountName := ""
if t.originalRelatedAccountColumnIndex >= 0 && t.originalRelatedAccountColumnIndex < len(items) {
relatedAccountName = items[t.originalRelatedAccountColumnIndex]
if t.hasOriginalColumn(t.columns.relatedAccountColumnName) {
relatedAccountName = dataRow.GetData(t.columns.relatedAccountColumnName)
}
statusName := ""
if t.originalStatusColumnIndex >= 0 && t.originalStatusColumnIndex < len(items) {
statusName = items[t.originalStatusColumnIndex]
if t.hasOriginalColumn(t.columns.statusColumnName) {
statusName = dataRow.GetData(t.columns.statusColumnName)
}
locale := user.Language
@@ -219,12 +195,13 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user
localeTextItems := locales.GetLocaleTextItems(locale)
if t.originalTypeColumnIndex >= 0 && t.originalTypeColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex]
if t.hasOriginalColumn(t.columns.typeColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(t.columns.typeColumnName)
if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
if dataRow.GetData(t.columns.typeColumnName) == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
if statusName == alipayTransactionDataStatusClosedName {
return nil, fmt.Sprintf("income transaction is closed")
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because income transaction is closed", rowId)
return nil, false, nil
}
if statusName == alipayTransactionDataStatusSuccessName {
@@ -234,20 +211,21 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
}
} else if items[t.originalTypeColumnIndex] == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
} else if dataRow.GetData(t.columns.typeColumnName) == alipayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
if statusName == alipayTransactionDataStatusClosedName {
return nil, fmt.Sprintf("non-income/expense transaction is closed")
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because non-income/expense transaction is closed", rowId)
return nil, false, nil
}
targetName := ""
productName := ""
if t.originalTargetNameColumnIndex >= 0 && t.originalTargetNameColumnIndex < len(items) {
targetName = items[t.originalTargetNameColumnIndex]
if t.hasOriginalColumn(t.columns.targetNameColumnName) {
targetName = dataRow.GetData(t.columns.targetNameColumnName)
}
if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) {
productName = items[t.originalProductNameColumnIndex]
if t.hasOriginalColumn(t.columns.productNameColumnName) {
productName = dataRow.GetData(t.columns.productNameColumnName)
}
if statusName == alipayTransactionDataStatusRefundSuccessName {
@@ -271,7 +249,8 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = targetName
} else {
return nil, fmt.Sprintf("product name (\"%s\") is unknown", productName)
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because product name (\"%s\") is unknown", rowId, productName)
return nil, false, nil
}
}
} else {
@@ -291,79 +270,33 @@ func (t *alipayTransactionDataTable) parseTransactionData(ctx core.Context, user
}
}
return data, ""
return data, true, nil
}
func createNewAlipayTransactionDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune, originalColumnNames alipayTransactionColumnNames) (*alipayTransactionDataTable, error) {
allOriginalLines, err := parseAllLinesFromAlipayTransactionPlainText(ctx, reader, fileHeaderLine, dataHeaderStartContent, dataBottomEndLineRune)
dataTable, err := createNewAlipayImportedDataTable(ctx, reader, fileHeaderLine, dataHeaderStartContent, dataBottomEndLineRune)
if err != nil {
return nil, err
}
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayTransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable)
originalHeaderItems := allOriginalLines[0]
originalHeaderItemMap := make(map[string]int)
for i := 0; i < len(originalHeaderItems); i++ {
originalHeaderItemMap[originalHeaderItems[i]] = i
}
timeColumnIdx, timeColumnExists := originalHeaderItemMap[originalColumnNames.timeColumnName]
categoryColumnIdx, categoryColumnExists := originalHeaderItemMap[originalColumnNames.categoryColumnName]
targetNameColumnIdx, targetNameColumnExists := originalHeaderItemMap[originalColumnNames.targetNameColumnName]
productNameColumnIdx, productNameColumnExists := originalHeaderItemMap[originalColumnNames.productNameColumnName]
amountColumnIdx, amountColumnExists := originalHeaderItemMap[originalColumnNames.amountColumnName]
typeColumnIdx, typeColumnExists := originalHeaderItemMap[originalColumnNames.typeColumnName]
relatedAccountColumnIdx, relatedAccountColumnExists := originalHeaderItemMap[originalColumnNames.relatedAccountColumnName]
statusColumnIdx, statusColumnExists := originalHeaderItemMap[originalColumnNames.statusColumnName]
descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap[originalColumnNames.descriptionColumnName]
if !timeColumnExists || !amountColumnExists || !typeColumnExists || !statusColumnExists {
log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayTransactionPlainTextDataTable] cannot parse alipay csv data, because missing essential columns in header row")
if !commonDataTable.HasColumn(originalColumnNames.timeColumnName) ||
!commonDataTable.HasColumn(originalColumnNames.amountColumnName) ||
!commonDataTable.HasColumn(originalColumnNames.typeColumnName) ||
!commonDataTable.HasColumn(originalColumnNames.statusColumnName) {
log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayTransactionDataTable] cannot parse alipay csv data, because missing essential columns in header row")
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
if originalColumnNames.categoryColumnName == "" || !categoryColumnExists {
categoryColumnIdx = -1
}
if originalColumnNames.targetNameColumnName == "" || !targetNameColumnExists {
targetNameColumnIdx = -1
}
if originalColumnNames.productNameColumnName == "" || !productNameColumnExists {
productNameColumnIdx = -1
}
if originalColumnNames.relatedAccountColumnName == "" || !relatedAccountColumnExists {
relatedAccountColumnIdx = -1
}
if originalColumnNames.descriptionColumnName == "" || !descriptionColumnExists {
descriptionColumnIdx = -1
}
return &alipayTransactionDataTable{
allOriginalLines: allOriginalLines,
originalHeaderLineColumnNames: originalHeaderItems,
originalTimeColumnIndex: timeColumnIdx,
originalCategoryColumnIndex: categoryColumnIdx,
originalTargetNameColumnIndex: targetNameColumnIdx,
originalProductNameColumnIndex: productNameColumnIdx,
originalAmountColumnIndex: amountColumnIdx,
originalTypeColumnIndex: typeColumnIdx,
originalRelatedAccountColumnIndex: relatedAccountColumnIdx,
originalStatusColumnIndex: statusColumnIdx,
originalDescriptionColumnIndex: descriptionColumnIdx,
innerDataTable: commonDataTable,
columns: originalColumnNames,
}, nil
}
func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) ([][]string, error) {
func createNewAlipayImportedDataTable(ctx core.Context, reader io.Reader, fileHeaderLine string, dataHeaderStartContent string, dataBottomEndLineRune rune) (datatable.ImportedDataTable, error) {
csvReader := csv.NewReader(reader)
csvReader.FieldsPerRecord = -1
@@ -379,7 +312,7 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea
}
if err != nil {
log.Errorf(ctx, "[alipay_transaction_csv_data_table.parseAllLinesFromAlipayTransactionPlainText] cannot parse alipay csv data, because %s", err.Error())
log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse alipay csv data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
@@ -390,7 +323,7 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea
hasFileHeader = true
continue
} else {
log.Warnf(ctx, "[alipay_transaction_csv_data_table.parseAllLinesFromAlipayTransactionPlainText] read unexpected line before read file header, line content is %s", strings.Join(items, ","))
log.Warnf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ","))
continue
}
}
@@ -418,7 +351,7 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea
}
if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) {
log.Errorf(ctx, "[alipay_transaction_csv_data_table.parseAllLinesFromAlipayTransactionPlainText] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0]))
log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0]))
return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow
}
@@ -430,5 +363,12 @@ func parseAllLinesFromAlipayTransactionPlainText(ctx core.Context, reader io.Rea
return nil, errs.ErrInvalidFileHeader
}
return allOriginalLines, nil
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[alipay_transaction_csv_data_table.createNewAlipayImportedDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines)
return dataTable, nil
}
pkg/converters/alipay/alipay_transaction_data_csv_file_importer_test.go
+70
View File
@@ -364,6 +364,56 @@ func TestAlipayCsvFileImporterParseImportedData_ParseCategory(t *testing.T) {
assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name)
}
func TestAlipayCsvFileImporterParseImportedData_ParseRelatedAccount(t *testing.T) {
converter := AlipayAppTransactionDataCsvFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
data1, err := simplifiedchinese.GB18030.NewEncoder().String("------------------------------------------------------------------------------------\n" +
"导出信息：\n" +
"姓名：xxx\n" +
"支付宝账户：xxx@xxx.xxx\n" +
"起始时间：[2024-01-01 00:00:00] 终止时间：[2024-09-01 23:59:59]\n" +
"导出交易类型：[全部]\n" +
"------------------------支付宝（中国）网络技术有限公司 电子客户回单------------------------\n" +
"交易时间,商品说明,收/支,金额,收/付款方式,交易状态,\n" +
"2024-09-01 03:45:07,余额宝-单次转入,不计收支,0.01,Test Account,交易成功,\n" +
"2024-09-01 05:07:29,信用卡还款,不计收支,0.02,Test Account2,交易成功,\n")
assert.Nil(t, err)
allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 2, len(allNewTransactions))
assert.Equal(t, 3, len(allNewAccounts))
assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid)
assert.Equal(t, int64(1), allNewTransactions[0].Amount)
assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[0].OriginalDestinationAccountName)
assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid)
assert.Equal(t, int64(2), allNewTransactions[1].Amount)
assert.Equal(t, "Test Account2", allNewTransactions[1].OriginalSourceAccountName)
assert.Equal(t, "", allNewTransactions[1].OriginalDestinationAccountName)
assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid)
assert.Equal(t, "Test Account", allNewAccounts[0].Name)
assert.Equal(t, "CNY", allNewAccounts[0].Currency)
assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid)
assert.Equal(t, "", allNewAccounts[1].Name)
assert.Equal(t, "CNY", allNewAccounts[1].Currency)
assert.Equal(t, int64(1234567890), allNewAccounts[2].Uid)
assert.Equal(t, "Test Account2", allNewAccounts[2].Name)
assert.Equal(t, "CNY", allNewAccounts[2].Currency)
}
func TestAlipayCsvFileImporterParseImportedData_ParseDescription(t *testing.T) {
converter := AlipayWebTransactionDataCsvFileImporter
context := core.NewNullContext()
@@ -479,3 +529,23 @@ func TestAlipayCsvFileImporterParseImportedData_MissingRequiredColumn(t *testing
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data4), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message)
}
func TestAlipayCsvFileImporterParseImportedData_NoTransactionData(t *testing.T) {
converter := AlipayWebTransactionDataCsvFileImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1,
DefaultCurrency: "CNY",
}
// Missing Time Column
data1, err := simplifiedchinese.GB18030.NewEncoder().String("支付宝交易记录明细查询\n" +
"账号:[xxx@xxx.xxx]\n" +
"起始日期:[2024-01-01 00:00:00] 终止日期:[2024-09-01 23:59:59]\n" +
"---------------------------------交易记录明细列表------------------------------------\n" +
"交易创建时间 ,金额（元）,收/支 ,交易状态 ,\n" +
"------------------------------------------------------------------------------------\n")
_, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte(data1), 0, nil, nil, nil, nil, nil)
assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message)
}
pkg/converters/csv/csv_file_imported_data_table.go
+15 -2
View File
@@ -2,6 +2,7 @@ package csv
import (
"encoding/csv"
"fmt"
"io"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
@@ -72,6 +73,11 @@ func (t *CsvFileImportedDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allLines)
}
// CurrentRowId returns current index
func (t *CsvFileImportedDataRowIterator) CurrentRowId() string {
return fmt.Sprintf("line#%d", t.currentIndex)
}
// Next returns the next imported data row
func (t *CsvFileImportedDataRowIterator) Next() datatable.ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allLines) {
@@ -88,11 +94,18 @@ func (t *CsvFileImportedDataRowIterator) Next() datatable.ImportedDataRow {
}
}
// CreateNewCsvDataTable returns comma separated values data table by io readers
func CreateNewCsvDataTable(ctx core.Context, reader io.Reader) (*CsvFileImportedDataTable, error) {
// CreateNewCsvImportedDataTable returns comma separated values data table by io readers
func CreateNewCsvImportedDataTable(ctx core.Context, reader io.Reader) (*CsvFileImportedDataTable, error) {
return createNewCsvFileDataTable(ctx, reader, ',')
}
// CreateNewCustomCsvImportedDataTable returns character separated values data table by io readers
func CreateNewCustomCsvImportedDataTable(allLines [][]string) *CsvFileImportedDataTable {
return &CsvFileImportedDataTable{
allLines: allLines,
}
}
func createNewCsvFileDataTable(ctx core.Context, reader io.Reader, separator rune) (*CsvFileImportedDataTable, error) {
csvReader := csv.NewReader(reader)
csvReader.Comma = separator
pkg/converters/datatable/common_data_table.go
+40
View File
@@ -0,0 +1,40 @@
package datatable
// CommonDataTable defines the structure of common data table
type CommonDataTable interface {
// HeaderColumnCount returns the total count of column in header row
HeaderColumnCount() int
// HasColumn returns whether the common data table has specified column name
HasColumn(columnName string) bool
// DataRowCount returns the total count of common data row
DataRowCount() int
// DataRowIterator returns the iterator of common data row
DataRowIterator() CommonDataRowIterator
}
// CommonDataRow defines the structure of common data row
type CommonDataRow interface {
// ColumnCount returns the total count of column in this data row
ColumnCount() int
// HasData returns whether the common data row has specified column data
HasData(columnName string) bool
// GetData returns the data in the specified column name
GetData(columnName string) string
}
// CommonDataRowIterator defines the structure of common data row iterator
type CommonDataRowIterator interface {
// HasNext returns whether the iterator does not reach the end
HasNext() bool
// CurrentRowId returns current row id
CurrentRowId() string
// Next returns the next common data row
Next() CommonDataRow
}
pkg/converters/datatable/imported_common_data_table.go
+107
View File
@@ -0,0 +1,107 @@
package datatable
// ImportedCommonDataTable defines the structure of imported common data table
type ImportedCommonDataTable struct {
innerDataTable ImportedDataTable
dataColumnIndexes map[string]int
}
// ImportedCommonDataRow defines the structure of imported common data row
type ImportedCommonDataRow struct {
rowData map[string]string
}
// ImportedCommonDataRowIterator defines the structure of imported common data row iterator
type ImportedCommonDataRowIterator struct {
commonDataTable *ImportedCommonDataTable
innerIterator ImportedDataRowIterator
}
// HeaderColumnCount returns the total count of column in header row
func (t *ImportedCommonDataTable) HeaderColumnCount() int {
return len(t.innerDataTable.HeaderColumnNames())
}
// HasColumn returns whether the data table has specified column name
func (t *ImportedCommonDataTable) HasColumn(columnName string) bool {
index, exists := t.dataColumnIndexes[columnName]
return exists && index >= 0
}
// DataRowCount returns the total count of common data row
func (t *ImportedCommonDataTable) DataRowCount() int {
return t.innerDataTable.DataRowCount()
}
// DataRowIterator returns the iterator of common data row
func (t *ImportedCommonDataTable) DataRowIterator() CommonDataRowIterator {
return &ImportedCommonDataRowIterator{
commonDataTable: t,
innerIterator: t.innerDataTable.DataRowIterator(),
}
}
// HasData returns whether the common data row has specified column data
func (r *ImportedCommonDataRow) HasData(columnName string) bool {
_, exists := r.rowData[columnName]
return exists
}
// ColumnCount returns the total count of column in this data row
func (r *ImportedCommonDataRow) ColumnCount() int {
return len(r.rowData)
}
// GetData returns the data in the specified column name
func (r *ImportedCommonDataRow) GetData(columnName string) string {
return r.rowData[columnName]
}
// HasNext returns whether the iterator does not reach the end
func (t *ImportedCommonDataRowIterator) HasNext() bool {
return t.innerIterator.HasNext()
}
// CurrentRowId returns current row id
func (t *ImportedCommonDataRowIterator) CurrentRowId() string {
return t.innerIterator.CurrentRowId()
}
// Next returns the next common data row
func (t *ImportedCommonDataRowIterator) Next() CommonDataRow {
importedRow := t.innerIterator.Next()
if importedRow == nil {
return nil
}
rowData := make(map[string]string, len(t.commonDataTable.dataColumnIndexes))
for column, columnIndex := range t.commonDataTable.dataColumnIndexes {
if columnIndex < 0 || columnIndex >= importedRow.ColumnCount() {
continue
}
value := importedRow.GetData(columnIndex)
rowData[column] = value
}
return &ImportedCommonDataRow{
rowData: rowData,
}
}
// CreateNewImportedCommonDataTable returns common data table from imported data table
func CreateNewImportedCommonDataTable(dataTable ImportedDataTable) *ImportedCommonDataTable {
headerLineItems := dataTable.HeaderColumnNames()
dataColumnIndexes := make(map[string]int, len(headerLineItems))
for i := 0; i < len(headerLineItems); i++ {
dataColumnIndexes[headerLineItems[i]] = i
}
return &ImportedCommonDataTable{
innerDataTable: dataTable,
dataColumnIndexes: dataColumnIndexes,
}
}
pkg/converters/datatable/imported_data_table.go
+3
View File
@@ -26,6 +26,9 @@ type ImportedDataRowIterator interface {
// HasNext returns whether the iterator does not reach the end
HasNext() bool
// CurrentRowId returns current row id
CurrentRowId() string
// Next returns the next imported data row
Next() ImportedDataRow
}
pkg/converters/datatable/imported_transaction_data_table.go
+5 -5
View File
@@ -143,13 +143,13 @@ func (t *ImportedTransactionDataRowIterator) Next(ctx core.Context, user *models
}, nil
}
// CreateImportedTransactionDataTable returns transaction data table from imported data table
func CreateImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable {
return CreateImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil)
// CreateNewImportedTransactionDataTable returns transaction data table from imported data table
func CreateNewImportedTransactionDataTable(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string) *ImportedTransactionDataTable {
return CreateNewImportedTransactionDataTableWithRowParser(dataTable, dataColumnMapping, nil)
}
// CreateImportedTransactionDataTableWithRowParser returns transaction data table from imported data table
func CreateImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable {
// CreateNewImportedTransactionDataTableWithRowParser returns transaction data table from imported data table
func CreateNewImportedTransactionDataTableWithRowParser(dataTable ImportedDataTable, dataColumnMapping map[TransactionDataTableColumn]string, rowParser TransactionDataRowParser) *ImportedTransactionDataTable {
headerLineItems := dataTable.HeaderColumnNames()
headerItemMap := make(map[string]int, len(headerLineItems))
pkg/converters/default/default_transaction_data_plain_text_converter.go
+1 -1
View File
@@ -93,7 +93,7 @@ func (c *defaultTransactionDataPlainTextConverter) ParseImportedData(ctx core.Co
return nil, nil, nil, nil, nil, nil, err
}
transactionDataTable := datatable.CreateImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping)
transactionDataTable := datatable.CreateNewImportedTransactionDataTable(dataTable, ezbookkeepingDataColumnNameMapping)
dataTableImporter := datatable.CreateNewImporter(
ezbookkeepingTransactionTypeNameMapping,
pkg/converters/default/default_transaction_plain_text_data_table.go
+5
View File
@@ -78,6 +78,11 @@ func (t *defaultPlainTextDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allLines)
}
// CurrentRowId returns current index
func (t *defaultPlainTextDataRowIterator) CurrentRowId() string {
return fmt.Sprintf("line#%d", t.currentIndex)
}
// Next returns the next imported data row
func (t *defaultPlainTextDataRowIterator) Next() datatable.ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allLines) {
pkg/converters/excel/excel_file_imported_data_table.go
+6
View File
@@ -2,6 +2,7 @@ package excel
import (
"bytes"
"fmt"
"github.com/shakinm/xlsReader/xls"
@@ -115,6 +116,11 @@ func (t *ExcelFileDataRowIterator) HasNext() bool {
return false
}
// CurrentRowId returns current index
func (t *ExcelFileDataRowIterator) CurrentRowId() string {
return fmt.Sprintf("table#%d-row#%d", t.currentTableIndex, t.currentRowIndexInTable)
}
// Next returns the next imported data row
func (t *ExcelFileDataRowIterator) Next() datatable.ImportedDataRow {
allSheets := t.dataTable.workbook.GetSheets()
pkg/converters/feidee/feidee_mymoney_app_transaction_data_csv_file_importer.go
+167 -209
View File
@@ -5,6 +5,7 @@ import (
"io"
"strings"
csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
@@ -12,14 +13,35 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/models"
)
const feideeMymoneyTransactionDataCsvFileHeader = "随手记导出文件(headers:v5;"
const feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + feideeMymoneyTransactionDataCsvFileHeader
const feideeMymoneyAppTransactionDataCsvFileHeader = "随手记导出文件(headers:v5;"
const feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + feideeMymoneyAppTransactionDataCsvFileHeader
const feideeMymoneyCsvFileTransactionTypeModifyBalanceText = "余额变更"
const feideeMymoneyCsvFileTransactionTypeIncomeText = "收入"
const feideeMymoneyCsvFileTransactionTypeExpenseText = "支出"
const feideeMymoneyCsvFileTransactionTypeTransferInText = "转入"
const feideeMymoneyCsvFileTransactionTypeTransferOutText = "转出"
const feideeMymoneyAppTransactionTimeColumnName = "日期"
const feideeMymoneyAppTransactionTypeColumnName = "交易类型"
const feideeMymoneyAppTransactionCategoryColumnName = "类别"
const feideeMymoneyAppTransactionSubCategoryColumnName = "子类别"
const feideeMymoneyAppTransactionAccountNameColumnName = "账户"
const feideeMymoneyAppTransactionAccountCurrencyColumnName = "账户币种"
const feideeMymoneyAppTransactionAmountColumnName = "金额"
const feideeMymoneyAppTransactionDescriptionColumnName = "备注"
const feideeMymoneyAppTransactionRelatedIdColumnName = "关联Id"
const feideeMymoneyAppTransactionTypeModifyBalanceText = "余额变更"
const feideeMymoneyAppTransactionTypeIncomeText = "收入"
const feideeMymoneyAppTransactionTypeExpenseText = "支出"
const feideeMymoneyAppTransactionTypeTransferInText = "转入"
const feideeMymoneyAppTransactionTypeTransferOutText = "转出"
var feideeMymoneyAppDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: feideeMymoneyAppTransactionTimeColumnName,
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: feideeMymoneyAppTransactionTypeColumnName,
datatable.TRANSACTION_DATA_TABLE_CATEGORY: feideeMymoneyAppTransactionCategoryColumnName,
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: feideeMymoneyAppTransactionSubCategoryColumnName,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: feideeMymoneyAppTransactionAccountNameColumnName,
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: feideeMymoneyAppTransactionAccountCurrencyColumnName,
datatable.TRANSACTION_DATA_TABLE_AMOUNT: feideeMymoneyAppTransactionAmountColumnName,
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: feideeMymoneyAppTransactionDescriptionColumnName,
}
// feideeMymoneyAppTransactionDataCsvFileImporter defines the structure of feidee mymoney app csv importer for transaction data
type feideeMymoneyAppTransactionDataCsvFileImporter struct{}
@@ -32,173 +54,44 @@ var (
// ParseImportedData returns the imported data by parsing the feidee mymoney app transaction csv data
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
content := string(data)
if strings.Index(content, feideeMymoneyTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom) != 0 {
return nil, nil, nil, nil, nil, nil, errs.ErrInvalidFileHeader
}
allLines, err := c.parseAllLinesFromCsvData(ctx, content)
dataTable, err := c.createNewFeideeMymoneyAppImportedDataTable(ctx, content)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
if len(allLines) < 2 {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because data table row count is less 1", user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrNotFoundTransactionDataInFile
}
commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable)
headerLineItems := allLines[0]
headerItemMap := make(map[string]int)
for i := 0; i < len(headerLineItems); i++ {
headerItemMap[headerLineItems[i]] = i
}
timeColumnIdx, timeColumnExists := headerItemMap["日期"]
typeColumnIdx, typeColumnExists := headerItemMap["交易类型"]
categoryColumnIdx, categoryColumnExists := headerItemMap["类别"]
subCategoryColumnIdx, subCategoryColumnExists := headerItemMap["子类别"]
accountColumnIdx, accountColumnExists := headerItemMap["账户"]
accountCurrencyColumnIdx, accountCurrencyColumnExists := headerItemMap["账户币种"]
amountColumnIdx, amountColumnExists := headerItemMap["金额"]
descriptionColumnIdx, descriptionColumnExists := headerItemMap["备注"]
relatedIdColumnIdx, relatedIdColumnExists := headerItemMap["关联Id"]
if !timeColumnExists || !typeColumnExists || !subCategoryColumnExists ||
!accountColumnExists || !amountColumnExists || !relatedIdColumnExists {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data for user \"uid:%d\", because missing essential columns in header row", user.Uid)
if !commonDataTable.HasColumn(feideeMymoneyAppTransactionTimeColumnName) ||
!commonDataTable.HasColumn(feideeMymoneyAppTransactionTypeColumnName) ||
!commonDataTable.HasColumn(feideeMymoneyAppTransactionSubCategoryColumnName) ||
!commonDataTable.HasColumn(feideeMymoneyAppTransactionAccountNameColumnName) ||
!commonDataTable.HasColumn(feideeMymoneyAppTransactionAmountColumnName) ||
!commonDataTable.HasColumn(feideeMymoneyAppTransactionRelatedIdColumnName) {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse import data, because missing essential columns in header row")
return nil, nil, nil, nil, nil, nil, errs.ErrMissingRequiredFieldInHeaderRow
}
newColumns := make([]datatable.TransactionDataTableColumn, 0, 11)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME)
transactionDataTable, err := c.createNewFeideeMymoneyAppTransactionDataTable(ctx, commonDataTable)
if categoryColumnExists {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_CATEGORY)
}
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
if accountCurrencyColumnExists {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY)
}
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_AMOUNT)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME)
if accountCurrencyColumnExists {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY)
}
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT)
if descriptionColumnExists {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_DESCRIPTION)
}
transactionRowParser := createFeideeMymoneyTransactionDataRowParser()
dataTable := datatable.CreateNewWritableTransactionDataTableWithRowParser(newColumns, transactionRowParser)
transferTransactionsMap := make(map[string]map[datatable.TransactionDataTableColumn]string, 0)
for i := 1; i < len(allLines); i++ {
items := allLines[i]
if len(items) < len(headerLineItems) {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse row \"index:%d\" for user \"uid:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", i, user.Uid, len(items), len(headerLineItems))
return nil, nil, nil, nil, nil, nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow
}
data, relatedId := c.parseTransactionData(items,
timeColumnIdx,
timeColumnExists,
typeColumnIdx,
typeColumnExists,
categoryColumnIdx,
categoryColumnExists,
subCategoryColumnIdx,
subCategoryColumnExists,
accountColumnIdx,
accountColumnExists,
accountCurrencyColumnIdx,
accountCurrencyColumnExists,
amountColumnIdx,
amountColumnExists,
descriptionColumnIdx,
descriptionColumnExists,
relatedIdColumnIdx,
relatedIdColumnExists,
)
transactionType := data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]
if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText || transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText || transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText {
if transactionType == feideeMymoneyCsvFileTransactionTypeModifyBalanceText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
} else if transactionType == feideeMymoneyCsvFileTransactionTypeIncomeText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
} else if transactionType == feideeMymoneyCsvFileTransactionTypeExpenseText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
}
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = ""
dataTable.Add(data)
} else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText || transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText {
if relatedId == "" {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] transfer transaction has blank related id in row \"index:%d\" for user \"uid:%d\"", i, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrRelatedIdCannotBeBlank
}
relatedData, exists := transferTransactionsMap[relatedId]
if !exists {
transferTransactionsMap[relatedId] = data
continue
}
if transactionType == feideeMymoneyCsvFileTransactionTypeTransferInText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferOutText {
relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = data[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
dataTable.Add(relatedData)
delete(transferTransactionsMap, relatedId)
} else if transactionType == feideeMymoneyCsvFileTransactionTypeTransferOutText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyCsvFileTransactionTypeTransferInText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
dataTable.Add(data)
delete(transferTransactionsMap, relatedId)
} else {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] transfer transaction type \"%s\" is not expected in row \"index:%d\" for user \"uid:%d\"", transactionType, i, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTypeInvalid
}
} else {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] cannot parse transaction type \"%s\" in row \"index:%d\" for user \"uid:%d\"", transactionType, i, user.Uid)
return nil, nil, nil, nil, nil, nil, errs.ErrTransactionTypeInvalid
}
}
if len(transferTransactionsMap) > 0 {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.ParseImportedData] there are %d transactions (related id is %s) which don't have related records", len(transferTransactionsMap), c.getRelatedIds(transferTransactionsMap))
return nil, nil, nil, nil, nil, nil, errs.ErrFoundRecordNotHasRelatedRecord
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping)
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) parseAllLinesFromCsvData(ctx core.Context, content string) ([][]string, error) {
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppImportedDataTable(ctx core.Context, content string) (datatable.ImportedDataTable, error) {
if strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeader) != 0 && strings.Index(content, feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) != 0 {
return nil, errs.ErrInvalidFileHeader
}
csvReader := csv.NewReader(strings.NewReader(content))
csvReader.FieldsPerRecord = -1
allLines := make([][]string, 0)
allOriginalLines := make([][]string, 0)
hasFileHeader := false
for {
@@ -209,91 +102,156 @@ func (c *feideeMymoneyAppTransactionDataCsvFileImporter) parseAllLinesFromCsvDat
}
if err != nil {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.parseAllLinesFromCsvData] cannot parse feidee mymoney csv data, because %s", err.Error())
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse feidee mymoney csv data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
if !hasFileHeader {
if len(items) <= 0 {
continue
} else if strings.Index(items[0], feideeMymoneyTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], feideeMymoneyTransactionDataCsvFileHeaderWithUtf8Bom) == 0 {
} else if strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeader) == 0 || strings.Index(items[0], feideeMymoneyAppTransactionDataCsvFileHeaderWithUtf8Bom) == 0 {
hasFileHeader = true
continue
} else {
log.Warnf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.parseAllLinesFromCsvData] read unexpected line before read file header, line content is %s", strings.Join(items, ","))
log.Warnf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ","))
}
}
allLines = append(allLines, items)
allOriginalLines = append(allOriginalLines, items)
}
return allLines, nil
if !hasFileHeader {
return nil, errs.ErrInvalidFileHeader
}
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines)
return dataTable, nil
}
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) parseTransactionData(
items []string,
timeColumnIdx int,
timeColumnExists bool,
typeColumnIdx int,
typeColumnExists bool,
categoryColumnIdx int,
categoryColumnExists bool,
subCategoryColumnIdx int,
subCategoryColumnExists bool,
accountColumnIdx int,
accountColumnExists bool,
accountCurrencyColumnIdx int,
accountCurrencyColumnExists bool,
amountColumnIdx int,
amountColumnExists bool,
descriptionColumnIdx int,
descriptionColumnExists bool,
relatedIdColumnIdx int,
relatedIdColumnExists bool,
) (map[datatable.TransactionDataTableColumn]string, string) {
data := make(map[datatable.TransactionDataTableColumn]string, 11)
relatedId := ""
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) createNewFeideeMymoneyAppTransactionDataTable(ctx core.Context, commonDataTable datatable.CommonDataTable) (datatable.TransactionDataTable, error) {
newColumns := make([]datatable.TransactionDataTableColumn, 0, 11)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME)
if timeColumnExists && timeColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[timeColumnIdx]
if commonDataTable.HasColumn(feideeMymoneyAppTransactionCategoryColumnName) {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_CATEGORY)
}
if typeColumnExists && typeColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[typeColumnIdx]
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME)
if commonDataTable.HasColumn(feideeMymoneyAppTransactionAccountCurrencyColumnName) {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY)
}
if categoryColumnExists && categoryColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_CATEGORY] = items[categoryColumnIdx]
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_AMOUNT)
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME)
if commonDataTable.HasColumn(feideeMymoneyAppTransactionAccountCurrencyColumnName) {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY)
}
if subCategoryColumnExists && subCategoryColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[subCategoryColumnIdx]
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT)
if commonDataTable.HasColumn(feideeMymoneyAppTransactionDescriptionColumnName) {
newColumns = append(newColumns, datatable.TRANSACTION_DATA_TABLE_DESCRIPTION)
}
if accountColumnExists && accountColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = items[accountColumnIdx]
transactionRowParser := createFeideeMymoneyTransactionDataRowParser()
transactionDataTable := datatable.CreateNewWritableTransactionDataTableWithRowParser(newColumns, transactionRowParser)
transferTransactionsMap := make(map[string]map[datatable.TransactionDataTableColumn]string, 0)
commonDataTableIterator := commonDataTable.DataRowIterator()
for commonDataTableIterator.HasNext() {
dataRow := commonDataTableIterator.Next()
rowId := commonDataTableIterator.CurrentRowId()
if dataRow.ColumnCount() < commonDataTable.HeaderColumnCount() {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse row \"%s\", because may missing some columns (column count %d in data row is less than header column count %d)", rowId, dataRow.ColumnCount(), commonDataTable.HeaderColumnCount())
return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow
}
data := make(map[datatable.TransactionDataTableColumn]string, 11)
relatedId := ""
for columnType, columnName := range feideeMymoneyAppDataColumnNameMapping {
if dataRow.HasData(columnName) {
data[columnType] = dataRow.GetData(columnName)
}
}
if dataRow.HasData(feideeMymoneyAppTransactionRelatedIdColumnName) {
relatedId = dataRow.GetData(feideeMymoneyAppTransactionRelatedIdColumnName)
}
transactionType := data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]
if transactionType == feideeMymoneyAppTransactionTypeModifyBalanceText || transactionType == feideeMymoneyAppTransactionTypeIncomeText || transactionType == feideeMymoneyAppTransactionTypeExpenseText {
if transactionType == feideeMymoneyAppTransactionTypeModifyBalanceText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE]
} else if transactionType == feideeMymoneyAppTransactionTypeIncomeText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME]
} else if transactionType == feideeMymoneyAppTransactionTypeExpenseText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE]
}
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = ""
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = ""
transactionDataTable.Add(data)
} else if transactionType == feideeMymoneyAppTransactionTypeTransferInText || transactionType == feideeMymoneyAppTransactionTypeTransferOutText {
if relatedId == "" {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] transfer transaction has blank related id in row \"%s\"", rowId)
return nil, errs.ErrRelatedIdCannotBeBlank
}
relatedData, exists := transferTransactionsMap[relatedId]
if !exists {
transferTransactionsMap[relatedId] = data
continue
}
if transactionType == feideeMymoneyAppTransactionTypeTransferInText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyAppTransactionTypeTransferOutText {
relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
relatedData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = data[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
transactionDataTable.Add(relatedData)
delete(transferTransactionsMap, relatedId)
} else if transactionType == feideeMymoneyAppTransactionTypeTransferOutText && relatedData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == feideeMymoneyAppTransactionTypeTransferInText {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = feideeMymoneyTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = relatedData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = relatedData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
transactionDataTable.Add(data)
delete(transferTransactionsMap, relatedId)
} else {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] transfer transaction type \"%s\" is not expected in row \"%s\"", transactionType, rowId)
return nil, errs.ErrTransactionTypeInvalid
}
} else {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] cannot parse transaction type \"%s\" in row \"%s\"", transactionType, rowId)
return nil, errs.ErrTransactionTypeInvalid
}
}
if accountCurrencyColumnExists && accountCurrencyColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = items[accountCurrencyColumnIdx]
if len(transferTransactionsMap) > 0 {
log.Errorf(ctx, "[feidee_mymoney_app_transaction_data_csv_file_importer.createNewFeideeMymoneyAppTransactionDataTable] there are %d transactions (related id is %s) which don't have related records", len(transferTransactionsMap), c.getFeideeMymoneyAppRelatedTransactionIds(transferTransactionsMap))
return nil, errs.ErrFoundRecordNotHasRelatedRecord
}
if amountColumnExists && amountColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[amountColumnIdx]
}
if descriptionColumnExists && descriptionColumnIdx < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[descriptionColumnIdx]
}
if relatedIdColumnExists && relatedIdColumnIdx < len(items) {
relatedId = items[relatedIdColumnIdx]
}
return data, relatedId
return transactionDataTable, nil
}
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) getRelatedIds(transferTransactionsMap map[string]map[datatable.TransactionDataTableColumn]string) string {
func (c *feideeMymoneyAppTransactionDataCsvFileImporter) getFeideeMymoneyAppRelatedTransactionIds(transferTransactionsMap map[string]map[datatable.TransactionDataTableColumn]string) string {
builder := strings.Builder{}
for relatedId := range transferTransactionsMap {
pkg/converters/feidee/feidee_mymoney_transaction_data_file_importer.go
-24
View File
@@ -1,24 +0,0 @@
package feidee
import (
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
var feideeMymoneyDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "日期",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "交易类型",
datatable.TRANSACTION_DATA_TABLE_CATEGORY: "分类",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "子分类",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "账户1",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "金额",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "备注",
}
var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "余额变更",
models.TRANSACTION_TYPE_INCOME: "收入",
models.TRANSACTION_TYPE_EXPENSE: "支出",
models.TRANSACTION_TYPE_TRANSFER: "转账",
}
pkg/converters/feidee/feidee_mymoney_transaction_data_row_parser.go
+7
View File
@@ -7,6 +7,13 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
var feideeMymoneyTransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "余额变更",
models.TRANSACTION_TYPE_INCOME: "收入",
models.TRANSACTION_TYPE_EXPENSE: "支出",
models.TRANSACTION_TYPE_TRANSFER: "转账",
}
// feideeMymoneyTransactionDataRowParser defines the structure of feidee mymoney transaction data row parser
type feideeMymoneyTransactionDataRowParser struct {
}
pkg/converters/feidee/feidee_mymoney_web_transaction_data_xls_file_importer.go
+12 -1
View File
@@ -7,6 +7,17 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/models"
)
var feideeMymoneyWebDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "日期",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "交易类型",
datatable.TRANSACTION_DATA_TABLE_CATEGORY: "分类",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "子分类",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "账户1",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "金额",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "账户2",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "备注",
}
// feideeMymoneyWebTransactionDataXlsFileImporter defines the structure of feidee mymoney (web) xls importer for transaction data
type feideeMymoneyWebTransactionDataXlsFileImporter struct {
datatable.DataTableTransactionDataImporter
@@ -26,7 +37,7 @@ func (c *feideeMymoneyWebTransactionDataXlsFileImporter) ParseImportedData(ctx c
}
transactionRowParser := createFeideeMymoneyTransactionDataRowParser()
transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyDataColumnNameMapping, transactionRowParser)
transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, feideeMymoneyWebDataColumnNameMapping, transactionRowParser)
dataTableImporter := datatable.CreateNewSimpleImporter(feideeMymoneyTransactionTypeNameMapping)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
pkg/converters/fireflyIII/fireflyiii_transaction_data_csv_file_importer.go
+2 -2
View File
@@ -41,14 +41,14 @@ var (
// ParseImportedData returns the imported data by parsing the firefly III transaction csv data
func (c *fireflyIIITransactionDataCsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
reader := bytes.NewReader(data)
dataTable, err := csv.CreateNewCsvDataTable(ctx, reader)
dataTable, err := csv.CreateNewCsvImportedDataTable(ctx, reader)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
transactionRowParser := createFireflyIIITransactionDataRowParser()
transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser)
transactionDataTable := datatable.CreateNewImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser)
dataTableImporter := datatable.CreateNewImporter(fireflyIIITransactionTypeNameMapping, "", ",")
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
pkg/converters/wechat/wechat_pay_transaction_csv_data_table.go
+89 -139
View File
@@ -2,10 +2,10 @@ package wechat
import (
"encoding/csv"
"fmt"
"io"
"strings"
csvdatatable "github.com/mayswind/ezbookkeeping/pkg/converters/csv"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
@@ -19,6 +19,15 @@ const wechatPayTransactionDataCsvFileHeader = "微信支付账单明细"
const wechatPayTransactionDataCsvFileHeaderWithUtf8Bom = "\xEF\xBB\xBF" + wechatPayTransactionDataCsvFileHeader
const wechatPayTransactionDataHeaderStartContentBeginning = "----------------------微信支付账单明细列表--------------------"
const wechatPayTransactionTimeColumnName = "交易时间"
const wechatPayTransactionCategoryColumnName = "交易类型"
const wechatPayTransactionProductNameColumnName = "商品"
const wechatPayTransactionTypeColumnName = "收/支"
const wechatPayTransactionAmountColumnName = "金额(元)"
const wechatPayTransactionRelatedAccountColumnName = "支付方式"
const wechatPayTransactionStatusColumnName = "当前状态"
const wechatPayTransactionDescriptionColumnName = "备注"
const wechatPayTransactionDataCategoryTransferToWeChatWallet = "零钱充值"
const wechatPayTransactionDataCategoryTransferFromWeChatWallet = "零钱提现"
@@ -34,33 +43,21 @@ var wechatPayTransactionSupportedColumns = map[datatable.TransactionDataTableCol
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true,
}
// wechatPayTransactionDataTable defines the structure of wechatPay transaction plain text data table
// wechatPayTransactionDataTable defines the structure of wechat pay transaction plain text data table
type wechatPayTransactionDataTable struct {
allOriginalLines [][]string
originalHeaderLineColumnNames []string
originalTimeColumnIndex int
originalCategoryColumnIndex int
originalTargetNameColumnIndex int
originalProductNameColumnIndex int
originalTypeColumnIndex int
originalAmountColumnIndex int
originalRelatedAccountColumnIndex int
originalStatusColumnIndex int
originalDescriptionColumnIndex int
innerDataTable datatable.CommonDataTable
}
// wechatPayTransactionDataRow defines the structure of wechatPay transaction plain text data row
// wechatPayTransactionDataRow defines the structure of wechat pay transaction plain text data row
type wechatPayTransactionDataRow struct {
dataTable *wechatPayTransactionDataTable
isValid bool
originalItems []string
finalItems map[datatable.TransactionDataTableColumn]string
isValid bool
finalItems map[datatable.TransactionDataTableColumn]string
}
// wechatPayTransactionDataRowIterator defines the structure of wechatPay transaction plain text data row iterator
// wechatPayTransactionDataRowIterator defines the structure of wechat pay transaction plain text data row iterator
type wechatPayTransactionDataRowIterator struct {
dataTable *wechatPayTransactionDataTable
currentIndex int
dataTable *wechatPayTransactionDataTable
innerIterator datatable.CommonDataRowIterator
}
// HasColumn returns whether the transaction data table has specified column
@@ -71,18 +68,14 @@ func (t *wechatPayTransactionDataTable) HasColumn(column datatable.TransactionDa
// TransactionRowCount returns the total count of transaction data row
func (t *wechatPayTransactionDataTable) TransactionRowCount() int {
if len(t.allOriginalLines) < 1 {
return 0
}
return len(t.allOriginalLines) - 1
return t.innerDataTable.DataRowCount()
}
// TransactionRowIterator returns the iterator of transaction data row
func (t *wechatPayTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator {
return &wechatPayTransactionDataRowIterator{
dataTable: t,
currentIndex: 0,
dataTable: t,
innerIterator: t.innerDataTable.DataRowIterator(),
}
}
@@ -104,89 +97,80 @@ func (r *wechatPayTransactionDataRow) GetData(column datatable.TransactionDataTa
// HasNext returns whether the iterator does not reach the end
func (t *wechatPayTransactionDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
return t.innerIterator.HasNext()
}
// Next returns the next imported data row
func (t *wechatPayTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) {
if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
importedRow := t.innerIterator.Next()
if importedRow == nil {
return nil, nil
}
t.currentIndex++
finalItems, isValid, err := t.dataTable.parseTransactionData(ctx, user, importedRow, t.innerIterator.CurrentRowId())
rowItems := t.dataTable.allOriginalLines[t.currentIndex]
isValid := true
if t.dataTable.originalTypeColumnIndex >= 0 &&
rowItems[t.dataTable.originalTypeColumnIndex] != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] &&
rowItems[t.dataTable.originalTypeColumnIndex] != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] &&
rowItems[t.dataTable.originalTypeColumnIndex] != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because type is \"%s\"", t.currentIndex, rowItems[t.dataTable.originalTypeColumnIndex])
isValid = false
}
var finalItems map[datatable.TransactionDataTableColumn]string
var errMsg string
if isValid {
finalItems, errMsg = t.dataTable.parseTransactionData(ctx, user, rowItems)
if finalItems == nil {
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.Next] skip parsing transaction in row \"index:%d\", because %s", t.currentIndex, errMsg)
isValid = false
}
if err != nil {
return nil, err
}
return &wechatPayTransactionDataRow{
dataTable: t.dataTable,
isValid: isValid,
originalItems: rowItems,
finalItems: finalItems,
isValid: isValid,
finalItems: finalItems,
}, nil
}
func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, items []string) (map[datatable.TransactionDataTableColumn]string, string) {
func (t *wechatPayTransactionDataTable) hasOriginalColumn(columnName string) bool {
return columnName != "" && t.innerDataTable.HasColumn(columnName)
}
func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, user *models.User, dataRow datatable.CommonDataRow, rowId string) (map[datatable.TransactionDataTableColumn]string, bool, error) {
if dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] &&
dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_EXPENSE] &&
dataRow.GetData(wechatPayTransactionTypeColumnName) != wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because type is \"%s\"", rowId, dataRow.GetData(wechatPayTransactionTypeColumnName))
return nil, false, nil
}
data := make(map[datatable.TransactionDataTableColumn]string, len(wechatPayTransactionSupportedColumns))
if t.originalTimeColumnIndex >= 0 && t.originalTimeColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = items[t.originalTimeColumnIndex]
if t.hasOriginalColumn(wechatPayTransactionTimeColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = dataRow.GetData(wechatPayTransactionTimeColumnName)
}
if t.originalCategoryColumnIndex >= 0 && t.originalCategoryColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = items[t.originalCategoryColumnIndex]
} else {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
if t.hasOriginalColumn(wechatPayTransactionCategoryColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = dataRow.GetData(wechatPayTransactionCategoryColumnName)
}
if t.originalAmountColumnIndex >= 0 && t.originalAmountColumnIndex < len(items) {
amount, success := utils.ParseFirstConsecutiveNumber(items[t.originalAmountColumnIndex])
if t.hasOriginalColumn(wechatPayTransactionAmountColumnName) {
amount, success := utils.ParseFirstConsecutiveNumber(dataRow.GetData(wechatPayTransactionAmountColumnName))
if success {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = amount
} else {
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = items[t.originalAmountColumnIndex]
if !success {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] cannot parse amount \"%s\" of transaction in row \"%s\"", dataRow.GetData(wechatPayTransactionAmountColumnName), rowId)
return nil, false, errs.ErrAmountInvalid
}
data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = amount
}
if t.originalDescriptionColumnIndex >= 0 && t.originalDescriptionColumnIndex < len(items) && items[t.originalDescriptionColumnIndex] != "" && items[t.originalDescriptionColumnIndex] != "/" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalDescriptionColumnIndex]
} else if t.originalProductNameColumnIndex >= 0 && t.originalProductNameColumnIndex < len(items) && items[t.originalProductNameColumnIndex] != "" && items[t.originalProductNameColumnIndex] != "/" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = items[t.originalProductNameColumnIndex]
if t.hasOriginalColumn(wechatPayTransactionDescriptionColumnName) && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "" && dataRow.GetData(wechatPayTransactionDescriptionColumnName) != "/" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(wechatPayTransactionDescriptionColumnName)
} else if t.hasOriginalColumn(wechatPayTransactionProductNameColumnName) && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "" && dataRow.GetData(wechatPayTransactionProductNameColumnName) != "/" {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = dataRow.GetData(wechatPayTransactionProductNameColumnName)
} else {
data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = ""
}
relatedAccountName := ""
if t.originalRelatedAccountColumnIndex >= 0 && t.originalRelatedAccountColumnIndex < len(items) {
relatedAccountName = items[t.originalRelatedAccountColumnIndex]
if t.hasOriginalColumn(wechatPayTransactionRelatedAccountColumnName) {
relatedAccountName = dataRow.GetData(wechatPayTransactionRelatedAccountColumnName)
}
statusName := ""
if t.originalStatusColumnIndex >= 0 && t.originalStatusColumnIndex < len(items) {
statusName = items[t.originalStatusColumnIndex]
if t.hasOriginalColumn(wechatPayTransactionStatusColumnName) {
statusName = dataRow.GetData(wechatPayTransactionStatusColumnName)
}
locale := user.Language
@@ -197,17 +181,17 @@ func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, u
localeTextItems := locales.GetLocaleTextItems(locale)
if t.originalTypeColumnIndex >= 0 && t.originalTypeColumnIndex < len(items) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = items[t.originalTypeColumnIndex]
if t.hasOriginalColumn(wechatPayTransactionTypeColumnName) {
data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = dataRow.GetData(wechatPayTransactionTypeColumnName)
if items[t.originalTypeColumnIndex] == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
if dataRow.GetData(wechatPayTransactionTypeColumnName) == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
if relatedAccountName == "" || relatedAccountName == "/" {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.WeChatWallet
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
} else {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
}
} else if items[t.originalTypeColumnIndex] == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
} else if dataRow.GetData(wechatPayTransactionTypeColumnName) == wechatPayTransactionTypeNameMapping[models.TRANSACTION_TYPE_TRANSFER] {
if data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == wechatPayTransactionDataCategoryTransferToWeChatWallet {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.WeChatWallet
@@ -215,7 +199,8 @@ func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, u
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = localeTextItems.DataConverterTextItems.WeChatWallet
data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = relatedAccountName
} else {
return nil, fmt.Sprintf("unkown transfer transaction category")
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseTransactionData] skip parsing transaction in row \"%s\", because unkown transfer transaction category \"%s\"", rowId, data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY])
return nil, false, nil
}
} else {
data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = relatedAccountName
@@ -234,75 +219,33 @@ func (t *wechatPayTransactionDataTable) parseTransactionData(ctx core.Context, u
}
}
return data, ""
return data, true, nil
}
func createNewWeChatPayTransactionDataTable(ctx core.Context, reader io.Reader) (*wechatPayTransactionDataTable, error) {
allOriginalLines, err := parseAllLinesFromWechatPayTransactionPlainText(ctx, reader)
dataTable, err := createNewWeChatPayImportedDataTable(ctx, reader)
if err != nil {
return nil, err
}
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewwechatPayTransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
commonDataTable := datatable.CreateNewImportedCommonDataTable(dataTable)
originalHeaderItems := allOriginalLines[0]
originalHeaderItemMap := make(map[string]int)
for i := 0; i < len(originalHeaderItems); i++ {
originalHeaderItemMap[originalHeaderItems[i]] = i
}
timeColumnIdx, timeColumnExists := originalHeaderItemMap["交易时间"]
categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["交易类型"]
targetNameColumnIdx, targetNameColumnExists := originalHeaderItemMap["交易对方"]
productNameColumnIdx, productNameColumnExists := originalHeaderItemMap["商品"]
typeColumnIdx, typeColumnExists := originalHeaderItemMap["收/支"]
amountColumnIdx, amountColumnExists := originalHeaderItemMap["金额(元)"]
relatedAccountColumnIdx, relatedAccountColumnExists := originalHeaderItemMap["支付方式"]
statusColumnIdx, statusColumnExists := originalHeaderItemMap["当前状态"]
descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["备注"]
if !timeColumnExists || !categoryColumnExists || !typeColumnExists || !amountColumnExists || !statusColumnExists {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewwechatPayTransactionPlainTextDataTable] cannot parse wechat pay csv data, because missing essential columns in header row")
if !commonDataTable.HasColumn(wechatPayTransactionTimeColumnName) ||
!commonDataTable.HasColumn(wechatPayTransactionCategoryColumnName) ||
!commonDataTable.HasColumn(wechatPayTransactionTypeColumnName) ||
!commonDataTable.HasColumn(wechatPayTransactionAmountColumnName) ||
!commonDataTable.HasColumn(wechatPayTransactionStatusColumnName) {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayTransactionDataTable] cannot parse wechat pay csv data, because missing essential columns in header row")
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
if !targetNameColumnExists {
targetNameColumnIdx = -1
}
if !productNameColumnExists {
productNameColumnIdx = -1
}
if !relatedAccountColumnExists {
relatedAccountColumnIdx = -1
}
if !descriptionColumnExists {
descriptionColumnIdx = -1
}
return &wechatPayTransactionDataTable{
allOriginalLines: allOriginalLines,
originalHeaderLineColumnNames: originalHeaderItems,
originalTimeColumnIndex: timeColumnIdx,
originalCategoryColumnIndex: categoryColumnIdx,
originalTargetNameColumnIndex: targetNameColumnIdx,
originalProductNameColumnIndex: productNameColumnIdx,
originalAmountColumnIndex: amountColumnIdx,
originalTypeColumnIndex: typeColumnIdx,
originalRelatedAccountColumnIndex: relatedAccountColumnIdx,
originalStatusColumnIndex: statusColumnIdx,
originalDescriptionColumnIndex: descriptionColumnIdx,
innerDataTable: commonDataTable,
}, nil
}
func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) {
func createNewWeChatPayImportedDataTable(ctx core.Context, reader io.Reader) (datatable.ImportedDataTable, error) {
csvReader := csv.NewReader(reader)
csvReader.FieldsPerRecord = -1
@@ -318,7 +261,7 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io.
}
if err != nil {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.parseAllLinesFromWechatPayTransactionPlainText] cannot parse wechat pay csv data, because %s", err.Error())
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] cannot parse wechat pay csv data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
@@ -329,7 +272,7 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io.
hasFileHeader = true
continue
} else {
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.parseAllLinesFromWechatPayTransactionPlainText] read unexpected line before read file header, line content is %s", strings.Join(items, ","))
log.Warnf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] read unexpected line before read file header, line content is %s", strings.Join(items, ","))
continue
}
}
@@ -355,7 +298,7 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io.
}
if len(allOriginalLines) > 0 && len(items) < len(allOriginalLines[0]) {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.parseAllLinesFromWechatPayTransactionPlainText] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0]))
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] cannot parse row \"index:%d\", because may missing some columns (column count %d in data row is less than header column count %d)", len(allOriginalLines), len(items), len(allOriginalLines[0]))
return nil, errs.ErrFewerFieldsInDataRowThanInHeaderRow
}
@@ -367,5 +310,12 @@ func parseAllLinesFromWechatPayTransactionPlainText(ctx core.Context, reader io.
return nil, errs.ErrInvalidFileHeader
}
return allOriginalLines, nil
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[wechat_pay_transaction_csv_data_table.createNewWeChatPayImportedDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
dataTable := csvdatatable.CreateNewCustomCsvImportedDataTable(allOriginalLines)
return dataTable, nil
}