code refactor

This commit is contained in:
MaysWind
2024-10-13 19:44:29 +08:00
parent 5ac9eb5d5c
commit d9b819d1a1
26 changed files with 1331 additions and 1226 deletions
@@ -8,7 +8,28 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// fireflyiiiTransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data
var fireflyIIITransactionDataColumnNameMapping = map[datatable.TransactionDataTableColumn]string{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: "date",
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: "type",
datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: "category",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: "source_name",
datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: "currency_code",
datatable.TRANSACTION_DATA_TABLE_AMOUNT: "amount",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: "destination_name",
datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: "foreign_currency_code",
datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: "foreign_amount",
datatable.TRANSACTION_DATA_TABLE_TAGS: "tags",
datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: "description",
}
var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance",
models.TRANSACTION_TYPE_INCOME: "Deposit",
models.TRANSACTION_TYPE_EXPENSE: "Withdrawal",
models.TRANSACTION_TYPE_TRANSFER: "Transfer",
}
// fireflyIIITransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data
type fireflyIIITransactionDataCsvImporter struct{}
// Initialize a firefly III transaction data csv file importer singleton instance
@@ -16,25 +37,18 @@ var (
FireflyIIITransactionDataCsvImporter = &fireflyIIITransactionDataCsvImporter{}
)
// ParseImportedData returns the imported data by parsing the firefly iii transaction csv data
// ParseImportedData returns the imported data by parsing the firefly III transaction csv data
func (c *fireflyIIITransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
reader := bytes.NewReader(data)
dataTable, err := createNewFireflyIIITransactionPlainTextDataTable(
ctx,
reader,
)
dataTable, err := datatable.CreateNewDefaultCsvDataTable(ctx, reader)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewImporter(
dataTable.GetDataColumnMapping(),
fireflyIIITransactionTypeNameMapping,
"",
",",
)
transactionRowParser := createFireflyIIITransactionDataRowParser()
transactionDataTable := datatable.CreateImportedTransactionDataTableWithRowParser(dataTable, fireflyIIITransactionDataColumnNameMapping, transactionRowParser)
dataTableImporter := datatable.CreateNewImporter(fireflyIIITransactionTypeNameMapping, "", ",")
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
@@ -114,6 +114,34 @@ func TestFireFlyIIICsvFileConverterParseImportedData_ParseInvalidType(t *testing
assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message)
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidTimezone(t *testing.T) {
converter := FireflyIIITransactionDataCsvImporter
context := core.NewNullContext()
user := &models.User{
Uid: 1234567890,
DefaultCurrency: "CNY",
}
allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56-10:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+00:00,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte("type,amount,date,source_name,destination_name,category\n"+
"Withdrawal,-1.00,2024-09-01T12:34:56+12:45,\"Test Account\",\"A expense account\",\"Test Category\""), 0, nil, nil, nil, nil, nil)
assert.Nil(t, err)
assert.Equal(t, 1, len(allNewTransactions))
assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime))
}
func TestFireFlyIIICsvFileConverterParseImportedData_ParseValidAccountCurrency(t *testing.T) {
converter := FireflyIIITransactionDataCsvImporter
context := core.NewNullContext()
@@ -1,304 +0,0 @@
package fireflyIII
import (
"encoding/csv"
"io"
"time"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
var fireflyIIITransactionSupportedColumns = []datatable.DataTableColumn{
datatable.DATA_TABLE_TRANSACTION_TIME,
datatable.DATA_TABLE_TRANSACTION_TYPE,
datatable.DATA_TABLE_SUB_CATEGORY,
datatable.DATA_TABLE_ACCOUNT_NAME,
datatable.DATA_TABLE_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_AMOUNT,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_RELATED_AMOUNT,
datatable.DATA_TABLE_TAGS,
datatable.DATA_TABLE_DESCRIPTION,
}
var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance",
models.TRANSACTION_TYPE_INCOME: "Deposit",
models.TRANSACTION_TYPE_EXPENSE: "Withdrawal",
models.TRANSACTION_TYPE_TRANSFER: "Transfer",
}
// fireflyIIITransactionPlainTextDataTable defines the structure of firefly III transaction plain text data table
type fireflyIIITransactionPlainTextDataTable struct {
allOriginalLines [][]string
originalHeaderLineColumnNames []string
originalColumnIndex map[datatable.DataTableColumn]int
}
// fireflyIIITransactionPlainTextDataRow defines the structure of firefly III transaction plain text data row
type fireflyIIITransactionPlainTextDataRow struct {
dataTable *fireflyIIITransactionPlainTextDataTable
originalItems []string
finalItems map[datatable.DataTableColumn]string
}
// fireflyIIITransactionPlainTextDataRowIterator defines the structure of firefly III transaction plain text data row iterator
type fireflyIIITransactionPlainTextDataRowIterator struct {
dataTable *fireflyIIITransactionPlainTextDataTable
currentIndex int
}
// DataRowCount returns the total count of data row
func (t *fireflyIIITransactionPlainTextDataTable) DataRowCount() int {
if len(t.allOriginalLines) < 1 {
return 0
}
return len(t.allOriginalLines) - 1
}
// GetDataColumnMapping returns data column map for data importer
func (t *fireflyIIITransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string {
dataColumnMapping := make(map[datatable.DataTableColumn]string, len(fireflyIIITransactionSupportedColumns))
for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
column := fireflyIIITransactionSupportedColumns[i]
dataColumnMapping[column] = utils.IntToString(int(column))
}
return dataColumnMapping
}
// HeaderLineColumnNames returns the header column name list
func (t *fireflyIIITransactionPlainTextDataTable) HeaderLineColumnNames() []string {
columnIndexes := make([]string, len(fireflyIIITransactionSupportedColumns))
for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
column := fireflyIIITransactionSupportedColumns[i]
if t.originalColumnIndex[column] >= 0 {
columnIndexes[i] = utils.IntToString(int(column))
} else {
columnIndexes[i] = "-1"
}
}
return columnIndexes
}
// DataRowIterator returns the iterator of data row
func (t *fireflyIIITransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &fireflyIIITransactionPlainTextDataRowIterator{
dataTable: t,
currentIndex: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *fireflyIIITransactionPlainTextDataRow) IsValid() bool {
return true
}
// ColumnCount returns the total count of column in this data row
func (r *fireflyIIITransactionPlainTextDataRow) ColumnCount() int {
return len(fireflyIIITransactionSupportedColumns)
}
// GetData returns the data in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetData(columnIndex int) string {
if columnIndex >= len(fireflyIIITransactionSupportedColumns) {
return ""
}
dataColumn := fireflyIIITransactionSupportedColumns[columnIndex]
return r.finalItems[dataColumn]
}
// GetTime returns the time in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
return utils.ParseFromLongDateTimeWithTimezone(r.GetData(columnIndex))
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return nil, errs.ErrNotSupported
}
// HasNext returns whether the iterator does not reach the end
func (t *fireflyIIITransactionPlainTextDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
}
// Next returns the next imported data row
func (t *fireflyIIITransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
return nil
}
t.currentIndex++
rowItems := t.dataTable.allOriginalLines[t.currentIndex]
finalItems := t.dataTable.parseTransactionData(rowItems)
return &fireflyIIITransactionPlainTextDataRow{
dataTable: t.dataTable,
originalItems: rowItems,
finalItems: finalItems,
}
}
func (t *fireflyIIITransactionPlainTextDataTable) parseTransactionData(items []string) map[datatable.DataTableColumn]string {
data := make(map[datatable.DataTableColumn]string, 12)
data[datatable.DATA_TABLE_SUB_CATEGORY] = ""
for column, index := range t.originalColumnIndex {
if index >= 0 && index < len(items) {
data[column] = items[index]
}
}
// trim trailing zero in decimal
if data[datatable.DATA_TABLE_AMOUNT] != "" {
data[datatable.DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
}
if data[datatable.DATA_TABLE_RELATED_AMOUNT] != "" {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_RELATED_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_RELATED_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount)
}
} else {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT]
}
// the related account currency field is foreign currency in firefly iii actually
if data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY]
}
// the destination account of modify balance transaction in firefly iii is the asset account
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// the destination account of income transaction in firefly iii is the asset account
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
}
return data
}
func createNewFireflyIIITransactionPlainTextDataTable(ctx core.Context, reader io.Reader) (*fireflyIIITransactionPlainTextDataTable, error) {
allOriginalLines, err := parseAllLinesFromFireflyIIITransactionPlainText(ctx, reader)
if err != nil {
return nil, err
}
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
originalHeaderItems := allOriginalLines[0]
originalHeaderItemMap := make(map[string]int)
for i := 0; i < len(originalHeaderItems); i++ {
originalHeaderItemMap[originalHeaderItems[i]] = i
}
typeColumnIdx, typeColumnExists := originalHeaderItemMap["type"]
amountColumnIdx, amountColumnExists := originalHeaderItemMap["amount"]
foreignAmountColumnIdx, foreignAmountColumnExists := originalHeaderItemMap["foreign_amount"]
currencyColumnIdx, currencyColumnExists := originalHeaderItemMap["currency_code"]
foreignCurrencyColumnIdx, foreignCurrencyColumnExists := originalHeaderItemMap["foreign_currency_code"]
descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["description"]
dateColumnIdx, dateColumnExists := originalHeaderItemMap["date"]
sourceNameColumnIdx, sourceNameColumnExists := originalHeaderItemMap["source_name"]
destinationNameColumnIdx, destinationNameColumnExists := originalHeaderItemMap["destination_name"]
categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["category"]
tagsColumnIdx, tagsColumnExists := originalHeaderItemMap["tags"]
if !typeColumnExists || !amountColumnExists || !dateColumnExists || !sourceNameColumnExists || !destinationNameColumnExists || !categoryColumnExists {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse firefly III csv data, because missing essential columns in header row")
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
if !foreignAmountColumnExists {
foreignAmountColumnIdx = -1
}
if !currencyColumnExists {
currencyColumnIdx = -1
}
if !foreignCurrencyColumnExists {
foreignCurrencyColumnIdx = -1
}
if !descriptionColumnExists {
descriptionColumnIdx = -1
}
if !tagsColumnExists {
tagsColumnIdx = -1
}
return &fireflyIIITransactionPlainTextDataTable{
allOriginalLines: allOriginalLines,
originalHeaderLineColumnNames: originalHeaderItems,
originalColumnIndex: map[datatable.DataTableColumn]int{
datatable.DATA_TABLE_TRANSACTION_TIME: dateColumnIdx,
datatable.DATA_TABLE_TRANSACTION_TYPE: typeColumnIdx,
datatable.DATA_TABLE_SUB_CATEGORY: categoryColumnIdx,
datatable.DATA_TABLE_ACCOUNT_NAME: sourceNameColumnIdx,
datatable.DATA_TABLE_ACCOUNT_CURRENCY: currencyColumnIdx,
datatable.DATA_TABLE_AMOUNT: amountColumnIdx,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: destinationNameColumnIdx,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: foreignCurrencyColumnIdx,
datatable.DATA_TABLE_RELATED_AMOUNT: foreignAmountColumnIdx,
datatable.DATA_TABLE_TAGS: tagsColumnIdx,
datatable.DATA_TABLE_DESCRIPTION: descriptionColumnIdx,
},
}, nil
}
func parseAllLinesFromFireflyIIITransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) {
csvReader := csv.NewReader(reader)
csvReader.FieldsPerRecord = -1
allOriginalLines := make([][]string, 0)
for {
items, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.parseAllLinesFromFireflyIIITransactionPlainText] cannot parse firefly III csv data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
allOriginalLines = append(allOriginalLines, items)
}
return allOriginalLines, nil
}
@@ -0,0 +1,89 @@
package fireflyIII
import (
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
// fireflyIIITransactionDataRowParser defines the structure of firefly III transaction data row parser
type fireflyIIITransactionDataRowParser struct {
}
// GetAddedColumns returns the added columns after converting the data row
func (p *fireflyIIITransactionDataRowParser) GetAddedColumns() []datatable.TransactionDataTableColumn {
return []datatable.TransactionDataTableColumn{
datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE,
}
}
// Parse returns the converted transaction data row
func (p *fireflyIIITransactionDataRowParser) Parse(data map[datatable.TransactionDataTableColumn]string) (rowData map[datatable.TransactionDataTableColumn]string, rowDataValid bool, err error) {
rowData = make(map[datatable.TransactionDataTableColumn]string, len(data))
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
for column, value := range data {
rowData[column] = value
}
// parse long date time and timezone
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" {
dateTime, err := utils.ParseFromLongDateTimeWithTimezone(rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME])
if err != nil {
return nil, false, errs.ErrTransactionTimeInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location())
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location())
}
// trim trailing zero in decimal
if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] != "" {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
if err != nil {
return nil, false, errs.ErrAmountInvalid
}
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount)
} else {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]
}
// the related account currency field is foreign currency in firefly III actually
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY]
}
// the destination account of modify balance transaction in firefly III is the asset account
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// the destination account of income transaction in firefly III is the asset account
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]
}
return rowData, true, nil
}
// createFireflyIIITransactionDataRowParser returns firefly III transaction data row parser
func createFireflyIIITransactionDataRowParser() datatable.TransactionDataRowParser {
return &fireflyIIITransactionDataRowParser{}
}