import transaction from firefly iii

This commit is contained in:
MaysWind
2024-10-12 01:17:56 +08:00
parent f75e078fed
commit bd66408c3d
11 changed files with 448 additions and 2 deletions
@@ -0,0 +1,40 @@
package fireflyIII
import (
"bytes"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/models"
)
// fireflyiiiTransactionDataCsvImporter defines the structure of firefly III csv importer for transaction data
type fireflyIIITransactionDataCsvImporter struct{}
// Initialize a firefly III transaction data csv file importer singleton instance
var (
FireflyIIITransactionDataCsvImporter = &fireflyIIITransactionDataCsvImporter{}
)
// ParseImportedData returns the imported data by parsing the firefly iii transaction csv data
func (c *fireflyIIITransactionDataCsvImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
reader := bytes.NewReader(data)
dataTable, err := createNewFireflyIIITransactionPlainTextDataTable(
ctx,
reader,
)
if err != nil {
return nil, nil, nil, nil, nil, nil, err
}
dataTableImporter := datatable.CreateNewImporter(
dataTable.GetDataColumnMapping(),
fireflyIIITransactionTypeNameMapping,
"",
",",
)
return dataTableImporter.ParseImportedData(ctx, user, dataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
}
@@ -0,0 +1 @@
package fireflyIII
@@ -0,0 +1,311 @@
package fireflyIII
import (
"encoding/csv"
"io"
"time"
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
"github.com/mayswind/ezbookkeeping/pkg/core"
"github.com/mayswind/ezbookkeeping/pkg/errs"
"github.com/mayswind/ezbookkeeping/pkg/log"
"github.com/mayswind/ezbookkeeping/pkg/models"
"github.com/mayswind/ezbookkeeping/pkg/utils"
)
var fireflyIIITransactionSupportedColumns = []datatable.DataTableColumn{
datatable.DATA_TABLE_TRANSACTION_TIME,
datatable.DATA_TABLE_TRANSACTION_TYPE,
datatable.DATA_TABLE_SUB_CATEGORY,
datatable.DATA_TABLE_ACCOUNT_NAME,
datatable.DATA_TABLE_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_AMOUNT,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY,
datatable.DATA_TABLE_RELATED_AMOUNT,
datatable.DATA_TABLE_TAGS,
datatable.DATA_TABLE_DESCRIPTION,
}
var fireflyIIITransactionTypeNameMapping = map[models.TransactionType]string{
models.TRANSACTION_TYPE_MODIFY_BALANCE: "Opening balance",
models.TRANSACTION_TYPE_INCOME: "Deposit",
models.TRANSACTION_TYPE_EXPENSE: "Withdrawal",
models.TRANSACTION_TYPE_TRANSFER: "Transfer",
}
// fireflyIIITransactionPlainTextDataTable defines the structure of firefly III transaction plain text data table
type fireflyIIITransactionPlainTextDataTable struct {
allOriginalLines [][]string
originalHeaderLineColumnNames []string
originalColumnIndex map[datatable.DataTableColumn]int
}
// fireflyIIITransactionPlainTextDataRow defines the structure of firefly III transaction plain text data row
type fireflyIIITransactionPlainTextDataRow struct {
dataTable *fireflyIIITransactionPlainTextDataTable
originalItems []string
finalItems map[datatable.DataTableColumn]string
}
// fireflyIIITransactionPlainTextDataRowIterator defines the structure of firefly III transaction plain text data row iterator
type fireflyIIITransactionPlainTextDataRowIterator struct {
dataTable *fireflyIIITransactionPlainTextDataTable
currentIndex int
}
// DataRowCount returns the total count of data row
func (t *fireflyIIITransactionPlainTextDataTable) DataRowCount() int {
if len(t.allOriginalLines) < 1 {
return 0
}
return len(t.allOriginalLines) - 1
}
// GetDataColumnMapping returns data column map for data importer
func (t *fireflyIIITransactionPlainTextDataTable) GetDataColumnMapping() map[datatable.DataTableColumn]string {
dataColumnMapping := make(map[datatable.DataTableColumn]string, len(fireflyIIITransactionSupportedColumns))
for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
column := fireflyIIITransactionSupportedColumns[i]
if t.originalColumnIndex[column] < 0 {
continue
}
dataColumnMapping[column] = utils.IntToString(int(column))
}
return dataColumnMapping
}
// HeaderLineColumnNames returns the header column name list
func (t *fireflyIIITransactionPlainTextDataTable) HeaderLineColumnNames() []string {
columnIndexes := make([]string, len(fireflyIIITransactionSupportedColumns))
for i := 0; i < len(fireflyIIITransactionSupportedColumns); i++ {
column := fireflyIIITransactionSupportedColumns[i]
if t.originalColumnIndex[column] < 0 {
continue
}
columnIndexes[i] = utils.IntToString(int(column))
}
return columnIndexes
}
// DataRowIterator returns the iterator of data row
func (t *fireflyIIITransactionPlainTextDataTable) DataRowIterator() datatable.ImportedDataRowIterator {
return &fireflyIIITransactionPlainTextDataRowIterator{
dataTable: t,
currentIndex: 0,
}
}
// IsValid returns whether this row contains valid data for importing
func (r *fireflyIIITransactionPlainTextDataRow) IsValid() bool {
return true
}
// ColumnCount returns the total count of column in this data row
func (r *fireflyIIITransactionPlainTextDataRow) ColumnCount() int {
return len(r.finalItems)
}
// GetData returns the data in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetData(columnIndex int) string {
if columnIndex >= len(fireflyIIITransactionSupportedColumns) {
return ""
}
dataColumn := fireflyIIITransactionSupportedColumns[columnIndex]
return r.finalItems[dataColumn]
}
// GetTime returns the time in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetTime(columnIndex int, timezoneOffset int16) (time.Time, error) {
return utils.ParseFromLongDateTimeWithTimezone(r.GetData(columnIndex))
}
// GetTimezoneOffset returns the time zone offset in the specified column index
func (r *fireflyIIITransactionPlainTextDataRow) GetTimezoneOffset(columnIndex int) (*time.Location, error) {
return nil, errs.ErrNotSupported
}
// HasNext returns whether the iterator does not reach the end
func (t *fireflyIIITransactionPlainTextDataRowIterator) HasNext() bool {
return t.currentIndex+1 < len(t.dataTable.allOriginalLines)
}
// Next returns the next imported data row
func (t *fireflyIIITransactionPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) datatable.ImportedDataRow {
if t.currentIndex+1 >= len(t.dataTable.allOriginalLines) {
return nil
}
t.currentIndex++
rowItems := t.dataTable.allOriginalLines[t.currentIndex]
finalItems := t.dataTable.parseTransactionData(rowItems)
return &fireflyIIITransactionPlainTextDataRow{
dataTable: t.dataTable,
originalItems: rowItems,
finalItems: finalItems,
}
}
func (t *fireflyIIITransactionPlainTextDataTable) parseTransactionData(items []string) map[datatable.DataTableColumn]string {
data := make(map[datatable.DataTableColumn]string, 12)
data[datatable.DATA_TABLE_SUB_CATEGORY] = ""
for column, index := range t.originalColumnIndex {
data[column] = items[index]
}
// trim trailing zero in decimal
if data[datatable.DATA_TABLE_AMOUNT] != "" {
data[datatable.DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_AMOUNT] = utils.FormatAmount(-amount)
}
}
if data[datatable.DATA_TABLE_RELATED_AMOUNT] != "" {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(data[datatable.DATA_TABLE_RELATED_AMOUNT])
amount, err := utils.ParseAmount(data[datatable.DATA_TABLE_RELATED_AMOUNT])
if err == nil {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(-amount)
}
} else {
data[datatable.DATA_TABLE_RELATED_AMOUNT] = data[datatable.DATA_TABLE_AMOUNT]
}
// the related account currency field is foreign currency in firefly iii actually
if data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] == "" {
data[datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = data[datatable.DATA_TABLE_ACCOUNT_CURRENCY]
}
// the destination account of modify balance transaction in firefly iii is the asset account
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_MODIFY_BALANCE] {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
}
// the destination account of income transaction in firefly iii is the asset account
if data[datatable.DATA_TABLE_TRANSACTION_TYPE] == fireflyIIITransactionTypeNameMapping[models.TRANSACTION_TYPE_INCOME] {
data[datatable.DATA_TABLE_ACCOUNT_NAME] = data[datatable.DATA_TABLE_RELATED_ACCOUNT_NAME]
}
return data
}
func createNewFireflyIIITransactionPlainTextDataTable(ctx core.Context, reader io.Reader) (*fireflyIIITransactionPlainTextDataTable, error) {
allOriginalLines, err := parseAllLinesFromFireflyIIITransactionPlainText(ctx, reader)
if err != nil {
return nil, err
}
if len(allOriginalLines) < 2 {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse import data, because data table row count is less 1")
return nil, errs.ErrNotFoundTransactionDataInFile
}
originalHeaderItems := allOriginalLines[0]
originalHeaderItemMap := make(map[string]int)
for i := 0; i < len(originalHeaderItems); i++ {
originalHeaderItemMap[originalHeaderItems[i]] = i
}
typeColumnIdx, typeColumnExists := originalHeaderItemMap["type"]
amountColumnIdx, amountColumnExists := originalHeaderItemMap["amount"]
foreignAmountColumnIdx, foreignAmountColumnExists := originalHeaderItemMap["foreign_amount"]
currencyColumnIdx, currencyColumnExists := originalHeaderItemMap["currency_code"]
foreignCurrencyColumnIdx, foreignCurrencyColumnExists := originalHeaderItemMap["foreign_currency_code"]
descriptionColumnIdx, descriptionColumnExists := originalHeaderItemMap["description"]
dateColumnIdx, dateColumnExists := originalHeaderItemMap["date"]
sourceNameColumnIdx, sourceNameColumnExists := originalHeaderItemMap["source_name"]
destinationNameColumnIdx, destinationNameColumnExists := originalHeaderItemMap["destination_name"]
categoryColumnIdx, categoryColumnExists := originalHeaderItemMap["category"]
tagsColumnIdx, tagsColumnExists := originalHeaderItemMap["tags"]
if !typeColumnExists || !amountColumnExists || !dateColumnExists || !sourceNameColumnExists || !destinationNameColumnExists {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.createNewFireflyIIITransactionPlainTextDataTable] cannot parse firefly III csv data, because missing essential columns in header row")
return nil, errs.ErrMissingRequiredFieldInHeaderRow
}
if !foreignAmountColumnExists {
foreignAmountColumnIdx = -1
}
if !currencyColumnExists {
currencyColumnIdx = -1
}
if !foreignCurrencyColumnExists {
foreignCurrencyColumnIdx = -1
}
if !descriptionColumnExists {
descriptionColumnIdx = -1
}
if !categoryColumnExists {
categoryColumnIdx = -1
}
if !tagsColumnExists {
tagsColumnIdx = -1
}
return &fireflyIIITransactionPlainTextDataTable{
allOriginalLines: allOriginalLines,
originalHeaderLineColumnNames: originalHeaderItems,
originalColumnIndex: map[datatable.DataTableColumn]int{
datatable.DATA_TABLE_TRANSACTION_TIME: dateColumnIdx,
datatable.DATA_TABLE_TRANSACTION_TYPE: typeColumnIdx,
datatable.DATA_TABLE_SUB_CATEGORY: categoryColumnIdx,
datatable.DATA_TABLE_ACCOUNT_NAME: sourceNameColumnIdx,
datatable.DATA_TABLE_ACCOUNT_CURRENCY: currencyColumnIdx,
datatable.DATA_TABLE_AMOUNT: amountColumnIdx,
datatable.DATA_TABLE_RELATED_ACCOUNT_NAME: destinationNameColumnIdx,
datatable.DATA_TABLE_RELATED_ACCOUNT_CURRENCY: foreignCurrencyColumnIdx,
datatable.DATA_TABLE_RELATED_AMOUNT: foreignAmountColumnIdx,
datatable.DATA_TABLE_TAGS: tagsColumnIdx,
datatable.DATA_TABLE_DESCRIPTION: descriptionColumnIdx,
},
}, nil
}
func parseAllLinesFromFireflyIIITransactionPlainText(ctx core.Context, reader io.Reader) ([][]string, error) {
csvReader := csv.NewReader(reader)
csvReader.FieldsPerRecord = -1
allOriginalLines := make([][]string, 0)
for {
items, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Errorf(ctx, "[fireflyiii_transaction_data_plain_text_data_table.parseAllLinesFromFireflyIIITransactionPlainText] cannot parse firefly III csv data, because %s", err.Error())
return nil, errs.ErrInvalidCSVFile
}
allOriginalLines = append(allOriginalLines, items)
}
return allOriginalLines, nil
}
@@ -5,6 +5,7 @@ import (
"github.com/mayswind/ezbookkeeping/pkg/converters/base"
"github.com/mayswind/ezbookkeeping/pkg/converters/default"
"github.com/mayswind/ezbookkeeping/pkg/converters/feidee"
"github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII"
"github.com/mayswind/ezbookkeeping/pkg/errs"
)
@@ -25,6 +26,8 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter,
return _default.EzBookKeepingTransactionDataCSVFileConverter, nil
} else if fileType == "ezbookkeeping_tsv" {
return _default.EzBookKeepingTransactionDataTSVFileConverter, nil
} else if fileType == "firefly_iii_csv" {
return fireflyIII.FireflyIIITransactionDataCsvImporter, nil
} else if fileType == "feidee_mymoney_csv" {
return feidee.FeideeMymoneyTransactionDataCsvImporter, nil
} else if fileType == "feidee_mymoney_xls" {
+6
View File
@@ -10,6 +10,7 @@ import (
const (
longDateTimeFormat = "2006-01-02 15:04:05"
longDateTimeWithTimezoneFormat = "2006-01-02T15:04:05Z07:00"
longDateTimeWithoutSecondFormat = "2006-01-02 15:04"
shortDateTimeFormat = "2006-1-2 15:4:5"
yearMonthDateTimeFormat = "2006-01"
@@ -135,6 +136,11 @@ func ParseFromLongDateTime(t string, utcOffset int16) (time.Time, error) {
return time.ParseInLocation(longDateTimeFormat, t, timezone)
}
// ParseFromLongDateTimeWithTimezone parses a formatted string in long date time format
func ParseFromLongDateTimeWithTimezone(t string) (time.Time, error) {
return time.Parse(longDateTimeWithTimezoneFormat, t)
}
// ParseFromLongDateTimeWithoutSecond parses a formatted string in long date time format (no second)
func ParseFromLongDateTimeWithoutSecond(t string, utcOffset int16) (time.Time, error) {
timezone := time.FixedZone("Timezone", int(utcOffset)*60)
+9
View File
@@ -131,6 +131,15 @@ func TestParseFromLongDateTime(t *testing.T) {
assert.Equal(t, expectedValue, actualValue)
}
func TestParseFromLongDateTimeWithTimezone(t *testing.T) {
expectedValue := int64(1617238883)
actualTime, err := ParseFromLongDateTimeWithTimezone("2021-04-01T06:01:23+05:00")
assert.Equal(t, nil, err)
actualValue := actualTime.Unix()
assert.Equal(t, expectedValue, actualValue)
}
func TestParseFromLongDateTimeWithoutSecond(t *testing.T) {
expectedValue := int64(1691947440)
actualTime, err := ParseFromLongDateTimeWithoutSecond("2023-08-13 17:24", 0)
+30
View File
@@ -3,6 +3,7 @@ package utils
import (
"crypto/rand"
"math/big"
"strings"
)
// GetRandomInteger returns a random number, the max parameter represents upper limit
@@ -15,3 +16,32 @@ func GetRandomInteger(max int) (int, error) {
return int(result.Int64()), nil
}
// TrimTrailingZerosInDecimal returns a textual number without trailing zeros in decimal
func TrimTrailingZerosInDecimal(num string) string {
if len(num) < 1 {
return num
}
dotPosition := strings.Index(num, ".")
if dotPosition < 0 {
return num
}
lastNonZeroPosition := len(num)
for i := len(num) - 1; i > dotPosition+1; i-- {
if num[i] == '0' {
lastNonZeroPosition = i
} else {
break
}
}
if lastNonZeroPosition >= len(num) {
return num
}
return num[0:lastNonZeroPosition]
}
+33
View File
@@ -0,0 +1,33 @@
package utils
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestTrimTrailingZerosInDecimal(t *testing.T) {
expectedValue := "123.45"
actualValue := TrimTrailingZerosInDecimal("123.45000000000")
assert.Equal(t, expectedValue, actualValue)
expectedValue = "0.12"
actualValue = TrimTrailingZerosInDecimal("0.12000000000")
assert.Equal(t, expectedValue, actualValue)
expectedValue = "0.120000000001"
actualValue = TrimTrailingZerosInDecimal("0.120000000001")
assert.Equal(t, expectedValue, actualValue)
expectedValue = ".12"
actualValue = TrimTrailingZerosInDecimal(".12000000000")
assert.Equal(t, expectedValue, actualValue)
expectedValue = "12345000000000"
actualValue = TrimTrailingZerosInDecimal("12345000000000")
assert.Equal(t, expectedValue, actualValue)
expectedValue = ""
actualValue = TrimTrailingZerosInDecimal("")
assert.Equal(t, expectedValue, actualValue)
}
+9
View File
@@ -19,6 +19,15 @@ const supportedImportFileTypes = [
anchor: 'export-transactions'
}
},
{
type: 'firefly_iii_csv',
name: 'Firefly III Data Export File',
extensions: '.csv',
document: {
supportMultiLanguages: true,
anchor: 'how-to-get-firefly-iii-data-export-file'
}
},
{
type: 'feidee_mymoney_csv',
name: 'Feidee MyMoney (App) Data Export File',
+3 -1
View File
@@ -1188,7 +1188,8 @@
"document": {
"anchor": {
"export_and_import": {
"export-transactions": "export-transactions"
"export-transactions": "export-transactions",
"how-to-get-firefly-iii-data-export-file": "how-to-get-firefly-iii-data-export-file"
}
}
},
@@ -1516,6 +1517,7 @@
"How to export this file?": "How to export this file?",
"ezbookkeeping Data Export File (CSV)": "ezbookkeeping Data Export File (CSV)",
"ezbookkeeping Data Export File (TSV)": "ezbookkeeping Data Export File (TSV)",
"Firefly III Data Export File": "Firefly III Data Export File",
"Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App) Data Export File",
"Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web) Data Export File",
"Alipay (App) Data Export File": "Alipay (App) Data Export File",
+3 -1
View File
@@ -1188,7 +1188,8 @@
"document": {
"anchor": {
"export_and_import": {
"export-transactions": "导出交易"
"export-transactions": "导出交易",
"how-to-get-firefly-iii-data-export-file": "如何获取firefly-iii数据导出文件"
}
}
},
@@ -1516,6 +1517,7 @@
"How to export this file?": "如何导出该文件?",
"ezbookkeeping Data Export File (CSV)": "ezbookkeeping 数据导出文件 (CSV)",
"ezbookkeeping Data Export File (TSV)": "ezbookkeeping 数据导出文件 (TSV)",
"Firefly III Data Export File": "Firefly III 数据导出文件",
"Feidee MyMoney (App) Data Export File": "金蝶随手记 (App) 数据导出文件",
"Feidee MyMoney (Web) Data Export File": "金蝶随手记 (Web版) 数据导出文件",
"Alipay (App) Data Export File": "支付宝 (App) 数据导出文件",