diff --git a/README.md b/README.md index 77a30532..6dd45fc3 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Online Demo: [https://ezbookkeeping-demo.mayswind.net](https://ezbookkeeping-dem 7. Multi-language support 8. Two-factor authentication 9. Application lock (PIN code / WebAuthn) -10. Data export & import (OFX, QFX, QIF, IIF, CSV, GnuCash, FireFly III, etc.) +10. Data export & import (OFX, QFX, QIF, IIF, CSV, GnuCash, FireFly III, Beancount, etc.) ## Screenshots ### Desktop Version diff --git a/pkg/converters/beancount/beancount_amount_expression_evaluator.go b/pkg/converters/beancount/beancount_amount_expression_evaluator.go new file mode 100644 index 00000000..c7ccea3b --- /dev/null +++ b/pkg/converters/beancount/beancount_amount_expression_evaluator.go @@ -0,0 +1,197 @@ +package beancount + +import ( + "fmt" + "strconv" + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" +) + +var operatorPriority = map[rune]int{ + '+': 1, + '-': 1, + '*': 2, + '/': 2, +} + +func toPostfixExprTokens(ctx core.Context, expr string) ([]string, error) { + finalTokens := make([]string, 0) + operatorStack := make([]rune, 0) + currentNumberBuilder := strings.Builder{} + isLastTokenOperator := true + + expr = strings.ReplaceAll(expr, " ", "") + + for i := 0; i < len(expr); i++ { + ch := rune(expr[i]) + + // number + if '0' <= ch && ch <= '9' || ch == '.' { + currentNumberBuilder.WriteRune(ch) + continue + } else if ch == '-' && i+1 < len(expr) && '0' <= expr[i+1] && expr[i+1] <= '9' && currentNumberBuilder.Len() == 0 && isLastTokenOperator { + currentNumberBuilder.WriteRune(ch) + continue + } + + // operator or parenthesis + if currentNumberBuilder.Len() > 0 { + finalTokens = append(finalTokens, currentNumberBuilder.String()) + currentNumberBuilder.Reset() + isLastTokenOperator = false + } + + switch ch { + case '+', '-', '*', '/': + if ch == '-' && isLastTokenOperator { + currentNumberBuilder.WriteRune(ch) + continue + } + + for len(operatorStack) > 0 { + topOperator := operatorStack[len(operatorStack)-1] + + if topOperator == '(' { + break + } + + if operatorPriority[topOperator] >= operatorPriority[ch] { + finalTokens = append(finalTokens, string(topOperator)) + operatorStack = operatorStack[:len(operatorStack)-1] + } else { + break + } + } + + operatorStack = append(operatorStack, ch) + isLastTokenOperator = true + case '(': + operatorStack = append(operatorStack, ch) + isLastTokenOperator = true + case ')': + hasLeftParenthesis := false + + for len(operatorStack) > 0 { + topOperator := operatorStack[len(operatorStack)-1] + operatorStack = operatorStack[:len(operatorStack)-1] + + if topOperator == '(' { + hasLeftParenthesis = true + break + } + + finalTokens = append(finalTokens, string(topOperator)) + } + + if !hasLeftParenthesis { + log.Warnf(ctx, "[beancount_amount_expression_evaluator.toPostfixExprTokens] cannot parse expression \"%s\", because missing left parenthesis", expr) + return nil, errs.ErrInvalidAmountExpression + } + + isLastTokenOperator = false + default: + log.Warnf(ctx, "[beancount_amount_expression_evaluator.toPostfixExprTokens] cannot parse expression \"%s\", because containing unknown token \"%c\"", expr, ch) + return nil, errs.ErrInvalidAmountExpression + } + } + + if currentNumberBuilder.Len() > 0 { + finalTokens = append(finalTokens, currentNumberBuilder.String()) + } + + for len(operatorStack) > 0 { + topOperator := operatorStack[len(operatorStack)-1] + operatorStack = operatorStack[:len(operatorStack)-1] + + if topOperator == '(' { + log.Warnf(ctx, "[beancount_amount_expression_evaluator.toPostfixExprTokens] cannot parse expression \"%s\", because missing right parenthesis", expr) + return nil, errs.ErrInvalidAmountExpression + } + + finalTokens = append(finalTokens, string(topOperator)) + } + + return finalTokens, nil +} + +func evaluatePostfixExpr(ctx core.Context, tokens []string) (float64, error) { + stack := make([]float64, 0) + + for i := 0; i < len(tokens); i++ { + token := tokens[i] + + switch token { + case "+", "-", "*", "/": // operators + if len(stack) < 2 { + log.Warnf(ctx, "[beancount_amount_expression_evaluator.evaluatePostfixExpr] cannot evaluate expression \"%s\", because not enough operands", strings.Join(tokens, " ")) + return 0, errs.ErrInvalidAmountExpression + } + + // pop the top two operands + b := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + a := stack[len(stack)-1] + stack = stack[:len(stack)-1] + + // evaluate the operation + var result float64 + switch token { + case "+": + result = a + b + case "-": + result = a - b + case "*": + result = a * b + case "/": + if b == 0 { + log.Warnf(ctx, "[beancount_amount_expression_evaluator.evaluatePostfixExpr] cannot evaluate expression \"%s\", because division by zero", strings.Join(tokens, " ")) + return 0, errs.ErrInvalidAmountExpression + } + result = a / b + } + + // push the result back to the stack + stack = append(stack, result) + default: // operands + num, err := strconv.ParseFloat(token, 64) + + if err != nil { + log.Warnf(ctx, "[beancount_amount_expression_evaluator.evaluatePostfixExpr] cannot evaluate expression \"%s\", because containing invalid number", strings.Join(tokens, " ")) + return 0, errs.ErrInvalidAmountExpression + } + + stack = append(stack, num) + } + } + + if len(stack) != 1 { + log.Warnf(ctx, "[beancount_amount_expression_evaluator.evaluatePostfixExpr] cannot evaluate expression \"%s\", because missing operator", strings.Join(tokens, " ")) + return 0, errs.ErrInvalidAmountExpression + } + + return stack[0], nil +} + +func evaluateBeancountAmountExpression(ctx core.Context, expr string) (string, error) { + if expr == "" { + return "", nil + } + + postfixExprTokens, err := toPostfixExprTokens(ctx, expr) + + if err != nil { + return "", err + } + + result, err := evaluatePostfixExpr(ctx, postfixExprTokens) + + if err != nil { + return "", err + } + + return fmt.Sprintf("%.2f", result), nil +} diff --git a/pkg/converters/beancount/beancount_amount_expression_evaluator_test.go b/pkg/converters/beancount/beancount_amount_expression_evaluator_test.go new file mode 100644 index 00000000..c49acaea --- /dev/null +++ b/pkg/converters/beancount/beancount_amount_expression_evaluator_test.go @@ -0,0 +1,216 @@ +package beancount + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" +) + +func TestToPostfixExprTokens_ValidExpression(t *testing.T) { + context := core.NewNullContext() + + result, err := toPostfixExprTokens(context, "1+2") + assert.Nil(t, err) + assert.Equal(t, []string{"1", "2", "+"}, result) + + result, err = toPostfixExprTokens(context, "3-4") + assert.Nil(t, err) + assert.Equal(t, []string{"3", "4", "-"}, result) + + result, err = toPostfixExprTokens(context, "5*6") + assert.Nil(t, err) + assert.Equal(t, []string{"5", "6", "*"}, result) + + result, err = toPostfixExprTokens(context, "8/2") + assert.Nil(t, err) + assert.Equal(t, []string{"8", "2", "/"}, result) + + result, err = toPostfixExprTokens(context, "1+2*3-(4/2)") + assert.Nil(t, err) + assert.Equal(t, []string{"1", "2", "3", "*", "+", "4", "2", "/", "-"}, result) + + result, err = toPostfixExprTokens(context, "1 + 2 * 3") + assert.Nil(t, err) + assert.Equal(t, []string{"1", "2", "3", "*", "+"}, result) + + result, err = toPostfixExprTokens(context, "-1+2") + assert.Nil(t, err) + assert.Equal(t, []string{"-1", "2", "+"}, result) + + result, err = toPostfixExprTokens(context, "1.5+2.3") + assert.Nil(t, err) + assert.Equal(t, []string{"1.5", "2.3", "+"}, result) + + result, err = toPostfixExprTokens(context, "(1+2)-3") + assert.Nil(t, err) + assert.Equal(t, []string{"1", "2", "+", "3", "-"}, result) + + result, err = toPostfixExprTokens(context, "2*-3-3/-2") + assert.Nil(t, err) + assert.Equal(t, []string{"2", "-3", "*", "3", "-2", "/", "-"}, result) + + result, err = toPostfixExprTokens(context, "-1.2-3.4*(-5.6/7.8*(9.0-1.2))") + assert.Nil(t, err) + assert.Equal(t, []string{"-1.2", "3.4", "-5.6", "7.8", "/", "9.0", "1.2", "-", "*", "*", "-"}, result) + + result, err = toPostfixExprTokens(context, "((((((1+2)*(3+4))))))") + assert.Nil(t, err) + assert.Equal(t, []string{"1", "2", "+", "3", "4", "+", "*"}, result) + + result, err = toPostfixExprTokens(context, "(((())))") + assert.Nil(t, err) + assert.Equal(t, []string{}, result) + + result, err = toPostfixExprTokens(context, "+-*/") + assert.Nil(t, err) + assert.Equal(t, []string{"-", "*", "/", "+"}, result) + + result, err = toPostfixExprTokens(context, "") + assert.Nil(t, err) + assert.Equal(t, []string{}, result) +} + +func TestToPostfixExprTokens_InvalidExpression(t *testing.T) { + context := core.NewNullContext() + + _, err := toPostfixExprTokens(context, "1=2") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = toPostfixExprTokens(context, "(1") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = toPostfixExprTokens(context, "2)") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = toPostfixExprTokens(context, "((((1+2)))") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = toPostfixExprTokens(context, ")(") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) +} + +func TestEvaluatePostfixExpr_ValidExpression(t *testing.T) { + context := core.NewNullContext() + + result, err := evaluatePostfixExpr(context, []string{"1", "2", "+"}) + assert.Nil(t, err) + assert.Equal(t, float64(3), result) + + result, err = evaluatePostfixExpr(context, []string{"5", "3", "-"}) + assert.Nil(t, err) + assert.Equal(t, float64(2), result) + + result, err = evaluatePostfixExpr(context, []string{"4", "3", "*"}) + assert.Nil(t, err) + assert.Equal(t, float64(12), result) + + result, err = evaluatePostfixExpr(context, []string{"6", "2", "/"}) + assert.Nil(t, err) + assert.Equal(t, float64(3), result) + + result, err = evaluatePostfixExpr(context, []string{"1", "2", "3", "*", "+", "4", "2", "/", "-"}) + assert.Nil(t, err) + assert.Equal(t, float64(5), result) +} + +func TestEvaluatePostfixExpr_InvalidExpression(t *testing.T) { + context := core.NewNullContext() + + _, err := evaluatePostfixExpr(context, []string{"1", "0", "/"}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluatePostfixExpr(context, []string{"1", "+"}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluatePostfixExpr(context, []string{"1", "="}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluatePostfixExpr(context, []string{"1", "("}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluatePostfixExpr(context, []string{"1", ")"}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluatePostfixExpr(context, []string{"1", "2", "+", "3"}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluatePostfixExpr(context, []string{"abc"}) + assert.Equal(t, errs.ErrInvalidAmountExpression, err) +} + +func TestEvaluateBeancountAmountExpression_ValidExpression(t *testing.T) { + context := core.NewNullContext() + + result, err := evaluateBeancountAmountExpression(context, "") + assert.Nil(t, err) + assert.Equal(t, "", result) + + result, err = evaluateBeancountAmountExpression(context, "1+2") + assert.Nil(t, err) + assert.Equal(t, "3.00", result) + + result, err = evaluateBeancountAmountExpression(context, "(1+2)*3") + assert.Nil(t, err) + assert.Equal(t, "9.00", result) + + result, err = evaluateBeancountAmountExpression(context, "-1+2") + assert.Nil(t, err) + assert.Equal(t, "1.00", result) + + result, err = evaluateBeancountAmountExpression(context, "1.5+2.5") + assert.Nil(t, err) + assert.Equal(t, "4.00", result) + + result, err = evaluateBeancountAmountExpression(context, "1+2*3-(4/2)") + assert.Nil(t, err) + assert.Equal(t, "5.00", result) + + result, err = evaluateBeancountAmountExpression(context, "2*-3-3/-2") + assert.Nil(t, err) + assert.Equal(t, "-4.50", result) + + result, err = evaluateBeancountAmountExpression(context, "-1.2-3.4*(-5.6/7.8*(9.0-1.2))") + assert.Nil(t, err) + assert.Equal(t, "17.84", result) + + result, err = evaluateBeancountAmountExpression(context, "(((2+3)))*(((((-5+7)))))") + assert.Nil(t, err) + assert.Equal(t, "10.00", result) +} + +func TestEvaluateBeancountAmountExpression_InvalidExpression(t *testing.T) { + context := core.NewNullContext() + + _, err := evaluateBeancountAmountExpression(context, "1++2") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1^2") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "+-*/") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "a+b") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1/0") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1+(2*3") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1+2*3)") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1+((((2*3)))") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1+2(3)") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) + + _, err = evaluateBeancountAmountExpression(context, "1)*(2") + assert.Equal(t, errs.ErrInvalidAmountExpression, err) +} diff --git a/pkg/converters/beancount/beancount_data.go b/pkg/converters/beancount/beancount_data.go new file mode 100644 index 00000000..da03c5e6 --- /dev/null +++ b/pkg/converters/beancount/beancount_data.go @@ -0,0 +1,93 @@ +package beancount + +import "strings" + +const beancountEquityAccountNameOpeningBalance = "Opening-Balances" + +// beancountDirective represents the Beancount directive +type beancountDirective string + +// Beancount directives +const ( + beancountDirectiveOpen beancountDirective = "open" + beancountDirectiveClose beancountDirective = "close" + beancountDirectiveTransaction beancountDirective = "txn" + beancountDirectiveCompletedTransaction beancountDirective = "*" + beancountDirectiveInCompleteTransaction beancountDirective = "!" + beancountDirectivePaddingTransaction beancountDirective = "P" + beancountDirectiveCommodity beancountDirective = "commodity" + beancountDirectivePrice beancountDirective = "price" + beancountDirectiveNote beancountDirective = "note" + beancountDirectiveDocument beancountDirective = "document" + beancountDirectiveEvent beancountDirective = "event" + beancountDirectiveBalance beancountDirective = "balance" + beancountDirectivePad beancountDirective = "pad" + beancountDirectiveQuery beancountDirective = "query" + beancountDirectiveCustom beancountDirective = "custom" +) + +// beancountAccountType represents the Beancount account type +type beancountAccountType byte + +// Beancount account types +const ( + beancountUnknownAccountType beancountAccountType = 0 + beancountAssetsAccountType beancountAccountType = 1 + beancountLiabilitiesAccountType beancountAccountType = 2 + beancountEquityAccountType beancountAccountType = 3 + beancountIncomeAccountType beancountAccountType = 4 + beancountExpensesAccountType beancountAccountType = 5 +) + +// beancountData defines the structure of beancount data +type beancountData struct { + accounts map[string]*beancountAccount + transactions []*beancountTransactionEntry +} + +// beancountAccount defines the structure of beancount account +type beancountAccount struct { + name string + accountType beancountAccountType + openDate string + closeDate string +} + +// beancountTransactionEntry defines the structure of beancount transaction entry +type beancountTransactionEntry struct { + date string + directive beancountDirective + payee string + narration string + postings []*beancountPosting + tags []string + links []string + metadata map[string]string +} + +// beancountPosting defines the structure of beancount transaction posting +type beancountPosting struct { + account string + amount string + originalAmount string + commodity string + totalCost string + totalCostCommodity string + price string + priceCommodity string + metadata map[string]string +} + +func (a *beancountAccount) isOpeningBalanceEquityAccount() bool { + if a.accountType != beancountEquityAccountType { + return false + } + + nameItems := strings.Split(a.name, string(beancountMetadataKeySuffix)) + + if len(nameItems) != 2 { + return false + } + + return nameItems[1] == beancountEquityAccountNameOpeningBalance +} diff --git a/pkg/converters/beancount/beancount_data_reader.go b/pkg/converters/beancount/beancount_data_reader.go new file mode 100644 index 00000000..105c51eb --- /dev/null +++ b/pkg/converters/beancount/beancount_data_reader.go @@ -0,0 +1,655 @@ +package beancount + +import ( + "bytes" + "encoding/csv" + "io" + "strings" + + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +const beancountDefaultAssetsAccountTypeName = "Assets" +const beancountDefaultLiabilitiesAccountTypeName = "Liabilities" +const beancountDefaultEquityAccountTypeName = "Equity" +const beancountDefaultIncomeAccountTypeName = "Income" +const beancountDefaultExpenseAccountTypeName = "Expenses" + +const beancountOptionAssetsAccountTypeName = "name_assets" +const beancountOptionLiabilitiesAccountTypeName = "name_liabilities" +const beancountOptionEquityAccountTypeName = "name_equity" +const beancountOptionIncomeAccountTypeName = "name_income" +const beancountOptionExpenseAccountTypeName = "name_expenses" + +const beancountCommentPrefix = ';' +const beancountAccountNameItemsSeparator = ":" +const beancountMetadataKeySuffix = ':' +const beancountPricePrefix = '@' +const beancountLinkPrefix = '^' +const beancountTagPrefix = '#' + +// beancountDataReader defines the structure of Beancount data reader +type beancountDataReader struct { + accountTypeNameMap map[string]beancountAccountType + accountTypeNameReversedMap map[beancountAccountType]string + allData [][]string +} + +// read returns the imported Beancount data +// Reference: https://beancount.github.io/docs/beancount_language_syntax.html +func (r *beancountDataReader) read(ctx core.Context) (*beancountData, error) { + if len(r.allData) < 1 { + return nil, errs.ErrNotFoundTransactionDataInFile + } + + data := &beancountData{ + accounts: make(map[string]*beancountAccount), + transactions: make([]*beancountTransactionEntry, 0), + } + + var err error + var currentTransactionEntry *beancountTransactionEntry + var currentTransactionPosting *beancountPosting + var currentTags []string + + for i := 0; i < len(r.allData); i++ { + items := r.allData[i] + + if len(items) == 0 || (len(items) == 1 && len(items[0]) == 0) || (len(r.getNotEmptyItemByIndex(items, 0)) > 0 && r.getNotEmptyItemByIndex(items, 0)[0] == beancountCommentPrefix) { // skip empty or comment lines + continue + } + + if r.getNotEmptyItemsCount(items) < 2 { + log.Warnf(ctx, "[beancount_data_reader.read] cannot parse line#%d \"%s\", because not enough items in line", i, strings.Join(items, " ")) + continue + } + + firstItem := items[0] + + if firstItem == "include" { // not support include directive + return nil, errs.ErrBeancountFileNotSupportInclude + } else if firstItem == "plugin" { // skip plugin directive lines + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + continue + } else if firstItem == "option" { + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + r.readAndSetOption(ctx, i, items) + continue + } else if firstItem == "pushtag" { + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + currentTags = r.readAndSetTags(ctx, i, items, currentTags, true) + continue + } else if firstItem == "poptag" { + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + currentTags = r.readAndSetTags(ctx, i, items, currentTags, false) + continue + } + + if len(firstItem) == 0 { // original line has space prefix, maybe transaction posting or metadata line + actualFirstItem := r.getNotEmptyItemByIndex(items, 0) + + if len(actualFirstItem) == 0 { // skip empty lines + continue + } + + if ('A' <= actualFirstItem[0] && actualFirstItem[0] <= 'Z') || actualFirstItem[0] == '!' { // transaction posting + if currentTransactionEntry != nil && currentTransactionPosting != nil { + currentTransactionEntry.postings = append(currentTransactionEntry.postings, currentTransactionPosting) + currentTransactionPosting = nil + } + + currentTransactionPosting, err = r.readTransactionPostingLine(ctx, i, items, data, actualFirstItem[0] == '!') + + if err != nil { + return nil, err + } + } else if 'a' <= actualFirstItem[0] && actualFirstItem[0] <= 'z' { // metadata + metadata := r.readTransactionMetadataLine(ctx, i, items) + + if metadata == nil { + continue + } + + metadataKey := metadata[0] + metadataValue := metadata[1] + + if currentTransactionPosting != nil { + if _, exists := currentTransactionPosting.metadata[metadataKey]; !exists { + currentTransactionPosting.metadata[metadataKey] = metadataValue + } + } else if currentTransactionEntry != nil { + if _, exists := currentTransactionEntry.metadata[metadataKey]; !exists { + currentTransactionEntry.metadata[metadataKey] = metadataValue + } + } + } else { + log.Warnf(ctx, "[beancount_data_reader.read] cannot parse line#%d \"%s\", because line prefix is invalid", i, strings.Join(items, " ")) + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + continue + } + } else if _, err := utils.ParseFromLongDateFirstTime(firstItem, 0); err == nil { // original line has date as first item + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + + directive := r.getNotEmptyItemByIndex(items, 1) + + if directive == string(beancountDirectiveOpen) || + directive == string(beancountDirectiveClose) { + _, err := r.readAccountLine(ctx, i, items, firstItem, beancountDirective(directive), data) + + if err != nil { + return nil, err + } + } else if directive == string(beancountDirectiveTransaction) || + directive == string(beancountDirectiveCompletedTransaction) || + directive == string(beancountDirectiveInCompleteTransaction) || + directive == string(beancountDirectivePaddingTransaction) { + currentTransactionEntry = r.readTransactionLine(ctx, i, items, firstItem, beancountDirective(directive), currentTags) + } else if directive == string(beancountDirectiveCommodity) || + directive == string(beancountDirectivePrice) || + directive == string(beancountDirectiveNote) || + directive == string(beancountDirectiveDocument) || + directive == string(beancountDirectiveEvent) || + directive == string(beancountDirectiveBalance) || + directive == string(beancountDirectivePad) || + directive == string(beancountDirectiveQuery) || + directive == string(beancountDirectiveCustom) { // skip commodity / price / note / document / event / balance / pad / query / custom lines + continue + } else { + log.Warnf(ctx, "[beancount_data_reader.read] cannot parse line#%d \"%s\", because directive is unknown", i, strings.Join(items, " ")) + continue + } + } else { // first item not start with date or space + currentTransactionEntry, currentTransactionPosting = r.updateCurrentState(data, currentTransactionEntry, currentTransactionPosting) + continue + } + } + + if currentTransactionEntry != nil { + if currentTransactionPosting != nil { + currentTransactionEntry.postings = append(currentTransactionEntry.postings, currentTransactionPosting) + currentTransactionPosting = nil + } + + data.transactions = append(data.transactions, currentTransactionEntry) + currentTransactionEntry = nil + } + + return data, nil +} + +func (r *beancountDataReader) updateCurrentState(data *beancountData, currentTransactionEntry *beancountTransactionEntry, currentTransactionPosting *beancountPosting) (*beancountTransactionEntry, *beancountPosting) { + if currentTransactionEntry != nil { + if currentTransactionPosting != nil { + currentTransactionEntry.postings = append(currentTransactionEntry.postings, currentTransactionPosting) + currentTransactionPosting = nil + } + + data.transactions = append(data.transactions, currentTransactionEntry) + currentTransactionEntry = nil + currentTransactionPosting = nil + } + + return currentTransactionEntry, currentTransactionPosting +} + +func (r *beancountDataReader) readAndSetOption(ctx core.Context, lineIndex int, items []string) { + if r.getNotEmptyItemsCount(items) != 3 { + log.Warnf(ctx, "[beancount_data_reader.readAndSetOption] cannot parse account type name option line#%d \"%s\", because items count in line not correct", lineIndex, strings.Join(items, " ")) + return + } + + optionName := r.getNotEmptyItemByIndex(items, 1) + optionValue := r.getNotEmptyItemByIndex(items, 2) + + switch optionName { + case beancountOptionAssetsAccountTypeName: + r.setAccountTypeNameMap(beancountAssetsAccountType, optionValue) + break + case beancountOptionLiabilitiesAccountTypeName: + r.setAccountTypeNameMap(beancountLiabilitiesAccountType, optionValue) + break + case beancountOptionEquityAccountTypeName: + r.setAccountTypeNameMap(beancountEquityAccountType, optionValue) + break + case beancountOptionIncomeAccountTypeName: + r.setAccountTypeNameMap(beancountIncomeAccountType, optionValue) + break + case beancountOptionExpenseAccountTypeName: + r.setAccountTypeNameMap(beancountExpensesAccountType, optionValue) + break + default: + log.Warnf(ctx, "[beancount_data_reader.readAndSetOption] skip option line#%d \"%s\"", lineIndex, strings.Join(items, " ")) + break + } +} + +func (r *beancountDataReader) readAndSetTags(ctx core.Context, lineIndex int, items []string, currentTags []string, pushTag bool) []string { + if r.getNotEmptyItemsCount(items) != 2 { + log.Warnf(ctx, "[beancount_data_reader.readAndSetTags] cannot parse push/pop tag line#%d \"%s\", because items count in line not correct", lineIndex, strings.Join(items, " ")) + return currentTags + } + + tag := r.getNotEmptyItemByIndex(items, 1) + + if len(tag) < 2 || tag[0] != beancountTagPrefix { + log.Warnf(ctx, "[beancount_data_reader.readAndSetTags] cannot parse push/pop tag line#%d \"%s\", because tag is invalid", lineIndex, strings.Join(items, " ")) + return currentTags + } + + tag = tag[1:] + + if pushTag { + for i := 0; i < len(currentTags); i++ { + if currentTags[i] == tag { + return currentTags + } + } + + return append(currentTags, tag) + } else { // pop tag + for i := 0; i < len(currentTags); i++ { + if currentTags[i] == tag { + return append(currentTags[:i], currentTags[i+1:]...) + } + } + + return currentTags + } +} + +func (r *beancountDataReader) setAccountTypeNameMap(accountType beancountAccountType, accountTypeName string) { + delete(r.accountTypeNameMap, r.accountTypeNameReversedMap[accountType]) + r.accountTypeNameMap[accountTypeName] = accountType + r.accountTypeNameReversedMap[accountType] = accountTypeName +} + +func (r *beancountDataReader) readAccountLine(ctx core.Context, lineIndex int, items []string, date string, directive beancountDirective, data *beancountData) (*beancountAccount, error) { + if r.getNotEmptyItemsCount(items) < 3 { + log.Warnf(ctx, "[beancount_data_reader.parseAccount] cannot parse account line#%d \"%s\", because items count in line not correct", lineIndex, strings.Join(items, " ")) + return nil, nil + } + + var err error + accountName := r.getNotEmptyItemByIndex(items, 2) + account, exists := data.accounts[accountName] + + if !exists { + account, err = r.createAccount(ctx, data, accountName) + + if err != nil { + return nil, err + } + } + + if directive == beancountDirectiveOpen { + account.openDate = date + return account, nil + } else if directive == beancountDirectiveClose { + account.closeDate = date + return account, nil + } else { + log.Warnf(ctx, "[beancount_data_reader.parseAccount] cannot parse account line#%d \"%s\", because directive is invalid", lineIndex, strings.Join(items, " ")) + return nil, nil + } +} + +func (r *beancountDataReader) createAccount(ctx core.Context, data *beancountData, accountName string) (*beancountAccount, error) { + account := &beancountAccount{ + name: accountName, + accountType: beancountUnknownAccountType, + } + + accountNameItems := strings.Split(accountName, beancountAccountNameItemsSeparator) + + if len(accountNameItems) > 1 { + accountType, exists := r.accountTypeNameMap[accountNameItems[0]] + + if exists { + account.accountType = accountType + } else { + log.Warnf(ctx, "[beancount_data_reader.createAccount] cannot parse account \"%s\", because account type \"%s\" is invalid", accountName, accountNameItems[0]) + return nil, errs.ErrInvalidBeancountFile + } + } + + data.accounts[accountName] = account + return account, nil +} + +func (r *beancountDataReader) readTransactionLine(ctx core.Context, lineIndex int, items []string, date string, directive beancountDirective, tags []string) *beancountTransactionEntry { + transactionEntry := &beancountTransactionEntry{ + date: date, + directive: directive, + tags: make([]string, 0), + links: make([]string, 0), + metadata: make(map[string]string), + } + + transactionEntry.tags = append(transactionEntry.tags, tags...) + + allTags := make(map[string]bool, len(transactionEntry.tags)) + + for _, tag := range transactionEntry.tags { + allTags[tag] = true + } + + // YYYY-MM-DD [txn|Flag] [[Payee] Narration] [#tag] [ˆlink] + payeeNarrationFirstIndex := 2 + payeeNarrationLastIndex := len(items) - 1 + + // parse remain items + for i := payeeNarrationFirstIndex; i < len(items); i++ { + item := items[i] + + if len(item) == 0 { + continue + } + + if item[0] == beancountCommentPrefix { // ; comment + if i-1 < payeeNarrationLastIndex { + payeeNarrationLastIndex = i - 1 + } + + break + } + + if item[0] == beancountTagPrefix { // [#tag] + tagName := item[1:] + + if _, exists := allTags[tagName]; !exists { + transactionEntry.tags = append(transactionEntry.tags, tagName) + allTags[tagName] = true + } + + if i-1 < payeeNarrationLastIndex { + payeeNarrationLastIndex = i - 1 + } + } else if item[0] == beancountLinkPrefix { // [ˆlink] + transactionEntry.links = append(transactionEntry.links, item[1:]) + + if i-1 < payeeNarrationLastIndex { + payeeNarrationLastIndex = i - 1 + } + } + } + + if payeeNarrationLastIndex-payeeNarrationFirstIndex >= 1 { + transactionEntry.payee = items[payeeNarrationFirstIndex] + transactionEntry.narration = items[payeeNarrationFirstIndex+1] + } else if payeeNarrationLastIndex-payeeNarrationFirstIndex >= 0 { + transactionEntry.narration = items[payeeNarrationFirstIndex] + } + + return transactionEntry +} + +func (r *beancountDataReader) readTransactionPostingLine(ctx core.Context, lineIndex int, items []string, data *beancountData, hasFlag bool) (*beancountPosting, error) { + // [Flag] Account Amount [{Cost}] [@ Price] + accountNameExpectedIndex := 0 + + if hasFlag { + accountNameExpectedIndex = 1 + } + + if r.getNotEmptyItemsCount(items) <= accountNameExpectedIndex { + log.Warnf(ctx, "[beancount_data_reader.readTransactionPostingLine] cannot parse transaction posting line#%d \"%s\", because items count in line not correct", lineIndex, strings.Join(items, " ")) + return nil, nil + } + + accountName, accountNameActualIndex := r.getNotEmptyItemAndIndexByIndex(items, accountNameExpectedIndex) + + if accountName == "" || accountNameActualIndex < 0 { + log.Warnf(ctx, "[beancount_data_reader.readTransactionPostingLine] cannot parse transaction posting line#%d \"%s\", because missing account name", lineIndex, strings.Join(items, " ")) + return nil, errs.ErrMissingAccountData + } + + transactionPositing := &beancountPosting{ + account: accountName, + metadata: make(map[string]string), + } + + amountActualLastIndex := -1 + transactionPositing.originalAmount, amountActualLastIndex = r.getOriginalAmountAndLastIndexFromIndex(items, accountNameActualIndex+1) + + if transactionPositing.originalAmount == "" || amountActualLastIndex < 0 { + log.Warnf(ctx, "[beancount_data_reader.readTransactionPostingLine] cannot parse transaction posting line#%d \"%s\", because missing amount", lineIndex, strings.Join(items, " ")) + return nil, errs.ErrAmountInvalid + } + + finalAmount, err := evaluateBeancountAmountExpression(ctx, transactionPositing.originalAmount) + + if err != nil { + log.Warnf(ctx, "[beancount_data_reader.readTransactionPostingLine] cannot evaluate amount expression in line#%d \"%s\", because %s", lineIndex, strings.Join(items, " "), err.Error()) + return nil, errs.ErrAmountInvalid + } else { + transactionPositing.amount = finalAmount + } + + commodityActualIndex := -1 + transactionPositing.commodity, commodityActualIndex = r.getNotEmptyItemAndIndexFromIndex(items, amountActualLastIndex+1) + + if transactionPositing.commodity == "" || commodityActualIndex < 0 { + log.Warnf(ctx, "[beancount_data_reader.readTransactionPostingLine] cannot parse transaction posting line#%d \"%s\", because missing commodity", lineIndex, strings.Join(items, " ")) + return nil, errs.ErrInvalidBeancountFile + } + + if strings.ToUpper(transactionPositing.commodity) != transactionPositing.commodity { // The syntax for a currency is a word all in capital letters + log.Warnf(ctx, "[beancount_data_reader.readTransactionPostingLine] cannot parse transaction posting line#%d \"%s\", because commodity name is not capital letters", lineIndex, strings.Join(items, " ")) + return nil, errs.ErrInvalidBeancountFile + } + + // parse remain items + if commodityActualIndex > 0 { + for i := commodityActualIndex + 1; i < len(items); i++ { + item := items[i] + + if len(item) == 0 { + continue + } + + if item[0] == beancountCommentPrefix { // ; comment + break + } + + if len(item) == 2 && item[0] == beancountPricePrefix && item[1] == beancountPricePrefix { // [@@ TotalCost] + totalCost, totalCostActualIndex := r.getNotEmptyItemAndIndexFromIndex(items, i+1) + + if totalCostActualIndex > 0 { + transactionPositing.totalCost = totalCost + i = totalCostActualIndex + + totalCostCommodity, totalCostCommodityActualIndex := r.getNotEmptyItemAndIndexFromIndex(items, totalCostActualIndex+1) + + if totalCostCommodityActualIndex > 0 { + transactionPositing.totalCostCommodity = totalCostCommodity + i = totalCostCommodityActualIndex + } + } + } else if len(item) == 1 && item[0] == beancountPricePrefix { // [@ Price] + price, priceActualIndex := r.getNotEmptyItemAndIndexFromIndex(items, i+1) + + if priceActualIndex > 0 { + transactionPositing.price = price + i = priceActualIndex + + priceCommodity, priceCommodityActualIndex := r.getNotEmptyItemAndIndexFromIndex(items, priceActualIndex+1) + + if priceCommodityActualIndex > 0 { + transactionPositing.priceCommodity = priceCommodity + i = priceCommodityActualIndex + } + } + } + } + } + + if transactionPositing.account != "" { + _, exists := data.accounts[transactionPositing.account] + + if !exists { + _, err := r.createAccount(ctx, data, transactionPositing.account) + + if err != nil { + return nil, err + } + } + } + + return transactionPositing, nil +} + +func (r *beancountDataReader) readTransactionMetadataLine(ctx core.Context, lineIndex int, items []string) []string { + key := r.getNotEmptyItemByIndex(items, 0) + value := r.getNotEmptyItemByIndex(items, 1) + + if key == "" || value == "" { + log.Warnf(ctx, "[beancount_data_reader.readTransactionMetadataLine] cannot parse metadata line#%d \"%s\", because key or value is empty", lineIndex, strings.Join(items, " ")) + return nil + } + + if len(key) == 0 || key[len(key)-1] != beancountMetadataKeySuffix { + log.Warnf(ctx, "[beancount_data_reader.readTransactionMetadataLine] cannot parse metadata line#%d \"%s\", because key is invalid correct", lineIndex, strings.Join(items, " ")) + return nil + } + + key = key[:len(key)-1] + + return []string{key, value} +} + +func (r *beancountDataReader) getNotEmptyItemByIndex(items []string, index int) string { + item, _ := r.getNotEmptyItemAndIndexByIndex(items, index) + return item +} + +func (r *beancountDataReader) getNotEmptyItemAndIndexByIndex(items []string, index int) (string, int) { + count := -1 + + for i := 0; i < len(items); i++ { + item := items[i] + + if len(item) == 0 { + continue + } + + count++ + + if count == index { + return items[i], i + } + } + + return "", -1 +} + +func (r *beancountDataReader) getNotEmptyItemAndIndexFromIndex(items []string, startIndex int) (string, int) { + for i := startIndex; i < len(items); i++ { + item := items[i] + + if len(item) == 0 { + continue + } + + return item, i + } + + return "", -1 +} + +func (r *beancountDataReader) getNotEmptyItemsCount(items []string) int { + count := 0 + + for i := 0; i < len(items); i++ { + if len(items[i]) > 0 { + count++ + } + } + + return count +} + +func (r *beancountDataReader) getOriginalAmountAndLastIndexFromIndex(items []string, startIndex int) (string, int) { + amountBuilder := strings.Builder{} + lastIndex := -1 + + for i := startIndex; i < len(items); i++ { + item := items[i] + + if len(item) == 0 { + continue + } + + valid := true + + // The Amount in “Postings” can also be an arithmetic expression using ( ) * / - + + for j := 0; j < len(item); j++ { + if !(item[j] >= '0' && item[j] <= '9') && item[j] != '.' && item[j] != '(' && item[j] != ')' && + item[j] != '*' && item[j] != '/' && item[j] != '-' && item[j] != '+' { + valid = false + break + } + } + + if !valid { + break + } + + if amountBuilder.Len() > 0 { + amountBuilder.WriteRune(' ') + } + + amountBuilder.WriteString(item) + lastIndex = i + } + + return amountBuilder.String(), lastIndex +} + +func createNewBeancountDataReader(ctx core.Context, data []byte) (*beancountDataReader, error) { + fallback := unicode.UTF8.NewDecoder() + reader := transform.NewReader(bytes.NewReader(data), unicode.BOMOverride(fallback)) + csvReader := csv.NewReader(reader) + csvReader.Comma = ' ' + csvReader.FieldsPerRecord = -1 + + allData := make([][]string, 0) + + for { + items, err := csvReader.Read() + + if err == io.EOF { + break + } + + if err != nil { + log.Errorf(ctx, "[beancount_data_reader.createNewBeancountDataReader] cannot parse data, because %s", err.Error()) + return nil, errs.ErrInvalidBeancountFile + } + + allData = append(allData, items) + } + + return &beancountDataReader{ + accountTypeNameMap: map[string]beancountAccountType{ + beancountDefaultAssetsAccountTypeName: beancountAssetsAccountType, + beancountDefaultLiabilitiesAccountTypeName: beancountLiabilitiesAccountType, + beancountDefaultEquityAccountTypeName: beancountEquityAccountType, + beancountDefaultIncomeAccountTypeName: beancountIncomeAccountType, + beancountDefaultExpenseAccountTypeName: beancountExpensesAccountType, + }, + accountTypeNameReversedMap: map[beancountAccountType]string{ + beancountAssetsAccountType: beancountDefaultAssetsAccountTypeName, + beancountLiabilitiesAccountType: beancountDefaultLiabilitiesAccountTypeName, + beancountEquityAccountType: beancountDefaultEquityAccountTypeName, + beancountIncomeAccountType: beancountDefaultIncomeAccountTypeName, + beancountExpensesAccountType: beancountDefaultExpenseAccountTypeName, + }, + allData: allData, + }, nil +} diff --git a/pkg/converters/beancount/beancount_data_reader_test.go b/pkg/converters/beancount/beancount_data_reader_test.go new file mode 100644 index 00000000..e33222ed --- /dev/null +++ b/pkg/converters/beancount/beancount_data_reader_test.go @@ -0,0 +1,520 @@ +package beancount + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" +) + +func TestBeancountDataReaderRead(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "; Test Beancount Data\n"+ + "option \"name_assets\" \"AssetsAccount\"\n"+ + "option \"name_liabilities\" \"LiabilitiesAccount\"\n"+ + "option \"name_equity\" \"EquityAccount\"\n"+ + "option \"name_income\" \"IncomeAccount\"\n"+ + "option \"name_expenses\" \"ExpensesAccount\"\n"+ + "\n"+ + "2024-01-01 open AssetsAccount:TestAccount\n"+ + "2024-01-02 open LiabilitiesAccount:TestAccount2\n"+ + "2024-01-03 open EquityAccount:Opening-Balances\n"+ + "\n"+ + "; The following transactions with tag1 and tag2\n"+ + "pushtag #tag1\n"+ + "pushtag #tag2\n"+ + "\n"+ + "2024-01-05 * \"Payee Name\" \"Foo Bar\" #tag3 #tag4 ^test-link\n"+ + " IncomeAccount:TestCategory -123.45 CNY\n"+ + " AssetsAccount:TestAccount 123.45 CNY\n"+ + "; The following transactions with tag2\n"+ + "poptag #tag1\n"+ + "2024-01-06 * \"test\n#test2\" #tag5 #tag6 ^test-link2\n"+ + " LiabilitiesAccount:TestAccount2 -0.12 USD\n"+ + " ExpensesAccount:TestCategory2 0.12 USD\n"+ + "2024-01-07 close AssetsAccount:TestAccount\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 5, len(actualData.accounts)) + assert.Equal(t, "AssetsAccount:TestAccount", actualData.accounts["AssetsAccount:TestAccount"].name) + assert.Equal(t, beancountAssetsAccountType, actualData.accounts["AssetsAccount:TestAccount"].accountType) + assert.Equal(t, "2024-01-01", actualData.accounts["AssetsAccount:TestAccount"].openDate) + assert.Equal(t, "2024-01-07", actualData.accounts["AssetsAccount:TestAccount"].closeDate) + + assert.Equal(t, "LiabilitiesAccount:TestAccount2", actualData.accounts["LiabilitiesAccount:TestAccount2"].name) + assert.Equal(t, beancountLiabilitiesAccountType, actualData.accounts["LiabilitiesAccount:TestAccount2"].accountType) + assert.Equal(t, "2024-01-02", actualData.accounts["LiabilitiesAccount:TestAccount2"].openDate) + + assert.Equal(t, 2, len(actualData.transactions)) + + assert.Equal(t, "2024-01-05", actualData.transactions[0].date) + assert.Equal(t, "Payee Name", actualData.transactions[0].payee) + assert.Equal(t, "Foo Bar", actualData.transactions[0].narration) + assert.Equal(t, 2, len(actualData.transactions[0].postings)) + assert.Equal(t, "IncomeAccount:TestCategory", actualData.transactions[0].postings[0].account) + assert.Equal(t, "-123.45", actualData.transactions[0].postings[0].amount) + assert.Equal(t, "CNY", actualData.transactions[0].postings[0].commodity) + assert.Equal(t, "AssetsAccount:TestAccount", actualData.transactions[0].postings[1].account) + assert.Equal(t, "123.45", actualData.transactions[0].postings[1].amount) + assert.Equal(t, "CNY", actualData.transactions[0].postings[1].commodity) + + assert.Equal(t, 4, len(actualData.transactions[0].tags)) + assert.Equal(t, actualData.transactions[0].tags[0], "tag1") + assert.Equal(t, actualData.transactions[0].tags[1], "tag2") + assert.Equal(t, actualData.transactions[0].tags[2], "tag3") + assert.Equal(t, actualData.transactions[0].tags[3], "tag4") + + assert.Equal(t, 1, len(actualData.transactions[0].links)) + assert.Equal(t, actualData.transactions[0].links[0], "test-link") + + assert.Equal(t, "2024-01-06", actualData.transactions[1].date) + assert.Equal(t, "", actualData.transactions[1].payee) + assert.Equal(t, "test\n#test2", actualData.transactions[1].narration) + assert.Equal(t, 2, len(actualData.transactions[1].postings)) + assert.Equal(t, "LiabilitiesAccount:TestAccount2", actualData.transactions[1].postings[0].account) + assert.Equal(t, "-0.12", actualData.transactions[1].postings[0].amount) + assert.Equal(t, "USD", actualData.transactions[1].postings[0].commodity) + assert.Equal(t, "ExpensesAccount:TestCategory2", actualData.transactions[1].postings[1].account) + assert.Equal(t, "0.12", actualData.transactions[1].postings[1].amount) + assert.Equal(t, "USD", actualData.transactions[1].postings[1].commodity) + + assert.Equal(t, 3, len(actualData.transactions[1].tags)) + assert.Equal(t, actualData.transactions[1].tags[0], "tag2") + assert.Equal(t, actualData.transactions[1].tags[1], "tag5") + assert.Equal(t, actualData.transactions[1].tags[2], "tag6") + + assert.Equal(t, 1, len(actualData.transactions[1].links)) + assert.Equal(t, actualData.transactions[1].links[0], "test-link2") +} + +func TestBeancountDataReaderRead_EmptyContent(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte("")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) +} + +func TestBeancountDataReaderRead_UnsupportedInclude(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte("include \"other.beancount\"")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrBeancountFileNotSupportInclude.Message) +} + +func TestBeancountDataReaderRead_SkipUnsupportedDirective(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "plugin \"beancount.plugins.plugin_name\"\n"+ + "unknown directive\n"+ + "2024-01-01 commodity USD\n"+ + "2024-01-01 price USD 1.08 CAD\n"+ + "2024-01-01 note Assets:Test \"some text\"\n"+ + "2024-01-01 document Assets:Test \"scheme://path\"\n"+ + "2024-01-01 event \"location\" \"address\"\n"+ + "2024-01-01 balance Assets:Test 100.00 USD\n"+ + "2024-01-01 pad Assets:Test Equity:Opening-Balances\n"+ + "2024-01-01 query \"Name\" \"\nSELECT FIELDS FROM TABLE\"\n"+ + "2024-01-01 custom \"Type\" \"Value\"\n"+ + "2024-01-01 unknown directive\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.Nil(t, err) +} + +func TestBeancountDataReaderReadAndSetOption_AccountTypeName(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "option \"name_assets\" \"A\"\n"+ + "option \"name_liabilities\" \"L\"\n"+ + "option \"name_equity\" \"E\"\n"+ + "\n"+ + "2024-01-01 open A:TestAccount\n"+ + "2024-01-02 open L:TestAccount2\n"+ + "2024-01-03 open E:Opening-Balances\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 3, len(actualData.accounts)) + + assert.Equal(t, "A:TestAccount", actualData.accounts["A:TestAccount"].name) + assert.Equal(t, beancountAssetsAccountType, actualData.accounts["A:TestAccount"].accountType) + + assert.Equal(t, "L:TestAccount2", actualData.accounts["L:TestAccount2"].name) + assert.Equal(t, beancountLiabilitiesAccountType, actualData.accounts["L:TestAccount2"].accountType) + + assert.Equal(t, "E:Opening-Balances", actualData.accounts["E:Opening-Balances"].name) + assert.Equal(t, beancountEquityAccountType, actualData.accounts["E:Opening-Balances"].accountType) + assert.True(t, actualData.accounts["E:Opening-Balances"].isOpeningBalanceEquityAccount()) +} + +func TestBeancountDataReaderReadAndSetOption_InvalidLineOrUnsupportedOption(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "option \"test\" \"Test\" \"Test2\"\n"+ + "option \"test\" \"Test\"\n"+ + "option \"test\"\n"+ + "option \n"+ + "option\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.Nil(t, err) +} + +func TestBeancountDataReaderReadAndSetTags(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "pushtag #tag1\n"+ + "pushtag #tag2\n"+ + "pushtag #tag2\n"+ + "pushtag #tag1\n"+ + "\n"+ + "2024-01-01 * #tag3 #tag4\n"+ + "poptag #tag1\n"+ + "poptag #tag2\n"+ + "pushtag\n"+ + "pushtag \n"+ + "pushtag tag\n"+ + "2024-01-02 * #tag5 #tag6\n"+ + "poptag #tag1\n"+ + "poptag #tag2\n"+ + "poptag\n"+ + "poptag \n"+ + "2024-01-03 * #tag5 #tag6\n"+ + "pushtag #tag3\n"+ + "pushtag #tag6\n"+ + "2024-01-04 * #tag5 #tag6\n"+ + "2024-01-05 * #tag5 #tag6 #tag6 #tag5\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 5, len(actualData.transactions)) + + assert.Equal(t, 4, len(actualData.transactions[0].tags)) + assert.Equal(t, actualData.transactions[0].tags[0], "tag1") + assert.Equal(t, actualData.transactions[0].tags[1], "tag2") + assert.Equal(t, actualData.transactions[0].tags[2], "tag3") + assert.Equal(t, actualData.transactions[0].tags[3], "tag4") + + assert.Equal(t, 2, len(actualData.transactions[1].tags)) + assert.Equal(t, actualData.transactions[1].tags[0], "tag5") + assert.Equal(t, actualData.transactions[1].tags[1], "tag6") + + assert.Equal(t, 2, len(actualData.transactions[2].tags)) + assert.Equal(t, actualData.transactions[2].tags[0], "tag5") + assert.Equal(t, actualData.transactions[2].tags[1], "tag6") + + assert.Equal(t, 3, len(actualData.transactions[3].tags)) + assert.Equal(t, actualData.transactions[3].tags[0], "tag3") + assert.Equal(t, actualData.transactions[3].tags[1], "tag6") + assert.Equal(t, actualData.transactions[3].tags[2], "tag5") + + assert.Equal(t, 3, len(actualData.transactions[4].tags)) + assert.Equal(t, actualData.transactions[4].tags[0], "tag3") + assert.Equal(t, actualData.transactions[4].tags[1], "tag6") + assert.Equal(t, actualData.transactions[4].tags[2], "tag5") +} + +func TestBeancountDataReaderReadAccountLine_InvalidLine(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 open\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + assert.Equal(t, 0, len(actualData.accounts)) +} + +func TestBeancountDataReaderReadAccountLine_InvalidAccountType(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 open Test:TestAccount\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) + + reader, err = createNewBeancountDataReader(context, []byte(""+ + "option \"name_assets\" \"A\"\n"+ + "\n"+ + "2024-01-01 open Assets:TestAccount\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) +} + +func TestBeancountDataReaderReadTransactionLine(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + "2024-01-02 * \"test\ttest2\ntest3\" ; some comment\n"+ + "2024-01-03 ! \"test\" \"test2\"\n"+ + "2024-01-04 P \"test\" #tag #tag2 ; some comment\n"+ + "2024-01-05 txn \"test\" ^scheme://path/to/test/link ; some comment\n"+ + "2024-01-06 txn ; \"test\" \"test2\" #tag ^link\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 6, len(actualData.transactions)) + + assert.Equal(t, "2024-01-01", actualData.transactions[0].date) + assert.Equal(t, beancountDirectiveCompletedTransaction, actualData.transactions[0].directive) + assert.Equal(t, "", actualData.transactions[0].payee) + assert.Equal(t, "", actualData.transactions[0].narration) + + assert.Equal(t, "2024-01-02", actualData.transactions[1].date) + assert.Equal(t, beancountDirectiveCompletedTransaction, actualData.transactions[1].directive) + assert.Equal(t, "", actualData.transactions[1].payee) + assert.Equal(t, "test\ttest2\ntest3", actualData.transactions[1].narration) + + assert.Equal(t, "2024-01-03", actualData.transactions[2].date) + assert.Equal(t, beancountDirectiveInCompleteTransaction, actualData.transactions[2].directive) + assert.Equal(t, "test", actualData.transactions[2].payee) + assert.Equal(t, "test2", actualData.transactions[2].narration) + + assert.Equal(t, "2024-01-04", actualData.transactions[3].date) + assert.Equal(t, beancountDirectivePaddingTransaction, actualData.transactions[3].directive) + assert.Equal(t, "", actualData.transactions[3].payee) + assert.Equal(t, "test", actualData.transactions[3].narration) + + assert.Equal(t, 2, len(actualData.transactions[3].tags)) + assert.Equal(t, actualData.transactions[3].tags[0], "tag") + assert.Equal(t, actualData.transactions[3].tags[1], "tag2") + + assert.Equal(t, "2024-01-05", actualData.transactions[4].date) + assert.Equal(t, beancountDirectiveTransaction, actualData.transactions[4].directive) + assert.Equal(t, "", actualData.transactions[4].payee) + assert.Equal(t, "test", actualData.transactions[4].narration) + + assert.Equal(t, 1, len(actualData.transactions[4].links)) + assert.Equal(t, actualData.transactions[4].links[0], "scheme://path/to/test/link") + + assert.Equal(t, "2024-01-06", actualData.transactions[5].date) + assert.Equal(t, beancountDirectiveTransaction, actualData.transactions[5].directive) + assert.Equal(t, "", actualData.transactions[5].payee) + assert.Equal(t, "", actualData.transactions[5].narration) +} + +func TestBeancountDataReaderReadTransactionPostingLine(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Income:TestCategory -123.45 CNY ; some comment\n"+ + " Assets:TestAccount 123.45 CNY\n"+ + "2024-01-02 *\n"+ + " Liabilities:TestAccount2 -0.23 USD ; some comment\n"+ + " Expenses:TestCategory2 0.12 USD @@ 0.84 CNY\n"+ + " Expenses:TestCategory3 0.11 USD @ 7.12 CNY\n"+ + " ! Expenses:TestCategory4 0.00 USD {0.00 CNY}\n"+ + " Expenses:TestCategory5 \n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 2, len(actualData.transactions)) + + assert.Equal(t, "2024-01-01", actualData.transactions[0].date) + assert.Equal(t, 2, len(actualData.transactions[0].postings)) + assert.Equal(t, "Income:TestCategory", actualData.transactions[0].postings[0].account) + assert.Equal(t, "-123.45", actualData.transactions[0].postings[0].amount) + assert.Equal(t, "CNY", actualData.transactions[0].postings[0].commodity) + + assert.Equal(t, "Assets:TestAccount", actualData.transactions[0].postings[1].account) + assert.Equal(t, "123.45", actualData.transactions[0].postings[1].amount) + assert.Equal(t, "CNY", actualData.transactions[0].postings[1].commodity) + + assert.Equal(t, "2024-01-02", actualData.transactions[1].date) + assert.Equal(t, 4, len(actualData.transactions[1].postings)) + + assert.Equal(t, "Liabilities:TestAccount2", actualData.transactions[1].postings[0].account) + assert.Equal(t, "-0.23", actualData.transactions[1].postings[0].amount) + assert.Equal(t, "USD", actualData.transactions[1].postings[0].commodity) + assert.Equal(t, "Expenses:TestCategory2", actualData.transactions[1].postings[1].account) + + assert.Equal(t, "0.12", actualData.transactions[1].postings[1].amount) + assert.Equal(t, "USD", actualData.transactions[1].postings[1].commodity) + assert.Equal(t, "0.84", actualData.transactions[1].postings[1].totalCost) + assert.Equal(t, "CNY", actualData.transactions[1].postings[1].totalCostCommodity) + assert.Equal(t, "Expenses:TestCategory3", actualData.transactions[1].postings[2].account) + + assert.Equal(t, "0.11", actualData.transactions[1].postings[2].amount) + assert.Equal(t, "USD", actualData.transactions[1].postings[2].commodity) + assert.Equal(t, "7.12", actualData.transactions[1].postings[2].price) + assert.Equal(t, "CNY", actualData.transactions[1].postings[2].priceCommodity) + + assert.Equal(t, "0.00", actualData.transactions[1].postings[3].amount) + assert.Equal(t, "USD", actualData.transactions[1].postings[3].commodity) +} + +func TestBeancountDataReaderReadTransactionPostingLine_AmountExpression(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Income:TestCategory (1.2-3.4) * 5.6 / 7.8 CNY\n"+ + " Assets:TestAccount 1.2 * 3.4/-5.6 - 7.8 CNY\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 1, len(actualData.transactions)) + + assert.Equal(t, "2024-01-01", actualData.transactions[0].date) + assert.Equal(t, 2, len(actualData.transactions[0].postings)) + assert.Equal(t, "Income:TestCategory", actualData.transactions[0].postings[0].account) + assert.Equal(t, "(1.2-3.4) * 5.6 / 7.8", actualData.transactions[0].postings[0].originalAmount) + assert.Equal(t, "-1.58", actualData.transactions[0].postings[0].amount) + assert.Equal(t, "CNY", actualData.transactions[0].postings[0].commodity) + + assert.Equal(t, "Assets:TestAccount", actualData.transactions[0].postings[1].account) + assert.Equal(t, "1.2 * 3.4/-5.6 - 7.8", actualData.transactions[0].postings[1].originalAmount) + assert.Equal(t, "-8.53", actualData.transactions[0].postings[1].amount) + assert.Equal(t, "CNY", actualData.transactions[0].postings[1].commodity) +} + +func TestBeancountDataReaderReadTransactionPostingLine_InvalidAmountExpression(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Income:TestCategory (1.2-3.4)*5.6/0 CNY\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) + + reader, err = createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Assets:TestAccount abc CNY\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} + +func TestBeancountDataReaderReadTransactionPostingLine_InvalidAccountType(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Income:TestCategory -123.45 CNY\n"+ + " Test:TestAccount 123.45 CNY\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) +} + +func TestBeancountDataReaderReadTransactionPostingLine_InvalidCommodity(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Income:TestCategory -123.45 cny\n"+ + " Assets:TestAccount 123.45 cny\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) +} + +func TestBeancountDataReaderReadTransactionPostingLine_MissingAmount(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Assets:TestAccount\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + assert.Equal(t, 1, len(actualData.transactions)) + assert.Equal(t, 0, len(actualData.transactions[0].postings)) + + reader, err = createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Assets:TestAccount \n")) + assert.Nil(t, err) + + actualData, err = reader.read(context) + assert.Nil(t, err) + assert.Equal(t, 1, len(actualData.transactions)) + assert.Equal(t, 0, len(actualData.transactions[0].postings)) +} + +func TestBeancountDataReaderReadTransactionPostingLine_MissingCommodity(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Assets:TestAccount 123.45\n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) + + reader, err = createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " Assets:TestAccount 123.45 \n")) + assert.Nil(t, err) + + _, err = reader.read(context) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) +} + +func TestBeancountDataReaderReadTransactionMetadataLine(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewBeancountDataReader(context, []byte(""+ + "2024-01-01 *\n"+ + " key: value\n"+ + " key2: \"value 2\"\n"+ + " key3: \n"+ + " key4: \"\"\n"+ + " key5 : \"\"\n"+ + " key2: \"new value\"\n"+ + " Income:TestCategory -123.45 CNY\n"+ + " Assets:TestAccount 123.45 CNY\n"+ + "2024-01-02 *\n"+ + " Liabilities:TestAccount2 -0.23 USD\n"+ + " key6: value6\n"+ + " key7: \"value 7\"\n"+ + " key8: \n"+ + " key9: \"\"\n"+ + " key0 : \"\"\n"+ + " key6: \"new value\"\n"+ + " Expenses:TestCategory2 0.12 USD\n")) + assert.Nil(t, err) + + actualData, err := reader.read(context) + assert.Nil(t, err) + + assert.Equal(t, 2, len(actualData.transactions)) + + assert.Equal(t, "2024-01-01", actualData.transactions[0].date) + assert.Equal(t, 2, len(actualData.transactions[0].postings)) + assert.Equal(t, 2, len(actualData.transactions[0].metadata)) + assert.Equal(t, "value", actualData.transactions[0].metadata["key"]) + assert.Equal(t, "value 2", actualData.transactions[0].metadata["key2"]) + + assert.Equal(t, "2024-01-02", actualData.transactions[1].date) + assert.Equal(t, 2, len(actualData.transactions[1].postings)) + assert.Equal(t, 2, len(actualData.transactions[1].postings[0].metadata)) + assert.Equal(t, "value6", actualData.transactions[1].postings[0].metadata["key6"]) + assert.Equal(t, "value 7", actualData.transactions[1].postings[0].metadata["key7"]) + assert.Equal(t, 0, len(actualData.transactions[1].postings[1].metadata)) +} diff --git a/pkg/converters/beancount/beancount_data_test.go b/pkg/converters/beancount/beancount_data_test.go new file mode 100644 index 00000000..7d251cc1 --- /dev/null +++ b/pkg/converters/beancount/beancount_data_test.go @@ -0,0 +1,41 @@ +package beancount + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBeancountAccount_IsOpeningBalanceEquityAccount_True(t *testing.T) { + account := beancountAccount{ + accountType: beancountEquityAccountType, + name: "Equity:Opening-Balances", + } + assert.True(t, account.isOpeningBalanceEquityAccount()) + + account = beancountAccount{ + accountType: beancountEquityAccountType, + name: "E:Opening-Balances", + } + assert.True(t, account.isOpeningBalanceEquityAccount()) +} + +func TestBeancountAccount_IsOpeningBalanceEquityAccount_False(t *testing.T) { + account := beancountAccount{ + accountType: beancountAssetsAccountType, + name: "Equity:Opening-Balances", + } + assert.False(t, account.isOpeningBalanceEquityAccount()) + + account = beancountAccount{ + accountType: beancountEquityAccountType, + name: "Opening-Balances", + } + assert.False(t, account.isOpeningBalanceEquityAccount()) + + account = beancountAccount{ + accountType: beancountEquityAccountType, + name: "Equity:Other", + } + assert.False(t, account.isOpeningBalanceEquityAccount()) +} diff --git a/pkg/converters/beancount/beancount_transaction_data_file_importer.go b/pkg/converters/beancount/beancount_transaction_data_file_importer.go new file mode 100644 index 00000000..f09adc54 --- /dev/null +++ b/pkg/converters/beancount/beancount_transaction_data_file_importer.go @@ -0,0 +1,49 @@ +package beancount + +import ( + "github.com/mayswind/ezbookkeeping/pkg/converters/converter" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +var beancountTransactionTypeNameMapping = map[models.TransactionType]string{ + models.TRANSACTION_TYPE_MODIFY_BALANCE: utils.IntToString(int(models.TRANSACTION_TYPE_MODIFY_BALANCE)), + models.TRANSACTION_TYPE_INCOME: utils.IntToString(int(models.TRANSACTION_TYPE_INCOME)), + models.TRANSACTION_TYPE_EXPENSE: utils.IntToString(int(models.TRANSACTION_TYPE_EXPENSE)), + models.TRANSACTION_TYPE_TRANSFER: utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)), +} + +// beancountTransactionDataImporter defines the structure of Beancount importer for transaction data +type beancountTransactionDataImporter struct { +} + +// Initialize a beancount transaction data importer singleton instance +var ( + BeancountTransactionDataImporter = &beancountTransactionDataImporter{} +) + +// ParseImportedData returns the imported data by parsing the Beancount transaction data +func (c *beancountTransactionDataImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { + beancountDataReader, err := createNewBeancountDataReader(ctx, data) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + beancountData, err := beancountDataReader.read(ctx) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + transactionDataTable, err := createNewBeancountTransactionDataTable(beancountData) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + dataTableImporter := converter.CreateNewImporterWithTypeNameMapping(beancountTransactionTypeNameMapping, "", BEANCOUNT_TRANSACTION_TAG_SEPARATOR) + + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) +} diff --git a/pkg/converters/beancount/beancount_transaction_data_file_importer_test.go b/pkg/converters/beancount/beancount_transaction_data_file_importer_test.go new file mode 100644 index 00000000..d145347d --- /dev/null +++ b/pkg/converters/beancount/beancount_transaction_data_file_importer_test.go @@ -0,0 +1,358 @@ +package beancount + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +func TestBeancountTransactionDataFileParseImportedData_MinimumValidData(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 *\n"+ + " Equity:Opening-Balances -123.45 CNY\n"+ + " Assets:TestAccount 123.45 CNY\n"+ + "2024-09-02 *\n"+ + " Income:TestCategory -0.12 CNY\n"+ + " Assets:TestAccount 0.12 CNY\n"+ + "2024-09-03 *\n"+ + " Assets:TestAccount -1.00 CNY\n"+ + " Expenses:TestCategory2 1.00 CNY\n"+ + "2024-09-04 *\n"+ + " Assets:TestAccount -0.05 CNY\n"+ + " Assets:TestAccount2 0.05 CNY\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) + assert.Equal(t, 0, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[1].OriginalSourceAccountName) + assert.Equal(t, "Income:TestCategory", allNewTransactions[1].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[2].OriginalSourceAccountName) + assert.Equal(t, "Expenses:TestCategory2", allNewTransactions[2].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[3].OriginalSourceAccountName) + assert.Equal(t, "Assets:TestAccount2", allNewTransactions[3].OriginalDestinationAccountName) + assert.Equal(t, "", allNewTransactions[3].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Assets:TestAccount", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Assets:TestAccount2", allNewAccounts[1].Name) + assert.Equal(t, "CNY", allNewAccounts[1].Currency) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Expenses:TestCategory2", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Income:TestCategory", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "", allNewSubTransferCategories[0].Name) +} + +func TestBeancountTransactionDataFileParseImportedData_MinimumValidData2(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 *\n"+ + " Assets:TestAccount 123.45 CNY\n"+ + " Equity:Opening-Balances -123.45 CNY\n"+ + "2024-09-02 *\n"+ + " Assets:TestAccount 0.12 CNY\n"+ + " Income:TestCategory -0.12 CNY\n"+ + "2024-09-03 *\n"+ + " Expenses:TestCategory2 1.00 CNY\n"+ + " Assets:TestAccount -1.00 CNY\n"+ + "2024-09-04 *\n"+ + " Assets:TestAccount2 0.05 CNY\n"+ + " Assets:TestAccount -0.05 CNY\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) + assert.Equal(t, 0, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725235200), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[1].OriginalSourceAccountName) + assert.Equal(t, "Income:TestCategory", allNewTransactions[1].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725321600), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[2].OriginalSourceAccountName) + assert.Equal(t, "Expenses:TestCategory2", allNewTransactions[2].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725408000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[3].OriginalSourceAccountName) + assert.Equal(t, "Assets:TestAccount2", allNewTransactions[3].OriginalDestinationAccountName) + assert.Equal(t, "", allNewTransactions[3].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Assets:TestAccount", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Assets:TestAccount2", allNewAccounts[1].Name) + assert.Equal(t, "CNY", allNewAccounts[1].Currency) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Expenses:TestCategory2", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Income:TestCategory", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "", allNewSubTransferCategories[0].Name) +} + +func TestBeancountTransactionDataFileParseImportedData_ParseInvalidTime(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024/09/01 *\n"+ + " Equity:Opening-Balances -123.45 CNY\n"+ + " Assets:TestAccount 123.45 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) +} + +func TestBeancountTransactionDataFileParseImportedData_ParseValidCurrency(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Assets:TestAccount -0.12 USD\n"+ + " Assets:TestAccount2 0.84 CNY\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, int64(12), allNewTransactions[0].Amount) + assert.Equal(t, int64(84), allNewTransactions[0].RelatedAccountAmount) + assert.Equal(t, "Assets:TestAccount", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "USD", allNewTransactions[0].OriginalSourceAccountCurrency) + assert.Equal(t, "Assets:TestAccount2", allNewTransactions[0].OriginalDestinationAccountName) + assert.Equal(t, "CNY", allNewTransactions[0].OriginalDestinationAccountCurrency) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Assets:TestAccount", allNewAccounts[0].Name) + assert.Equal(t, "USD", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Assets:TestAccount2", allNewAccounts[1].Name) + assert.Equal(t, "CNY", allNewAccounts[1].Currency) +} + +func TestBeancountTransactionDataFileParseImportedData_ParseInvalidAmount(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 *\n"+ + " Equity:Opening-Balances -abc CNY\n"+ + " Assets:TestAccount abc CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 *\n"+ + " Equity:Opening-Balances -1/0 CNY\n"+ + " Assets:TestAccount 1/0 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} + +func TestBeancountTransactionDataFileParseImportedData_ParseDescription(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 * \"foo bar\t#test\n\"\n"+ + " Equity:Opening-Balances -123.45 CNY\n"+ + " Assets:TestAccount 123.45 CNY\n"+ + "2024-09-02 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Income:TestCategory -0.12 CNY\n"+ + " Assets:TestAccount 0.12 CNY\n"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 2, len(allNewTransactions)) + + assert.Equal(t, "foo bar\t#test\n", allNewTransactions[0].Comment) + assert.Equal(t, "Hello\nWorld", allNewTransactions[1].Comment) +} + +func TestBeancountTransactionDataFileParseImportedData_InvalidTransaction(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-02 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Assets:TestAccount 0.11 CNY\n"+ + " Assets:TestAccount2 0.11 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-02 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Expenses:TestCategory -0.11 CNY\n"+ + " Expenses:TestCategory2 0.11 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrThereAreNotSupportedTransactionType.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-02 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Income:TestCategory -0.11 CNY\n"+ + " Income:TestCategory2 0.11 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrThereAreNotSupportedTransactionType.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-02 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Equity:TestCategory -0.11 CNY\n"+ + " Equity:TestCategory2 0.11 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrThereAreNotSupportedTransactionType.Message) +} + +func TestBeancountTransactionDataFileParseImportedData_NotSupportedToParseSplitTransaction(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-02 * \"Payee Name\" \"Hello\nWorld\"\n"+ + " Assets:TestAccount -0.23 CNY\n"+ + " Assets:TestAccount2 0.11 CNY\n"+ + " Assets:TestAccount3 0.12 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrNotSupportedSplitTransactions.Message) +} + +func TestBeancountTransactionDataFileParseImportedData_MissingTransactionRequiredData(t *testing.T) { + converter := BeancountTransactionDataImporter + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + // Missing Transaction Time + _, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "* \"narration\"\n"+ + " Equity:Opening-Balances -123.45 CNY\n"+ + " Assets:TestAccount 123.45 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) + + // Missing Account Name + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 * \"narration\"\n"+ + " Equity:Opening-Balances -123.45 CNY\n"+ + " 123.45 CNY\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) + + // Missing Amount + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 * \"narration\"\n"+ + " Equity:Opening-Balances\n"+ + " Assets:TestAccount\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) + + // Missing Commodity + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 * \"narration\"\n"+ + " Equity:Opening-Balances -123.45\n"+ + " Assets:TestAccount 123.45\n"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrInvalidBeancountFile.Message) +} diff --git a/pkg/converters/beancount/beancount_transaction_data_table.go b/pkg/converters/beancount/beancount_transaction_data_table.go new file mode 100644 index 00000000..4b2a9a6a --- /dev/null +++ b/pkg/converters/beancount/beancount_transaction_data_table.go @@ -0,0 +1,248 @@ +package beancount + +import ( + "strings" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +var beancountTransactionSupportedColumns = map[datatable.TransactionDataTableColumn]bool{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: true, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: true, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: true, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: true, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: true, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: true, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: true, +} + +var BEANCOUNT_TRANSACTION_TAG_SEPARATOR = "#" + +// beancountTransactionDataTable defines the structure of Beancount transaction data table +type beancountTransactionDataTable struct { + allData []*beancountTransactionEntry + accountMap map[string]*beancountAccount +} + +// beancountTransactionDataRow defines the structure of Beancount transaction data row +type beancountTransactionDataRow struct { + dataTable *beancountTransactionDataTable + data *beancountTransactionEntry + finalItems map[datatable.TransactionDataTableColumn]string +} + +// beancountTransactionDataRowIterator defines the structure of Beancount transaction data row iterator +type beancountTransactionDataRowIterator struct { + dataTable *beancountTransactionDataTable + currentIndex int +} + +// HasColumn returns whether the transaction data table has specified column +func (t *beancountTransactionDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool { + _, exists := beancountTransactionSupportedColumns[column] + return exists +} + +// TransactionRowCount returns the total count of transaction data row +func (t *beancountTransactionDataTable) TransactionRowCount() int { + return len(t.allData) +} + +// TransactionRowIterator returns the iterator of transaction data row +func (t *beancountTransactionDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator { + return &beancountTransactionDataRowIterator{ + dataTable: t, + currentIndex: -1, + } +} + +// IsValid returns whether this row is valid data for importing +func (r *beancountTransactionDataRow) IsValid() bool { + return true +} + +// GetData returns the data in the specified column type +func (r *beancountTransactionDataRow) GetData(column datatable.TransactionDataTableColumn) string { + _, exists := beancountTransactionSupportedColumns[column] + + if exists { + return r.finalItems[column] + } + + return "" +} + +// HasNext returns whether the iterator does not reach the end +func (t *beancountTransactionDataRowIterator) HasNext() bool { + return t.currentIndex+1 < len(t.dataTable.allData) +} + +// Next returns the next imported data row +func (t *beancountTransactionDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { + if t.currentIndex+1 >= len(t.dataTable.allData) { + return nil, nil + } + + t.currentIndex++ + + data := t.dataTable.allData[t.currentIndex] + rowItems, err := t.parseTransaction(ctx, user, data) + + if err != nil { + return nil, err + } + + return &beancountTransactionDataRow{ + dataTable: t.dataTable, + data: data, + finalItems: rowItems, + }, nil +} + +func (t *beancountTransactionDataRowIterator) parseTransaction(ctx core.Context, user *models.User, beancountEntry *beancountTransactionEntry) (map[datatable.TransactionDataTableColumn]string, error) { + data := make(map[datatable.TransactionDataTableColumn]string, len(beancountTransactionSupportedColumns)) + + if beancountEntry.date == "" { + return nil, errs.ErrMissingTransactionTime + } + + // Beancount supports the international ISO 8601 standard format for dates, with dashes or the same ordering with slashes + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = strings.ReplaceAll(beancountEntry.date, "/", "-") + " 00:00:00" + + if len(beancountEntry.postings) == 2 { + splitData1 := beancountEntry.postings[0] + splitData2 := beancountEntry.postings[1] + + account1 := t.dataTable.accountMap[splitData1.account] + account2 := t.dataTable.accountMap[splitData2.account] + + if account1 == nil || account2 == nil { + return nil, errs.ErrMissingAccountData + } + + amount1, err := utils.ParseAmount(splitData1.amount) + + if err != nil { + log.Errorf(ctx, "[beancount_transaction_data_table.parseTransaction] cannot parse amount \"%s\", because %s", splitData1.amount, err.Error()) + return nil, errs.ErrAmountInvalid + } + + amount2, err := utils.ParseAmount(splitData2.amount) + + if err != nil { + log.Errorf(ctx, "[beancount_transaction_data_table.parseTransaction] cannot parse amount \"%s\", because %s", splitData2.amount, err.Error()) + return nil, errs.ErrAmountInvalid + } + + if ((account1.accountType == beancountEquityAccountType || account1.accountType == beancountIncomeAccountType) && (account2.accountType == beancountAssetsAccountType || account2.accountType == beancountLiabilitiesAccountType)) || + ((account2.accountType == beancountEquityAccountType || account2.accountType == beancountIncomeAccountType) && (account1.accountType == beancountAssetsAccountType || account1.accountType == beancountLiabilitiesAccountType)) { // income + fromAccount := account1 + toAccount := account2 + toCurrency := splitData2.commodity + toAmount := amount2 + + if (account2.accountType == beancountEquityAccountType || account2.accountType == beancountIncomeAccountType) && (account1.accountType == beancountAssetsAccountType || account1.accountType == beancountLiabilitiesAccountType) { + fromAccount = account2 + toAccount = account1 + toCurrency = splitData1.commodity + toAmount = amount1 + } + + if fromAccount.isOpeningBalanceEquityAccount() { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = utils.IntToString(int(models.TRANSACTION_TYPE_MODIFY_BALANCE)) + } else { + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = utils.IntToString(int(models.TRANSACTION_TYPE_INCOME)) + } + + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = fromAccount.name + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = toAccount.name + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = toCurrency + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(toAmount) + } else if account1.accountType == beancountExpensesAccountType && (account2.accountType == beancountAssetsAccountType || account2.accountType == beancountLiabilitiesAccountType) || + (account2.accountType == beancountExpensesAccountType && (account1.accountType == beancountAssetsAccountType || account1.accountType == beancountLiabilitiesAccountType)) { // expense + fromAccount := account1 + fromCurrency := splitData1.commodity + fromAmount := amount1 + toAccount := account2 + + if account1.accountType == beancountExpensesAccountType && (account2.accountType == beancountAssetsAccountType || account2.accountType == beancountLiabilitiesAccountType) { + fromAccount = account2 + fromCurrency = splitData2.commodity + fromAmount = amount2 + toAccount = account1 + } + + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = utils.IntToString(int(models.TRANSACTION_TYPE_EXPENSE)) + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = toAccount.name + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = fromAccount.name + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = fromCurrency + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(-fromAmount) + } else if (account1.accountType == beancountAssetsAccountType || account1.accountType == beancountLiabilitiesAccountType) && + (account2.accountType == beancountAssetsAccountType || account2.accountType == beancountLiabilitiesAccountType) { + var fromAccount, toAccount *beancountAccount + var fromAmount, toAmount int64 + var fromCurrency, toCurrency string + + if amount1 < 0 { + fromAccount = account1 + fromCurrency = splitData1.commodity + fromAmount = -amount1 + toAccount = account2 + toCurrency = splitData2.commodity + toAmount = amount2 + } else if amount2 < 0 { + fromAccount = account2 + fromCurrency = splitData2.commodity + fromAmount = -amount2 + toAccount = account1 + toCurrency = splitData1.commodity + toAmount = amount1 + } else { + log.Errorf(ctx, "[beancount_transaction_data_table.parseTransaction] cannot parse transfer transaction, because unexcepted account amounts \"%d\" and \"%d\"", amount1, amount2) + return nil, errs.ErrInvalidBeancountFile + } + + data[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)) + data[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = fromAccount.name + data[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY] = fromCurrency + data[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(fromAmount) + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = toAccount.name + data[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY] = toCurrency + data[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(toAmount) + } else { + log.Errorf(ctx, "[beancount_transaction_data_table.parseTransaction] cannot parse transaction, because unexcepted account types \"%d\" and \"%d\"", account1.accountType, account2.accountType) + return nil, errs.ErrThereAreNotSupportedTransactionType + } + } else if len(beancountEntry.postings) <= 1 { + log.Errorf(ctx, "[beancount_transaction_data_table.parseTransaction] cannot parse transaction, because postings count is %d", len(beancountEntry.postings)) + return nil, errs.ErrInvalidBeancountFile + } else { + log.Errorf(ctx, "[beancount_transaction_data_table.parseTransaction] cannot parse split transaction, because postings count is %d", len(beancountEntry.postings)) + return nil, errs.ErrNotSupportedSplitTransactions + } + + data[datatable.TRANSACTION_DATA_TABLE_TAGS] = strings.Join(beancountEntry.tags, BEANCOUNT_TRANSACTION_TAG_SEPARATOR) + data[datatable.TRANSACTION_DATA_TABLE_DESCRIPTION] = beancountEntry.narration + + return data, nil +} + +func createNewBeancountTransactionDataTable(beancountData *beancountData) (*beancountTransactionDataTable, error) { + if beancountData == nil { + return nil, errs.ErrNotFoundTransactionDataInFile + } + + return &beancountTransactionDataTable{ + allData: beancountData.transactions, + accountMap: beancountData.accounts, + }, nil +} diff --git a/pkg/converters/transaction_data_converters.go b/pkg/converters/transaction_data_converters.go index 3c14d6b7..e42d6f90 100644 --- a/pkg/converters/transaction_data_converters.go +++ b/pkg/converters/transaction_data_converters.go @@ -2,6 +2,7 @@ package converters import ( "github.com/mayswind/ezbookkeeping/pkg/converters/alipay" + "github.com/mayswind/ezbookkeeping/pkg/converters/beancount" "github.com/mayswind/ezbookkeeping/pkg/converters/converter" "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/converters/default" @@ -50,6 +51,8 @@ func GetTransactionDataImporter(fileType string) (converter.TransactionDataImpor return gnucash.GnuCashTransactionDataImporter, nil } else if fileType == "firefly_iii_csv" { return fireflyIII.FireflyIIITransactionDataCsvFileImporter, nil + } else if fileType == "beancount" { + return beancount.BeancountTransactionDataImporter, nil } else if fileType == "feidee_mymoney_csv" { return feidee.FeideeMymoneyAppTransactionDataCsvFileImporter, nil } else if fileType == "feidee_mymoney_xls" { diff --git a/pkg/errs/converter.go b/pkg/errs/converter.go index 1f61d9fc..426ad966 100644 --- a/pkg/errs/converter.go +++ b/pkg/errs/converter.go @@ -25,4 +25,7 @@ var ( ErrInvalidIIFFile = NewNormalError(NormalSubcategoryConverter, 18, http.StatusBadRequest, "invalid iif file") ErrInvalidOFXFile = NewNormalError(NormalSubcategoryConverter, 19, http.StatusBadRequest, "invalid ofx file") ErrInvalidSGMLFile = NewNormalError(NormalSubcategoryConverter, 20, http.StatusBadRequest, "invalid sgml file") + ErrInvalidBeancountFile = NewNormalError(NormalSubcategoryConverter, 21, http.StatusBadRequest, "invalid beancount file") + ErrBeancountFileNotSupportInclude = NewNormalError(NormalSubcategoryConverter, 22, http.StatusBadRequest, "not support include directive for beancount file") + ErrInvalidAmountExpression = NewNormalError(NormalSubcategoryConverter, 23, http.StatusBadRequest, "invalid amount expression") ) diff --git a/src/consts/file.ts b/src/consts/file.ts index 9755dcee..463e13cc 100644 --- a/src/consts/file.ts +++ b/src/consts/file.ts @@ -171,6 +171,11 @@ export const SUPPORTED_IMPORT_FILE_TYPES: ImportFileType[] = [ anchor: 'how-to-get-firefly-iii-data-export-file' } }, + { + type: 'beancount', + name: 'Beancount Data File', + extensions: '.beancount' + }, { type: 'feidee_mymoney_csv', name: 'Feidee MyMoney (App) Data Export File', diff --git a/src/locales/de.json b/src/locales/de.json index 95a0ea85..82e9e2bb 100644 --- a/src/locales/de.json +++ b/src/locales/de.json @@ -1159,6 +1159,9 @@ "invalid iif file": "Ungültige IIF-Datei", "invalid ofx file": "Ungültige OFX-Datei", "invalid sgml file": "Ungültige SGML-Datei", + "invalid beancount file": "Invalid Beancount file", + "not support include directive for beancount file": "Not support \"include\" directive for Beancount file", + "invalid amount expression": "Amount expression is invalid", "query items cannot be blank": "Abfrageelemente dürfen nicht leer sein", "query items too much": "Zu viele Abfrageelemente", "query items have invalid item": "Ungültiges Element in Abfrageelementen", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "GnuCash XML-Datenbankdatei", "Firefly III Data Export File": "Firefly III-Datenexportdatei", + "Beancount Data File": "Beancount Data File", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App)-Datenexportdatei", "Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web)-Datenexportdatei", "Feidee MyMoney (Elecloud) Data Export File": "Feidee MyMoney (Elecloud) Data Export File", diff --git a/src/locales/en.json b/src/locales/en.json index 96c2067f..e06b0d85 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1159,6 +1159,9 @@ "invalid iif file": "Invalid IIF file", "invalid ofx file": "Invalid OFX file", "invalid sgml file": "Invalid SGML file", + "invalid beancount file": "Invalid Beancount file", + "not support include directive for beancount file": "Not support \"include\" directive for Beancount file", + "invalid amount expression": "Amount expression is invalid", "query items cannot be blank": "There are no query items", "query items too much": "There are too many query items", "query items have invalid item": "There is invalid item in query items", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "GnuCash XML Database File", "Firefly III Data Export File": "Firefly III Data Export File", + "Beancount Data File": "Beancount Data File", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App) Data Export File", "Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web) Data Export File", "Feidee MyMoney (Elecloud) Data Export File": "Feidee MyMoney (Elecloud) Data Export File", diff --git a/src/locales/es.json b/src/locales/es.json index 20a9664b..49108eb1 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -1159,6 +1159,9 @@ "invalid iif file": "Archivo IIF no válido", "invalid ofx file": "Archivo OFX no válido", "invalid sgml file": "Archivo SGML no válido", + "invalid beancount file": "Invalid Beancount file", + "not support include directive for beancount file": "Not support \"include\" directive for Beancount file", + "invalid amount expression": "Amount expression is invalid", "query items cannot be blank": "--", "query items too much": "--", "query items have invalid item": "Hay un elemento no válido en los elementos de consulta", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "Archivo de base de datos XML GnuCash", "Firefly III Data Export File": "Archivo de exportación de datos de Firefly III", + "Beancount Data File": "Beancount Data File", "Feidee MyMoney (App) Data Export File": "Archivo de exportación de datos Feidee MyMoney (aplicación)", "Feidee MyMoney (Web) Data Export File": "Archivo de exportación de datos Feidee MyMoney (Web)", "Feidee MyMoney (Elecloud) Data Export File": "Feidee MyMoney (Elecloud) Data Export File", diff --git a/src/locales/ja.json b/src/locales/ja.json index 489d0a37..2c1a54e7 100644 --- a/src/locales/ja.json +++ b/src/locales/ja.json @@ -1159,6 +1159,9 @@ "invalid iif file": "無効なIIFファイルです", "invalid ofx file": "無効なOFXファイルです", "invalid sgml file": "無効なSGMLファイル", + "invalid beancount file": "Invalid Beancount file", + "not support include directive for beancount file": "Not support \"include\" directive for Beancount file", + "invalid amount expression": "Amount expression is invalid", "query items cannot be blank": "クエリ項目がありません", "query items too much": "クエリ項目が多すぎます", "query items have invalid item": "クエリ項目に無効な項目があります", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) データ", "GnuCash XML Database File": "GnuCash XMLデータベースファイル", "Firefly III Data Export File": "Firefly III データエクスポートファイル", + "Beancount Data File": "Beancount Data File", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App) データベースファイル", "Feidee MyMoney (Web) Data Export File": "Feidee MyMoney (Web) データベースファイル", "Feidee MyMoney (Elecloud) Data Export File": "Feidee MyMoney (Elecloud) Data Export File", diff --git a/src/locales/ru.json b/src/locales/ru.json index b028a537..b6dfd375 100644 --- a/src/locales/ru.json +++ b/src/locales/ru.json @@ -1159,6 +1159,9 @@ "invalid iif file": "Недопустимый IIF-файл", "invalid ofx file": "Недопустимый OFX-файл", "invalid sgml file": "Недопустимый SGML-файл", + "invalid beancount file": "Invalid Beancount file", + "not support include directive for beancount file": "Not support \"include\" directive for Beancount file", + "invalid amount expression": "Amount expression is invalid", "query items cannot be blank": "Нет элементов запроса", "query items too much": "Слишком много элементов запроса", "query items have invalid item": "В элементах запроса присутствует недопустимый элемент", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "Файл базы данных GnuCash XML", "Firefly III Data Export File": "Файл экспорта данных Firefly III", + "Beancount Data File": "Beancount Data File", "Feidee MyMoney (App) Data Export File": "Файл экспорта данных Feidee MyMoney (приложение)", "Feidee MyMoney (Web) Data Export File": "Файл экспорта данных Feidee MyMoney (веб)", "Feidee MyMoney (Elecloud) Data Export File": "Feidee MyMoney (Elecloud) Data Export File", diff --git a/src/locales/vi.json b/src/locales/vi.json index 059259df..88064f63 100644 --- a/src/locales/vi.json +++ b/src/locales/vi.json @@ -1159,6 +1159,9 @@ "invalid iif file": "Tệp IIF không hợp lệ", "invalid ofx file": "Tệp OFX không hợp lệ", "invalid sgml file": "Tệp SGML không hợp lệ", + "invalid beancount file": "Invalid Beancount file", + "not support include directive for beancount file": "Not support \"include\" directive for Beancount file", + "invalid amount expression": "Amount expression is invalid", "query items cannot be blank": "Không có mục truy vấn", "query items too much": "Có quá nhiều mục truy vấn", "query items have invalid item": "Có mục không hợp lệ trong các mục truy vấn", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "Tệp cơ sở dữ liệu XML GnuCash", "Firefly III Data Export File": "Tệp xuất dữ liệu Firefly III", + "Beancount Data File": "Beancount Data File", "Feidee MyMoney (App) Data Export File": "Tệp xuất dữ liệu Feidee MyMoney (Ứng dụng)", "Feidee MyMoney (Web) Data Export File": "Tệp xuất dữ liệu Feidee MyMoney (Web)", "Feidee MyMoney (Elecloud) Data Export File": "Feidee MyMoney (Elecloud) Data Export File", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index 4d7d0d3b..f88d7dea 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -1159,6 +1159,9 @@ "invalid iif file": "无效的 IIF 文件", "invalid ofx file": "无效的 OFX 文件", "invalid sgml file": "无效的 SGML 文件", + "invalid beancount file": "无效的 Beancount 文件", + "not support include directive for beancount file": "不支持 Beancount 文件的 \"include\" 指令", + "invalid amount expression": "金额表达式无效", "query items cannot be blank": "请求项目不能为空", "query items too much": "请求项目过多", "query items have invalid item": "请求项目中有非法项目", @@ -1651,6 +1654,7 @@ "Delimiter-separated Values (DSV) Data": "分隔符分隔值 (DSV) 数据", "GnuCash XML Database File": "GnuCash XML 数据库文件", "Firefly III Data Export File": "Firefly III 数据导出文件", + "Beancount Data File": "Beancount 数据文件", "Feidee MyMoney (App) Data Export File": "随手记 (App) 数据导出文件", "Feidee MyMoney (Web) Data Export File": "随手记 (Web版) 数据导出文件", "Feidee MyMoney (Elecloud) Data Export File": "随手记 (神象云账本) 数据导出文件",