mirror of
https://github.com/mayswind/ezbookkeeping.git
synced 2026-05-17 00:12:11 +08:00
import transaction from custom delimiter-separated values file
This commit is contained in:
MaysWind
+159
-1
@@ -1,6 +1,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -8,6 +9,8 @@ import (
|
||||
orderedmap "github.com/wk8/go-ordered-map/v2"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters"
|
||||
baseconverters "github.com/mayswind/ezbookkeeping/pkg/converters/base"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/duplicatechecker"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
@@ -1030,6 +1033,83 @@ func (a *TransactionsApi) TransactionDeleteHandler(c *core.WebContext) (any, *er
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// TransactionParseImportDsvFileDataHandler returns the parsed file data by request parameters for current user
|
||||
func (a *TransactionsApi) TransactionParseImportDsvFileDataHandler(c *core.WebContext) (any, *errs.Error) {
|
||||
uid := c.GetCurrentUid()
|
||||
form, err := c.MultipartForm()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to get multi-part form data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrParameterInvalid
|
||||
}
|
||||
|
||||
fileTypes := form.Value["fileType"]
|
||||
|
||||
if len(fileTypes) < 1 || fileTypes[0] == "" {
|
||||
return nil, errs.ErrImportFileTypeIsEmpty
|
||||
}
|
||||
|
||||
fileType := fileTypes[0]
|
||||
|
||||
if !converters.IsCustomDelimiterSeparatedValuesFileType(fileType) {
|
||||
return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported)
|
||||
}
|
||||
|
||||
fileEncodings := form.Value["fileEncoding"]
|
||||
|
||||
if len(fileEncodings) < 1 || fileEncodings[0] == "" {
|
||||
return nil, errs.ErrImportFileEncodingIsEmpty
|
||||
}
|
||||
|
||||
fileEncoding := fileEncodings[0]
|
||||
dataParser, err := converters.CreateNewDelimiterSeparatedValuesDataParser(fileType, fileEncoding)
|
||||
|
||||
if err != nil {
|
||||
return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported)
|
||||
}
|
||||
|
||||
importFiles := form.File["file"]
|
||||
|
||||
if len(importFiles) < 1 {
|
||||
log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] there is no import file in request for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrNoFilesUpload
|
||||
}
|
||||
|
||||
if importFiles[0].Size < 1 {
|
||||
log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] the size of import file in request is zero for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrUploadedFileEmpty
|
||||
}
|
||||
|
||||
if importFiles[0].Size > int64(a.CurrentConfig().MaxImportFileSize) {
|
||||
log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] the upload file size \"%d\" exceeds the maximum size \"%d\" of import file for user \"uid:%d\"", importFiles[0].Size, a.CurrentConfig().MaxImportFileSize, uid)
|
||||
return nil, errs.ErrExceedMaxUploadFileSize
|
||||
}
|
||||
|
||||
importFile, err := importFiles[0].Open()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to get import file from request for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
defer importFile.Close()
|
||||
fileData, err := io.ReadAll(importFile)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to read import file data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
allLines, err := dataParser.ParseDsvFileLines(c, fileData)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to parse import file data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
return allLines, nil
|
||||
}
|
||||
|
||||
// TransactionParseImportFileHandler returns the parsed transaction data by request parameters for current user
|
||||
func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext) (any, *errs.Error) {
|
||||
uid := c.GetCurrentUid()
|
||||
@@ -1054,7 +1134,84 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext)
|
||||
}
|
||||
|
||||
fileType := fileTypes[0]
|
||||
dataImporter, err := converters.GetTransactionDataImporter(fileType)
|
||||
|
||||
var dataImporter baseconverters.TransactionDataImporter
|
||||
|
||||
if converters.IsCustomDelimiterSeparatedValuesFileType(fileType) {
|
||||
fileEncodings := form.Value["fileEncoding"]
|
||||
|
||||
if len(fileEncodings) < 1 || fileEncodings[0] == "" {
|
||||
return nil, errs.ErrImportFileEncodingIsEmpty
|
||||
}
|
||||
|
||||
fileEncoding := fileEncodings[0]
|
||||
|
||||
columnMappings := form.Value["columnMapping"]
|
||||
|
||||
if len(columnMappings) < 1 || columnMappings[0] == "" {
|
||||
return nil, errs.ErrImportFileColumnMappingInvalid
|
||||
}
|
||||
|
||||
var columnIndexMapping = map[datatable.TransactionDataTableColumn]int{}
|
||||
err = json.Unmarshal([]byte(columnMappings[0]), &columnIndexMapping)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportFileHandler] failed to parse column mapping for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrImportFileColumnMappingInvalid
|
||||
}
|
||||
|
||||
transactionTypeMappings := form.Value["transactionTypeMapping"]
|
||||
|
||||
if len(transactionTypeMappings) < 1 || transactionTypeMappings[0] == "" {
|
||||
return nil, errs.ErrImportFileTransactionTypeMappingInvalid
|
||||
}
|
||||
|
||||
var transactionTypeNameMapping = map[string]models.TransactionType{}
|
||||
err = json.Unmarshal([]byte(transactionTypeMappings[0]), &transactionTypeNameMapping)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportFileHandler] failed to parse transaction type mapping for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrImportFileTransactionTypeMappingInvalid
|
||||
}
|
||||
|
||||
hasHeaderLines := form.Value["hasHeaderLine"]
|
||||
hasHeaderLine := false
|
||||
|
||||
if len(hasHeaderLines) > 0 {
|
||||
hasHeaderLine = hasHeaderLines[0] == "true"
|
||||
}
|
||||
|
||||
timeFormats := form.Value["timeFormat"]
|
||||
|
||||
if len(timeFormats) < 1 || timeFormats[0] == "" {
|
||||
return nil, errs.ErrImportFileTransactionTimeFormatInvalid
|
||||
}
|
||||
|
||||
timezoneFormats := form.Value["timezoneFormat"]
|
||||
timezoneFormat := ""
|
||||
|
||||
if len(timezoneFormats) > 0 {
|
||||
timezoneFormat = timezoneFormats[0]
|
||||
}
|
||||
|
||||
geoLocationSeparators := form.Value["geoSeparator"]
|
||||
geoLocationSeparator := ""
|
||||
|
||||
if len(geoLocationSeparators) > 0 {
|
||||
geoLocationSeparator = geoLocationSeparators[0]
|
||||
}
|
||||
|
||||
transactionTagSeparators := form.Value["tagSeparator"]
|
||||
transactionTagSeparator := ""
|
||||
|
||||
if len(transactionTagSeparators) > 0 {
|
||||
transactionTagSeparator = transactionTagSeparators[0]
|
||||
}
|
||||
|
||||
dataImporter, err = converters.CreateNewDelimiterSeparatedValuesDataImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormats[0], timezoneFormat, geoLocationSeparator, transactionTagSeparator)
|
||||
} else {
|
||||
dataImporter, err = converters.GetTransactionDataImporter(fileType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported)
|
||||
@@ -1084,6 +1241,7 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext)
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
defer importFile.Close()
|
||||
fileData, err := io.ReadAll(importFile)
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -418,7 +418,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
|
||||
geoLongitude := float64(0)
|
||||
geoLatitude := float64(0)
|
||||
|
||||
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION) {
|
||||
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION) && c.geoLocationSeparator != "" {
|
||||
geoLocationItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), c.geoLocationSeparator)
|
||||
|
||||
if len(geoLocationItems) == 2 {
|
||||
@@ -442,7 +442,13 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u
|
||||
var tagNames []string
|
||||
|
||||
if dataTable.HasColumn(TRANSACTION_DATA_TABLE_TAGS) {
|
||||
tagNameItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS), c.transactionTagSeparator)
|
||||
var tagNameItems []string
|
||||
|
||||
if c.transactionTagSeparator != "" {
|
||||
tagNameItems = strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS), c.transactionTagSeparator)
|
||||
} else {
|
||||
tagNameItems = append(tagNameItems, dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS))
|
||||
}
|
||||
|
||||
for i := 0; i < len(tagNameItems); i++ {
|
||||
tagName := tagNameItems[i]
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
package dsv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/encoding"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"golang.org/x/text/encoding/japanese"
|
||||
"golang.org/x/text/encoding/korean"
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"golang.org/x/text/encoding/traditionalchinese"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
"golang.org/x/text/transform"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/base"
|
||||
csvconverter "github.com/mayswind/ezbookkeeping/pkg/converters/csv"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/models"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/utils"
|
||||
)
|
||||
|
||||
var supportedFileTypeSeparators = map[string]rune{
|
||||
"custom_csv": ',',
|
||||
"custom_tsv": '\t',
|
||||
}
|
||||
|
||||
var supportedFileEncodings = map[string]encoding.Encoding{
|
||||
"utf-8": unicode.UTF8, // UTF-8
|
||||
"utf-8-bom": unicode.UTF8BOM, // UTF-8 with BOM
|
||||
"utf-16le": unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), // UTF-16 Little Endian
|
||||
"utf-16be": unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), // UTF-16 Big Endian
|
||||
"cp437": charmap.CodePage437, // OEM United States (CP-437)
|
||||
"cp863": charmap.CodePage863, // OEM Canadian French (CP-863)
|
||||
"cp037": charmap.CodePage037, // IBM EBCDIC US/Canada (CP-037)
|
||||
"cp1047": charmap.CodePage1047, // IBM EBCDIC Open Systems (CP-1047)
|
||||
"cp1140": charmap.CodePage1140, // IBM EBCDIC US/Canada with Euro (CP-1140)
|
||||
"iso-8859-1": charmap.ISO8859_1, // Western European (ISO-8859-1)
|
||||
"cp850": charmap.CodePage850, // Western European (CP-850)
|
||||
"cp858": charmap.CodePage858, // Western European with Euro (CP-858)
|
||||
"windows-1252": charmap.Windows1252, // Western European (Windows-1252)
|
||||
"iso-8859-15": charmap.ISO8859_15, // Western European (ISO-8859-15)
|
||||
"iso-8859-4": charmap.ISO8859_4, // North European (ISO-8859-4)
|
||||
"iso-8859-10": charmap.ISO8859_10, // North European (ISO-8859-10)
|
||||
"cp865": charmap.CodePage865, // North European (CP-865)
|
||||
"iso-8859-2": charmap.ISO8859_2, // Central European (ISO-8859-2)
|
||||
"cp852": charmap.CodePage852, // Central European (CP-852)
|
||||
"windows-1250": charmap.Windows1250, // Central European (Windows-1250)
|
||||
"iso-8859-14": charmap.ISO8859_14, // Celtic (ISO-8859-14)
|
||||
"iso-8859-3": charmap.ISO8859_3, // South European (ISO-8859-3)
|
||||
"cp860": charmap.CodePage860, // Portuguese (CP-860)
|
||||
"iso-8859-7": charmap.ISO8859_7, // Greek (ISO-8859-7)
|
||||
"windows-1253": charmap.Windows1253, // Greek (Windows-1253)
|
||||
"iso-8859-9": charmap.ISO8859_9, // Turkish (ISO-8859-9)
|
||||
"windows-1254": charmap.Windows1254, // Turkish (Windows-1254)
|
||||
"iso-8859-13": charmap.ISO8859_13, // Baltic (ISO-8859-13)
|
||||
"windows-1257": charmap.Windows1257, // Baltic (Windows-1257)
|
||||
"iso-8859-16": charmap.ISO8859_16, // South-Eastern European (ISO-8859-16)
|
||||
"iso-8859-5": charmap.ISO8859_5, // Cyrillic (ISO-8859-5)
|
||||
"cp855": charmap.CodePage855, // Cyrillic (CP-855)
|
||||
"cp866": charmap.CodePage866, // Cyrillic (CP-866)
|
||||
"windows-1251": charmap.Windows1251, // Cyrillic (Windows-1251)
|
||||
"koi8r": charmap.KOI8R, // Cyrillic (KOI8-R)
|
||||
"koi8u": charmap.KOI8U, // Cyrillic (KOI8-U)
|
||||
"iso-8859-6": charmap.ISO8859_6, // Arabic (ISO-8859-6)
|
||||
"windows-1256": charmap.Windows1256, // Arabic (Windows-1256)
|
||||
"iso-8859-8": charmap.ISO8859_8, // Hebrew (ISO-8859-8)
|
||||
"cp862": charmap.CodePage862, // Hebrew (CP-862)
|
||||
"windows-1255": charmap.Windows1255, // Hebrew (Windows-1255)
|
||||
"windows-874": charmap.Windows874, // Thai (Windows-874)
|
||||
"windows-1258": charmap.Windows1258, // Vietnamese (Windows-1258)
|
||||
"gb18030": simplifiedchinese.GB18030, // Simplified Chinese (GB18030)
|
||||
"gbk": simplifiedchinese.GBK, // Simplified Chinese (GBK)
|
||||
"big5": traditionalchinese.Big5, // Traditional Chinese (Big5)
|
||||
"euc-kr": korean.EUCKR, // Korean (EUC-KR)
|
||||
"euc-jp": japanese.EUCJP, // Japanese (EUC-JP)
|
||||
"iso-2022-jp": japanese.ISO2022JP, // Japanese (ISO-2022-JP)
|
||||
"shift_jis": japanese.ShiftJIS, // Japanese (Shift JIS)
|
||||
}
|
||||
|
||||
var customTransactionTypeNameMapping = map[models.TransactionType]string{
|
||||
models.TRANSACTION_TYPE_MODIFY_BALANCE: utils.IntToString(int(models.TRANSACTION_TYPE_MODIFY_BALANCE)),
|
||||
models.TRANSACTION_TYPE_INCOME: utils.IntToString(int(models.TRANSACTION_TYPE_INCOME)),
|
||||
models.TRANSACTION_TYPE_EXPENSE: utils.IntToString(int(models.TRANSACTION_TYPE_EXPENSE)),
|
||||
models.TRANSACTION_TYPE_TRANSFER: utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)),
|
||||
}
|
||||
|
||||
type CustomTransactionDataDsvFileParser interface {
|
||||
ParseDsvFileLines(ctx core.Context, data []byte) ([][]string, error)
|
||||
}
|
||||
|
||||
// customTransactionDataDsvFileImporter defines the structure of custom dsv importer for transaction data
|
||||
type customTransactionDataDsvFileImporter struct {
|
||||
fileEncoding encoding.Encoding
|
||||
separator rune
|
||||
columnIndexMapping map[datatable.TransactionDataTableColumn]int
|
||||
transactionTypeNameMapping map[string]models.TransactionType
|
||||
hasHeaderLine bool
|
||||
timeFormat string
|
||||
timezoneFormat string
|
||||
geoLocationSeparator string
|
||||
transactionTagSeparator string
|
||||
}
|
||||
|
||||
// ParseDsvFileLines returns the parsed file lines for specified the dsv file data
|
||||
func (c *customTransactionDataDsvFileImporter) ParseDsvFileLines(ctx core.Context, data []byte) ([][]string, error) {
|
||||
reader := transform.NewReader(bytes.NewReader(data), c.fileEncoding.NewDecoder())
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.Comma = c.separator
|
||||
csvReader.FieldsPerRecord = -1
|
||||
|
||||
allLines := make([][]string, 0)
|
||||
|
||||
for {
|
||||
items, err := csvReader.Read()
|
||||
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "[custom_transaction_data_dsv_file_importer.ParseDsvFileLines] cannot parse dsv data, because %s", err.Error())
|
||||
return nil, errs.ErrInvalidCSVFile
|
||||
}
|
||||
|
||||
if len(items) == 1 && items[0] == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
for index := range items {
|
||||
items[index] = strings.Trim(items[index], " ")
|
||||
}
|
||||
|
||||
allLines = append(allLines, items)
|
||||
}
|
||||
|
||||
return allLines, nil
|
||||
}
|
||||
|
||||
// ParseImportedData returns the imported data by parsing the custom transaction dsv data
|
||||
func (c *customTransactionDataDsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
|
||||
allLines, err := c.ParseDsvFileLines(ctx, data)
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
if !c.hasHeaderLine {
|
||||
allLines = append([][]string{{}}, allLines...)
|
||||
}
|
||||
|
||||
dataTable := csvconverter.CreateNewCustomCsvImportedDataTable(allLines)
|
||||
transactionDataTable := CreateNewCustomPlainTextDataTable(dataTable, c.columnIndexMapping, c.transactionTypeNameMapping, c.timeFormat, c.timezoneFormat)
|
||||
dataTableImporter := datatable.CreateNewImporter(customTransactionTypeNameMapping, c.geoLocationSeparator, c.transactionTagSeparator)
|
||||
|
||||
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
|
||||
}
|
||||
|
||||
// IsDelimiterSeparatedValuesFileType returns whether the file type is the delimiter-separated values file type
|
||||
func IsDelimiterSeparatedValuesFileType(fileType string) bool {
|
||||
_, exists := supportedFileTypeSeparators[fileType]
|
||||
return exists
|
||||
}
|
||||
|
||||
// CreateNewCustomTransactionDataDsvFileParser returns a new custom dsv parser for transaction data
|
||||
func CreateNewCustomTransactionDataDsvFileParser(fileType string, fileEncoding string) (CustomTransactionDataDsvFileParser, error) {
|
||||
separator, exists := supportedFileTypeSeparators[fileType]
|
||||
|
||||
if !exists {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
enc, exists := supportedFileEncodings[fileEncoding]
|
||||
|
||||
if !exists {
|
||||
return nil, errs.ErrImportFileEncodingNotSupported
|
||||
}
|
||||
|
||||
return &customTransactionDataDsvFileImporter{
|
||||
fileEncoding: enc,
|
||||
separator: separator,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateNewCustomTransactionDataDsvFileImporter returns a new custom dsv importer for transaction data
|
||||
func CreateNewCustomTransactionDataDsvFileImporter(fileType string, fileEncoding string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, geoLocationSeparator string, transactionTagSeparator string) (base.TransactionDataImporter, error) {
|
||||
separator, exists := supportedFileTypeSeparators[fileType]
|
||||
|
||||
if !exists {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
enc, exists := supportedFileEncodings[fileEncoding]
|
||||
|
||||
if !exists {
|
||||
return nil, errs.ErrImportFileEncodingNotSupported
|
||||
}
|
||||
|
||||
if _, exists = columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME]; !exists {
|
||||
return nil, errs.ErrMissingRequiredFieldInHeaderRow
|
||||
}
|
||||
|
||||
if _, exists = columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]; !exists {
|
||||
return nil, errs.ErrMissingRequiredFieldInHeaderRow
|
||||
}
|
||||
|
||||
if _, exists = columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_AMOUNT]; !exists {
|
||||
return nil, errs.ErrMissingRequiredFieldInHeaderRow
|
||||
}
|
||||
|
||||
return &customTransactionDataDsvFileImporter{
|
||||
fileEncoding: enc,
|
||||
separator: separator,
|
||||
columnIndexMapping: columnIndexMapping,
|
||||
transactionTypeNameMapping: transactionTypeNameMapping,
|
||||
hasHeaderLine: hasHeaderLine,
|
||||
timeFormat: timeFormat,
|
||||
timezoneFormat: timezoneFormat,
|
||||
geoLocationSeparator: geoLocationSeparator,
|
||||
transactionTagSeparator: transactionTagSeparator,
|
||||
}, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,275 @@
|
||||
package dsv
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/models"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/utils"
|
||||
)
|
||||
|
||||
// customPlainTextDataTable defines the structure of custom plain text transaction data table
|
||||
type customPlainTextDataTable struct {
|
||||
innerDataTable datatable.ImportedDataTable
|
||||
columnIndexMapping map[datatable.TransactionDataTableColumn]int
|
||||
transactionTypeNameMapping map[string]models.TransactionType
|
||||
timeFormat string
|
||||
timezoneFormat string
|
||||
timeFormatIncludeTimezone bool
|
||||
}
|
||||
|
||||
// customPlainTextDataRow defines the structure of custom plain text transaction data row
|
||||
type customPlainTextDataRow struct {
|
||||
transactionDataTable *customPlainTextDataTable
|
||||
rowData map[datatable.TransactionDataTableColumn]string
|
||||
isValid bool
|
||||
}
|
||||
|
||||
// customPlainTextDataRowIterator defines the structure of custom plain text transaction data row iterator
|
||||
type customPlainTextDataRowIterator struct {
|
||||
transactionDataTable *customPlainTextDataTable
|
||||
innerIterator datatable.ImportedDataRowIterator
|
||||
}
|
||||
|
||||
// HasColumn returns whether the data table has specified column
|
||||
func (t *customPlainTextDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool {
|
||||
// custom dsv file allows no sub category, account name and related account name column mapping, but data table converter needs these columns
|
||||
if column == datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY ||
|
||||
column == datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME ||
|
||||
column == datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME {
|
||||
return true
|
||||
}
|
||||
|
||||
// timezone column will be added when original time format contains timezone
|
||||
if t.timeFormatIncludeTimezone && column == datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE {
|
||||
return true
|
||||
}
|
||||
|
||||
_, exists := t.columnIndexMapping[column]
|
||||
return exists
|
||||
}
|
||||
|
||||
// TransactionRowCount returns the total count of transaction data row
|
||||
func (t *customPlainTextDataTable) TransactionRowCount() int {
|
||||
return t.innerDataTable.DataRowCount()
|
||||
}
|
||||
|
||||
// TransactionRowIterator returns the iterator of transaction data row
|
||||
func (t *customPlainTextDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator {
|
||||
return &customPlainTextDataRowIterator{
|
||||
transactionDataTable: t,
|
||||
innerIterator: t.innerDataTable.DataRowIterator(),
|
||||
}
|
||||
}
|
||||
|
||||
// IsValid returns whether this row is valid data for importing
|
||||
func (r *customPlainTextDataRow) IsValid() bool {
|
||||
return r.isValid
|
||||
}
|
||||
|
||||
// GetData returns the data in the specified column type
|
||||
func (r *customPlainTextDataRow) GetData(column datatable.TransactionDataTableColumn) string {
|
||||
return r.rowData[column]
|
||||
}
|
||||
|
||||
// HasNext returns whether the iterator does not reach the end
|
||||
func (t *customPlainTextDataRowIterator) HasNext() bool {
|
||||
return t.innerIterator.HasNext()
|
||||
}
|
||||
|
||||
// Next returns the next transaction data row
|
||||
func (t *customPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) {
|
||||
importedRow := t.innerIterator.Next()
|
||||
|
||||
if importedRow == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rowData, isValid, err := t.parseTransaction(ctx, user, importedRow)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "[custom_transaction_plain_text_data_table.Next] cannot parsing transaction in row \"%s\", because %s", t.innerIterator.CurrentRowId(), err.Error())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &customPlainTextDataRow{
|
||||
transactionDataTable: t.transactionDataTable,
|
||||
rowData: rowData,
|
||||
isValid: isValid,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *customPlainTextDataRowIterator) parseTransaction(ctx core.Context, user *models.User, row datatable.ImportedDataRow) (map[datatable.TransactionDataTableColumn]string, bool, error) {
|
||||
rowData := make(map[datatable.TransactionDataTableColumn]string, len(t.transactionDataTable.columnIndexMapping))
|
||||
|
||||
for column, columnIndex := range t.transactionDataTable.columnIndexMapping {
|
||||
if columnIndex < 0 || columnIndex >= row.ColumnCount() {
|
||||
continue
|
||||
}
|
||||
|
||||
value := row.GetData(columnIndex)
|
||||
rowData[column] = value
|
||||
}
|
||||
|
||||
// parse transaction type
|
||||
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] != "" {
|
||||
transactionType, exists := t.transactionDataTable.transactionTypeNameMapping[rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]]
|
||||
|
||||
if !exists {
|
||||
log.Warnf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] skip parsing this transaction, because transaction type \"%s\" mapping not defined", rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE])
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
mappedTransactionType, exists := customTransactionTypeNameMapping[transactionType]
|
||||
|
||||
if !exists {
|
||||
log.Errorf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] cannot parsing transaction type \"%s\", because type \"%d\" is invalid", rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE], transactionType)
|
||||
return nil, false, errs.ErrTransactionTypeInvalid
|
||||
}
|
||||
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = mappedTransactionType
|
||||
}
|
||||
|
||||
// parse date time
|
||||
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" {
|
||||
dateTime, err := time.Parse(t.transactionDataTable.timeFormat, rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME])
|
||||
|
||||
if err != nil {
|
||||
return nil, false, errs.ErrTransactionTimeInvalid
|
||||
}
|
||||
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location())
|
||||
|
||||
if t.transactionDataTable.timeFormatIncludeTimezone {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location())
|
||||
}
|
||||
}
|
||||
|
||||
// parse timezone
|
||||
if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] != "" {
|
||||
if t.transactionDataTable.timezoneFormat == "Z" || t.transactionDataTable.timezoneFormat == "" { // -HH:mm
|
||||
// Do Nothing
|
||||
} else if t.transactionDataTable.timezoneFormat == "ZZ" { // -HHmm
|
||||
timezone := rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE]
|
||||
|
||||
if len(timezone) != 5 {
|
||||
return nil, false, errs.ErrTransactionTimeZoneInvalid
|
||||
}
|
||||
|
||||
timezone = timezone[:3] + ":" + timezone[3:]
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = timezone
|
||||
} else {
|
||||
return nil, false, errs.ErrImportFileTransactionTimezoneFormatInvalid
|
||||
}
|
||||
}
|
||||
|
||||
// use primary category if sub category is empty
|
||||
if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" && rowData[datatable.TRANSACTION_DATA_TABLE_CATEGORY] != "" {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = rowData[datatable.TRANSACTION_DATA_TABLE_CATEGORY]
|
||||
}
|
||||
|
||||
// trim trailing zero in decimal
|
||||
if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
|
||||
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT])
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] cannot parsing transaction amount \"%s\", because %s", rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT], err.Error())
|
||||
return nil, false, errs.ErrAmountInvalid
|
||||
}
|
||||
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount)
|
||||
}
|
||||
|
||||
if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] != "" {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
|
||||
amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT])
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] cannot parsing transaction related amount \"%s\", because %s", rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT], err.Error())
|
||||
return nil, false, errs.ErrAmountInvalid
|
||||
}
|
||||
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(amount)
|
||||
}
|
||||
|
||||
if _, exists := rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY]; !exists {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = ""
|
||||
}
|
||||
|
||||
if _, exists := rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]; !exists {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = ""
|
||||
}
|
||||
|
||||
if _, exists := rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]; !exists {
|
||||
rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = ""
|
||||
}
|
||||
|
||||
return rowData, true, nil
|
||||
}
|
||||
|
||||
// CreateNewCustomPlainTextDataTable returns transaction data table from imported data table
|
||||
func CreateNewCustomPlainTextDataTable(dataTable datatable.ImportedDataTable, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, timeFormat string, timezoneFormat string) *customPlainTextDataTable {
|
||||
timeFormatIncludeTimezone := strings.Contains(timeFormat, "z") || strings.Contains(timeFormat, "Z")
|
||||
|
||||
return &customPlainTextDataTable{
|
||||
innerDataTable: dataTable,
|
||||
columnIndexMapping: columnIndexMapping,
|
||||
transactionTypeNameMapping: transactionTypeNameMapping,
|
||||
timeFormat: getDateTimeFormat(timeFormat),
|
||||
timezoneFormat: timezoneFormat,
|
||||
timeFormatIncludeTimezone: timeFormatIncludeTimezone,
|
||||
}
|
||||
}
|
||||
|
||||
func getDateTimeFormat(format string) string {
|
||||
// convert moment.js format to Go format
|
||||
|
||||
format = strings.ReplaceAll(format, "YYYY", "2006")
|
||||
format = strings.ReplaceAll(format, "YY", "06")
|
||||
|
||||
format = strings.ReplaceAll(format, "MMMM", "January")
|
||||
format = strings.ReplaceAll(format, "MMM", "Jan")
|
||||
format = strings.ReplaceAll(format, "MM", "01")
|
||||
format = strings.ReplaceAll(format, "M", "1")
|
||||
|
||||
format = strings.ReplaceAll(format, "DD", "02")
|
||||
format = strings.ReplaceAll(format, "D", "2")
|
||||
|
||||
format = strings.ReplaceAll(format, "dddd", "Monday")
|
||||
format = strings.ReplaceAll(format, "ddd", "Mon")
|
||||
|
||||
format = strings.ReplaceAll(format, "HH", "15")
|
||||
format = strings.ReplaceAll(format, "H", "15")
|
||||
|
||||
format = strings.ReplaceAll(format, "hh", "03")
|
||||
format = strings.ReplaceAll(format, "h", "3")
|
||||
|
||||
format = strings.ReplaceAll(format, "mm", "04")
|
||||
format = strings.ReplaceAll(format, "m", "4")
|
||||
|
||||
format = strings.ReplaceAll(format, "ss", "05")
|
||||
format = strings.ReplaceAll(format, "s", "5")
|
||||
|
||||
for i := 9; i >= 1; i-- {
|
||||
format = strings.ReplaceAll(format, "."+strings.Repeat("S", i), "."+strings.Repeat("9", i))
|
||||
}
|
||||
|
||||
format = strings.ReplaceAll(format, "A", "PM")
|
||||
format = strings.ReplaceAll(format, "a", "pm")
|
||||
|
||||
format = strings.ReplaceAll(format, "zz", "MST")
|
||||
format = strings.ReplaceAll(format, "z", "MST")
|
||||
|
||||
if strings.Contains(format, "ZZ") {
|
||||
format = strings.ReplaceAll(format, "ZZ", "Z0700")
|
||||
} else if strings.Contains(format, "Z") {
|
||||
format = strings.ReplaceAll(format, "Z", "Z07:00")
|
||||
}
|
||||
|
||||
return format
|
||||
}
|
||||
@@ -3,7 +3,9 @@ package converters
|
||||
import (
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/alipay"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/base"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/default"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/dsv"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/feidee"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/gnucash"
|
||||
@@ -12,6 +14,7 @@ import (
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/qif"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/wechat"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/models"
|
||||
)
|
||||
|
||||
// GetTransactionDataExporter returns the transaction data exporter according to the file type
|
||||
@@ -61,3 +64,18 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter,
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
}
|
||||
|
||||
// IsCustomDelimiterSeparatedValuesFileType returns whether the file type is the delimiter-separated values file type
|
||||
func IsCustomDelimiterSeparatedValuesFileType(fileType string) bool {
|
||||
return dsv.IsDelimiterSeparatedValuesFileType(fileType)
|
||||
}
|
||||
|
||||
// CreateNewDelimiterSeparatedValuesDataParser returns a new delimiter-separated values data parser according to the file type and encoding
|
||||
func CreateNewDelimiterSeparatedValuesDataParser(fileType string, fileEncoding string) (dsv.CustomTransactionDataDsvFileParser, error) {
|
||||
return dsv.CreateNewCustomTransactionDataDsvFileParser(fileType, fileEncoding)
|
||||
}
|
||||
|
||||
// CreateNewDelimiterSeparatedValuesDataImporter returns a new delimiter-separated values data importer according to the file type and encoding
|
||||
func CreateNewDelimiterSeparatedValuesDataImporter(fileType string, fileEncoding string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, geoLocationSeparator string, transactionTagSeparator string) (base.TransactionDataImporter, error) {
|
||||
return dsv.CreateNewCustomTransactionDataDsvFileImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormat, timezoneFormat, geoLocationSeparator, transactionTagSeparator)
|
||||
}
|
||||
|
||||
@@ -35,4 +35,10 @@ var (
|
||||
ErrCannotAddTransactionBeforeBalanceModificationTransaction = NewSystemError(NormalSubcategoryTransaction, 28, http.StatusBadRequest, "cannot add transaction before balance modification transaction")
|
||||
ErrBalanceModificationTransactionCannotModifyTime = NewSystemError(NormalSubcategoryTransaction, 29, http.StatusBadRequest, "balance modification transaction cannot modify transaction time")
|
||||
ErrTransferTransactionAmountCannotBeLessThanZero = NewNormalError(NormalSubcategoryTransaction, 30, http.StatusBadRequest, "transfer transaction amount cannot be less than zero")
|
||||
ErrImportFileEncodingIsEmpty = NewSystemError(NormalSubcategoryTransaction, 31, http.StatusBadRequest, "import file encoding is empty")
|
||||
ErrImportFileEncodingNotSupported = NewSystemError(NormalSubcategoryTransaction, 32, http.StatusBadRequest, "import file encoding not supported")
|
||||
ErrImportFileColumnMappingInvalid = NewSystemError(NormalSubcategoryTransaction, 33, http.StatusBadRequest, "column mapping invalid")
|
||||
ErrImportFileTransactionTypeMappingInvalid = NewSystemError(NormalSubcategoryTransaction, 34, http.StatusBadRequest, "transaction type mapping invalid")
|
||||
ErrImportFileTransactionTimeFormatInvalid = NewSystemError(NormalSubcategoryTransaction, 35, http.StatusBadRequest, "transaction time format invalid")
|
||||
ErrImportFileTransactionTimezoneFormatInvalid = NewSystemError(NormalSubcategoryTransaction, 36, http.StatusBadRequest, "transaction time zone format invalid")
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user