diff --git a/README.md b/README.md index 0b6d05b0..450fe7d1 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Online Demo: [https://ezbookkeeping-demo.mayswind.net](https://ezbookkeeping-dem 7. Multi-language support 8. Two-factor authentication 9. Application lock (PIN code / WebAuthn) -10. Data import & export (OFX, QFX, QIF, IIF, GnuCash, FireFly III, etc.) +10. Data import & export (OFX, QFX, QIF, IIF, CSV, GnuCash, FireFly III, etc.) ## Screenshots ### Desktop Version diff --git a/cmd/webserver.go b/cmd/webserver.go index 185a0790..08bfb4df 100644 --- a/cmd/webserver.go +++ b/cmd/webserver.go @@ -315,6 +315,7 @@ func startWebServer(c *core.CliContext) error { apiV1Route.POST("/transactions/delete.json", bindApi(api.Transactions.TransactionDeleteHandler)) if config.EnableDataImport { + apiV1Route.POST("/transactions/parse_dsv_file.json", bindApi(api.Transactions.TransactionParseImportDsvFileDataHandler)) apiV1Route.POST("/transactions/parse_import.json", bindApi(api.Transactions.TransactionParseImportFileHandler)) apiV1Route.POST("/transactions/import.json", bindApi(api.Transactions.TransactionImportHandler)) } diff --git a/pkg/api/transactions.go b/pkg/api/transactions.go index 585f1fbd..89050b6e 100644 --- a/pkg/api/transactions.go +++ b/pkg/api/transactions.go @@ -1,6 +1,7 @@ package api import ( + "encoding/json" "io" "sort" "strings" @@ -8,6 +9,8 @@ import ( orderedmap "github.com/wk8/go-ordered-map/v2" "github.com/mayswind/ezbookkeeping/pkg/converters" + baseconverters "github.com/mayswind/ezbookkeeping/pkg/converters/base" + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/duplicatechecker" "github.com/mayswind/ezbookkeeping/pkg/errs" @@ -1030,6 +1033,83 @@ func (a *TransactionsApi) TransactionDeleteHandler(c *core.WebContext) (any, *er return true, nil } +// TransactionParseImportDsvFileDataHandler returns the parsed file data by request parameters for current user +func (a *TransactionsApi) TransactionParseImportDsvFileDataHandler(c *core.WebContext) (any, *errs.Error) { + uid := c.GetCurrentUid() + form, err := c.MultipartForm() + + if err != nil { + log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to get multi-part form data for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.ErrParameterInvalid + } + + fileTypes := form.Value["fileType"] + + if len(fileTypes) < 1 || fileTypes[0] == "" { + return nil, errs.ErrImportFileTypeIsEmpty + } + + fileType := fileTypes[0] + + if !converters.IsCustomDelimiterSeparatedValuesFileType(fileType) { + return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported) + } + + fileEncodings := form.Value["fileEncoding"] + + if len(fileEncodings) < 1 || fileEncodings[0] == "" { + return nil, errs.ErrImportFileEncodingIsEmpty + } + + fileEncoding := fileEncodings[0] + dataParser, err := converters.CreateNewDelimiterSeparatedValuesDataParser(fileType, fileEncoding) + + if err != nil { + return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported) + } + + importFiles := form.File["file"] + + if len(importFiles) < 1 { + log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] there is no import file in request for user \"uid:%d\"", uid) + return nil, errs.ErrNoFilesUpload + } + + if importFiles[0].Size < 1 { + log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] the size of import file in request is zero for user \"uid:%d\"", uid) + return nil, errs.ErrUploadedFileEmpty + } + + if importFiles[0].Size > int64(a.CurrentConfig().MaxImportFileSize) { + log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] the upload file size \"%d\" exceeds the maximum size \"%d\" of import file for user \"uid:%d\"", importFiles[0].Size, a.CurrentConfig().MaxImportFileSize, uid) + return nil, errs.ErrExceedMaxUploadFileSize + } + + importFile, err := importFiles[0].Open() + + if err != nil { + log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to get import file from request for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.ErrOperationFailed + } + + defer importFile.Close() + fileData, err := io.ReadAll(importFile) + + if err != nil { + log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to read import file data for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.Or(err, errs.ErrOperationFailed) + } + + allLines, err := dataParser.ParseDsvFileLines(c, fileData) + + if err != nil { + log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to parse import file data for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.Or(err, errs.ErrOperationFailed) + } + + return allLines, nil +} + // TransactionParseImportFileHandler returns the parsed transaction data by request parameters for current user func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext) (any, *errs.Error) { uid := c.GetCurrentUid() @@ -1054,7 +1134,84 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext) } fileType := fileTypes[0] - dataImporter, err := converters.GetTransactionDataImporter(fileType) + + var dataImporter baseconverters.TransactionDataImporter + + if converters.IsCustomDelimiterSeparatedValuesFileType(fileType) { + fileEncodings := form.Value["fileEncoding"] + + if len(fileEncodings) < 1 || fileEncodings[0] == "" { + return nil, errs.ErrImportFileEncodingIsEmpty + } + + fileEncoding := fileEncodings[0] + + columnMappings := form.Value["columnMapping"] + + if len(columnMappings) < 1 || columnMappings[0] == "" { + return nil, errs.ErrImportFileColumnMappingInvalid + } + + var columnIndexMapping = map[datatable.TransactionDataTableColumn]int{} + err = json.Unmarshal([]byte(columnMappings[0]), &columnIndexMapping) + + if err != nil { + log.Errorf(c, "[transactions.TransactionParseImportFileHandler] failed to parse column mapping for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.ErrImportFileColumnMappingInvalid + } + + transactionTypeMappings := form.Value["transactionTypeMapping"] + + if len(transactionTypeMappings) < 1 || transactionTypeMappings[0] == "" { + return nil, errs.ErrImportFileTransactionTypeMappingInvalid + } + + var transactionTypeNameMapping = map[string]models.TransactionType{} + err = json.Unmarshal([]byte(transactionTypeMappings[0]), &transactionTypeNameMapping) + + if err != nil { + log.Errorf(c, "[transactions.TransactionParseImportFileHandler] failed to parse transaction type mapping for user \"uid:%d\", because %s", uid, err.Error()) + return nil, errs.ErrImportFileTransactionTypeMappingInvalid + } + + hasHeaderLines := form.Value["hasHeaderLine"] + hasHeaderLine := false + + if len(hasHeaderLines) > 0 { + hasHeaderLine = hasHeaderLines[0] == "true" + } + + timeFormats := form.Value["timeFormat"] + + if len(timeFormats) < 1 || timeFormats[0] == "" { + return nil, errs.ErrImportFileTransactionTimeFormatInvalid + } + + timezoneFormats := form.Value["timezoneFormat"] + timezoneFormat := "" + + if len(timezoneFormats) > 0 { + timezoneFormat = timezoneFormats[0] + } + + geoLocationSeparators := form.Value["geoSeparator"] + geoLocationSeparator := "" + + if len(geoLocationSeparators) > 0 { + geoLocationSeparator = geoLocationSeparators[0] + } + + transactionTagSeparators := form.Value["tagSeparator"] + transactionTagSeparator := "" + + if len(transactionTagSeparators) > 0 { + transactionTagSeparator = transactionTagSeparators[0] + } + + dataImporter, err = converters.CreateNewDelimiterSeparatedValuesDataImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormats[0], timezoneFormat, geoLocationSeparator, transactionTagSeparator) + } else { + dataImporter, err = converters.GetTransactionDataImporter(fileType) + } if err != nil { return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported) @@ -1084,6 +1241,7 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext) return nil, errs.ErrOperationFailed } + defer importFile.Close() fileData, err := io.ReadAll(importFile) if err != nil { diff --git a/pkg/converters/datatable/data_table_transaction_data_converter.go b/pkg/converters/datatable/data_table_transaction_data_converter.go index 3f80464f..8c90458d 100644 --- a/pkg/converters/datatable/data_table_transaction_data_converter.go +++ b/pkg/converters/datatable/data_table_transaction_data_converter.go @@ -418,7 +418,7 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u geoLongitude := float64(0) geoLatitude := float64(0) - if dataTable.HasColumn(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION) { + if dataTable.HasColumn(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION) && c.geoLocationSeparator != "" { geoLocationItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION), c.geoLocationSeparator) if len(geoLocationItems) == 2 { @@ -442,7 +442,13 @@ func (c *DataTableTransactionDataImporter) ParseImportedData(ctx core.Context, u var tagNames []string if dataTable.HasColumn(TRANSACTION_DATA_TABLE_TAGS) { - tagNameItems := strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS), c.transactionTagSeparator) + var tagNameItems []string + + if c.transactionTagSeparator != "" { + tagNameItems = strings.Split(dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS), c.transactionTagSeparator) + } else { + tagNameItems = append(tagNameItems, dataRow.GetData(TRANSACTION_DATA_TABLE_TAGS)) + } for i := 0; i < len(tagNameItems); i++ { tagName := tagNameItems[i] diff --git a/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go new file mode 100644 index 00000000..5bd48b8b --- /dev/null +++ b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go @@ -0,0 +1,227 @@ +package dsv + +import ( + "bytes" + "encoding/csv" + "io" + "strings" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" + + "github.com/mayswind/ezbookkeeping/pkg/converters/base" + csvconverter "github.com/mayswind/ezbookkeeping/pkg/converters/csv" + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +var supportedFileTypeSeparators = map[string]rune{ + "custom_csv": ',', + "custom_tsv": '\t', +} + +var supportedFileEncodings = map[string]encoding.Encoding{ + "utf-8": unicode.UTF8, // UTF-8 + "utf-8-bom": unicode.UTF8BOM, // UTF-8 with BOM + "utf-16le": unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), // UTF-16 Little Endian + "utf-16be": unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), // UTF-16 Big Endian + "cp437": charmap.CodePage437, // OEM United States (CP-437) + "cp863": charmap.CodePage863, // OEM Canadian French (CP-863) + "cp037": charmap.CodePage037, // IBM EBCDIC US/Canada (CP-037) + "cp1047": charmap.CodePage1047, // IBM EBCDIC Open Systems (CP-1047) + "cp1140": charmap.CodePage1140, // IBM EBCDIC US/Canada with Euro (CP-1140) + "iso-8859-1": charmap.ISO8859_1, // Western European (ISO-8859-1) + "cp850": charmap.CodePage850, // Western European (CP-850) + "cp858": charmap.CodePage858, // Western European with Euro (CP-858) + "windows-1252": charmap.Windows1252, // Western European (Windows-1252) + "iso-8859-15": charmap.ISO8859_15, // Western European (ISO-8859-15) + "iso-8859-4": charmap.ISO8859_4, // North European (ISO-8859-4) + "iso-8859-10": charmap.ISO8859_10, // North European (ISO-8859-10) + "cp865": charmap.CodePage865, // North European (CP-865) + "iso-8859-2": charmap.ISO8859_2, // Central European (ISO-8859-2) + "cp852": charmap.CodePage852, // Central European (CP-852) + "windows-1250": charmap.Windows1250, // Central European (Windows-1250) + "iso-8859-14": charmap.ISO8859_14, // Celtic (ISO-8859-14) + "iso-8859-3": charmap.ISO8859_3, // South European (ISO-8859-3) + "cp860": charmap.CodePage860, // Portuguese (CP-860) + "iso-8859-7": charmap.ISO8859_7, // Greek (ISO-8859-7) + "windows-1253": charmap.Windows1253, // Greek (Windows-1253) + "iso-8859-9": charmap.ISO8859_9, // Turkish (ISO-8859-9) + "windows-1254": charmap.Windows1254, // Turkish (Windows-1254) + "iso-8859-13": charmap.ISO8859_13, // Baltic (ISO-8859-13) + "windows-1257": charmap.Windows1257, // Baltic (Windows-1257) + "iso-8859-16": charmap.ISO8859_16, // South-Eastern European (ISO-8859-16) + "iso-8859-5": charmap.ISO8859_5, // Cyrillic (ISO-8859-5) + "cp855": charmap.CodePage855, // Cyrillic (CP-855) + "cp866": charmap.CodePage866, // Cyrillic (CP-866) + "windows-1251": charmap.Windows1251, // Cyrillic (Windows-1251) + "koi8r": charmap.KOI8R, // Cyrillic (KOI8-R) + "koi8u": charmap.KOI8U, // Cyrillic (KOI8-U) + "iso-8859-6": charmap.ISO8859_6, // Arabic (ISO-8859-6) + "windows-1256": charmap.Windows1256, // Arabic (Windows-1256) + "iso-8859-8": charmap.ISO8859_8, // Hebrew (ISO-8859-8) + "cp862": charmap.CodePage862, // Hebrew (CP-862) + "windows-1255": charmap.Windows1255, // Hebrew (Windows-1255) + "windows-874": charmap.Windows874, // Thai (Windows-874) + "windows-1258": charmap.Windows1258, // Vietnamese (Windows-1258) + "gb18030": simplifiedchinese.GB18030, // Simplified Chinese (GB18030) + "gbk": simplifiedchinese.GBK, // Simplified Chinese (GBK) + "big5": traditionalchinese.Big5, // Traditional Chinese (Big5) + "euc-kr": korean.EUCKR, // Korean (EUC-KR) + "euc-jp": japanese.EUCJP, // Japanese (EUC-JP) + "iso-2022-jp": japanese.ISO2022JP, // Japanese (ISO-2022-JP) + "shift_jis": japanese.ShiftJIS, // Japanese (Shift JIS) +} + +var customTransactionTypeNameMapping = map[models.TransactionType]string{ + models.TRANSACTION_TYPE_MODIFY_BALANCE: utils.IntToString(int(models.TRANSACTION_TYPE_MODIFY_BALANCE)), + models.TRANSACTION_TYPE_INCOME: utils.IntToString(int(models.TRANSACTION_TYPE_INCOME)), + models.TRANSACTION_TYPE_EXPENSE: utils.IntToString(int(models.TRANSACTION_TYPE_EXPENSE)), + models.TRANSACTION_TYPE_TRANSFER: utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)), +} + +type CustomTransactionDataDsvFileParser interface { + ParseDsvFileLines(ctx core.Context, data []byte) ([][]string, error) +} + +// customTransactionDataDsvFileImporter defines the structure of custom dsv importer for transaction data +type customTransactionDataDsvFileImporter struct { + fileEncoding encoding.Encoding + separator rune + columnIndexMapping map[datatable.TransactionDataTableColumn]int + transactionTypeNameMapping map[string]models.TransactionType + hasHeaderLine bool + timeFormat string + timezoneFormat string + geoLocationSeparator string + transactionTagSeparator string +} + +// ParseDsvFileLines returns the parsed file lines for specified the dsv file data +func (c *customTransactionDataDsvFileImporter) ParseDsvFileLines(ctx core.Context, data []byte) ([][]string, error) { + reader := transform.NewReader(bytes.NewReader(data), c.fileEncoding.NewDecoder()) + csvReader := csv.NewReader(reader) + csvReader.Comma = c.separator + csvReader.FieldsPerRecord = -1 + + allLines := make([][]string, 0) + + for { + items, err := csvReader.Read() + + if err == io.EOF { + break + } + + if err != nil { + log.Errorf(ctx, "[custom_transaction_data_dsv_file_importer.ParseDsvFileLines] cannot parse dsv data, because %s", err.Error()) + return nil, errs.ErrInvalidCSVFile + } + + if len(items) == 1 && items[0] == "" { + continue + } + + for index := range items { + items[index] = strings.Trim(items[index], " ") + } + + allLines = append(allLines, items) + } + + return allLines, nil +} + +// ParseImportedData returns the imported data by parsing the custom transaction dsv data +func (c *customTransactionDataDsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezoneOffset int16, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) { + allLines, err := c.ParseDsvFileLines(ctx, data) + + if err != nil { + return nil, nil, nil, nil, nil, nil, err + } + + if !c.hasHeaderLine { + allLines = append([][]string{{}}, allLines...) + } + + dataTable := csvconverter.CreateNewCustomCsvImportedDataTable(allLines) + transactionDataTable := CreateNewCustomPlainTextDataTable(dataTable, c.columnIndexMapping, c.transactionTypeNameMapping, c.timeFormat, c.timezoneFormat) + dataTableImporter := datatable.CreateNewImporter(customTransactionTypeNameMapping, c.geoLocationSeparator, c.transactionTagSeparator) + + return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezoneOffset, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap) +} + +// IsDelimiterSeparatedValuesFileType returns whether the file type is the delimiter-separated values file type +func IsDelimiterSeparatedValuesFileType(fileType string) bool { + _, exists := supportedFileTypeSeparators[fileType] + return exists +} + +// CreateNewCustomTransactionDataDsvFileParser returns a new custom dsv parser for transaction data +func CreateNewCustomTransactionDataDsvFileParser(fileType string, fileEncoding string) (CustomTransactionDataDsvFileParser, error) { + separator, exists := supportedFileTypeSeparators[fileType] + + if !exists { + return nil, errs.ErrImportFileTypeNotSupported + } + + enc, exists := supportedFileEncodings[fileEncoding] + + if !exists { + return nil, errs.ErrImportFileEncodingNotSupported + } + + return &customTransactionDataDsvFileImporter{ + fileEncoding: enc, + separator: separator, + }, nil +} + +// CreateNewCustomTransactionDataDsvFileImporter returns a new custom dsv importer for transaction data +func CreateNewCustomTransactionDataDsvFileImporter(fileType string, fileEncoding string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, geoLocationSeparator string, transactionTagSeparator string) (base.TransactionDataImporter, error) { + separator, exists := supportedFileTypeSeparators[fileType] + + if !exists { + return nil, errs.ErrImportFileTypeNotSupported + } + + enc, exists := supportedFileEncodings[fileEncoding] + + if !exists { + return nil, errs.ErrImportFileEncodingNotSupported + } + + if _, exists = columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME]; !exists { + return nil, errs.ErrMissingRequiredFieldInHeaderRow + } + + if _, exists = columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]; !exists { + return nil, errs.ErrMissingRequiredFieldInHeaderRow + } + + if _, exists = columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_AMOUNT]; !exists { + return nil, errs.ErrMissingRequiredFieldInHeaderRow + } + + return &customTransactionDataDsvFileImporter{ + fileEncoding: enc, + separator: separator, + columnIndexMapping: columnIndexMapping, + transactionTypeNameMapping: transactionTypeNameMapping, + hasHeaderLine: hasHeaderLine, + timeFormat: timeFormat, + timezoneFormat: timezoneFormat, + geoLocationSeparator: geoLocationSeparator, + transactionTagSeparator: transactionTagSeparator, + }, nil +} diff --git a/pkg/converters/dsv/custom_transaction_data_dsv_file_importer_test.go b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer_test.go new file mode 100644 index 00000000..427bfa23 --- /dev/null +++ b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer_test.go @@ -0,0 +1,1085 @@ +package dsv + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +func TestIsDelimiterSeparatedValuesFileType(t *testing.T) { + assert.True(t, IsDelimiterSeparatedValuesFileType("custom_csv")) + assert.True(t, IsDelimiterSeparatedValuesFileType("custom_tsv")) + + assert.False(t, IsDelimiterSeparatedValuesFileType("dsv")) + assert.False(t, IsDelimiterSeparatedValuesFileType("csv")) + assert.False(t, IsDelimiterSeparatedValuesFileType("tsv")) +} + +func TestCustomTransactionDataDsvFileParser_ParseDsvFileLines(t *testing.T) { + converter, err := CreateNewCustomTransactionDataDsvFileParser("custom_csv", "utf-8") + assert.Nil(t, err) + + context := core.NewNullContext() + + allLines, err := converter.ParseDsvFileLines(context, []byte( + "2024-09-01 00:00:00,B,123.45\n"+ + "2024-09-01 01:23:45,I,0.12\n")) + assert.Nil(t, err) + + assert.Equal(t, 2, len(allLines)) + + assert.Equal(t, 3, len(allLines[0])) + assert.Equal(t, "2024-09-01 00:00:00", allLines[0][0]) + assert.Equal(t, "B", allLines[0][1]) + assert.Equal(t, "123.45", allLines[0][2]) + + assert.Equal(t, 3, len(allLines[1])) + assert.Equal(t, "2024-09-01 01:23:45", allLines[1][0]) + assert.Equal(t, "I", allLines[1][1]) + assert.Equal(t, "0.12", allLines[1][2]) + + converter, err = CreateNewCustomTransactionDataDsvFileParser("custom_tsv", "utf-8") + assert.Nil(t, err) + + allLines, err = converter.ParseDsvFileLines(context, []byte( + "2024-09-01 12:34:56\tE\t1.00\n"+ + "2024-09-01 23:59:59\tT\t0.05")) + assert.Nil(t, err) + + assert.Equal(t, 2, len(allLines)) + + assert.Equal(t, 3, len(allLines[0])) + assert.Equal(t, "2024-09-01 12:34:56", allLines[0][0]) + assert.Equal(t, "E", allLines[0][1]) + assert.Equal(t, "1.00", allLines[0][2]) + + assert.Equal(t, 3, len(allLines[1])) + assert.Equal(t, "2024-09-01 23:59:59", allLines[1][0]) + assert.Equal(t, "T", allLines[1][1]) + assert.Equal(t, "0.05", allLines[1][2]) +} + +func TestCustomTransactionDataDsvFileImporter_MinimumValidData(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "I": models.TRANSACTION_TYPE_INCOME, + "E": models.TRANSACTION_TYPE_EXPENSE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 00:00:00,B,123.45\n"+ + "2024-09-01 01:23:45,I,0.12\n"+ + "2024-09-01 12:34:56,E,1.00\n"+ + "2024-09-01 23:59:59,T,0.05"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 1, len(allNewAccounts)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) + assert.Equal(t, 0, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, "", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725153825), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + assert.Equal(t, "", allNewTransactions[1].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[1].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + assert.Equal(t, "", allNewTransactions[2].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[2].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725235199), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, "", allNewTransactions[3].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[3].OriginalDestinationAccountName) + assert.Equal(t, "", allNewTransactions[3].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "", allNewSubTransferCategories[0].Name) +} + +func TestCustomTransactionDataDsvFileImporter_WithAllSupportedColumns(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: 1, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 2, + datatable.TRANSACTION_DATA_TABLE_CATEGORY: 3, + datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY: 4, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: 5, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: 6, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 7, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: 8, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: 9, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: 10, + datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION: 11, + datatable.TRANSACTION_DATA_TABLE_TAGS: 12, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: 13, + } + transactionTypeMapping := map[string]models.TransactionType{ + "Balance Modification": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "Income": models.TRANSACTION_TYPE_INCOME, + "Expense": models.TRANSACTION_TYPE_EXPENSE, + "Transfer": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, true, "YYYY-MM-DD HH:mm:ss", "", " ", ";") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "\"Time\",\"Timezone\",\"Type\",\"Category\",\"Sub Category\",\"Account\",\"Account Currency\",\"Amount\",\"Account2\",\"Account2 Currency\",\"Account2 Amount\",\"Geographic Location\",\"Tags\",\"Description\"\n"+ + "\"2024-09-01 00:00:00\",\"+08:00\",\"Balance Modification\",\"\",\"\",\"Test Account\",\"CNY\",\"123.45\",\"\",\"\",\"\",\"\",\"\",\"\"\n"+ + "\"2024-09-01 01:23:45\",\"+08:00\",\"Income\",\"Test Category\",\"Test Sub Category\",\"Test Account\",\"CNY\",\"0.12\",\"\",\"\",\"\",\"123.450000 45.670000\",\"Test Tag;Test Tag2\",\"Hello World\"\n"+ + "\"2024-09-01 12:34:56\",\"+00:00\",\"Expense\",\"Test Category2\",\"Test Sub Category2\",\"Test Account\",\"CNY\",\"1.00\",\"\",\"\",\"\",\"\",\"Test Tag\",\"Foo#Bar\"\n"+ + "\"2024-09-01 23:59:59\",\"-05:00\",\"Transfer\",\"Test Category3\",\"Test Sub Category3\",\"Test Account\",\"CNY\",\"0.05\",\"Test Account2\",\"USD\",\"0.35\",\"\",\"Test Tag2\",\"foo\tbar\""), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) + assert.Equal(t, 2, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725120000), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, "Test Account", allNewTransactions[0].OriginalSourceAccountName) + assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725125025), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + assert.Equal(t, "Test Account", allNewTransactions[1].OriginalSourceAccountName) + assert.Equal(t, "Test Sub Category", allNewTransactions[1].OriginalCategoryName) + assert.Equal(t, 123.45, allNewTransactions[1].GeoLongitude) + assert.Equal(t, 45.67, allNewTransactions[1].GeoLatitude) + assert.Equal(t, "Hello World", allNewTransactions[1].Comment) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + assert.Equal(t, "Test Account", allNewTransactions[2].OriginalSourceAccountName) + assert.Equal(t, "Test Sub Category2", allNewTransactions[2].OriginalCategoryName) + assert.Equal(t, "Foo#Bar", allNewTransactions[2].Comment) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725253199), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, "Test Account", allNewTransactions[3].OriginalSourceAccountName) + assert.Equal(t, "Test Account2", allNewTransactions[3].OriginalDestinationAccountName) + assert.Equal(t, "Test Sub Category3", allNewTransactions[3].OriginalCategoryName) + assert.Equal(t, "foo\tbar", allNewTransactions[3].Comment) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "CNY", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Test Account2", allNewAccounts[1].Name) + assert.Equal(t, "USD", allNewAccounts[1].Currency) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Test Sub Category2", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Test Sub Category", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "Test Sub Category3", allNewSubTransferCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewTags[0].Uid) + assert.Equal(t, "Test Tag", allNewTags[0].Name) + + assert.Equal(t, int64(1234567890), allNewTags[1].Uid) + assert.Equal(t, "Test Tag2", allNewTags[1].Name) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidTime(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01T12:34:56,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "09/01/2024 12:34:56,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseTransactionWithoutType(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "I": models.TRANSACTION_TYPE_INCOME, + "E": models.TRANSACTION_TYPE_EXPENSE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,A,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrNotFoundTransactionDataInFile.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidType(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": 0, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,B,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTypeInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseTimeWithTimezone(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ssZ", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56-10:00,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56+00:00,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56+12:45,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) +} + +func TestCustomTransactionDataDsvFileImporter_ParseTimeWithTimezone2(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ssZZ", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56-1000,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56+0000,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56+1245,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) +} + +func TestCustomTransactionDataDsvFileImporter_ParseValidTimezone(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: 1, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,-10:00,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,+00:00,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,+12:45,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) +} + +func TestCustomTransactionDataDsvFileImporter_ParseValidTimezone2(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: 1, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "ZZ", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,-1000,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725230096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,+0000,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,+1245,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, int64(1725148196), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidTimezoneFormat(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: 1, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "z", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,CST,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrImportFileTransactionTimezoneFormatInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidTimezone(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: 1, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,Asia/Shanghai,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeZoneInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,-0700,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeZoneInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidTimezone2(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE: 1, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "ZZ", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,Asia/Shanghai,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeZoneInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,0700,E,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrTransactionTimeZoneInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParsePrimaryCategory(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_CATEGORY: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "I": models.TRANSACTION_TYPE_INCOME, + "E": models.TRANSACTION_TYPE_EXPENSE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, allNewSubExpenseCategories, allNewSubIncomeCategories, allNewSubTransferCategories, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 00:00:00,B,,123.45\n"+ + "2024-09-01 01:23:45,I,Test Category,0.12\n"+ + "2024-09-01 12:34:56,E,Test Category2,1.00\n"+ + "2024-09-01 23:59:59,T,Test Category3,0.05"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + assert.Equal(t, 1, len(allNewSubExpenseCategories)) + assert.Equal(t, 1, len(allNewSubIncomeCategories)) + assert.Equal(t, 1, len(allNewSubTransferCategories)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_MODIFY_BALANCE, allNewTransactions[0].Type) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, "", allNewTransactions[0].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_INCOME, allNewTransactions[1].Type) + assert.Equal(t, int64(1725153825), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, "Test Category", allNewTransactions[1].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_EXPENSE, allNewTransactions[2].Type) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, "Test Category2", allNewTransactions[2].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, models.TRANSACTION_DB_TYPE_TRANSFER_OUT, allNewTransactions[3].Type) + assert.Equal(t, int64(1725235199), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, "Test Category3", allNewTransactions[3].OriginalCategoryName) + + assert.Equal(t, int64(1234567890), allNewSubExpenseCategories[0].Uid) + assert.Equal(t, "Test Category2", allNewSubExpenseCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubIncomeCategories[0].Uid) + assert.Equal(t, "Test Category", allNewSubIncomeCategories[0].Name) + + assert.Equal(t, int64(1234567890), allNewSubTransferCategories[0].Uid) + assert.Equal(t, "Test Category3", allNewSubTransferCategories[0].Name) +} + +func TestCustomTransactionDataDsvFileImporter_ParseValidAccountCurrency(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: 2, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: 3, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 4, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: 5, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: 6, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: 7, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, allNewAccounts, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 01:23:45,B,Test Account,USD,123.45,,,\n"+ + "2024-09-01 12:34:56,T,Test Account,USD,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 2, len(allNewTransactions)) + assert.Equal(t, 2, len(allNewAccounts)) + + assert.Equal(t, int64(1234567890), allNewAccounts[0].Uid) + assert.Equal(t, "Test Account", allNewAccounts[0].Name) + assert.Equal(t, "USD", allNewAccounts[0].Currency) + + assert.Equal(t, int64(1234567890), allNewAccounts[1].Uid) + assert.Equal(t, "Test Account2", allNewAccounts[1].Name) + assert.Equal(t, "EUR", allNewAccounts[1].Currency) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidAccountCurrency(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: 2, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: 3, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 4, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: 5, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: 6, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: 7, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 01:23:45,B,Test Account,USD,123.45,,,\n"+ + "2024-09-01 12:34:56,T,Test Account,CNY,1.23,Test Account2,EUR,1.10"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 01:23:45,B,Test Account,USD,123.45,,,\n"+ + "2024-09-01 12:34:56,T,Test Account2,CNY,1.23,Test Account,EUR,1.10"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseNotSupportedCurrency(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: 2, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_CURRENCY: 3, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 4, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: 5, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_CURRENCY: 6, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: 7, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 01:23:45,B,Test Account,XXX,123.45,,,"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 01:23:45,T,Test Account,USD,123.45,Test Account2,XXX,123.45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAccountCurrencyInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseValidAmount(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + "I": models.TRANSACTION_TYPE_INCOME, + "E": models.TRANSACTION_TYPE_EXPENSE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 00:00:00,B,123.45000000,\n"+ + "2024-09-01 01:23:45,I,0.12000000,\n"+ + "2024-09-01 12:34:56,E,1.00000000,\n"+ + "2024-09-01 23:59:59,T,0.05000000,0.35000000"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTransactions)) + + assert.Equal(t, int64(1234567890), allNewTransactions[0].Uid) + assert.Equal(t, int64(1725148800), utils.GetUnixTimeFromTransactionTime(allNewTransactions[0].TransactionTime)) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + + assert.Equal(t, int64(1234567890), allNewTransactions[1].Uid) + assert.Equal(t, int64(1725153825), utils.GetUnixTimeFromTransactionTime(allNewTransactions[1].TransactionTime)) + assert.Equal(t, int64(12), allNewTransactions[1].Amount) + + assert.Equal(t, int64(1234567890), allNewTransactions[2].Uid) + assert.Equal(t, int64(1725194096), utils.GetUnixTimeFromTransactionTime(allNewTransactions[2].TransactionTime)) + assert.Equal(t, int64(100), allNewTransactions[2].Amount) + + assert.Equal(t, int64(1234567890), allNewTransactions[3].Uid) + assert.Equal(t, int64(1725235199), utils.GetUnixTimeFromTransactionTime(allNewTransactions[3].TransactionTime)) + assert.Equal(t, int64(5), allNewTransactions[3].Amount) + assert.Equal(t, int64(35), allNewTransactions[3].RelatedAccountAmount) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidAmount(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: 4, + datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT: 5, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,E,Test Account,123 45,,"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,T,Test Account,123.45,Test Account2,123 45"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrAmountInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseNoAmount2(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME: 2, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 3, + datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME: 4, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,E,Test Account,123.45,"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, int64(0), allNewTransactions[0].RelatedAccountAmount) + + allNewTransactions, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,T,Test Account,123.45,Test Account2"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, int64(12345), allNewTransactions[0].Amount) + assert.Equal(t, int64(12345), allNewTransactions[0].RelatedAccountAmount) +} + +func TestCustomTransactionDataDsvFileImporter_ParseValidGeographicLocation(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", ";", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,E,123.45,123.45;45.56"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, 123.45, allNewTransactions[0].GeoLongitude) + assert.Equal(t, 45.56, allNewTransactions[0].GeoLatitude) +} + +func TestCustomTransactionDataDsvFileImporter_ParseInvalidGeographicLocation(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + datatable.TRANSACTION_DATA_TABLE_GEOGRAPHIC_LOCATION: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", " ", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,E,123.45,,,1"), 0, nil, nil, nil, nil, nil) + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, float64(0), allNewTransactions[0].GeoLongitude) + assert.Equal(t, float64(0), allNewTransactions[0].GeoLatitude) + + _, _, _, _, _, _, err = converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,E,123.45,a b"), 0, nil, nil, nil, nil, nil) + assert.EqualError(t, err, errs.ErrGeographicLocationInvalid.Message) +} + +func TestCustomTransactionDataDsvFileImporter_ParseTag(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + datatable.TRANSACTION_DATA_TABLE_TAGS: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", ";") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 00:00:00,E,123.45,foo;;bar.;#test;hello\tworld;;"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 4, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTags[0].Uid) + assert.Equal(t, "foo", allNewTags[0].Name) + + assert.Equal(t, int64(1234567890), allNewTags[1].Uid) + assert.Equal(t, "bar.", allNewTags[1].Name) + + assert.Equal(t, int64(1234567890), allNewTags[2].Uid) + assert.Equal(t, "#test", allNewTags[2].Name) + + assert.Equal(t, int64(1234567890), allNewTags[3].Uid) + assert.Equal(t, "hello\tworld", allNewTags[3].Name) +} + +func TestCustomTransactionDataDsvFileImporter_ParseTagWithoutSeparator(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + datatable.TRANSACTION_DATA_TABLE_TAGS: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "E": models.TRANSACTION_TYPE_EXPENSE, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + _, _, _, _, _, allNewTags, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 00:00:00,E,123.45,foo;;bar.;#test;hello\tworld;;"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + + assert.Equal(t, 1, len(allNewTags)) + + assert.Equal(t, int64(1234567890), allNewTags[0].Uid) + assert.Equal(t, "foo;;bar.;#test;hello world;;", allNewTags[0].Name) +} + +func TestCustomTransactionDataDsvFileImporter_ParseDescription(t *testing.T) { + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + datatable.TRANSACTION_DATA_TABLE_DESCRIPTION: 3, + } + transactionTypeMapping := map[string]models.TransactionType{ + "T": models.TRANSACTION_TYPE_TRANSFER, + } + converter, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.Nil(t, err) + + context := core.NewNullContext() + + user := &models.User{ + Uid: 1234567890, + DefaultCurrency: "CNY", + } + + allNewTransactions, _, _, _, _, _, err := converter.ParseImportedData(context, user, []byte( + "2024-09-01 12:34:56,T,123.45,foo bar\t#test"), 0, nil, nil, nil, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, 1, len(allNewTransactions)) + assert.Equal(t, "foo bar\t#test", allNewTransactions[0].Comment) +} + +func TestCustomTransactionDataDsvFileImporter_InvalidSeparator(t *testing.T) { + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + } + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + _, err := CreateNewCustomTransactionDataDsvFileImporter("test", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.EqualError(t, err, errs.ErrImportFileTypeNotSupported.Message) +} + +func TestCustomTransactionDataDsvFileImporter_InvalidFileEncoding(t *testing.T) { + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + } + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 2, + } + _, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "ascii", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.EqualError(t, err, errs.ErrImportFileEncodingNotSupported.Message) +} + +func TestCustomTransactionDataDsvFileImporter_MissingRequiredColumn(t *testing.T) { + transactionTypeMapping := map[string]models.TransactionType{ + "B": models.TRANSACTION_TYPE_MODIFY_BALANCE, + } + + // Missing Time Column + columnIndexMapping := map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 0, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 1, + } + _, err := CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) + + // Missing Type Column + columnIndexMapping = map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_AMOUNT: 1, + } + _, err = CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) + + // Missing Amount Column + columnIndexMapping = map[datatable.TransactionDataTableColumn]int{ + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME: 0, + datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE: 1, + } + _, err = CreateNewCustomTransactionDataDsvFileImporter("custom_csv", "utf-8", columnIndexMapping, transactionTypeMapping, false, "YYYY-MM-DD HH:mm:ss", "", "", "") + assert.EqualError(t, err, errs.ErrMissingRequiredFieldInHeaderRow.Message) +} diff --git a/pkg/converters/dsv/custom_transaction_plain_text_data_table.go b/pkg/converters/dsv/custom_transaction_plain_text_data_table.go new file mode 100644 index 00000000..8b443a95 --- /dev/null +++ b/pkg/converters/dsv/custom_transaction_plain_text_data_table.go @@ -0,0 +1,275 @@ +package dsv + +import ( + "strings" + "time" + + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/models" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +// customPlainTextDataTable defines the structure of custom plain text transaction data table +type customPlainTextDataTable struct { + innerDataTable datatable.ImportedDataTable + columnIndexMapping map[datatable.TransactionDataTableColumn]int + transactionTypeNameMapping map[string]models.TransactionType + timeFormat string + timezoneFormat string + timeFormatIncludeTimezone bool +} + +// customPlainTextDataRow defines the structure of custom plain text transaction data row +type customPlainTextDataRow struct { + transactionDataTable *customPlainTextDataTable + rowData map[datatable.TransactionDataTableColumn]string + isValid bool +} + +// customPlainTextDataRowIterator defines the structure of custom plain text transaction data row iterator +type customPlainTextDataRowIterator struct { + transactionDataTable *customPlainTextDataTable + innerIterator datatable.ImportedDataRowIterator +} + +// HasColumn returns whether the data table has specified column +func (t *customPlainTextDataTable) HasColumn(column datatable.TransactionDataTableColumn) bool { + // custom dsv file allows no sub category, account name and related account name column mapping, but data table converter needs these columns + if column == datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY || + column == datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME || + column == datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME { + return true + } + + // timezone column will be added when original time format contains timezone + if t.timeFormatIncludeTimezone && column == datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE { + return true + } + + _, exists := t.columnIndexMapping[column] + return exists +} + +// TransactionRowCount returns the total count of transaction data row +func (t *customPlainTextDataTable) TransactionRowCount() int { + return t.innerDataTable.DataRowCount() +} + +// TransactionRowIterator returns the iterator of transaction data row +func (t *customPlainTextDataTable) TransactionRowIterator() datatable.TransactionDataRowIterator { + return &customPlainTextDataRowIterator{ + transactionDataTable: t, + innerIterator: t.innerDataTable.DataRowIterator(), + } +} + +// IsValid returns whether this row is valid data for importing +func (r *customPlainTextDataRow) IsValid() bool { + return r.isValid +} + +// GetData returns the data in the specified column type +func (r *customPlainTextDataRow) GetData(column datatable.TransactionDataTableColumn) string { + return r.rowData[column] +} + +// HasNext returns whether the iterator does not reach the end +func (t *customPlainTextDataRowIterator) HasNext() bool { + return t.innerIterator.HasNext() +} + +// Next returns the next transaction data row +func (t *customPlainTextDataRowIterator) Next(ctx core.Context, user *models.User) (daraRow datatable.TransactionDataRow, err error) { + importedRow := t.innerIterator.Next() + + if importedRow == nil { + return nil, nil + } + + rowData, isValid, err := t.parseTransaction(ctx, user, importedRow) + + if err != nil { + log.Errorf(ctx, "[custom_transaction_plain_text_data_table.Next] cannot parsing transaction in row \"%s\", because %s", t.innerIterator.CurrentRowId(), err.Error()) + return nil, err + } + + return &customPlainTextDataRow{ + transactionDataTable: t.transactionDataTable, + rowData: rowData, + isValid: isValid, + }, nil +} + +func (t *customPlainTextDataRowIterator) parseTransaction(ctx core.Context, user *models.User, row datatable.ImportedDataRow) (map[datatable.TransactionDataTableColumn]string, bool, error) { + rowData := make(map[datatable.TransactionDataTableColumn]string, len(t.transactionDataTable.columnIndexMapping)) + + for column, columnIndex := range t.transactionDataTable.columnIndexMapping { + if columnIndex < 0 || columnIndex >= row.ColumnCount() { + continue + } + + value := row.GetData(columnIndex) + rowData[column] = value + } + + // parse transaction type + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] != "" { + transactionType, exists := t.transactionDataTable.transactionTypeNameMapping[rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]] + + if !exists { + log.Warnf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] skip parsing this transaction, because transaction type \"%s\" mapping not defined", rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]) + return nil, false, nil + } + + mappedTransactionType, exists := customTransactionTypeNameMapping[transactionType] + + if !exists { + log.Errorf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] cannot parsing transaction type \"%s\", because type \"%d\" is invalid", rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE], transactionType) + return nil, false, errs.ErrTransactionTypeInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE] = mappedTransactionType + } + + // parse date time + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] != "" { + dateTime, err := time.Parse(t.transactionDataTable.timeFormat, rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME]) + + if err != nil { + return nil, false, errs.ErrTransactionTimeInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME] = utils.FormatUnixTimeToLongDateTime(dateTime.Unix(), dateTime.Location()) + + if t.transactionDataTable.timeFormatIncludeTimezone { + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = utils.FormatTimezoneOffset(dateTime.Location()) + } + } + + // parse timezone + if rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] != "" { + if t.transactionDataTable.timezoneFormat == "Z" || t.transactionDataTable.timezoneFormat == "" { // -HH:mm + // Do Nothing + } else if t.transactionDataTable.timezoneFormat == "ZZ" { // -HHmm + timezone := rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] + + if len(timezone) != 5 { + return nil, false, errs.ErrTransactionTimeZoneInvalid + } + + timezone = timezone[:3] + ":" + timezone[3:] + rowData[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIMEZONE] = timezone + } else { + return nil, false, errs.ErrImportFileTransactionTimezoneFormatInvalid + } + } + + // use primary category if sub category is empty + if rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] == "" && rowData[datatable.TRANSACTION_DATA_TABLE_CATEGORY] != "" { + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = rowData[datatable.TRANSACTION_DATA_TABLE_CATEGORY] + } + + // trim trailing zero in decimal + if rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] != "" { + rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) + amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT]) + + if err != nil { + log.Errorf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] cannot parsing transaction amount \"%s\", because %s", rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT], err.Error()) + return nil, false, errs.ErrAmountInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_AMOUNT] = utils.FormatAmount(amount) + } + + if rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] != "" { + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.TrimTrailingZerosInDecimal(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT]) + amount, err := utils.ParseAmount(rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT]) + + if err != nil { + log.Errorf(ctx, "[custom_transaction_plain_text_data_table.parseTransaction] cannot parsing transaction related amount \"%s\", because %s", rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT], err.Error()) + return nil, false, errs.ErrAmountInvalid + } + + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_AMOUNT] = utils.FormatAmount(amount) + } + + if _, exists := rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY]; !exists { + rowData[datatable.TRANSACTION_DATA_TABLE_SUB_CATEGORY] = "" + } + + if _, exists := rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME]; !exists { + rowData[datatable.TRANSACTION_DATA_TABLE_ACCOUNT_NAME] = "" + } + + if _, exists := rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME]; !exists { + rowData[datatable.TRANSACTION_DATA_TABLE_RELATED_ACCOUNT_NAME] = "" + } + + return rowData, true, nil +} + +// CreateNewCustomPlainTextDataTable returns transaction data table from imported data table +func CreateNewCustomPlainTextDataTable(dataTable datatable.ImportedDataTable, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, timeFormat string, timezoneFormat string) *customPlainTextDataTable { + timeFormatIncludeTimezone := strings.Contains(timeFormat, "z") || strings.Contains(timeFormat, "Z") + + return &customPlainTextDataTable{ + innerDataTable: dataTable, + columnIndexMapping: columnIndexMapping, + transactionTypeNameMapping: transactionTypeNameMapping, + timeFormat: getDateTimeFormat(timeFormat), + timezoneFormat: timezoneFormat, + timeFormatIncludeTimezone: timeFormatIncludeTimezone, + } +} + +func getDateTimeFormat(format string) string { + // convert moment.js format to Go format + + format = strings.ReplaceAll(format, "YYYY", "2006") + format = strings.ReplaceAll(format, "YY", "06") + + format = strings.ReplaceAll(format, "MMMM", "January") + format = strings.ReplaceAll(format, "MMM", "Jan") + format = strings.ReplaceAll(format, "MM", "01") + format = strings.ReplaceAll(format, "M", "1") + + format = strings.ReplaceAll(format, "DD", "02") + format = strings.ReplaceAll(format, "D", "2") + + format = strings.ReplaceAll(format, "dddd", "Monday") + format = strings.ReplaceAll(format, "ddd", "Mon") + + format = strings.ReplaceAll(format, "HH", "15") + format = strings.ReplaceAll(format, "H", "15") + + format = strings.ReplaceAll(format, "hh", "03") + format = strings.ReplaceAll(format, "h", "3") + + format = strings.ReplaceAll(format, "mm", "04") + format = strings.ReplaceAll(format, "m", "4") + + format = strings.ReplaceAll(format, "ss", "05") + format = strings.ReplaceAll(format, "s", "5") + + for i := 9; i >= 1; i-- { + format = strings.ReplaceAll(format, "."+strings.Repeat("S", i), "."+strings.Repeat("9", i)) + } + + format = strings.ReplaceAll(format, "A", "PM") + format = strings.ReplaceAll(format, "a", "pm") + + format = strings.ReplaceAll(format, "zz", "MST") + format = strings.ReplaceAll(format, "z", "MST") + + if strings.Contains(format, "ZZ") { + format = strings.ReplaceAll(format, "ZZ", "Z0700") + } else if strings.Contains(format, "Z") { + format = strings.ReplaceAll(format, "Z", "Z07:00") + } + + return format +} diff --git a/pkg/converters/transaction_data_converters.go b/pkg/converters/transaction_data_converters.go index 43bdee6d..e6171fae 100644 --- a/pkg/converters/transaction_data_converters.go +++ b/pkg/converters/transaction_data_converters.go @@ -3,7 +3,9 @@ package converters import ( "github.com/mayswind/ezbookkeeping/pkg/converters/alipay" "github.com/mayswind/ezbookkeeping/pkg/converters/base" + "github.com/mayswind/ezbookkeeping/pkg/converters/datatable" "github.com/mayswind/ezbookkeeping/pkg/converters/default" + "github.com/mayswind/ezbookkeeping/pkg/converters/dsv" "github.com/mayswind/ezbookkeeping/pkg/converters/feidee" "github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII" "github.com/mayswind/ezbookkeeping/pkg/converters/gnucash" @@ -12,6 +14,7 @@ import ( "github.com/mayswind/ezbookkeeping/pkg/converters/qif" "github.com/mayswind/ezbookkeeping/pkg/converters/wechat" "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/models" ) // GetTransactionDataExporter returns the transaction data exporter according to the file type @@ -61,3 +64,18 @@ func GetTransactionDataImporter(fileType string) (base.TransactionDataImporter, return nil, errs.ErrImportFileTypeNotSupported } } + +// IsCustomDelimiterSeparatedValuesFileType returns whether the file type is the delimiter-separated values file type +func IsCustomDelimiterSeparatedValuesFileType(fileType string) bool { + return dsv.IsDelimiterSeparatedValuesFileType(fileType) +} + +// CreateNewDelimiterSeparatedValuesDataParser returns a new delimiter-separated values data parser according to the file type and encoding +func CreateNewDelimiterSeparatedValuesDataParser(fileType string, fileEncoding string) (dsv.CustomTransactionDataDsvFileParser, error) { + return dsv.CreateNewCustomTransactionDataDsvFileParser(fileType, fileEncoding) +} + +// CreateNewDelimiterSeparatedValuesDataImporter returns a new delimiter-separated values data importer according to the file type and encoding +func CreateNewDelimiterSeparatedValuesDataImporter(fileType string, fileEncoding string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, geoLocationSeparator string, transactionTagSeparator string) (base.TransactionDataImporter, error) { + return dsv.CreateNewCustomTransactionDataDsvFileImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormat, timezoneFormat, geoLocationSeparator, transactionTagSeparator) +} diff --git a/pkg/errs/transaction.go b/pkg/errs/transaction.go index 5c7d8e26..81c15456 100644 --- a/pkg/errs/transaction.go +++ b/pkg/errs/transaction.go @@ -35,4 +35,10 @@ var ( ErrCannotAddTransactionBeforeBalanceModificationTransaction = NewSystemError(NormalSubcategoryTransaction, 28, http.StatusBadRequest, "cannot add transaction before balance modification transaction") ErrBalanceModificationTransactionCannotModifyTime = NewSystemError(NormalSubcategoryTransaction, 29, http.StatusBadRequest, "balance modification transaction cannot modify transaction time") ErrTransferTransactionAmountCannotBeLessThanZero = NewNormalError(NormalSubcategoryTransaction, 30, http.StatusBadRequest, "transfer transaction amount cannot be less than zero") + ErrImportFileEncodingIsEmpty = NewSystemError(NormalSubcategoryTransaction, 31, http.StatusBadRequest, "import file encoding is empty") + ErrImportFileEncodingNotSupported = NewSystemError(NormalSubcategoryTransaction, 32, http.StatusBadRequest, "import file encoding not supported") + ErrImportFileColumnMappingInvalid = NewSystemError(NormalSubcategoryTransaction, 33, http.StatusBadRequest, "column mapping invalid") + ErrImportFileTransactionTypeMappingInvalid = NewSystemError(NormalSubcategoryTransaction, 34, http.StatusBadRequest, "transaction type mapping invalid") + ErrImportFileTransactionTimeFormatInvalid = NewSystemError(NormalSubcategoryTransaction, 35, http.StatusBadRequest, "transaction time format invalid") + ErrImportFileTransactionTimezoneFormatInvalid = NewSystemError(NormalSubcategoryTransaction, 36, http.StatusBadRequest, "transaction time zone format invalid") ) diff --git a/src/consts/file.ts b/src/consts/file.ts index 86598cd6..02e19656 100644 --- a/src/consts/file.ts +++ b/src/consts/file.ts @@ -8,6 +8,59 @@ export const SUPPORTED_DOCUMENT_LANGUAGES_FOR_IMPORT_FILE: Record 0 ? result : undefined; + } + + public static detectMany(dateTimes: string[]): KnownDateTimeFormat[] | undefined { + const detectedCounts: Record = {}; + + for (const dateTime of dateTimes) { + const detectedFormats = KnownDateTimeFormat.detect(dateTime); + + if (detectedFormats) { + for (const format of detectedFormats) { + detectedCounts[format.format] = (detectedCounts[format.format] || 0) + 1; + } + } else { + return undefined; + } + } + + const result: KnownDateTimeFormat[] = []; + + for (const format of KnownDateTimeFormat.allInstances) { + if (detectedCounts[format.format] === dateTimes.length) { + result.push(format); + } + } + + return result.length > 0 ? result : undefined; + } +} + export const LANGUAGE_DEFAULT_DATE_TIME_FORMAT_VALUE: number = 0; export interface DateFormat { diff --git a/src/core/file.ts b/src/core/file.ts index 2ee13b8a..e089c46b 100644 --- a/src/core/file.ts +++ b/src/core/file.ts @@ -8,6 +8,8 @@ export interface ImportFileType extends ImportFileTypeAndExtensions { readonly name: string; readonly extensions: string; readonly subTypes?: ImportFileTypeSubType[]; + readonly supportedEncodings?: string[]; + readonly dataFromTextbox?: boolean; readonly document?: { readonly supportMultiLanguages: boolean | string; readonly anchor: string; @@ -25,6 +27,8 @@ export interface LocalizedImportFileType extends ImportFileTypeAndExtensions { readonly displayName: string; readonly extensions: string; readonly subTypes?: LocalizedImportFileTypeSubType[]; + readonly supportedEncodings?: LocalizedImportFileTypeSupportedEncodings[]; + readonly dataFromTextbox?: boolean; readonly document?: LocalizedImportFileDocument; } @@ -34,6 +38,11 @@ export interface LocalizedImportFileTypeSubType extends ImportFileTypeAndExtensi readonly extensions?: string; } +export interface LocalizedImportFileTypeSupportedEncodings { + readonly encoding: string; + readonly displayName: string; +} + export interface LocalizedImportFileDocument { readonly language: string; readonly displayLanguageName: string; diff --git a/src/core/timezone.ts b/src/core/timezone.ts index f7581172..21a6cf15 100644 --- a/src/core/timezone.ts +++ b/src/core/timezone.ts @@ -1,4 +1,4 @@ -import type { TypeAndName } from './base.ts'; +import type { NameValue, TypeAndName } from './base.ts'; export interface TimezoneInfo { readonly displayName: string; @@ -13,6 +13,77 @@ export interface LocalizedTimezoneInfo { readonly displayNameWithUtcOffset: string; } +export class KnownDateTimezoneFormat implements NameValue { + private static readonly allInstances: KnownDateTimezoneFormat[] = []; + private static readonly allInstancesByValue: Record = {}; + + public static readonly HHColonMM = new KnownDateTimezoneFormat('±HH:mm', 'Z', /^[+-]?([0-1][0-9]|2[0-3]):[0-5][0-9]$/); + public static readonly HHMM = new KnownDateTimezoneFormat('±HHmm', 'ZZ', /^[+-]?([0-1][0-9]|2[0-3])[0-5][0-9]$/); + + public readonly name: string; + public readonly value: string; + private readonly regex: RegExp; + + private constructor(name: string, value: string, regex: RegExp) { + this.name = name; + this.value = value; + this.regex = regex; + + KnownDateTimezoneFormat.allInstances.push(this); + KnownDateTimezoneFormat.allInstancesByValue[value] = this; + } + + public isValid(dateTime: string): boolean { + return this.regex.test(dateTime); + } + + public static values(): KnownDateTimezoneFormat[] { + return KnownDateTimezoneFormat.allInstances; + } + + public static valueOf(value: string): KnownDateTimezoneFormat | undefined { + return KnownDateTimezoneFormat.allInstancesByValue[value]; + } + + public static detect(dateTime: string): KnownDateTimezoneFormat[] | undefined { + const result: KnownDateTimezoneFormat[] = []; + + for (const format of KnownDateTimezoneFormat.allInstances) { + if (format.isValid(dateTime)) { + result.push(format); + } + } + + return result.length > 0 ? result : undefined; + } + + public static detectMany(dateTimes: string[]): KnownDateTimezoneFormat[] | undefined { + const detectedCounts: Record = {}; + + for (const dateTime of dateTimes) { + const detectedFormats = KnownDateTimezoneFormat.detect(dateTime); + + if (detectedFormats) { + for (const format of detectedFormats) { + detectedCounts[format.value] = (detectedCounts[format.value] || 0) + 1; + } + } else { + return undefined; + } + } + + const result: KnownDateTimezoneFormat[] = []; + + for (const format of KnownDateTimezoneFormat.allInstances) { + if (detectedCounts[format.value] === dateTimes.length) { + result.push(format); + } + } + + return result.length > 0 ? result : undefined; + } +} + export class TimezoneTypeForStatistics implements TypeAndName { public static readonly ApplicationTimezone = new TimezoneTypeForStatistics(0, 'Application Timezone'); public static readonly TransactionTimezone = new TimezoneTypeForStatistics(1, 'Transaction Timezone'); diff --git a/src/core/transaction.ts b/src/core/transaction.ts index ae16c3e7..851a4ce0 100644 --- a/src/core/transaction.ts +++ b/src/core/transaction.ts @@ -57,3 +57,36 @@ export class TransactionTagFilterType implements TypeAndName { return TransactionTagFilterType.allInstances; } } + +export class ImportTransactionColumnType implements TypeAndName { + private static readonly allInstances: ImportTransactionColumnType[] = []; + + public static readonly TransactionTime = new ImportTransactionColumnType(1, 'Transaction Time'); + public static readonly TransactionTimezone = new ImportTransactionColumnType(2, 'Transaction Timezone'); + public static readonly TransactionType = new ImportTransactionColumnType(3, 'Transaction Type'); + public static readonly Category = new ImportTransactionColumnType(4, 'Category'); + public static readonly SubCategory = new ImportTransactionColumnType(5, 'Secondary Category'); + public static readonly AccountName = new ImportTransactionColumnType(6, 'Account Name'); + public static readonly AccountCurrency = new ImportTransactionColumnType(7, 'Currency'); + public static readonly Amount = new ImportTransactionColumnType(8, 'Amount'); + public static readonly RelatedAccountName = new ImportTransactionColumnType(9, 'Transfer In Account Name'); + public static readonly RelatedAccountCurrency = new ImportTransactionColumnType(10, 'Transfer In Currency'); + public static readonly RelatedAmount = new ImportTransactionColumnType(11, 'Transfer In Amount'); + public static readonly GeographicLocation = new ImportTransactionColumnType(12, 'Geographic Location'); + public static readonly Tags = new ImportTransactionColumnType(13, 'Tags'); + public static readonly Description = new ImportTransactionColumnType(14, 'Description'); + + public readonly type: number; + public readonly name: string; + + private constructor(type: number, name: string) { + this.type = type; + this.name = name; + + ImportTransactionColumnType.allInstances.push(this); + } + + public static values(): ImportTransactionColumnType[] { + return ImportTransactionColumnType.allInstances; + } +} diff --git a/src/lib/services.ts b/src/lib/services.ts index 7f3aab64..29b706da 100644 --- a/src/lib/services.ts +++ b/src/lib/services.ts @@ -2,6 +2,10 @@ import axios, { type AxiosRequestConfig, type AxiosRequestHeaders, type AxiosRes import type { ApiResponse } from '@/core/api.ts'; +import { + TransactionType +} from '@/core/transaction.ts'; + import { BASE_API_URL_PATH, BASE_QRCODE_PATH, @@ -426,10 +430,43 @@ export default { deleteTransaction: (req: TransactionDeleteRequest): ApiResponsePromise => { return axios.post>('v1/transactions/delete.json', req); }, - parseImportTransaction: ({ fileType, importFile }: { fileType: string, importFile: File }): ApiResponsePromise => { + parseImportDsvFile: ({ fileType, fileEncoding, importFile }: { fileType: string, fileEncoding?: string, importFile: File }): ApiResponsePromise => { + return axios.postForm>('v1/transactions/parse_dsv_file.json', { + fileType: fileType, + fileEncoding: fileEncoding, + file: importFile + }, { + timeout: DEFAULT_UPLOAD_API_TIMEOUT + } as ApiRequestConfig); + }, + parseImportTransaction: ({ fileType, fileEncoding, importFile, columnMapping, transactionTypeMapping, hasHeaderLine, timeFormat, timezoneFormat, geoSeparator, tagSeparator }: { fileType: string, fileEncoding?: string, importFile: File, columnMapping?: Record, transactionTypeMapping?: Record, hasHeaderLine?: boolean, timeFormat?: string, timezoneFormat?: string, geoSeparator?: string, tagSeparator?: string }): ApiResponsePromise => { + let textualColumnMapping: string | undefined = undefined; + let textualTransactionTypeMapping: string | undefined = undefined; + let textualHasHeaderLine: string | undefined = undefined; + + if (columnMapping) { + textualColumnMapping = JSON.stringify(columnMapping); + } + + if (transactionTypeMapping) { + textualTransactionTypeMapping = JSON.stringify(transactionTypeMapping); + } + + if (hasHeaderLine) { + textualHasHeaderLine = 'true'; + } + return axios.postForm>('v1/transactions/parse_import.json', { fileType: fileType, - file: importFile + fileEncoding: fileEncoding, + file: importFile, + columnMapping: textualColumnMapping, + transactionTypeMapping: textualTransactionTypeMapping, + hasHeaderLine: textualHasHeaderLine, + timeFormat: timeFormat, + timezoneFormat: timezoneFormat, + geoSeparator: geoSeparator, + tagSeparator: tagSeparator }, { timeout: DEFAULT_UPLOAD_API_TIMEOUT } as ApiRequestConfig); diff --git a/src/locales/de.json b/src/locales/de.json index 12ff965c..4eeb306a 100644 --- a/src/locales/de.json +++ b/src/locales/de.json @@ -1097,6 +1097,12 @@ "cannot add transaction before balance modification transaction": "Transaktion kann nicht vor der Saldoänderungstransaktion hinzugefügt werden", "balance modification transaction cannot modify transaction time": "Transaktionszeit kann für Saldoänderungstransaktion nicht geändert werden", "transfer transaction amount cannot be less than zero": "Betrag für Überweisungstransaktion darf nicht kleiner als 0 sein", + "import file encoding is empty": "Import file encoding is empty", + "import file encoding not supported": "import file encoding is not supported", + "column mapping invalid": "Column mapping is invalid", + "transaction type mapping invalid": "Transaction type mapping is invalid", + "transaction time format invalid": "Transaction time format is invalid", + "transaction time zone format invalid": "Transaction time zone format is invalid", "transaction category id is invalid": "Transaktionskategorie-ID ist ungültig", "transaction category not found": "Transaktionskategorie nicht gefunden", "transaction category type is invalid": "Transaktionskategorietyp ist ungültig", @@ -1214,6 +1220,58 @@ "parameter invalid color": "{parameter} hat ein ungültiges Format", "parameter invalid amount filter": "{parameter} hat ein ungültiges Format" }, + "encoding": { + "utf-8": "UTF-8", + "utf-8-bom": "UTF-8 with BOM", + "utf-16le": "UTF-16 Little Endian", + "utf-16be": "UTF-16 Big Endian", + "cp437": "OEM United States (CP-437)", + "cp863": "OEM Canadian French (CP-863)", + "cp037": "IBM EBCDIC US/Canada (CP-037)", + "cp1047": "IBM EBCDIC Open Systems (CP-1047)", + "cp1140": "IBM EBCDIC US/Canada with Euro (CP-1140)", + "iso-8859-1": "Western European (ISO-8859-1)", + "cp850": "Western European (CP-850)", + "cp858": "Western European with Euro (CP-858)", + "windows-1252": "Western European (Windows-1252)", + "iso-8859-15": "Western European (ISO-8859-15)", + "iso-8859-4": "North European (ISO-8859-4)", + "iso-8859-10": "Nordic (ISO-8859-10)", + "cp865": "Nordic (CP-865)", + "iso-8859-2": "Central European (ISO-8859-2)", + "cp852": "Central European (CP-852)", + "windows-1250": "Central European (Windows-1250)", + "iso-8859-14": "Celtic (ISO-8859-14)", + "iso-8859-3": "South European (ISO-8859-3)", + "cp860": "Portuguese (CP-860)", + "iso-8859-7": "Greek (ISO-8859-7)", + "windows-1253": "Greek (Windows-1253)", + "iso-8859-9": "Turkish (ISO-8859-9)", + "windows-1254": "Turkish (Windows-1254)", + "iso-8859-13": "Baltic (ISO-8859-13)", + "windows-1257": "Baltic (Windows-1257)", + "iso-8859-16": "South-Eastern European (ISO-8859-16)", + "iso-8859-5": "Cyrillic (ISO-8859-5)", + "cp855": "Cyrillic (CP-855)", + "cp866": "Cyrillic (CP-866)", + "windows-1251": "Cyrillic (Windows-1251)", + "koi8r": "Cyrillic (KOI8-R)", + "koi8u": "Cyrillic (KOI8-U)", + "iso-8859-6": "Arabic (ISO-8859-6)", + "windows-1256": "Arabic (Windows-1256)", + "iso-8859-8": "Hebrew (ISO-8859-8)", + "cp862": "Hebrew (CP-862)", + "windows-1255": "Hebrew (Windows-1255)", + "windows-874": "Thai (Windows-874)", + "windows-1258": "Vietnamese (Windows-1258)", + "gb18030": "Simplified Chinese (GB18030)", + "gbk": "Simplified Chinese (GBK)", + "big5": "Traditional Chinese (Big5)", + "euc-kr": "Korean (EUC-KR)", + "euc-jp": "Japanese (EUC-JP)", + "iso-2022-jp": "Japanese (ISO-2022-JP)", + "shift_jis": "Japanese (Shift_JIS)" + }, "document": { "anchor": { "export_and_import": { @@ -1242,6 +1300,7 @@ "Not set": "Nicht festgelegt", "No results": "Keine Ergebnisse", "Unknown": "Unbekannt", + "Auto detect": "Auto detect", "Miscellaneous": "Verschiedenes", "Default": "Standard", "Done": "Fertig", @@ -1268,6 +1327,13 @@ "Color": "Farbe", "Type": "Typ", "Format": "Format", + "File Encoding": "File Encoding", + "Space": "Space", + "Comma": "Comma", + "Semicolon": "Semicolon", + "Tab": "Tab", + "Vertical Bar": "Vertical Bar", + "Slash": "Slash", "All Types": "Alle Typen", "More": "Mehr", "All": "Alle", @@ -1521,6 +1587,8 @@ "Income Amount": "Einnahmenbetrag", "Transfer Out Amount": "Überweisungsbetrag (Ausgang)", "Transfer In Amount": "Überweisungsbetrag (Eingang)", + "Transfer In Account Name": "Transfer In Account Name", + "Transfer In Currency": "Transfer In Currency", "Show Amount": "Betrag anzeigen", "Hide Amount": "Betrag verbergen", "Swap Account": "Konto tauschen", @@ -1530,6 +1598,7 @@ "Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)", "Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)", "Category": "Kategorie", + "Secondary Category": "Secondary Category", "Multiple Categories": "Mehrere Kategorien", "Account": "Konto", "Multiple Accounts": "Mehrere Konten", @@ -1545,6 +1614,7 @@ "Scheduled Transaction Frequency": "Häufigkeit der geplanten Transaktion", "Transaction Timezone": "Transaktionszeitzone", "Same time as default timezone": "Gleiche Zeit wie Standardzeitzone", + "Transaction Type": "Transaction Type", "Geographic Location": "Geografischer Standort", "No Location": "Kein Standort", "Getting Location...": "Standort wird ermittelt...", @@ -1559,6 +1629,8 @@ "Import Transactions": "Transaktionen importieren", "Upload File": "Datei hochladen", "Upload Transaction Data File": "Transaktionsdatendatei hochladen", + "Define Column": "Define Column", + "Define and Check Column Mapping": "Define and Check Column Mapping", "Check & Modify": "Überprüfen & Ändern", "Check and Modify Your Data": "Überprüfen und Ändern Sie Ihre Daten", "Data Import Completed": "Datenimport abgeschlossen", @@ -1573,6 +1645,8 @@ "Month-day-year format": "Monat-Tag-Jahr-Format", "Day-month-year format": "Tag-Monat-Jahr-Format", "Intuit Interchange Format (IIF) File": "Intuit Interchange Format (IIF)-Datei", + "Delimiter-separated Values (DSV) File": "Delimiter-separated Values (DSV) File", + "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "GnuCash XML-Datenbankdatei", "Firefly III Data Export File": "Firefly III-Datenexportdatei", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App)-Datenexportdatei", @@ -1581,8 +1655,19 @@ "Alipay (Web) Transaction Flow File": "Alipay (Web)-Transaktionsflussdatei", "WeChat Pay Billing File": "WeChat Pay-Abrechnungsdatei", "Data File": "Datendatei", + "Data to import": "Data to import", "Please select a file to import": "Bitte wählen Sie eine Datei zum Importieren aus", + "Include Header Line": "Include Header Line", + "Time Format": "Time Format", + "Transaction Type Mapping": "Transaction Type Mapping", + "Timezone Format": "Timezone Format", + "Geographic Location Separator": "Geographic Location Separator", + "Transaction Tags Separator": "Transaction Tags Separator", + "Lines Per Page": "Lines Per Page", "No data to import": "Keine Daten zum Importieren", + "Missing transaction time, transaction type, or amount column mapping": "Missing transaction time, transaction type, or amount column mapping", + "Transaction type mapping is not set": "Transaction type mapping is not set", + "Transaction time format is not set": "Transaction time format is not set", "Cannot import invalid transactions": "Ungültige Transaktionen können nicht importiert werden", "Unable to parse import file": "Importdatei kann nicht geparst werden", "Batch Replace Selected Expense Categories": "Ausgewählte Ausgabenkategorien im Batch ersetzen", diff --git a/src/locales/en.json b/src/locales/en.json index 3e57f493..21a98302 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1097,6 +1097,12 @@ "cannot add transaction before balance modification transaction": "You cannot add transaction before the balance modification transaction", "balance modification transaction cannot modify transaction time": "You cannot modify transaction time for balance modification transaction", "transfer transaction amount cannot be less than zero": "Amount cannot be less than 0 for transfer transaction", + "import file encoding is empty": "Import file encoding is empty", + "import file encoding not supported": "import file encoding is not supported", + "column mapping invalid": "Column mapping is invalid", + "transaction type mapping invalid": "Transaction type mapping is invalid", + "transaction time format invalid": "Transaction time format is invalid", + "transaction time zone format invalid": "Transaction time zone format is invalid", "transaction category id is invalid": "Transaction category ID is invalid", "transaction category not found": "Transaction category is not found", "transaction category type is invalid": "Transaction category type is invalid", @@ -1214,6 +1220,58 @@ "parameter invalid color": "{parameter} is invalid format", "parameter invalid amount filter": "{parameter} is invalid format" }, + "encoding": { + "utf-8": "UTF-8", + "utf-8-bom": "UTF-8 with BOM", + "utf-16le": "UTF-16 Little Endian", + "utf-16be": "UTF-16 Big Endian", + "cp437": "OEM United States (CP-437)", + "cp863": "OEM Canadian French (CP-863)", + "cp037": "IBM EBCDIC US/Canada (CP-037)", + "cp1047": "IBM EBCDIC Open Systems (CP-1047)", + "cp1140": "IBM EBCDIC US/Canada with Euro (CP-1140)", + "iso-8859-1": "Western European (ISO-8859-1)", + "cp850": "Western European (CP-850)", + "cp858": "Western European with Euro (CP-858)", + "windows-1252": "Western European (Windows-1252)", + "iso-8859-15": "Western European (ISO-8859-15)", + "iso-8859-4": "North European (ISO-8859-4)", + "iso-8859-10": "Nordic (ISO-8859-10)", + "cp865": "Nordic (CP-865)", + "iso-8859-2": "Central European (ISO-8859-2)", + "cp852": "Central European (CP-852)", + "windows-1250": "Central European (Windows-1250)", + "iso-8859-14": "Celtic (ISO-8859-14)", + "iso-8859-3": "South European (ISO-8859-3)", + "cp860": "Portuguese (CP-860)", + "iso-8859-7": "Greek (ISO-8859-7)", + "windows-1253": "Greek (Windows-1253)", + "iso-8859-9": "Turkish (ISO-8859-9)", + "windows-1254": "Turkish (Windows-1254)", + "iso-8859-13": "Baltic (ISO-8859-13)", + "windows-1257": "Baltic (Windows-1257)", + "iso-8859-16": "South-Eastern European (ISO-8859-16)", + "iso-8859-5": "Cyrillic (ISO-8859-5)", + "cp855": "Cyrillic (CP-855)", + "cp866": "Cyrillic (CP-866)", + "windows-1251": "Cyrillic (Windows-1251)", + "koi8r": "Cyrillic (KOI8-R)", + "koi8u": "Cyrillic (KOI8-U)", + "iso-8859-6": "Arabic (ISO-8859-6)", + "windows-1256": "Arabic (Windows-1256)", + "iso-8859-8": "Hebrew (ISO-8859-8)", + "cp862": "Hebrew (CP-862)", + "windows-1255": "Hebrew (Windows-1255)", + "windows-874": "Thai (Windows-874)", + "windows-1258": "Vietnamese (Windows-1258)", + "gb18030": "Simplified Chinese (GB18030)", + "gbk": "Simplified Chinese (GBK)", + "big5": "Traditional Chinese (Big5)", + "euc-kr": "Korean (EUC-KR)", + "euc-jp": "Japanese (EUC-JP)", + "iso-2022-jp": "Japanese (ISO-2022-JP)", + "shift_jis": "Japanese (Shift_JIS)" + }, "document": { "anchor": { "export_and_import": { @@ -1242,6 +1300,7 @@ "Not set": "Not set", "No results": "No results", "Unknown": "Unknown", + "Auto detect": "Auto detect", "Miscellaneous": "Miscellaneous", "Default": "Default", "Done": "Done", @@ -1268,6 +1327,13 @@ "Color": "Color", "Type": "Type", "Format": "Format", + "File Encoding": "File Encoding", + "Space": "Space", + "Comma": "Comma", + "Semicolon": "Semicolon", + "Tab": "Tab", + "Vertical Bar": "Vertical Bar", + "Slash": "Slash", "All Types": "All Types", "More": "More", "All": "All", @@ -1521,6 +1587,8 @@ "Income Amount": "Income Amount", "Transfer Out Amount": "Transfer Out Amount", "Transfer In Amount": "Transfer In Amount", + "Transfer In Account Name": "Transfer In Account Name", + "Transfer In Currency": "Transfer In Currency", "Show Amount": "Show Amount", "Hide Amount": "Hide Amount", "Swap Account": "Swap Account", @@ -1530,6 +1598,7 @@ "Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)", "Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)", "Category": "Category", + "Secondary Category": "Secondary Category", "Multiple Categories": "Multiple Categories", "Account": "Account", "Multiple Accounts": "Multiple Accounts", @@ -1545,6 +1614,7 @@ "Scheduled Transaction Frequency": "Scheduled Transaction Frequency", "Transaction Timezone": "Transaction Timezone", "Same time as default timezone": "Same time as default timezone", + "Transaction Type": "Transaction Type", "Geographic Location": "Geographic Location", "No Location": "No Location", "Getting Location...": "Getting Location...", @@ -1559,6 +1629,8 @@ "Import Transactions": "Import Transactions", "Upload File": "Upload File", "Upload Transaction Data File": "Upload Transaction Data File", + "Define Column": "Define Column", + "Define and Check Column Mapping": "Define and Check Column Mapping", "Check & Modify": "Check & Modify", "Check and Modify Your Data": "Check and Modify Your Data", "Data Import Completed": "Data Import Completed", @@ -1573,6 +1645,8 @@ "Month-day-year format": "Month-day-year format", "Day-month-year format": "Day-month-year format", "Intuit Interchange Format (IIF) File": "Intuit Interchange Format (IIF) File", + "Delimiter-separated Values (DSV) File": "Delimiter-separated Values (DSV) File", + "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "GnuCash XML Database File", "Firefly III Data Export File": "Firefly III Data Export File", "Feidee MyMoney (App) Data Export File": "Feidee MyMoney (App) Data Export File", @@ -1581,8 +1655,19 @@ "Alipay (Web) Transaction Flow File": "Alipay (Web) Transaction Flow File", "WeChat Pay Billing File": "WeChat Pay Billing File", "Data File": "Data File", + "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Include Header Line": "Include Header Line", + "Time Format": "Time Format", + "Transaction Type Mapping": "Transaction Type Mapping", + "Timezone Format": "Timezone Format", + "Geographic Location Separator": "Geographic Location Separator", + "Transaction Tags Separator": "Transaction Tags Separator", + "Lines Per Page": "Lines Per Page", "No data to import": "No data to import", + "Missing transaction time, transaction type, or amount column mapping": "Missing transaction time, transaction type, or amount column mapping", + "Transaction type mapping is not set": "Transaction type mapping is not set", + "Transaction time format is not set": "Transaction time format is not set", "Cannot import invalid transactions": "Cannot import invalid transactions", "Unable to parse import file": "Unable to parse import file", "Batch Replace Selected Expense Categories": "Batch Replace Selected Expense Categories", diff --git a/src/locales/es.json b/src/locales/es.json index 6b9da5a8..5abc4de7 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -1097,6 +1097,12 @@ "cannot add transaction before balance modification transaction": "No puede agregar una transacción antes de la transacción de modificación del saldo", "balance modification transaction cannot modify transaction time": "No puede modificar el tiempo de transacción para la transacción de modificación de saldo", "transfer transaction amount cannot be less than zero": "El Importe no puede ser menor que 0 para la transacción de transferencia", + "import file encoding is empty": "Import file encoding is empty", + "import file encoding not supported": "import file encoding is not supported", + "column mapping invalid": "Column mapping is invalid", + "transaction type mapping invalid": "Transaction type mapping is invalid", + "transaction time format invalid": "Transaction time format is invalid", + "transaction time zone format invalid": "Transaction time zone format is invalid", "transaction category id is invalid": "El ID de categoría de transacción no es válido", "transaction category not found": "No se encuentra la categoría de transacción", "transaction category type is invalid": "El tipo de categoría de transacción no es válido", @@ -1214,6 +1220,58 @@ "parameter invalid color": "{parameter} es un formato no válido", "parameter invalid amount filter": "{parameter} es un formato no válido" }, + "encoding": { + "utf-8": "UTF-8", + "utf-8-bom": "UTF-8 with BOM", + "utf-16le": "UTF-16 Little Endian", + "utf-16be": "UTF-16 Big Endian", + "cp437": "OEM United States (CP-437)", + "cp863": "OEM Canadian French (CP-863)", + "cp037": "IBM EBCDIC US/Canada (CP-037)", + "cp1047": "IBM EBCDIC Open Systems (CP-1047)", + "cp1140": "IBM EBCDIC US/Canada with Euro (CP-1140)", + "iso-8859-1": "Western European (ISO-8859-1)", + "cp850": "Western European (CP-850)", + "cp858": "Western European with Euro (CP-858)", + "windows-1252": "Western European (Windows-1252)", + "iso-8859-15": "Western European (ISO-8859-15)", + "iso-8859-4": "North European (ISO-8859-4)", + "iso-8859-10": "Nordic (ISO-8859-10)", + "cp865": "Nordic (CP-865)", + "iso-8859-2": "Central European (ISO-8859-2)", + "cp852": "Central European (CP-852)", + "windows-1250": "Central European (Windows-1250)", + "iso-8859-14": "Celtic (ISO-8859-14)", + "iso-8859-3": "South European (ISO-8859-3)", + "cp860": "Portuguese (CP-860)", + "iso-8859-7": "Greek (ISO-8859-7)", + "windows-1253": "Greek (Windows-1253)", + "iso-8859-9": "Turkish (ISO-8859-9)", + "windows-1254": "Turkish (Windows-1254)", + "iso-8859-13": "Baltic (ISO-8859-13)", + "windows-1257": "Baltic (Windows-1257)", + "iso-8859-16": "South-Eastern European (ISO-8859-16)", + "iso-8859-5": "Cyrillic (ISO-8859-5)", + "cp855": "Cyrillic (CP-855)", + "cp866": "Cyrillic (CP-866)", + "windows-1251": "Cyrillic (Windows-1251)", + "koi8r": "Cyrillic (KOI8-R)", + "koi8u": "Cyrillic (KOI8-U)", + "iso-8859-6": "Arabic (ISO-8859-6)", + "windows-1256": "Arabic (Windows-1256)", + "iso-8859-8": "Hebrew (ISO-8859-8)", + "cp862": "Hebrew (CP-862)", + "windows-1255": "Hebrew (Windows-1255)", + "windows-874": "Thai (Windows-874)", + "windows-1258": "Vietnamese (Windows-1258)", + "gb18030": "Simplified Chinese (GB18030)", + "gbk": "Simplified Chinese (GBK)", + "big5": "Traditional Chinese (Big5)", + "euc-kr": "Korean (EUC-KR)", + "euc-jp": "Japanese (EUC-JP)", + "iso-2022-jp": "Japanese (ISO-2022-JP)", + "shift_jis": "Japanese (Shift_JIS)" + }, "document": { "anchor": { "export_and_import": { @@ -1242,6 +1300,7 @@ "Not set": "No establecido", "No results": "Sin resultados", "Unknown": "Desconocido", + "Auto detect": "Auto detect", "Miscellaneous": "Misceláneas", "Default": "Por defecto", "Done": "Hecho", @@ -1268,6 +1327,13 @@ "Color": "Color", "Type": "Tipo", "Format": "Formato", + "File Encoding": "File Encoding", + "Space": "Space", + "Comma": "Comma", + "Semicolon": "Semicolon", + "Tab": "Tab", + "Vertical Bar": "Vertical Bar", + "Slash": "Slash", "All Types": "Todos los tipos", "More": "Más", "All": "Todo", @@ -1521,6 +1587,8 @@ "Income Amount": "Importe de ingresos", "Transfer Out Amount": "Importe de transferencias enviadas", "Transfer In Amount": "Importe de transferencias recibidas", + "Transfer In Account Name": "Transfer In Account Name", + "Transfer In Currency": "Transfer In Currency", "Show Amount": "Mostrar importe", "Hide Amount": "Ocultar importe", "Swap Account": "Intercambiar cuenta", @@ -1530,6 +1598,7 @@ "Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)", "Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)", "Category": "Categoría", + "Secondary Category": "Secondary Category", "Multiple Categories": "Múltiples categorías", "Account": "Cuenta", "Multiple Accounts": "Varias cuentas", @@ -1545,6 +1614,7 @@ "Scheduled Transaction Frequency": "Frecuencia de transacciones programadas", "Transaction Timezone": "Zona horaria de transacción", "Same time as default timezone": "Misma hora que la zona horaria predeterminada", + "Transaction Type": "Transaction Type", "Geographic Location": "Ubicación geográfica", "No Location": "Sin ubicación", "Getting Location...": "Obteniendo ubicación...", @@ -1559,6 +1629,8 @@ "Import Transactions": "Importar transacciones", "Upload File": "Cargar archivo", "Upload Transaction Data File": "Cargar archivo de datos de transacción", + "Define Column": "Define Column", + "Define and Check Column Mapping": "Define and Check Column Mapping", "Check & Modify": "Verificar y modificar", "Check and Modify Your Data": "Verifique y modifique sus datos", "Data Import Completed": "Importación de datos completada", @@ -1573,6 +1645,8 @@ "Month-day-year format": "Formato mes-día-año", "Day-month-year format": "Formato día-mes-año", "Intuit Interchange Format (IIF) File": "Archivo de formato de intercambio Intuit (IIF)", + "Delimiter-separated Values (DSV) File": "Delimiter-separated Values (DSV) File", + "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "Archivo de base de datos XML GnuCash", "Firefly III Data Export File": "Archivo de exportación de datos de Firefly III", "Feidee MyMoney (App) Data Export File": "Archivo de exportación de datos Feidee MyMoney (aplicación)", @@ -1581,8 +1655,19 @@ "Alipay (Web) Transaction Flow File": "Archivo de flujo de transacciones de Alipay (web)", "WeChat Pay Billing File": "Archivo de facturación de pago de WeChat", "Data File": "Archivo de datos", + "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Include Header Line": "Include Header Line", + "Time Format": "Time Format", + "Transaction Type Mapping": "Transaction Type Mapping", + "Timezone Format": "Timezone Format", + "Geographic Location Separator": "Geographic Location Separator", + "Transaction Tags Separator": "Transaction Tags Separator", + "Lines Per Page": "Lines Per Page", "No data to import": "No hay datos para importar", + "Missing transaction time, transaction type, or amount column mapping": "Missing transaction time, transaction type, or amount column mapping", + "Transaction type mapping is not set": "Transaction type mapping is not set", + "Transaction time format is not set": "Transaction time format is not set", "Cannot import invalid transactions": "No se pueden importar transacciones no válidas", "Unable to parse import file": "No se puede analizar el archivo de importación", "Batch Replace Selected Expense Categories": "Reemplazar por lotes categorías de gastos seleccionadas", diff --git a/src/locales/helpers.ts b/src/locales/helpers.ts index 8605f15a..7d3f5d48 100644 --- a/src/locales/helpers.ts +++ b/src/locales/helpers.ts @@ -66,7 +66,8 @@ import { import { TransactionEditScopeType, - TransactionTagFilterType + TransactionTagFilterType, + ImportTransactionColumnType } from '@/core/transaction.ts'; import { @@ -85,6 +86,7 @@ import { import { type LocalizedImportFileType, type LocalizedImportFileTypeSubType, + type LocalizedImportFileTypeSupportedEncodings, type LocalizedImportFileDocument, } from '@/core/file.ts'; @@ -1138,11 +1140,27 @@ export function useI18n() { } } + const supportedEncodings: LocalizedImportFileTypeSupportedEncodings[] = []; + + if (fileType.supportedEncodings) { + for (let i = 0; i < fileType.supportedEncodings.length; i++) { + const encoding = fileType.supportedEncodings[i]; + const localizedEncoding: LocalizedImportFileTypeSupportedEncodings = { + encoding: encoding, + displayName: t(`encoding.${encoding}`) + }; + + supportedEncodings.push(localizedEncoding); + } + } + const localizedFileType: LocalizedImportFileType = { type: fileType.type, displayName: t(fileType.name), extensions: fileType.extensions, subTypes: subTypes.length ? subTypes : undefined, + supportedEncodings: supportedEncodings.length ? supportedEncodings : undefined, + dataFromTextbox: fileType.dataFromTextbox, document: document }; allSupportedImportFileTypes.push(localizedFileType); @@ -1680,6 +1698,7 @@ export function useI18n() { getAllTransactionEditScopeTypes: () => getLocalizedDisplayNameAndType(TransactionEditScopeType.values()), getAllTransactionTagFilterTypes: () => getLocalizedDisplayNameAndType(TransactionTagFilterType.values()), getAllTransactionScheduledFrequencyTypes: () => getLocalizedDisplayNameAndType(ScheduledTemplateFrequencyType.values()), + getAllImportTransactionColumnTypes: () => getLocalizedDisplayNameAndType(ImportTransactionColumnType.values()), getAllTransactionDefaultCategories, getAllDisplayExchangeRates, getAllSupportedImportFileTypes, diff --git a/src/locales/ru.json b/src/locales/ru.json index 2cd3f27d..6646fdd5 100644 --- a/src/locales/ru.json +++ b/src/locales/ru.json @@ -1097,6 +1097,12 @@ "cannot add transaction before balance modification transaction": "Нельзя добавить транзакцию до транзакции изменения баланса", "balance modification transaction cannot modify transaction time": "Нельзя изменить время транзакции для транзакции изменения баланса", "transfer transaction amount cannot be less than zero": "Сумма перевода не может быть меньше нуля", + "import file encoding is empty": "Import file encoding is empty", + "import file encoding not supported": "import file encoding is not supported", + "column mapping invalid": "Column mapping is invalid", + "transaction type mapping invalid": "Transaction type mapping is invalid", + "transaction time format invalid": "Transaction time format is invalid", + "transaction time zone format invalid": "Transaction time zone format is invalid", "transaction category id is invalid": "ID категории транзакции недействителен", "transaction category not found": "Категория транзакции не найдена", "transaction category type is invalid": "Тип категории транзакции недействителен", @@ -1214,6 +1220,58 @@ "parameter invalid color": "{parameter} имеет неверный формат", "parameter invalid amount filter": "{parameter} имеет неверный формат" }, + "encoding": { + "utf-8": "UTF-8", + "utf-8-bom": "UTF-8 with BOM", + "utf-16le": "UTF-16 Little Endian", + "utf-16be": "UTF-16 Big Endian", + "cp437": "OEM United States (CP-437)", + "cp863": "OEM Canadian French (CP-863)", + "cp037": "IBM EBCDIC US/Canada (CP-037)", + "cp1047": "IBM EBCDIC Open Systems (CP-1047)", + "cp1140": "IBM EBCDIC US/Canada with Euro (CP-1140)", + "iso-8859-1": "Western European (ISO-8859-1)", + "cp850": "Western European (CP-850)", + "cp858": "Western European with Euro (CP-858)", + "windows-1252": "Western European (Windows-1252)", + "iso-8859-15": "Western European (ISO-8859-15)", + "iso-8859-4": "North European (ISO-8859-4)", + "iso-8859-10": "Nordic (ISO-8859-10)", + "cp865": "Nordic (CP-865)", + "iso-8859-2": "Central European (ISO-8859-2)", + "cp852": "Central European (CP-852)", + "windows-1250": "Central European (Windows-1250)", + "iso-8859-14": "Celtic (ISO-8859-14)", + "iso-8859-3": "South European (ISO-8859-3)", + "cp860": "Portuguese (CP-860)", + "iso-8859-7": "Greek (ISO-8859-7)", + "windows-1253": "Greek (Windows-1253)", + "iso-8859-9": "Turkish (ISO-8859-9)", + "windows-1254": "Turkish (Windows-1254)", + "iso-8859-13": "Baltic (ISO-8859-13)", + "windows-1257": "Baltic (Windows-1257)", + "iso-8859-16": "South-Eastern European (ISO-8859-16)", + "iso-8859-5": "Cyrillic (ISO-8859-5)", + "cp855": "Cyrillic (CP-855)", + "cp866": "Cyrillic (CP-866)", + "windows-1251": "Cyrillic (Windows-1251)", + "koi8r": "Cyrillic (KOI8-R)", + "koi8u": "Cyrillic (KOI8-U)", + "iso-8859-6": "Arabic (ISO-8859-6)", + "windows-1256": "Arabic (Windows-1256)", + "iso-8859-8": "Hebrew (ISO-8859-8)", + "cp862": "Hebrew (CP-862)", + "windows-1255": "Hebrew (Windows-1255)", + "windows-874": "Thai (Windows-874)", + "windows-1258": "Vietnamese (Windows-1258)", + "gb18030": "Simplified Chinese (GB18030)", + "gbk": "Simplified Chinese (GBK)", + "big5": "Traditional Chinese (Big5)", + "euc-kr": "Korean (EUC-KR)", + "euc-jp": "Japanese (EUC-JP)", + "iso-2022-jp": "Japanese (ISO-2022-JP)", + "shift_jis": "Japanese (Shift_JIS)" + }, "document": { "anchor": { "export_and_import": { @@ -1242,6 +1300,7 @@ "Not set": "Не установлено", "No results": "Нет результатов", "Unknown": "Неизвестно", + "Auto detect": "Auto detect", "Miscellaneous": "Разное", "Default": "По умолчанию", "Done": "Готово", @@ -1268,6 +1327,13 @@ "Color": "Цвет", "Type": "Тип", "Format": "Формат", + "File Encoding": "File Encoding", + "Space": "Space", + "Comma": "Comma", + "Semicolon": "Semicolon", + "Tab": "Tab", + "Vertical Bar": "Vertical Bar", + "Slash": "Slash", "All Types": "Все типы", "More": "Еще", "All": "Все", @@ -1521,6 +1587,8 @@ "Income Amount": "Сумма дохода", "Transfer Out Amount": "Сумма перевода (исходящий)", "Transfer In Amount": "Сумма перевода (входящий)", + "Transfer In Account Name": "Transfer In Account Name", + "Transfer In Currency": "Transfer In Currency", "Show Amount": "Показать сумму", "Hide Amount": "Скрыть сумму", "Swap Account": "Поменять счет", @@ -1530,6 +1598,7 @@ "Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)", "Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)", "Category": "Категория", + "Secondary Category": "Secondary Category", "Multiple Categories": "Несколько категорий", "Account": "Счет", "Multiple Accounts": "Несколько счетов", @@ -1545,6 +1614,7 @@ "Scheduled Transaction Frequency": "Частота запланированных транзакций", "Transaction Timezone": "Часовой пояс транзакции", "Same time as default timezone": "То же время, что и в часовом поясе по умолчанию", + "Transaction Type": "Transaction Type", "Geographic Location": "Географическое местоположение", "No Location": "Нет местоположения", "Getting Location...": "Получение местоположения...", @@ -1559,6 +1629,8 @@ "Import Transactions": "Импорт транзакций", "Upload File": "Загрузить файл", "Upload Transaction Data File": "Загрузить файл данных транзакций", + "Define Column": "Define Column", + "Define and Check Column Mapping": "Define and Check Column Mapping", "Check & Modify": "Проверить и изменить", "Check and Modify Your Data": "Проверьте и измените свои данные", "Data Import Completed": "Импорт данных завершен", @@ -1573,6 +1645,8 @@ "Month-day-year format": "Формат месяц-день-год", "Day-month-year format": "Формат день-месяц-год", "Intuit Interchange Format (IIF) File": "Файл Intuit Interchange Format (IIF)", + "Delimiter-separated Values (DSV) File": "Delimiter-separated Values (DSV) File", + "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "Файл базы данных GnuCash XML", "Firefly III Data Export File": "Файл экспорта данных Firefly III", "Feidee MyMoney (App) Data Export File": "Файл экспорта данных Feidee MyMoney (приложение)", @@ -1581,8 +1655,19 @@ "Alipay (Web) Transaction Flow File": "Файл потока транзакций Alipay (веб)", "WeChat Pay Billing File": "Файл выставления счетов WeChat Pay", "Data File": "Файл данных", + "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Include Header Line": "Include Header Line", + "Time Format": "Time Format", + "Transaction Type Mapping": "Transaction Type Mapping", + "Timezone Format": "Timezone Format", + "Geographic Location Separator": "Geographic Location Separator", + "Transaction Tags Separator": "Transaction Tags Separator", + "Lines Per Page": "Lines Per Page", "No data to import": "Нет данных для импорта", + "Missing transaction time, transaction type, or amount column mapping": "Missing transaction time, transaction type, or amount column mapping", + "Transaction type mapping is not set": "Transaction type mapping is not set", + "Transaction time format is not set": "Transaction time format is not set", "Cannot import invalid transactions": "Невозможно импортировать недействительные транзакции", "Unable to parse import file": "Не удалось обработать файл импорта", "Batch Replace Selected Expense Categories": "Пакетная замена выбранных категорий расходов", diff --git a/src/locales/vi.json b/src/locales/vi.json index 68ad37eb..812ef49a 100644 --- a/src/locales/vi.json +++ b/src/locales/vi.json @@ -1097,6 +1097,12 @@ "cannot add transaction before balance modification transaction": "Bạn không thể thêm giao dịch trước giao dịch sửa đổi số dư", "balance modification transaction cannot modify transaction time": "Bạn không thể sửa đổi thời gian giao dịch cho giao dịch sửa đổi số dư", "transfer transaction amount cannot be less than zero": "Số tiền không thể nhỏ hơn 0 đối với giao dịch chuyển khoản", + "import file encoding is empty": "Import file encoding is empty", + "import file encoding not supported": "import file encoding is not supported", + "column mapping invalid": "Column mapping is invalid", + "transaction type mapping invalid": "Transaction type mapping is invalid", + "transaction time format invalid": "Transaction time format is invalid", + "transaction time zone format invalid": "Transaction time zone format is invalid", "transaction category id is invalid": "ID danh mục giao dịch không hợp lệ", "transaction category not found": "Không tìm thấy danh mục giao dịch", "transaction category type is invalid": "Loại danh mục giao dịch không hợp lệ", @@ -1214,6 +1220,58 @@ "parameter invalid color": "{parameter} có định dạng không hợp lệ", "parameter invalid amount filter": "{parameter} có định dạng không hợp lệ" }, + "encoding": { + "utf-8": "UTF-8", + "utf-8-bom": "UTF-8 with BOM", + "utf-16le": "UTF-16 Little Endian", + "utf-16be": "UTF-16 Big Endian", + "cp437": "OEM United States (CP-437)", + "cp863": "OEM Canadian French (CP-863)", + "cp037": "IBM EBCDIC US/Canada (CP-037)", + "cp1047": "IBM EBCDIC Open Systems (CP-1047)", + "cp1140": "IBM EBCDIC US/Canada with Euro (CP-1140)", + "iso-8859-1": "Western European (ISO-8859-1)", + "cp850": "Western European (CP-850)", + "cp858": "Western European with Euro (CP-858)", + "windows-1252": "Western European (Windows-1252)", + "iso-8859-15": "Western European (ISO-8859-15)", + "iso-8859-4": "North European (ISO-8859-4)", + "iso-8859-10": "Nordic (ISO-8859-10)", + "cp865": "Nordic (CP-865)", + "iso-8859-2": "Central European (ISO-8859-2)", + "cp852": "Central European (CP-852)", + "windows-1250": "Central European (Windows-1250)", + "iso-8859-14": "Celtic (ISO-8859-14)", + "iso-8859-3": "South European (ISO-8859-3)", + "cp860": "Portuguese (CP-860)", + "iso-8859-7": "Greek (ISO-8859-7)", + "windows-1253": "Greek (Windows-1253)", + "iso-8859-9": "Turkish (ISO-8859-9)", + "windows-1254": "Turkish (Windows-1254)", + "iso-8859-13": "Baltic (ISO-8859-13)", + "windows-1257": "Baltic (Windows-1257)", + "iso-8859-16": "South-Eastern European (ISO-8859-16)", + "iso-8859-5": "Cyrillic (ISO-8859-5)", + "cp855": "Cyrillic (CP-855)", + "cp866": "Cyrillic (CP-866)", + "windows-1251": "Cyrillic (Windows-1251)", + "koi8r": "Cyrillic (KOI8-R)", + "koi8u": "Cyrillic (KOI8-U)", + "iso-8859-6": "Arabic (ISO-8859-6)", + "windows-1256": "Arabic (Windows-1256)", + "iso-8859-8": "Hebrew (ISO-8859-8)", + "cp862": "Hebrew (CP-862)", + "windows-1255": "Hebrew (Windows-1255)", + "windows-874": "Thai (Windows-874)", + "windows-1258": "Vietnamese (Windows-1258)", + "gb18030": "Simplified Chinese (GB18030)", + "gbk": "Simplified Chinese (GBK)", + "big5": "Traditional Chinese (Big5)", + "euc-kr": "Korean (EUC-KR)", + "euc-jp": "Japanese (EUC-JP)", + "iso-2022-jp": "Japanese (ISO-2022-JP)", + "shift_jis": "Japanese (Shift_JIS)" + }, "document": { "anchor": { "export_and_import": { @@ -1242,6 +1300,7 @@ "Not set": "Not set", "No results": "Không có kết quả", "Unknown": "Không rõ", + "Auto detect": "Auto detect", "Miscellaneous": "Linh tinh", "Default": "Mặc định", "Done": "Hoàn tất", @@ -1268,6 +1327,13 @@ "Color": "Màu sắc", "Type": "Loại", "Format": "Định dạng", + "File Encoding": "File Encoding", + "Space": "Space", + "Comma": "Comma", + "Semicolon": "Semicolon", + "Tab": "Tab", + "Vertical Bar": "Vertical Bar", + "Slash": "Slash", "All Types": "Tất cả các loại", "More": "Thêm", "All": "Tất cả", @@ -1521,6 +1587,8 @@ "Income Amount": "Số tiền thu nhập", "Transfer Out Amount": "Số tiền chuyển ra", "Transfer In Amount": "Số tiền chuyển vào", + "Transfer In Account Name": "Transfer In Account Name", + "Transfer In Currency": "Transfer In Currency", "Show Amount": "Hiển thị số tiền", "Hide Amount": "Ẩn số tiền", "Swap Account": "Hoán đổi tài khoản", @@ -1530,6 +1598,7 @@ "Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)", "Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)", "Category": "Danh mục", + "Secondary Category": "Secondary Category", "Multiple Categories": "Nhiều danh mục", "Account": "Tài khoản", "Multiple Accounts": "Nhiều tài khoản", @@ -1545,6 +1614,7 @@ "Scheduled Transaction Frequency": "Tần suất giao dịch theo lịch trình", "Transaction Timezone": "Múi giờ giao dịch", "Same time as default timezone": "Cùng thời gian với múi giờ mặc định", + "Transaction Type": "Transaction Type", "Geographic Location": "Vị trí địa lý", "No Location": "Không có vị trí", "Getting Location...": "Đang lấy vị trí...", @@ -1559,6 +1629,8 @@ "Import Transactions": "Nhập giao dịch", "Upload File": "Tải lên tệp", "Upload Transaction Data File": "Tải lên tệp dữ liệu giao dịch", + "Define Column": "Define Column", + "Define and Check Column Mapping": "Define and Check Column Mapping", "Check & Modify": "Kiểm tra & Sửa đổi", "Check and Modify Your Data": "Kiểm tra và sửa đổi dữ liệu của bạn", "Data Import Completed": "Nhập dữ liệu hoàn tất", @@ -1573,6 +1645,8 @@ "Month-day-year format": "Định dạng tháng-ngày-năm", "Day-month-year format": "Định dạng ngày-tháng-năm", "Intuit Interchange Format (IIF) File": "Tệp Intuit Interchange Format (IIF)", + "Delimiter-separated Values (DSV) File": "Delimiter-separated Values (DSV) File", + "Delimiter-separated Values (DSV) Data": "Delimiter-separated Values (DSV) Data", "GnuCash XML Database File": "Tệp cơ sở dữ liệu XML GnuCash", "Firefly III Data Export File": "Tệp xuất dữ liệu Firefly III", "Feidee MyMoney (App) Data Export File": "Tệp xuất dữ liệu Feidee MyMoney (Ứng dụng)", @@ -1581,8 +1655,19 @@ "Alipay (Web) Transaction Flow File": "Tệp luồng giao dịch Alipay (Web)", "WeChat Pay Billing File": "Tệp thanh toán WeChat Pay", "Data File": "Tệp dữ liệu", + "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Include Header Line": "Include Header Line", + "Time Format": "Time Format", + "Transaction Type Mapping": "Transaction Type Mapping", + "Timezone Format": "Timezone Format", + "Geographic Location Separator": "Geographic Location Separator", + "Transaction Tags Separator": "Transaction Tags Separator", + "Lines Per Page": "Lines Per Page", "No data to import": "Không có dữ liệu để nhập", + "Missing transaction time, transaction type, or amount column mapping": "Missing transaction time, transaction type, or amount column mapping", + "Transaction type mapping is not set": "Transaction type mapping is not set", + "Transaction time format is not set": "Transaction time format is not set", "Cannot import invalid transactions": "Không thể nhập giao dịch không hợp lệ", "Unable to parse import file": "Không thể phân tích tệp nhập", "Batch Replace Selected Expense Categories": "Thay thế hàng loạt các danh mục chi phí đã chọn", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index b3b828dd..9a9f34f8 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -1097,6 +1097,12 @@ "cannot add transaction before balance modification transaction": "不能添加早于修改余额的交易", "balance modification transaction cannot modify transaction time": "您无法对修改余额的交易修改交易时间", "transfer transaction amount cannot be less than zero": "转账交易的金额不能小于0", + "import file encoding is empty": "导入文件编码为空", + "import file encoding not supported": "导入文件编码不支持", + "column mapping invalid": "列映射无效", + "transaction type mapping invalid": "交易类型映射无效", + "transaction time format invalid": "交易时间格式无效", + "transaction time zone format invalid": "交易时区格式无效", "transaction category id is invalid": "交易分类ID无效", "transaction category not found": "交易分类不存在", "transaction category type is invalid": "交易分类类型无效", @@ -1214,6 +1220,58 @@ "parameter invalid color": "{parameter}格式错误", "parameter invalid amount filter": "{parameter}格式错误" }, + "encoding": { + "utf-8": "UTF-8", + "utf-8-bom": "UTF-8 带签名", + "utf-16le": "UTF-16 Little Endian", + "utf-16be": "UTF-16 Big Endian", + "cp437": "OEM 美国 (CP-437)", + "cp863": "OEM 加拿大法语 (CP-863)", + "cp037": "IBM EBCDIC 美国/加拿大 (CP-037)", + "cp1047": "IBM EBCDIC 开放系统 (CP-1047)", + "cp1140": "IBM EBCDIC 美国/加拿大 含欧元 (CP-1140)", + "iso-8859-1": "西欧 (ISO-8859-1)", + "cp850": "西欧 (CP-850)", + "cp858": "西欧 含欧元 (CP-858)", + "windows-1252": "西欧 (Windows-1252)", + "iso-8859-15": "西欧 (ISO-8859-15)", + "iso-8859-4": "北欧 (ISO-8859-4)", + "iso-8859-10": "北欧 (ISO-8859-10)", + "cp865": "北欧 (CP-865)", + "iso-8859-2": "中欧 (ISO-8859-2)", + "cp852": "中欧 (CP-852)", + "windows-1250": "中欧 (Windows-1250)", + "iso-8859-14": "凯尔特语族 (ISO-8859-14)", + "iso-8859-3": "南欧 (ISO-8859-3)", + "cp860": "葡萄牙语 (CP-860)", + "iso-8859-7": "希腊语 (ISO-8859-7)", + "windows-1253": "希腊语 (Windows-1253)", + "iso-8859-9": "土耳其语 (ISO-8859-9)", + "windows-1254": "土耳其语 (Windows-1254)", + "iso-8859-13": "波罗的语族 (ISO-8859-13)", + "windows-1257": "波罗的语族 (Windows-1257)", + "iso-8859-16": "东南欧 (ISO-8859-16)", + "iso-8859-5": "西里尔文 (ISO-8859-5)", + "cp855": "西里尔文 (CP-855)", + "cp866": "西里尔文 (CP-866)", + "windows-1251": "西里尔文 (Windows-1251)", + "koi8r": "西里尔文 (KOI8-R)", + "koi8u": "西里尔文 (KOI8-U)", + "iso-8859-6": "阿拉伯语 (ISO-8859-6)", + "windows-1256": "阿拉伯语 (Windows-1256)", + "iso-8859-8": "希伯来语 (ISO-8859-8)", + "cp862": "希伯来语 (CP-862)", + "windows-1255": "希伯来语 (Windows-1255)", + "windows-874": "泰语 (Windows-874)", + "windows-1258": "越南语 (Windows-1258)", + "gb18030": "简体中文 (GB18030)", + "gbk": "简体中文 (GBK)", + "big5": "繁体中文 (Big5)", + "euc-kr": "韩语 (EUC-KR)", + "euc-jp": "日语 (EUC-JP)", + "iso-2022-jp": "日语 (ISO-2022-JP)", + "shift_jis": "日语 (Shift_JIS)" + }, "document": { "anchor": { "export_and_import": { @@ -1242,6 +1300,7 @@ "Not set": "未设置", "No results": "无结果", "Unknown": "未知", + "Auto detect": "自动检测", "Miscellaneous": "杂项", "Default": "默认", "Done": "完成", @@ -1268,6 +1327,13 @@ "Color": "颜色", "Type": "类型", "Format": "格式", + "File Encoding": "文件编码", + "Space": "空格", + "Comma": "逗号", + "Semicolon": "分号", + "Tab": "制表符 Tab", + "Vertical Bar": "竖线", + "Slash": "Slash", "All Types": "全部类型", "More": "更多", "All": "全部", @@ -1521,6 +1587,8 @@ "Income Amount": "收入金额", "Transfer Out Amount": "转出金额", "Transfer In Amount": "转入金额", + "Transfer In Account Name": "转入账户名", + "Transfer In Currency": "转入货币", "Show Amount": "显示金额", "Hide Amount": "隐藏金额", "Swap Account": "交换账户", @@ -1530,6 +1598,7 @@ "Duplicate (With Geographic Location)": "复制 (含地理位置)", "Duplicate (With Time and Geographic Location)": "复制 (含时间和地理位置)", "Category": "分类", + "Secondary Category": "二级分类", "Multiple Categories": "多个分类", "Account": "账户", "Multiple Accounts": "多个账户", @@ -1545,6 +1614,7 @@ "Scheduled Transaction Frequency": "定时交易周期", "Transaction Timezone": "交易时区", "Same time as default timezone": "与默认时区时间相同", + "Transaction Type": "交易类型", "Geographic Location": "地理位置", "No Location": "没有位置", "Getting Location...": "正在获取位置...", @@ -1559,6 +1629,8 @@ "Import Transactions": "导入交易", "Upload File": "上传文件", "Upload Transaction Data File": "上传交易数据文件", + "Define Column": "定义列", + "Define and Check Column Mapping": "定义及检查列映射", "Check & Modify": "检查及修改", "Check and Modify Your Data": "检查及修改您的数据", "Data Import Completed": "数据导入完成", @@ -1573,6 +1645,8 @@ "Month-day-year format": "月-日-年 格式", "Day-month-year format": "日-月-年 格式", "Intuit Interchange Format (IIF) File": "Intuit Interchange Format (IIF) 文件", + "Delimiter-separated Values (DSV) File": "分隔符分隔值 (DSV) 文件", + "Delimiter-separated Values (DSV) Data": "分隔符分隔值 (DSV) 数据", "GnuCash XML Database File": "GnuCash XML 数据库文件", "Firefly III Data Export File": "Firefly III 数据导出文件", "Feidee MyMoney (App) Data Export File": "随手记 (App) 数据导出文件", @@ -1581,8 +1655,19 @@ "Alipay (Web) Transaction Flow File": "支付宝 (网页版) 交易流水文件", "WeChat Pay Billing File": "微信支付账单文件", "Data File": "数据文件", + "Data to import": "要导入的数据", "Please select a file to import": "请选择要导入的文件", + "Include Header Line": "包含标题行", + "Time Format": "时间格式", + "Transaction Type Mapping": "交易类型映射", + "Timezone Format": "时区格式", + "Geographic Location Separator": "地理位置分隔符", + "Transaction Tags Separator": "交易标签分隔符", + "Lines Per Page": "每页行数", "No data to import": "没有可以导入的数据", + "Missing transaction time, transaction type, or amount column mapping": "缺少交易时间、交易类型或金额列映射", + "Transaction type mapping is not set": "交易类型映射没有设置", + "Transaction time format is not set": "交易时间格式没有设置", "Cannot import invalid transactions": "不能导入无效的交易", "Unable to parse import file": "无法解析导入的文件", "Batch Replace Selected Expense Categories": "批量替换选中的支出分类", diff --git a/src/stores/transaction.ts b/src/stores/transaction.ts index ef29aa6a..8cd8b022 100644 --- a/src/stores/transaction.ts +++ b/src/stores/transaction.ts @@ -1056,9 +1056,34 @@ export const useTransactionsStore = defineStore('transactions', () => { }); } - function parseImportTransaction({ fileType, importFile }: { fileType: string, importFile: File }): Promise { + function parseImportDsvFile({ fileType, fileEncoding, importFile }: { fileType: string, fileEncoding?: string, importFile: File }): Promise { return new Promise((resolve, reject) => { - services.parseImportTransaction({ fileType, importFile }).then(response => { + services.parseImportDsvFile({ fileType, fileEncoding, importFile }).then(response => { + const data = response.data; + + if (!data || !data.success || !data.result) { + reject({ message: 'Unable to parse import file' }); + return; + } + + resolve(data.result); + }).catch(error => { + logger.error('Unable to parse import file', error); + + if (error.response && error.response.data && error.response.data.errorMessage) { + reject({ error: error.response.data }); + } else if (!error.processed) { + reject({ message: 'Unable to parse import file' }); + } else { + reject(error); + } + }); + }); + } + + function parseImportTransaction({ fileType, fileEncoding, importFile, columnMapping, transactionTypeMapping, hasHeaderLine, timeFormat, timezoneFormat, geoSeparator, tagSeparator }: { fileType: string, fileEncoding?: string, importFile: File, columnMapping?: Record, transactionTypeMapping?: Record, hasHeaderLine?: boolean, timeFormat?: string, timezoneFormat?: string, geoSeparator?: string, tagSeparator?: string }): Promise { + return new Promise((resolve, reject) => { + services.parseImportTransaction({ fileType, fileEncoding, importFile, columnMapping, transactionTypeMapping, hasHeaderLine, timeFormat, timezoneFormat, geoSeparator, tagSeparator }).then(response => { const data = response.data; if (!data || !data.success || !data.result) { @@ -1215,6 +1240,7 @@ export const useTransactionsStore = defineStore('transactions', () => { getTransaction, saveTransaction, deleteTransaction, + parseImportDsvFile, parseImportTransaction, importTransactions, uploadTransactionPicture, diff --git a/src/views/desktop/transactions/list/dialogs/ImportDialog.vue b/src/views/desktop/transactions/list/dialogs/ImportDialog.vue index eeb6c0b4..6e937834 100644 --- a/src/views/desktop/transactions/list/dialogs/ImportDialog.vue +++ b/src/views/desktop/transactions/list/dialogs/ImportDialog.vue @@ -179,7 +179,19 @@ /> - + + + + + + + + + @@ -202,6 +225,184 @@ + + + + + + {{ tt('Cancel') }} - + v-if="currentStep === 'defineColumn' || currentStep === 'uploadFile'"> {{ tt('Next') }} @@ -585,10 +786,12 @@ import { useTransactionsStore } from '@/stores/transaction.ts'; import { useOverviewStore } from '@/stores/overview.ts'; import { useStatisticsStore } from '@/stores/statistics.ts'; -import type { NameValue } from '@/core/base.ts'; +import type { NameValue, TypeAndDisplayName } from '@/core/base.ts'; +import { KnownDateTimeFormat } from '@/core/datetime.ts'; +import { KnownDateTimezoneFormat } from '@/core/timezone.ts'; import { CategoryType } from '@/core/category.ts'; -import { TransactionType } from '@/core/transaction.ts'; -import type { LocalizedImportFileType, LocalizedImportFileTypeSubType } from '@/core/file.ts'; +import { TransactionType, ImportTransactionColumnType } from '@/core/transaction.ts'; +import type { LocalizedImportFileType, LocalizedImportFileTypeSubType, LocalizedImportFileTypeSupportedEncodings } from '@/core/file.ts'; import { Account, type CategorizedAccountWithDisplayBalance } from '@/models/account.ts'; import type { TransactionCategory } from '@/models/transaction_category.ts'; import type { TransactionTag } from '@/models/transaction_tag.ts'; @@ -597,6 +800,9 @@ import { ImportTransaction } from '@/models/imported_transaction.ts'; import { isString, isNumber, + isObjectEmpty, + getObjectOwnFieldCount, + findDisplayNameByType, objectFieldToArrayItem } from '@/lib/common.ts'; import { @@ -636,6 +842,8 @@ type ConfirmDialogType = InstanceType; type SnackBarType = InstanceType; type BatchReplaceDialogType = InstanceType; +type ImportTransactionDialogStep = 'uploadFile' | 'defineColumn' | 'checkData' | 'finalResult'; + interface ImportTransactionDialogFilter { minDatetime: number | null; // minDatetime or maxDatetime is null for 'All Date Range', all are not null for 'Custom Date Range' maxDatetime: number | null; @@ -657,6 +865,7 @@ defineProps<{ const { tt, + getAllImportTransactionColumnTypes, getAllSupportedImportFileTypes, formatUnixTimeToLongDateTime, formatAmountWithCurrency, @@ -679,10 +888,20 @@ const fileInput = useTemplateRef('fileInput'); const showState = ref(false); const clientSessionId = ref(''); -const currentStep = ref('uploadFile'); +const currentStep = ref('uploadFile'); const fileType = ref('ezbookkeeping'); const fileSubType = ref('ezbookkeeping_csv'); +const fileEncoding = ref('utf-8'); const importFile = ref(null); +const importData = ref(''); +const parsedFileData = ref(undefined); +const parsedFileIncludeHeader = ref(true); +const parsedFileDataColumnMapping = ref>({}); +const parsedFileTransactionTypeMapping = ref>({}); +const parsedFileTimeFormat = ref(''); +const parsedFileTimezoneFormat = ref(''); +const parsedFileGeoLocationSeparator = ref(' '); +const parsedFileTagSeparator = ref(';'); const importTransactions = ref(undefined); const editingTransaction = ref(null); const editingTags = ref([]); @@ -713,26 +932,79 @@ const currentTimezoneOffsetMinutes = computed(() => getTimezoneOffsetMin const defaultCurrency = computed(() => userStore.currentUserDefaultCurrency); -const allSteps = computed(() => [ - { - name: 'uploadFile', - title: tt('Upload File'), - subTitle: tt('Upload Transaction Data File') - }, - { - name: 'checkData', - title: tt('Check & Modify'), - subTitle: tt('Check and Modify Your Data') - }, - { - name: 'finalResult', - title: tt('Complete'), - subTitle: tt('Data Import Completed') - } -]); +const allSteps = computed(() => { + const steps: StepBarItem[] = [ + { + name: 'uploadFile', + title: tt('Upload File'), + subTitle: tt('Upload Transaction Data File') + } + ]; + if (fileType.value === 'dsv' || fileType.value === 'dsv_data') { + steps.push({ + name: 'defineColumn', + title: tt('Define Column'), + subTitle: tt('Define and Check Column Mapping') + }); + } + + steps.push(...[ + { + name: 'checkData', + title: tt('Check & Modify'), + subTitle: tt('Check and Modify Your Data') + }, + { + name: 'finalResult', + title: tt('Complete'), + subTitle: tt('Data Import Completed') + } + ]); + + return steps; +}); + +const allImportTransactionColumnTypes = computed(() => getAllImportTransactionColumnTypes()); const allSupportedImportFileTypes = computed(() => getAllSupportedImportFileTypes()); +const allSeparators = computed(() => { + const separators: NameValue[] = [ + { + name: tt('Space'), + value: ' ' + }, + { + name: tt('Comma'), + value: ',' + }, + { + name: tt('Semicolon'), + value: ';' + }, + { + name: tt('Tab'), + value: '\t' + }, + { + name: tt('Vertical Bar'), + value: '|' + } + ]; + + return separators; +}); + +const isImportDataFromTextbox = computed(() => { + for (const importFileType of allSupportedImportFileTypes.value) { + if (importFileType.type === fileType.value) { + return !!importFileType.dataFromTextbox; + } + } + + return false; +}); + const allFileSubTypes = computed(() => { for (const importFileType of allSupportedImportFileTypes.value) { if (importFileType.type === fileType.value) { @@ -743,6 +1015,16 @@ const allFileSubTypes = computed(( return undefined; }); +const allSupportedEncodings = computed(() => { + for (const importFileType of allSupportedImportFileTypes.value) { + if (importFileType.type === fileType.value) { + return importFileType.supportedEncodings; + } + } + + return undefined; +}); + const allAccounts = computed(() => accountsStore.allPlainAccounts); const allVisibleAccounts = computed(() => accountsStore.allVisiblePlainAccounts); const allVisibleCategorizedAccounts = computed(() => getCategorizedAccountsWithDisplayBalance(allVisibleAccounts.value, showAccountBalance.value)); @@ -799,6 +1081,182 @@ const exportFileGuideDocumentLanguageName = computed(() => { const fileName = computed(() => importFile.value?.name || ''); +const parsedFileLines = computed[] | undefined>(() => { + if (!parsedFileData.value) { + return undefined; + } + + const allLines: Record[] = []; + const startIndex = parsedFileIncludeHeader.value ? 1 : 0; + + for (let i = startIndex, index = 1; i < parsedFileData.value.length; i++, index++) { + const line: Record = {}; + const columns = parsedFileData.value[i]; + + for (let j = 0; j < columns.length; j++) { + line['index'] = index.toString(); + line[`column${j + 1}`] = columns[j]; + } + + allLines.push(line); + } + + return allLines; +}); + +const parsedFileLinesTableHeight = computed(() => { + if (countPerPage.value <= 10 || !parsedFileLines.value || parsedFileLines.value.length <= 10) { + return undefined; + } else { + return 400; + } +}); + +const parsedFileLinesHeaders = computed(() => { + let maxColumnCount = 0; + + if (parsedFileData.value) { + for (let i = 0; i < parsedFileData.value.length; i++) { + if (parsedFileData.value[i].length > maxColumnCount) { + maxColumnCount = parsedFileData.value[i].length; + } + } + } + + const headers: object[] = []; + + headers.push({ key: 'index', value: 'index', title: '#', sortable: true, nowrap: true }); + + for (let i = 0; i < maxColumnCount; i++) { + let title = `#${i + 1}`; + + if (parsedFileIncludeHeader.value && parsedFileData.value && parsedFileData.value[0][i]) { + title = parsedFileData.value[0][i]; + } + + headers.push({ key: i.toString(), value: `column${i + 1}`, title: title, sortable: true, nowrap: true }); + } + + return headers; +}); + +const parsedFileLinesTablePageOptions = computed(() => getTablePageOptions(parsedFileLines.value?.length)); + +const parsedFileAllTransactionTypes = computed(() => { + if (!parsedFileData.value || !parsedFileData.value.length || !isNumber(parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionType.type])) { + return []; + } + + const allTypeMap: Record = {}; + const allTypes: string[] = []; + const typeColumnIndex = parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionType.type]; + + const startIndex = parsedFileIncludeHeader.value ? 1 : 0; + + for (let i = startIndex; i < parsedFileData.value.length; i++) { + if (parsedFileData.value[i].length <= typeColumnIndex) { + continue; + } + + const type = parsedFileData.value[i][typeColumnIndex]; + + if (type && !allTypeMap[type]) { + allTypes.push(type); + allTypeMap[type] = true; + } + } + + return allTypes; +}); + +const parsedFileValidMappedTransactionTypes = computed>(() => { + if (!parsedFileData.value || !parsedFileData.value.length || !isNumber(parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionType.type])) { + return {}; + } + + const result: Record = {}; + + if (!parsedFileTransactionTypeMapping.value) { + return result; + } + + for (const name in parsedFileTransactionTypeMapping.value) { + if (!Object.prototype.hasOwnProperty.call(parsedFileTransactionTypeMapping.value, name)) { + continue; + } + + const type = parsedFileTransactionTypeMapping.value[name]; + + if (TransactionType.ModifyBalance <= type && type <= TransactionType.Transfer) { + result[name] = type; + } + } + + return result; +}); + +const parsedFileAutoDetectedTimeFormat = computed(() => { + if (!parsedFileData.value || !parsedFileData.value.length || !isNumber(parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionTime.type])) { + return undefined; + } + + const allDateTimes: string[] = []; + const dateTimeColumnIndex = parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionTime.type]; + + const startIndex = parsedFileIncludeHeader.value ? 1 : 0; + + for (let i = startIndex; i < parsedFileData.value.length; i++) { + if (parsedFileData.value[i].length <= dateTimeColumnIndex) { + continue; + } + + const dateTime = parsedFileData.value[i][dateTimeColumnIndex]; + + if (dateTime) { + allDateTimes.push(dateTime); + } + } + + const detectedFormats = KnownDateTimeFormat.detectMany(allDateTimes); + + if (!detectedFormats || !detectedFormats.length || detectedFormats.length > 1) { + return undefined; + } + + return detectedFormats[0].format; +}); + +const parsedFileAutoDetectedTimezoneFormat = computed(() => { + if (!parsedFileData.value || !parsedFileData.value.length || !isNumber(parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionTimezone.type])) { + return undefined; + } + + const allTimezones: string[] = []; + const timezoneColumnIndex = parsedFileDataColumnMapping.value[ImportTransactionColumnType.TransactionTimezone.type]; + + const startIndex = parsedFileIncludeHeader.value ? 1 : 0; + + for (let i = startIndex; i < parsedFileData.value.length; i++) { + if (parsedFileData.value[i].length <= timezoneColumnIndex) { + continue; + } + + const timezone = parsedFileData.value[i][timezoneColumnIndex]; + + if (timezone) { + allTimezones.push(timezone); + } + } + + const detectedFormats = KnownDateTimezoneFormat.detectMany(allTimezones); + + if (!detectedFormats || !detectedFormats.length || detectedFormats.length > 1) { + return undefined; + } + + return detectedFormats[0].value; +}); + const importTransactionsTableHeight = computed(() => { if (countPerPage.value <= 10 || !importTransactions.value || importTransactions.value.length <= 10) { return undefined; @@ -821,30 +1279,7 @@ const importTransactionHeaders = computed(() => { ]; }); -const importTransactionsTablePageOptions = computed(() => { - const pageOptions: ImportTransactionsDialogTablePageOption[] = []; - - if (!importTransactions.value || importTransactions.value.length < 1) { - pageOptions.push({ value: -1, title: tt('All') }); - return pageOptions; - } - - const availableCountPerPage = [ 5, 10, 15, 20, 25, 30, 50 ]; - - for (let i = 0; i < availableCountPerPage.length; i++) { - const count = availableCountPerPage[i]; - - if (importTransactions.value.length < count) { - break; - } - - pageOptions.push({ value: count, title: count.toString() }); - } - - pageOptions.push({ value: -1, title: tt('All') }); - - return pageOptions; -}); +const importTransactionsTablePageOptions = computed(() => getTablePageOptions(importTransactions.value?.length)); const totalPageCount = computed(() => { if (!importTransactions.value || importTransactions.value.length < 1) { @@ -1052,6 +1487,55 @@ const displayFilterCustomDateRange = computed(() => { return `${minDisplayTime} - ${maxDisplayTime}` }); +function getTablePageOptions(linesCount?: number): ImportTransactionsDialogTablePageOption[] { + const pageOptions: ImportTransactionsDialogTablePageOption[] = []; + + if (!linesCount || linesCount < 1) { + pageOptions.push({ value: -1, title: tt('All') }); + return pageOptions; + } + + const availableCountPerPage = [ 5, 10, 15, 20, 25, 30, 50 ]; + + for (let i = 0; i < availableCountPerPage.length; i++) { + const count = availableCountPerPage[i]; + + if (linesCount < count) { + break; + } + + pageOptions.push({ value: count, title: count.toString() }); + } + + pageOptions.push({ value: -1, title: tt('All') }); + + return pageOptions; +} + +function getParseDataMappedColumnDisplayName(columnIndex: number): string { + for (const columnType in parsedFileDataColumnMapping.value) { + if (parsedFileDataColumnMapping.value[columnType] === columnIndex) { + return findDisplayNameByType(allImportTransactionColumnTypes.value, parseInt(columnType)) || tt('Unspecified'); + } + } + + return tt('Unspecified'); +} + +function updateParseDataMappedColumn(columnIndex: number, columnType: number): void { + if (parsedFileDataColumnMapping.value[columnType] === columnIndex) { + delete parsedFileDataColumnMapping.value[columnType]; + } else { + parsedFileDataColumnMapping.value[columnType] = columnIndex; + } + + for (const otherColumnType in parsedFileDataColumnMapping.value) { + if (otherColumnType !== columnType.toString() && parsedFileDataColumnMapping.value[otherColumnType] === columnIndex) { + delete parsedFileDataColumnMapping.value[otherColumnType]; + } + } +} + function isTransactionDisplayed(transaction: ImportTransaction): boolean { if (isNumber(filters.value.minDatetime) && isNumber(filters.value.maxDatetime) && (transaction.time < filters.value.minDatetime || transaction.time > filters.value.maxDatetime)) { return false; @@ -1328,8 +1812,18 @@ function getCurrentInvalidTagNames(): NameValue[] { function open(): Promise { fileType.value = 'ezbookkeeping'; fileSubType.value = 'ezbookkeeping_csv'; + fileEncoding.value = 'utf-8'; currentStep.value = 'uploadFile'; importFile.value = null; + importData.value = ''; + parsedFileData.value = undefined; + parsedFileIncludeHeader.value = true; + parsedFileDataColumnMapping.value = {}; + parsedFileTransactionTypeMapping.value = {}; + parsedFileTimeFormat.value = ''; + parsedFileTimezoneFormat.value = ''; + parsedFileGeoLocationSeparator.value = ' '; + parsedFileTagSeparator.value = ';'; importTransactions.value = undefined; editingTransaction.value = null; editingTags.value = []; @@ -1396,52 +1890,165 @@ function setImportFile(event: Event): void { } function parseData(): void { - if (!importFile.value) { - snackbar.value?.showError('Please select a file to import'); - return; - } - - submitting.value = true; - + let uploadFile: File; let type: string = fileType.value; + let encoding: string | undefined = undefined; if (allFileSubTypes.value) { type = fileSubType.value; } - transactionsStore.parseImportTransaction({ - fileType: type, - importFile: importFile.value - }).then(response => { - const parsedTransactions: ImportTransaction[] = []; + if (allSupportedEncodings.value) { + encoding = fileEncoding.value; + } - if (response.items) { - for (let i = 0; i < response.items.length; i++) { - const parsedTransaction = ImportTransaction.of(response.items[i], i); - parsedTransactions.push(parsedTransaction); + if (!isImportDataFromTextbox.value) { + if (!importFile.value) { + snackbar.value?.showError('Please select a file to import'); + return; + } + + uploadFile = importFile.value; + } else if (isImportDataFromTextbox.value) { + if (!importData.value) { + snackbar.value?.showError('No data to import'); + return; + } + + if (type === 'custom_csv') { + uploadFile = new File([importData.value], 'import.csv', { type: 'text/csv' }); + } else if (type === 'custom_tsv') { + uploadFile = new File([importData.value], 'import.tsv', { type: 'text/tab-separated-values' }); + } else { + snackbar.value?.showError('Parameter Invalid'); + return; + } + + encoding = 'utf-8'; + } else { // should not happen, but ts would check whether uploadFile has been assigned a value + snackbar.value?.showMessage('An error occurred'); + return; + } + + const isDsvFileType: boolean = fileType.value === 'dsv' || fileType.value === 'dsv_data'; + + if (isDsvFileType && currentStep.value === 'uploadFile') { + submitting.value = true; + + transactionsStore.parseImportDsvFile({ + fileType: type, + fileEncoding: encoding, + importFile: uploadFile + }).then(response => { + if (response && response.length) { + parsedFileData.value = response; + currentPage.value = 1; + countPerPage.value = 10; + currentStep.value = 'defineColumn'; + } else { + parsedFileData.value = undefined; + snackbar.value?.showError('No data to import'); + } + + submitting.value = false; + }).catch(error => { + submitting.value = false; + + if (!error.processed) { + snackbar.value?.showError(error); + } + }); + } else { + let columnMapping: Record | undefined = undefined; + let transactionTypeMapping: Record | undefined = undefined; + let hasHeaderLine: boolean | undefined = undefined; + let timeFormat: string | undefined = undefined; + let timezoneFormat: string | undefined = undefined; + let geoLocationSeparator: string | undefined = undefined; + let tagSeparator: string | undefined = undefined; + + if (isDsvFileType) { + columnMapping = parsedFileDataColumnMapping.value; + transactionTypeMapping = parsedFileValidMappedTransactionTypes.value; + hasHeaderLine = parsedFileIncludeHeader.value; + timeFormat = parsedFileTimeFormat.value; + timezoneFormat = parsedFileTimezoneFormat.value; + geoLocationSeparator = parsedFileGeoLocationSeparator.value; + tagSeparator = parsedFileTagSeparator.value; + + if (!columnMapping + || !isNumber(columnMapping[ImportTransactionColumnType.TransactionTime.type]) + || !isNumber(columnMapping[ImportTransactionColumnType.TransactionType.type]) + || !isNumber(columnMapping[ImportTransactionColumnType.Amount.type])) { + snackbar.value?.showError('Missing transaction time, transaction type, or amount column mapping'); + return; + } + + if (!transactionTypeMapping || isObjectEmpty(transactionTypeMapping)) { + snackbar.value?.showError('Transaction type mapping is not set'); + return; + } + + if (!parsedFileTimeFormat.value) { + timeFormat = parsedFileAutoDetectedTimeFormat.value; + } + + if (!parsedFileTimezoneFormat.value) { + timezoneFormat = parsedFileAutoDetectedTimezoneFormat.value; + } + + if (!timeFormat) { + snackbar.value?.showError('Transaction time format is not set'); + return; } } - importTransactions.value = parsedTransactions; - editingTransaction.value = null; - editingTags.value = []; - currentPage.value = 1; + submitting.value = true; - if (importTransactions.value && importTransactions.value.length >= 0 && importTransactions.value.length < 10) { - countPerPage.value = -1; - } else { + transactionsStore.parseImportTransaction({ + fileType: type, + fileEncoding: encoding, + importFile: uploadFile, + columnMapping: columnMapping, + transactionTypeMapping: transactionTypeMapping, + hasHeaderLine: hasHeaderLine, + timeFormat: timeFormat, + timezoneFormat: timezoneFormat, + geoSeparator: geoLocationSeparator, + tagSeparator: tagSeparator + }).then(response => { + const parsedTransactions: ImportTransaction[] = []; + + if (response.items) { + for (let i = 0; i < response.items.length; i++) { + const parsedTransaction = ImportTransaction.of(response.items[i], i); + parsedTransactions.push(parsedTransaction); + } + } + + importTransactions.value = parsedTransactions; + editingTransaction.value = null; + editingTags.value = []; + currentPage.value = 1; + + if (importTransactions.value && importTransactions.value.length >= 0 && importTransactions.value.length < 10) { + countPerPage.value = -1; + } else { + countPerPage.value = 10; + } + + currentPage.value = 1; countPerPage.value = 10; - } + currentStep.value = 'checkData'; + submitting.value = false; + }).catch(error => { + submitting.value = false; - currentStep.value = 'checkData'; - submitting.value = false; - }).catch(error => { - submitting.value = false; - - if (!error.processed) { - snackbar.value?.showError(error); - } - }); + if (!error.processed) { + snackbar.value?.showError(error); + } + }); + } } function submit(): void { @@ -1820,6 +2427,40 @@ defineExpose({