mirror of
https://github.com/mayswind/ezbookkeeping.git
synced 2026-05-19 09:14:27 +08:00
import transactions from custom xlsx/xls file
This commit is contained in:
MaysWind
+20
-21
@@ -1401,13 +1401,13 @@ func (a *TransactionsApi) TransactionDeleteHandler(c *core.WebContext) (any, *er
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// TransactionParseImportDsvFileDataHandler returns the parsed file data by request parameters for current user
|
||||
func (a *TransactionsApi) TransactionParseImportDsvFileDataHandler(c *core.WebContext) (any, *errs.Error) {
|
||||
// TransactionParseImportCustomFileDataHandler returns the parsed file data by request parameters for current user
|
||||
func (a *TransactionsApi) TransactionParseImportCustomFileDataHandler(c *core.WebContext) (any, *errs.Error) {
|
||||
uid := c.GetCurrentUid()
|
||||
form, err := c.MultipartForm()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to get multi-part form data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
log.Errorf(c, "[transactions.TransactionParseImportCustomFileDataHandler] failed to get multi-part form data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrParameterInvalid
|
||||
}
|
||||
|
||||
@@ -1419,18 +1419,18 @@ func (a *TransactionsApi) TransactionParseImportDsvFileDataHandler(c *core.WebCo
|
||||
|
||||
fileType := fileTypes[0]
|
||||
|
||||
if !converters.IsCustomDelimiterSeparatedValuesFileType(fileType) {
|
||||
if !converters.IsCustomFileFormatFileType(fileType) {
|
||||
return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported)
|
||||
}
|
||||
|
||||
fileEncodings := form.Value["fileEncoding"]
|
||||
fileEncoding := ""
|
||||
|
||||
if len(fileEncodings) < 1 || fileEncodings[0] == "" {
|
||||
return nil, errs.ErrImportFileEncodingIsEmpty
|
||||
if len(fileEncodings) > 0 {
|
||||
fileEncoding = fileEncodings[0]
|
||||
}
|
||||
|
||||
fileEncoding := fileEncodings[0]
|
||||
dataParser, err := converters.CreateNewDelimiterSeparatedValuesDataParser(fileType, fileEncoding)
|
||||
dataParser, err := converters.CreateNewCustomFileFormatTransactionDataParser(fileType, fileEncoding)
|
||||
|
||||
if err != nil {
|
||||
return nil, errs.Or(err, errs.ErrImportFileTypeNotSupported)
|
||||
@@ -1439,24 +1439,24 @@ func (a *TransactionsApi) TransactionParseImportDsvFileDataHandler(c *core.WebCo
|
||||
importFiles := form.File["file"]
|
||||
|
||||
if len(importFiles) < 1 {
|
||||
log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] there is no import file in request for user \"uid:%d\"", uid)
|
||||
log.Warnf(c, "[transactions.TransactionParseImportCustomFileDataHandler] there is no import file in request for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrNoFilesUpload
|
||||
}
|
||||
|
||||
if importFiles[0].Size < 1 {
|
||||
log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] the size of import file in request is zero for user \"uid:%d\"", uid)
|
||||
log.Warnf(c, "[transactions.TransactionParseImportCustomFileDataHandler] the size of import file in request is zero for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrUploadedFileEmpty
|
||||
}
|
||||
|
||||
if importFiles[0].Size > int64(a.CurrentConfig().MaxImportFileSize) {
|
||||
log.Warnf(c, "[transactions.TransactionParseImportDsvFileDataHandler] the upload file size \"%d\" exceeds the maximum size \"%d\" of import file for user \"uid:%d\"", importFiles[0].Size, a.CurrentConfig().MaxImportFileSize, uid)
|
||||
log.Warnf(c, "[transactions.TransactionParseImportCustomFileDataHandler] the upload file size \"%d\" exceeds the maximum size \"%d\" of import file for user \"uid:%d\"", importFiles[0].Size, a.CurrentConfig().MaxImportFileSize, uid)
|
||||
return nil, errs.ErrExceedMaxUploadFileSize
|
||||
}
|
||||
|
||||
importFile, err := importFiles[0].Open()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to get import file from request for user \"uid:%d\", because %s", uid, err.Error())
|
||||
log.Errorf(c, "[transactions.TransactionParseImportCustomFileDataHandler] failed to get import file from request for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
@@ -1464,14 +1464,14 @@ func (a *TransactionsApi) TransactionParseImportDsvFileDataHandler(c *core.WebCo
|
||||
fileData, err := io.ReadAll(importFile)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to read import file data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
log.Errorf(c, "[transactions.TransactionParseImportCustomFileDataHandler] failed to read import file data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
allLines, err := dataParser.ParseDsvFileLines(c, fileData)
|
||||
allLines, err := dataParser.ParseDataLines(c, fileData)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[transactions.TransactionParseImportDsvFileDataHandler] failed to parse import file data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
log.Errorf(c, "[transactions.TransactionParseImportCustomFileDataHandler] failed to parse import file data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
@@ -1514,15 +1514,14 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext)
|
||||
|
||||
var dataImporter converter.TransactionDataImporter
|
||||
|
||||
if converters.IsCustomDelimiterSeparatedValuesFileType(fileType) {
|
||||
if converters.IsCustomFileFormatFileType(fileType) {
|
||||
fileEncodings := form.Value["fileEncoding"]
|
||||
fileEncoding := ""
|
||||
|
||||
if len(fileEncodings) < 1 || fileEncodings[0] == "" {
|
||||
return nil, errs.ErrImportFileEncodingIsEmpty
|
||||
if len(fileEncodings) > 0 {
|
||||
fileEncoding = fileEncodings[0]
|
||||
}
|
||||
|
||||
fileEncoding := fileEncodings[0]
|
||||
|
||||
columnMappings := form.Value["columnMapping"]
|
||||
|
||||
if len(columnMappings) < 1 || columnMappings[0] == "" {
|
||||
@@ -1606,7 +1605,7 @@ func (a *TransactionsApi) TransactionParseImportFileHandler(c *core.WebContext)
|
||||
transactionTagSeparator = transactionTagSeparators[0]
|
||||
}
|
||||
|
||||
dataImporter, err = converters.CreateNewDelimiterSeparatedValuesDataImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormats[0], timezoneFormat, amountDecimalSeparator, amountDigitGroupingSymbol, geoLocationSeparator, geoLocationOrder, transactionTagSeparator)
|
||||
dataImporter, err = converters.CreateNewCustomTransactionDataImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormats[0], timezoneFormat, amountDecimalSeparator, amountDigitGroupingSymbol, geoLocationSeparator, geoLocationOrder, transactionTagSeparator)
|
||||
} else {
|
||||
dataImporter, err = converters.GetTransactionDataImporter(fileType)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
package custom
|
||||
|
||||
import "github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
|
||||
// CustomTransactionDataParser represents the parser for custom transaction data files
|
||||
type CustomTransactionDataParser interface {
|
||||
ParseDataLines(ctx core.Context, data []byte) ([][]string, error)
|
||||
}
|
||||
+15
-11
@@ -1,4 +1,4 @@
|
||||
package dsv
|
||||
package custom
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -94,10 +94,6 @@ var customTransactionTypeNameMapping = map[models.TransactionType]string{
|
||||
models.TRANSACTION_TYPE_TRANSFER: utils.IntToString(int(models.TRANSACTION_TYPE_TRANSFER)),
|
||||
}
|
||||
|
||||
type CustomTransactionDataDsvFileParser interface {
|
||||
ParseDsvFileLines(ctx core.Context, data []byte) ([][]string, error)
|
||||
}
|
||||
|
||||
// customTransactionDataDsvFileImporter defines the structure of custom dsv importer for transaction data
|
||||
type customTransactionDataDsvFileImporter struct {
|
||||
fileEncoding encoding.Encoding
|
||||
@@ -114,8 +110,8 @@ type customTransactionDataDsvFileImporter struct {
|
||||
transactionTagSeparator string
|
||||
}
|
||||
|
||||
// ParseDsvFileLines returns the parsed file lines for specified the dsv file data
|
||||
func (c *customTransactionDataDsvFileImporter) ParseDsvFileLines(ctx core.Context, data []byte) ([][]string, error) {
|
||||
// ParseDataLines returns the parsed file lines for specified the dsv file data
|
||||
func (c *customTransactionDataDsvFileImporter) ParseDataLines(ctx core.Context, data []byte) ([][]string, error) {
|
||||
reader := transform.NewReader(bytes.NewReader(data), c.fileEncoding.NewDecoder())
|
||||
csvReader := csv.NewReader(reader)
|
||||
csvReader.Comma = c.separator
|
||||
@@ -131,7 +127,7 @@ func (c *customTransactionDataDsvFileImporter) ParseDsvFileLines(ctx core.Contex
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(ctx, "[custom_transaction_data_dsv_file_importer.ParseDsvFileLines] cannot parse dsv data, because %s", err.Error())
|
||||
log.Errorf(ctx, "[custom_transaction_data_dsv_file_importer.ParseDataLines] cannot parse dsv data, because %s", err.Error())
|
||||
return nil, errs.ErrInvalidCSVFile
|
||||
}
|
||||
|
||||
@@ -151,7 +147,7 @@ func (c *customTransactionDataDsvFileImporter) ParseDsvFileLines(ctx core.Contex
|
||||
|
||||
// ParseImportedData returns the imported data by parsing the custom transaction dsv data
|
||||
func (c *customTransactionDataDsvFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezone *time.Location, additionalOptions converter.TransactionDataImporterOptions, accountMap map[string]*models.Account, expenseCategoryMap map[string]map[string]*models.TransactionCategory, incomeCategoryMap map[string]map[string]*models.TransactionCategory, transferCategoryMap map[string]map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
|
||||
allLines, err := c.ParseDsvFileLines(ctx, data)
|
||||
allLines, err := c.ParseDataLines(ctx, data)
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, err
|
||||
@@ -170,14 +166,18 @@ func IsDelimiterSeparatedValuesFileType(fileType string) bool {
|
||||
return exists
|
||||
}
|
||||
|
||||
// CreateNewCustomTransactionDataDsvFileParser returns a new custom dsv parser for transaction data
|
||||
func CreateNewCustomTransactionDataDsvFileParser(fileType string, fileEncoding string) (CustomTransactionDataDsvFileParser, error) {
|
||||
// CreateNewCustomTransactionDataDsvFileParser returns a new custom transaction data parser
|
||||
func CreateNewCustomTransactionDataDsvFileParser(fileType string, fileEncoding string) (CustomTransactionDataParser, error) {
|
||||
separator, exists := supportedFileTypeSeparators[fileType]
|
||||
|
||||
if !exists {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
if fileEncoding == "" {
|
||||
return nil, errs.ErrImportFileEncodingIsEmpty
|
||||
}
|
||||
|
||||
enc, exists := supportedFileEncodings[fileEncoding]
|
||||
|
||||
if !exists {
|
||||
@@ -198,6 +198,10 @@ func CreateNewCustomTransactionDataDsvFileImporter(fileType string, fileEncoding
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
if fileEncoding == "" {
|
||||
return nil, errs.ErrImportFileEncodingIsEmpty
|
||||
}
|
||||
|
||||
enc, exists := supportedFileEncodings[fileEncoding]
|
||||
|
||||
if !exists {
|
||||
+5
-5
@@ -1,4 +1,4 @@
|
||||
package dsv
|
||||
package custom
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -25,13 +25,13 @@ func TestIsDelimiterSeparatedValuesFileType(t *testing.T) {
|
||||
assert.False(t, IsDelimiterSeparatedValuesFileType("ssv"))
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataDsvFileParser_ParseDsvFileLines(t *testing.T) {
|
||||
func TestCustomTransactionDataDsvFileParser_ParseDataLines(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataDsvFileParser("custom_csv", "utf-8")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
allLines, err := importer.ParseDsvFileLines(context, []byte(
|
||||
allLines, err := importer.ParseDataLines(context, []byte(
|
||||
"2024-09-01 00:00:00,B,123.45\n"+
|
||||
"2024-09-01 01:23:45,I,0.12\n"))
|
||||
assert.Nil(t, err)
|
||||
@@ -51,7 +51,7 @@ func TestCustomTransactionDataDsvFileParser_ParseDsvFileLines(t *testing.T) {
|
||||
importer, err = CreateNewCustomTransactionDataDsvFileParser("custom_tsv", "utf-8")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err = importer.ParseDsvFileLines(context, []byte(
|
||||
allLines, err = importer.ParseDataLines(context, []byte(
|
||||
"2024-09-01 12:34:56\tE\t1.00\n"+
|
||||
"2024-09-01 23:59:59\tT\t0.05"))
|
||||
assert.Nil(t, err)
|
||||
@@ -71,7 +71,7 @@ func TestCustomTransactionDataDsvFileParser_ParseDsvFileLines(t *testing.T) {
|
||||
importer, err = CreateNewCustomTransactionDataDsvFileParser("custom_ssv", "utf-8")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err = importer.ParseDsvFileLines(context, []byte(
|
||||
allLines, err = importer.ParseDataLines(context, []byte(
|
||||
"2024-09-01 12:34:56;E;1.00\n"+
|
||||
"2024-09-01 23:59:59;T;0.05"))
|
||||
assert.Nil(t, err)
|
||||
@@ -0,0 +1,137 @@
|
||||
package custom
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/converter"
|
||||
csvconverter "github.com/mayswind/ezbookkeeping/pkg/converters/csv"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/excel"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/models"
|
||||
)
|
||||
|
||||
const customOOXMLExcelFileType = "custom_xlsx"
|
||||
const customMSCFBExcelFileType = "custom_xls"
|
||||
|
||||
// customTransactionDataExcelFileImporter defines the structure of custom excel importer for transaction data
|
||||
type customTransactionDataExcelFileImporter struct {
|
||||
fileType string
|
||||
columnIndexMapping map[datatable.TransactionDataTableColumn]int
|
||||
transactionTypeNameMapping map[string]models.TransactionType
|
||||
hasHeaderLine bool
|
||||
timeFormat string
|
||||
timezoneFormat string
|
||||
amountDecimalSeparator string
|
||||
amountDigitGroupingSymbol string
|
||||
geoLocationSeparator string
|
||||
geoLocationOrder converter.TransactionGeoLocationOrder
|
||||
transactionTagSeparator string
|
||||
}
|
||||
|
||||
// ParseDataLines returns the parsed file lines for specified the excel file data
|
||||
func (c *customTransactionDataExcelFileImporter) ParseDataLines(ctx core.Context, data []byte) ([][]string, error) {
|
||||
var excelDataTable datatable.BasicDataTable
|
||||
var err error
|
||||
|
||||
if c.fileType == customOOXMLExcelFileType {
|
||||
excelDataTable, err = excel.CreateNewExcelOOXMLFileBasicDataTable(data, false)
|
||||
} else if c.fileType == customMSCFBExcelFileType {
|
||||
excelDataTable, err = excel.CreateNewExcelMSCFBFileBasicDataTable(data, false)
|
||||
} else {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
iterator := excelDataTable.DataRowIterator()
|
||||
allLines := make([][]string, 0)
|
||||
|
||||
for iterator.HasNext() {
|
||||
row := iterator.Next()
|
||||
items := make([]string, row.ColumnCount())
|
||||
|
||||
for i := 0; i < row.ColumnCount(); i++ {
|
||||
items[i] = strings.Trim(row.GetData(i), " ")
|
||||
}
|
||||
|
||||
allLines = append(allLines, items)
|
||||
}
|
||||
|
||||
return allLines, nil
|
||||
}
|
||||
|
||||
// ParseImportedData returns the imported data by parsing the custom transaction dsv data
|
||||
func (c *customTransactionDataExcelFileImporter) ParseImportedData(ctx core.Context, user *models.User, data []byte, defaultTimezone *time.Location, additionalOptions converter.TransactionDataImporterOptions, accountMap map[string]*models.Account, expenseCategoryMap map[string]map[string]*models.TransactionCategory, incomeCategoryMap map[string]map[string]*models.TransactionCategory, transferCategoryMap map[string]map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (models.ImportedTransactionSlice, []*models.Account, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionCategory, []*models.TransactionTag, error) {
|
||||
allLines, err := c.ParseDataLines(ctx, data)
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
dataTable := csvconverter.CreateNewCustomCsvBasicDataTable(allLines, c.hasHeaderLine)
|
||||
transactionDataTable := CreateNewCustomPlainTextDataTable(dataTable, c.columnIndexMapping, c.transactionTypeNameMapping, c.timeFormat, c.timezoneFormat, c.amountDecimalSeparator, c.amountDigitGroupingSymbol)
|
||||
dataTableImporter := converter.CreateNewImporterWithTypeNameMapping(customTransactionTypeNameMapping, c.geoLocationSeparator, c.geoLocationOrder, c.transactionTagSeparator)
|
||||
|
||||
return dataTableImporter.ParseImportedData(ctx, user, transactionDataTable, defaultTimezone, additionalOptions, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
|
||||
}
|
||||
|
||||
// IsCustomExcelFileType returns whether the file type is the custom excel file type
|
||||
func IsCustomExcelFileType(fileType string) bool {
|
||||
return fileType == customOOXMLExcelFileType || fileType == customMSCFBExcelFileType
|
||||
}
|
||||
|
||||
// CreateNewCustomTransactionDataExcelFileParser returns a new custom transaction data parser
|
||||
func CreateNewCustomTransactionDataExcelFileParser(fileType string) (CustomTransactionDataParser, error) {
|
||||
if fileType != customOOXMLExcelFileType && fileType != customMSCFBExcelFileType {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
return &customTransactionDataExcelFileImporter{
|
||||
fileType: fileType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CreateNewCustomTransactionDataExcelFileImporter returns a new custom excel importer for transaction data
|
||||
func CreateNewCustomTransactionDataExcelFileImporter(fileType string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, amountDecimalSeparator string, amountDigitGroupingSymbol string, geoLocationSeparator string, geoLocationOrder string, transactionTagSeparator string) (converter.TransactionDataImporter, error) {
|
||||
if fileType != customOOXMLExcelFileType && fileType != customMSCFBExcelFileType {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
if geoLocationOrder == "" {
|
||||
geoLocationOrder = string(converter.TRANSACTION_GEO_LOCATION_ORDER_LONGITUDE_LATITUDE)
|
||||
} else if geoLocationOrder != string(converter.TRANSACTION_GEO_LOCATION_ORDER_LONGITUDE_LATITUDE) &&
|
||||
geoLocationOrder != string(converter.TRANSACTION_GEO_LOCATION_ORDER_LATITUDE_LONGITUDE) {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
|
||||
if _, exists := columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TIME]; !exists {
|
||||
return nil, errs.ErrMissingRequiredFieldInHeaderRow
|
||||
}
|
||||
|
||||
if _, exists := columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_TRANSACTION_TYPE]; !exists {
|
||||
return nil, errs.ErrMissingRequiredFieldInHeaderRow
|
||||
}
|
||||
|
||||
if _, exists := columnIndexMapping[datatable.TRANSACTION_DATA_TABLE_AMOUNT]; !exists {
|
||||
return nil, errs.ErrMissingRequiredFieldInHeaderRow
|
||||
}
|
||||
|
||||
return &customTransactionDataExcelFileImporter{
|
||||
fileType: fileType,
|
||||
columnIndexMapping: columnIndexMapping,
|
||||
transactionTypeNameMapping: transactionTypeNameMapping,
|
||||
hasHeaderLine: hasHeaderLine,
|
||||
timeFormat: timeFormat,
|
||||
timezoneFormat: timezoneFormat,
|
||||
amountDecimalSeparator: amountDecimalSeparator,
|
||||
amountDigitGroupingSymbol: amountDigitGroupingSymbol,
|
||||
geoLocationSeparator: geoLocationSeparator,
|
||||
geoLocationOrder: converter.TransactionGeoLocationOrder(geoLocationOrder),
|
||||
transactionTagSeparator: transactionTagSeparator,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package custom
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsCustomExcelFileType(t *testing.T) {
|
||||
assert.True(t, IsCustomExcelFileType("custom_xlsx"))
|
||||
assert.True(t, IsCustomExcelFileType("custom_xls"))
|
||||
|
||||
assert.False(t, IsCustomExcelFileType("xlsx"))
|
||||
assert.False(t, IsCustomExcelFileType("xls"))
|
||||
assert.False(t, IsCustomExcelFileType("excel"))
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseOOXMLExcelDataLines_EmptyData(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err := importer.ParseDataLines(context, testdata)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, 0, len(allLines))
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseOOXMLExcelDataLines_SingleSheet(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err := importer.ParseDataLines(context, testdata)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, 3, len(allLines))
|
||||
|
||||
assert.Equal(t, 3, len(allLines[0]))
|
||||
assert.Equal(t, "A1", allLines[0][0])
|
||||
assert.Equal(t, "B1", allLines[0][1])
|
||||
assert.Equal(t, "C1", allLines[0][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[1]))
|
||||
assert.Equal(t, "A2", allLines[1][0])
|
||||
assert.Equal(t, "B2", allLines[1][1])
|
||||
assert.Equal(t, "C2", allLines[1][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[2]))
|
||||
assert.Equal(t, "A3", allLines[2][0])
|
||||
assert.Equal(t, "B3", allLines[2][1])
|
||||
assert.Equal(t, "C3", allLines[2][2])
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseOOXMLExcelDataLines_MultipleSheet(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err := importer.ParseDataLines(context, testdata)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, 9, len(allLines))
|
||||
|
||||
assert.Equal(t, 3, len(allLines[0]))
|
||||
assert.Equal(t, "A1", allLines[0][0])
|
||||
assert.Equal(t, "B1", allLines[0][1])
|
||||
assert.Equal(t, "C1", allLines[0][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[1]))
|
||||
assert.Equal(t, "1-A2", allLines[1][0])
|
||||
assert.Equal(t, "1-B2", allLines[1][1])
|
||||
assert.Equal(t, "1-C2", allLines[1][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[2]))
|
||||
assert.Equal(t, "1-A3", allLines[2][0])
|
||||
assert.Equal(t, "1-B3", allLines[2][1])
|
||||
assert.Equal(t, "1-C3", allLines[2][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[3]))
|
||||
assert.Equal(t, "A1", allLines[3][0])
|
||||
assert.Equal(t, "B1", allLines[3][1])
|
||||
assert.Equal(t, "C1", allLines[3][2])
|
||||
|
||||
assert.Equal(t, 2, len(allLines[4]))
|
||||
assert.Equal(t, "3-A2", allLines[4][0])
|
||||
assert.Equal(t, "3-B2", allLines[4][1])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[5]))
|
||||
assert.Equal(t, "A1", allLines[5][0])
|
||||
assert.Equal(t, "B1", allLines[5][1])
|
||||
assert.Equal(t, "C1", allLines[5][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[6]))
|
||||
assert.Equal(t, "A1", allLines[6][0])
|
||||
assert.Equal(t, "B1", allLines[6][1])
|
||||
assert.Equal(t, "C1", allLines[6][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[7]))
|
||||
assert.Equal(t, "5-A2", allLines[7][0])
|
||||
assert.Equal(t, "5-B2", allLines[7][1])
|
||||
assert.Equal(t, "5-C2", allLines[7][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[8]))
|
||||
assert.Equal(t, "5-A3", allLines[8][0])
|
||||
assert.Equal(t, "5-B3", allLines[8][1])
|
||||
assert.Equal(t, "5-C3", allLines[8][2])
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseOOXMLExcelDataLines_MultipleSheetWithDifferentColumnCount(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/multiple_sheets_with_different_header_row_excel_file.xlsx")
|
||||
assert.Nil(t, err)
|
||||
|
||||
_, err = importer.ParseDataLines(context, testdata)
|
||||
assert.EqualError(t, err, errs.ErrFieldsInMultiTableAreDifferent.Message)
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseMSCFBExcelDataLines_EmptyData(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/empty_excel_file.xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err := importer.ParseDataLines(context, testdata)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, 0, len(allLines))
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseMSCFBExcelDataLines_SingleSheet(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/simple_excel_file.xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err := importer.ParseDataLines(context, testdata)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, 3, len(allLines))
|
||||
|
||||
assert.Equal(t, 3, len(allLines[0]))
|
||||
assert.Equal(t, "A1", allLines[0][0])
|
||||
assert.Equal(t, "B1", allLines[0][1])
|
||||
assert.Equal(t, "C1", allLines[0][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[1]))
|
||||
assert.Equal(t, "A2", allLines[1][0])
|
||||
assert.Equal(t, "B2", allLines[1][1])
|
||||
assert.Equal(t, "C2", allLines[1][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[2]))
|
||||
assert.Equal(t, "A3", allLines[2][0])
|
||||
assert.Equal(t, "B3", allLines[2][1])
|
||||
assert.Equal(t, "C3", allLines[2][2])
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseMSCFBExcelDataLines_MultipleSheet(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/multiple_sheets_excel_file.xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
allLines, err := importer.ParseDataLines(context, testdata)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, 9, len(allLines))
|
||||
|
||||
assert.Equal(t, 3, len(allLines[0]))
|
||||
assert.Equal(t, "A1", allLines[0][0])
|
||||
assert.Equal(t, "B1", allLines[0][1])
|
||||
assert.Equal(t, "C1", allLines[0][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[1]))
|
||||
assert.Equal(t, "1-A2", allLines[1][0])
|
||||
assert.Equal(t, "1-B2", allLines[1][1])
|
||||
assert.Equal(t, "1-C2", allLines[1][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[2]))
|
||||
assert.Equal(t, "1-A3", allLines[2][0])
|
||||
assert.Equal(t, "1-B3", allLines[2][1])
|
||||
assert.Equal(t, "1-C3", allLines[2][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[3]))
|
||||
assert.Equal(t, "A1", allLines[3][0])
|
||||
assert.Equal(t, "B1", allLines[3][1])
|
||||
assert.Equal(t, "C1", allLines[3][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[4]))
|
||||
assert.Equal(t, "3-A2", allLines[4][0])
|
||||
assert.Equal(t, "3-B2", allLines[4][1])
|
||||
assert.Equal(t, "", allLines[4][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[5]))
|
||||
assert.Equal(t, "A1", allLines[5][0])
|
||||
assert.Equal(t, "B1", allLines[5][1])
|
||||
assert.Equal(t, "C1", allLines[5][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[6]))
|
||||
assert.Equal(t, "A1", allLines[6][0])
|
||||
assert.Equal(t, "B1", allLines[6][1])
|
||||
assert.Equal(t, "C1", allLines[6][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[7]))
|
||||
assert.Equal(t, "5-A2", allLines[7][0])
|
||||
assert.Equal(t, "5-B2", allLines[7][1])
|
||||
assert.Equal(t, "5-C2", allLines[7][2])
|
||||
|
||||
assert.Equal(t, 3, len(allLines[8]))
|
||||
assert.Equal(t, "5-A3", allLines[8][0])
|
||||
assert.Equal(t, "5-B3", allLines[8][1])
|
||||
assert.Equal(t, "5-C3", allLines[8][2])
|
||||
}
|
||||
|
||||
func TestCustomTransactionDataParser_ParseMSCFBExcelDataLines_MultipleSheetWithDifferentColumnCount(t *testing.T) {
|
||||
importer, err := CreateNewCustomTransactionDataExcelFileParser("custom_xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
context := core.NewNullContext()
|
||||
|
||||
testdata, err := os.ReadFile("../../../testdata/multiple_sheets_with_different_header_row_excel_file.xls")
|
||||
assert.Nil(t, err)
|
||||
|
||||
_, err = importer.ParseDataLines(context, testdata)
|
||||
assert.EqualError(t, err, errs.ErrFieldsInMultiTableAreDifferent.Message)
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
package dsv
|
||||
package custom
|
||||
|
||||
import (
|
||||
"strings"
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/beancount"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/camt"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/converter"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/custom"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/datatable"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/default"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/dsv"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/feidee"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/fireflyIII"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/converters/gnucash"
|
||||
@@ -85,17 +85,29 @@ func GetTransactionDataImporter(fileType string) (converter.TransactionDataImpor
|
||||
}
|
||||
}
|
||||
|
||||
// IsCustomDelimiterSeparatedValuesFileType returns whether the file type is the delimiter-separated values file type
|
||||
func IsCustomDelimiterSeparatedValuesFileType(fileType string) bool {
|
||||
return dsv.IsDelimiterSeparatedValuesFileType(fileType)
|
||||
// IsCustomFileFormatFileType returns whether the file type is the custom file format
|
||||
func IsCustomFileFormatFileType(fileType string) bool {
|
||||
return custom.IsDelimiterSeparatedValuesFileType(fileType) || custom.IsCustomExcelFileType(fileType)
|
||||
}
|
||||
|
||||
// CreateNewDelimiterSeparatedValuesDataParser returns a new delimiter-separated values data parser according to the file type and encoding
|
||||
func CreateNewDelimiterSeparatedValuesDataParser(fileType string, fileEncoding string) (dsv.CustomTransactionDataDsvFileParser, error) {
|
||||
return dsv.CreateNewCustomTransactionDataDsvFileParser(fileType, fileEncoding)
|
||||
// CreateNewCustomFileFormatTransactionDataParser returns a new custom transaction data parser according to the file type and encoding
|
||||
func CreateNewCustomFileFormatTransactionDataParser(fileType string, fileEncoding string) (custom.CustomTransactionDataParser, error) {
|
||||
if custom.IsDelimiterSeparatedValuesFileType(fileType) {
|
||||
return custom.CreateNewCustomTransactionDataDsvFileParser(fileType, fileEncoding)
|
||||
} else if custom.IsCustomExcelFileType(fileType) {
|
||||
return custom.CreateNewCustomTransactionDataExcelFileParser(fileType)
|
||||
} else {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
}
|
||||
|
||||
// CreateNewDelimiterSeparatedValuesDataImporter returns a new delimiter-separated values data importer according to the file type and encoding
|
||||
func CreateNewDelimiterSeparatedValuesDataImporter(fileType string, fileEncoding string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, amountDecimalSeparator string, amountDigitGroupingSymbol string, geoLocationSeparator string, geoLocationOrder string, transactionTagSeparator string) (converter.TransactionDataImporter, error) {
|
||||
return dsv.CreateNewCustomTransactionDataDsvFileImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormat, timezoneFormat, amountDecimalSeparator, amountDigitGroupingSymbol, geoLocationSeparator, geoLocationOrder, transactionTagSeparator)
|
||||
// CreateNewCustomTransactionDataImporter returns a new custom transaction data importer according to the file type and encoding
|
||||
func CreateNewCustomTransactionDataImporter(fileType string, fileEncoding string, columnIndexMapping map[datatable.TransactionDataTableColumn]int, transactionTypeNameMapping map[string]models.TransactionType, hasHeaderLine bool, timeFormat string, timezoneFormat string, amountDecimalSeparator string, amountDigitGroupingSymbol string, geoLocationSeparator string, geoLocationOrder string, transactionTagSeparator string) (converter.TransactionDataImporter, error) {
|
||||
if custom.IsDelimiterSeparatedValuesFileType(fileType) {
|
||||
return custom.CreateNewCustomTransactionDataDsvFileImporter(fileType, fileEncoding, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormat, timezoneFormat, amountDecimalSeparator, amountDigitGroupingSymbol, geoLocationSeparator, geoLocationOrder, transactionTagSeparator)
|
||||
} else if custom.IsCustomExcelFileType(fileType) {
|
||||
return custom.CreateNewCustomTransactionDataExcelFileImporter(fileType, columnIndexMapping, transactionTypeNameMapping, hasHeaderLine, timeFormat, timezoneFormat, amountDecimalSeparator, amountDigitGroupingSymbol, geoLocationSeparator, geoLocationOrder, transactionTagSeparator)
|
||||
} else {
|
||||
return nil, errs.ErrImportFileTypeNotSupported
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user