diff --git a/pkg/converters/ofx/ofx_data.go b/pkg/converters/ofx/ofx_data.go index e641cebf..ee001e44 100644 --- a/pkg/converters/ofx/ofx_data.go +++ b/pkg/converters/ofx/ofx_data.go @@ -1,6 +1,8 @@ package ofx import ( + "encoding/xml" + "github.com/mayswind/ezbookkeeping/pkg/models" ) @@ -69,9 +71,10 @@ var ofxTransactionTypeMapping = map[ofxTransactionType]models.TransactionType{ // ofxFile represents the struct of open financial exchange (ofx) file type ofxFile struct { + XMLName xml.Name `xml:"OFX"` FileHeader *ofxFileHeader - BankMessageResponseV1 *ofxBankMessageResponseV1 - CreditCardMessageResponseV1 *ofxCreditCardMessageResponseV1 + BankMessageResponseV1 *ofxBankMessageResponseV1 `xml:"BANKMSGSRSV1"` + CreditCardMessageResponseV1 *ofxCreditCardMessageResponseV1 `xml:"CREDITCARDMSGSRSV1"` } // ofxFileHeader represents the struct of open financial exchange (ofx) file header @@ -85,101 +88,101 @@ type ofxFileHeader struct { // ofxBankMessageResponseV1 represents the struct of open financial exchange (ofx) bank message response v1 type ofxBankMessageResponseV1 struct { - StatementTransactionResponse *ofxBankStatementTransactionResponse + StatementTransactionResponse *ofxBankStatementTransactionResponse `xml:"STMTTRNRS"` } // ofxCreditCardMessageResponseV1 represents the struct of open financial exchange (ofx) credit card message response v1 type ofxCreditCardMessageResponseV1 struct { - StatementTransactionResponse *ofxCreditCardStatementTransactionResponse + StatementTransactionResponse *ofxCreditCardStatementTransactionResponse `xml:"CCSTMTTRNRS"` } // ofxBankStatementTransactionResponse represents the struct of open financial exchange (ofx) bank statement transaction response type ofxBankStatementTransactionResponse struct { - StatementResponse *ofxBankStatementResponse + StatementResponse *ofxBankStatementResponse `xml:"STMTRS"` } // ofxCreditCardStatementTransactionResponse represents the struct of open financial exchange (ofx) credit card statement transaction response type ofxCreditCardStatementTransactionResponse struct { - StatementResponse *ofxCreditCardStatementResponse + StatementResponse *ofxCreditCardStatementResponse `xml:"CCSTMTRS"` } // ofxBankStatementResponse represents the struct of open financial exchange (ofx) bank statement response type ofxBankStatementResponse struct { - DefaultCurrency string - AccountFrom *ofxBankAccount - TransactionList *ofxBankTransactionList + DefaultCurrency string `xml:"CURDEF"` + AccountFrom *ofxBankAccount `xml:"BANKACCTFROM"` + TransactionList *ofxBankTransactionList `xml:"BANKTRANLIST"` } // ofxCreditCardStatementResponse represents the struct of open financial exchange (ofx) credit card statement response type ofxCreditCardStatementResponse struct { - DefaultCurrency string - AccountFrom *ofxCreditCardAccount - TransactionList *ofxCreditCardTransactionList + DefaultCurrency string `xml:"CURDEF"` + AccountFrom *ofxCreditCardAccount `xml:"CCACCTFROM"` + TransactionList *ofxCreditCardTransactionList `xml:"BANKTRANLIST"` } // ofxBankAccount represents the struct of open financial exchange (ofx) bank account type ofxBankAccount struct { - BankId string - BranchId string - AccountId string - AccountType ofxAccountType - AccountKey string + BankId string `xml:"BANKID"` + BranchId string `xml:"BRANCHID"` + AccountId string `xml:"ACCTID"` + AccountType ofxAccountType `xml:"ACCTTYPE"` + AccountKey string `xml:"ACCTKEY"` } // ofxCreditCardAccount represents the struct of open financial exchange (ofx) credit card account type ofxCreditCardAccount struct { - AccountId string - AccountKey string + AccountId string `xml:"ACCTID"` + AccountKey string `xml:"ACCTKEY"` } // ofxBankTransactionList represents the struct of open financial exchange (ofx) bank transaction list type ofxBankTransactionList struct { - StartDate string - EndDate string - StatementTransactions []*ofxBankStatementTransaction + StartDate string `xml:"DTSTART"` + EndDate string `xml:"DTEND"` + StatementTransactions []*ofxBankStatementTransaction `xml:"STMTTRN"` } // ofxCreditCardTransactionList represents the struct of open financial exchange (ofx) credit card transaction list type ofxCreditCardTransactionList struct { - StartDate string - EndDate string - StatementTransactions []*ofxCreditCardStatementTransaction + StartDate string `xml:"DTSTART"` + EndDate string `xml:"DTEND"` + StatementTransactions []*ofxCreditCardStatementTransaction `xml:"STMTTRN"` } // ofxBaseStatementTransaction represents the struct of open financial exchange (ofx) base statement transaction type ofxBaseStatementTransaction struct { - TransactionId string - TransactionType ofxTransactionType - PostedDate string - Amount string - Name string - Payee *ofxPayee - Memo string - Currency string - OriginalCurrency string + TransactionId string `xml:"FITID"` + TransactionType ofxTransactionType `xml:"TRNTYPE"` + PostedDate string `xml:"DTPOSTED"` + Amount string `xml:"TRNAMT"` + Name string `xml:"NAME"` + Payee *ofxPayee `xml:"PAYEE"` + Memo string `xml:"MEMO"` + Currency string `xml:"CURRENCY"` + OriginalCurrency string `xml:"ORIGCURRENCY"` } // ofxBankStatementTransaction represents the struct of open financial exchange (ofx) bank statement transaction type ofxBankStatementTransaction struct { ofxBaseStatementTransaction - AccountTo *ofxBankAccount + AccountTo *ofxBankAccount `xml:"BANKACCTTO"` } // ofxCreditCardStatementTransaction represents the struct of open financial exchange (ofx) credit card statement transaction type ofxCreditCardStatementTransaction struct { ofxBaseStatementTransaction - AccountTo *ofxCreditCardAccount + AccountTo *ofxCreditCardAccount `xml:"CCACCTTO"` } // ofxPayee represents the struct of open financial exchange (ofx) payee info type ofxPayee struct { - Name string - Address1 string - Address2 string - Address3 string - City string - State string - PostalCode string - Country string - Phone string + Name string `xml:"NAME"` + Address1 string `xml:"ADDR1"` + Address2 string `xml:"ADDR2"` + Address3 string `xml:"ADDR3"` + City string `xml:"CITY"` + State string `xml:"STATE"` + PostalCode string `xml:"POSTALCODE"` + Country string `xml:"COUNTRY"` + Phone string `xml:"PHONE"` } diff --git a/pkg/converters/ofx/ofx_data_reader.go b/pkg/converters/ofx/ofx_data_reader.go index a5185e3f..db1f4a66 100644 --- a/pkg/converters/ofx/ofx_data_reader.go +++ b/pkg/converters/ofx/ofx_data_reader.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "encoding/xml" - "io" "regexp" "strings" @@ -14,6 +13,7 @@ import ( "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" + "github.com/mayswind/ezbookkeeping/pkg/converters/sgml" "github.com/mayswind/ezbookkeeping/pkg/core" "github.com/mayswind/ezbookkeeping/pkg/errs" "github.com/mayswind/ezbookkeeping/pkg/log" @@ -24,157 +24,35 @@ const ofxUnicodeEncoding = "unicode" const ofxUSAsciiEncoding = "usascii" const ofx1SGMLDataFormat = "OFXSGML" -const ofxDataElementName = "OFX" - -const ofxBankMessageResponseV1ElementName = "BANKMSGSRSV1" -const ofxCreditCardMessageResponseV1ElementName = "CREDITCARDMSGSRSV1" - -const ofxBankStatementTransactionResponseElementName = "STMTTRNRS" -const ofxCreditCardStatementTransactionResponseElementName = "CCSTMTTRNRS" - -const ofxBankStatementResponseElementName = "STMTRS" -const ofxCreditCardStatementResponseElementName = "CCSTMTRS" - -const ofxBankStatementResponseDefaultCurrencyName = "CURDEF" -const ofxBankStatementResponseBankAccountFromName = "BANKACCTFROM" -const ofxBankStatementResponseBankTransactionListName = "BANKTRANLIST" - -const ofxCreditCardStatementResponseDefaultCurrencyName = "CURDEF" -const ofxCreditCardStatementResponseCreditCardAccountFromName = "CCACCTFROM" -const ofxCreditCardStatementResponseCreditCardTransactionListName = "BANKTRANLIST" - -const ofxBankTransactionListStartDateName = "DTSTART" -const ofxBankTransactionListEndDateName = "DTEND" -const ofxBankTransactionListStatementTransactionsName = "STMTTRN" - -const ofxCreditCardTransactionListStartDateName = "DTSTART" -const ofxCreditCardTransactionListEndDateName = "DTEND" -const ofxCreditCardTransactionListStatementTransactionsName = "STMTTRN" - -const ofxBankAccountBankIdName = "BANKID" -const ofxBankAccountBranchIdName = "BRANCHID" -const ofxBankAccountAccountIdName = "ACCTID" -const ofxBankAccountAccountTypeName = "ACCTTYPE" -const ofxBankAccountAccountKeyName = "ACCTKEY" - -const ofxCreditCardAccountAccountIdName = "ACCTID" -const ofxCreditCardAccountAccountKeyName = "ACCTKEY" - -const ofxTransactionTransactionIdName = "FITID" -const ofxTransactionTransactionTypeName = "TRNTYPE" -const ofxTransactionPostedDateName = "DTPOSTED" -const ofxTransactionAmountName = "TRNAMT" -const ofxTransactionNameName = "NAME" -const ofxTransactionMemoName = "MEMO" -const ofxTransactionCurrencyName = "CURRENCY" -const ofxTransactionOriginalCurrencyName = "ORIGCURRENCY" -const ofxTransactionPayeeName = "PAYEE" -const ofxTransactionBankAccountToName = "BANKACCTTO" -const ofxTransactionCreditCardAccountToName = "CCACCTTO" - -const ofxPayeeNameName = "NAME" -const ofxPayeeAddress1Name = "ADDR1" -const ofxPayeeAddress2Name = "ADDR2" -const ofxPayeeAddress3Name = "ADDR3" -const ofxPayeeCityName = "CITY" -const ofxPayeeStateName = "STATE" -const ofxPayeePostalCodeName = "POSTALCODE" -const ofxPayeeCountryName = "COUNTRY" -const ofxPayeePhoneName = "PHONE" - -var ofxBankStatementResponseChildrenNames = map[string]bool{ - ofxBankStatementResponseDefaultCurrencyName: true, -} - -var ofxCreditCardStatementResponseChildrenNames = map[string]bool{ - ofxCreditCardStatementResponseDefaultCurrencyName: true, -} - -var ofxBankTransactionListChildrenNames = map[string]bool{ - ofxBankTransactionListStartDateName: true, - ofxBankTransactionListEndDateName: true, -} - -var ofxCreditCardTransactionListChildrenNames = map[string]bool{ - ofxCreditCardTransactionListStartDateName: true, - ofxCreditCardTransactionListEndDateName: true, -} - -var ofxBankAccountChildrenNames = map[string]bool{ - ofxBankAccountBankIdName: true, - ofxBankAccountBranchIdName: true, - ofxBankAccountAccountIdName: true, - ofxBankAccountAccountTypeName: true, - ofxBankAccountAccountKeyName: true, -} - -var ofxCreditCardAccountChildrenNames = map[string]bool{ - ofxCreditCardAccountAccountIdName: true, - ofxCreditCardAccountAccountKeyName: true, -} - -var ofxTransactionChildrenNames = map[string]bool{ - ofxTransactionTransactionIdName: true, - ofxTransactionTransactionTypeName: true, - ofxTransactionPostedDateName: true, - ofxTransactionAmountName: true, - ofxTransactionNameName: true, - ofxTransactionMemoName: true, - ofxTransactionCurrencyName: true, - ofxTransactionOriginalCurrencyName: true, -} - -var ofxPayeeChildrenNames = map[string]bool{ - ofxPayeeNameName: true, - ofxPayeeAddress1Name: true, - ofxPayeeAddress2Name: true, - ofxPayeeAddress3Name: true, - ofxPayeeCityName: true, - ofxPayeeStateName: true, - ofxPayeePostalCodeName: true, - ofxPayeeCountryName: true, - ofxPayeePhoneName: true, -} - var ofx2HeaderPattern = regexp.MustCompile("<\\?OFX( +[A-Z]+=\"[^=]*\")* *\\?>") var ofx2HeaderAttributePattern = regexp.MustCompile(" +([A-Z]+)=\"([^=]*)\"") // ofxFileReader defines the structure of open financial exchange (ofx) file reader -type ofxFileReader struct { +type ofxFileReader interface { + // read returns the imported open financial exchange (ofx) file + read(ctx core.Context) (*ofxFile, error) +} + +// ofxVersion1FileReader defines the structure of open financial exchange (ofx) declaration version 1.x file reader +type ofxVersion1FileReader struct { + fileHeader *ofxFileHeader + sgmlDecoder *sgml.Decoder +} + +// ofxVersion2FileReader defines the structure of open financial exchange (ofx) declaration version 2.x file reader +type ofxVersion2FileReader struct { fileHeader *ofxFileHeader xmlDecoder *xml.Decoder } // read returns the imported open financial exchange (ofx) file -func (r *ofxFileReader) read(ctx core.Context) (*ofxFile, error) { - var file *ofxFile - strictMode := true +func (r *ofxVersion1FileReader) read(ctx core.Context) (*ofxFile, error) { + file := &ofxFile{} - if r.fileHeader != nil && r.fileHeader.OFXDeclarationVersion == ofxVersion1 { - strictMode = false - } + err := r.sgmlDecoder.Decode(&file) - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if token.Name.Local == ofxDataElementName { - file, err = r.readOFXElement(ctx, strictMode, ofxDataElementName) - - if err != nil { - return nil, err - } - } - } - } - - if file == nil { - log.Errorf(ctx, "[ofxFileReader.read] cannot parse ofx file") + if err != nil { + log.Errorf(ctx, "[ofxVersion1FileReader.read] cannot read ofx 1.x file, because %s", err.Error()) return nil, errs.ErrInvalidOFXFile } @@ -183,859 +61,23 @@ func (r *ofxFileReader) read(ctx core.Context) (*ofxFile, error) { return file, nil } -func (r *ofxFileReader) readOFXElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxFile, error) { +// read returns the imported open financial exchange (ofx) file +func (r *ofxVersion2FileReader) read(ctx core.Context) (*ofxFile, error) { file := &ofxFile{} - hasEndElement := false - for { - token, err := r.xmlDecoder.RawToken() + err := r.xmlDecoder.Decode(&file) - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if token.Name.Local == ofxBankMessageResponseV1ElementName { - element, err := r.readBankMessageResponseV1Element(ctx, strictMode, ofxBankMessageResponseV1ElementName) - - if err != nil { - return nil, err - } - - file.BankMessageResponseV1 = element - } else if token.Name.Local == ofxCreditCardMessageResponseV1ElementName { - element, err := r.readCreditCardMessageResponseV1Element(ctx, strictMode, ofxCreditCardMessageResponseV1ElementName) - - if err != nil { - return nil, err - } - - file.CreditCardMessageResponseV1 = element - } - case xml.EndElement: - if token.Name.Local == parentElementName { - hasEndElement = true - break - } - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readOFXElement] not found element", parentElementName) + if err != nil { + log.Errorf(ctx, "[ofxVersion2FileReader.read] cannot read ofx 2.x file, because %s", err.Error()) return nil, errs.ErrInvalidOFXFile } + file.FileHeader = r.fileHeader + return file, nil } -func (r *ofxFileReader) readBankMessageResponseV1Element(ctx core.Context, strictMode bool, parentElementName string) (*ofxBankMessageResponseV1, error) { - response := &ofxBankMessageResponseV1{} - hasEndElement := false - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if token.Name.Local == ofxBankStatementTransactionResponseElementName { - element, err := r.readBankStatementTransactionResponseElement(ctx, strictMode, ofxBankStatementTransactionResponseElementName) - - if err != nil { - return nil, err - } - - response.StatementTransactionResponse = element - } - case xml.EndElement: - if token.Name.Local == parentElementName { - hasEndElement = true - break - } - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readBankMessageResponseV1Element] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - return response, nil -} - -func (r *ofxFileReader) readCreditCardMessageResponseV1Element(ctx core.Context, strictMode bool, parentElementName string) (*ofxCreditCardMessageResponseV1, error) { - response := &ofxCreditCardMessageResponseV1{} - hasEndElement := false - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if token.Name.Local == ofxCreditCardStatementTransactionResponseElementName { - element, err := r.readCreditCardStatementTransactionResponseElement(ctx, strictMode, ofxCreditCardStatementTransactionResponseElementName) - - if err != nil { - return nil, err - } - - response.StatementTransactionResponse = element - } - case xml.EndElement: - if token.Name.Local == parentElementName { - hasEndElement = true - break - } - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readCreditCardMessageResponseV1Element] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - return response, nil -} - -func (r *ofxFileReader) readBankStatementTransactionResponseElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxBankStatementTransactionResponse, error) { - response := &ofxBankStatementTransactionResponse{} - hasEndElement := false - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if token.Name.Local == ofxBankStatementResponseElementName { - element, err := r.readBankStatementResponseElement(ctx, strictMode, ofxBankStatementResponseElementName) - - if err != nil { - return nil, err - } - - response.StatementResponse = element - } - case xml.EndElement: - if token.Name.Local == parentElementName { - hasEndElement = true - break - } - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readBankStatementTransactionResponseElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - return response, nil -} - -func (r *ofxFileReader) readCreditCardStatementTransactionResponseElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxCreditCardStatementTransactionResponse, error) { - response := &ofxCreditCardStatementTransactionResponse{} - hasEndElement := false - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if token.Name.Local == ofxCreditCardStatementResponseElementName { - element, err := r.readCreditCardStatementResponseElement(ctx, strictMode, ofxCreditCardStatementResponseElementName) - - if err != nil { - return nil, err - } - - response.StatementResponse = element - } - case xml.EndElement: - if token.Name.Local == parentElementName { - hasEndElement = true - break - } - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readCreditCardStatementTransactionResponseElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - return response, nil -} - -func (r *ofxFileReader) readBankStatementResponseElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxBankStatementResponse, error) { - response := &ofxBankStatementResponse{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxBankStatementResponseChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } else if token.Name.Local == ofxBankStatementResponseBankAccountFromName { - element, err := r.readBankAccountElement(ctx, strictMode, ofxBankStatementResponseBankAccountFromName) - - if err != nil { - return nil, err - } - - response.AccountFrom = element - } else if token.Name.Local == ofxBankStatementResponseBankTransactionListName { - element, err := r.readBankTransactionListElement(ctx, strictMode, ofxBankStatementResponseBankTransactionListName) - - if err != nil { - return nil, err - } - - response.TransactionList = element - } - case xml.EndElement: - if ofxBankStatementResponseChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxBankStatementResponseChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readBankStatementResponseElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readBankStatementResponseElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxBankStatementResponseDefaultCurrencyName { - response.DefaultCurrency = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return response, nil -} - -func (r *ofxFileReader) readCreditCardStatementResponseElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxCreditCardStatementResponse, error) { - response := &ofxCreditCardStatementResponse{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxCreditCardStatementResponseChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } else if token.Name.Local == ofxCreditCardStatementResponseCreditCardAccountFromName { - element, err := r.readCreditAccountElement(ctx, strictMode, ofxCreditCardStatementResponseCreditCardAccountFromName) - - if err != nil { - return nil, err - } - - response.AccountFrom = element - } else if token.Name.Local == ofxCreditCardStatementResponseCreditCardTransactionListName { - element, err := r.readCreditCardTransactionListElement(ctx, strictMode, ofxCreditCardStatementResponseCreditCardTransactionListName) - - if err != nil { - return nil, err - } - - response.TransactionList = element - } - case xml.EndElement: - if ofxCreditCardStatementResponseChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxCreditCardStatementResponseChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readCreditCardStatementResponseElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readCreditCardStatementResponseElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxCreditCardStatementResponseDefaultCurrencyName { - response.DefaultCurrency = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return response, nil -} - -func (r *ofxFileReader) readBankAccountElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxBankAccount, error) { - account := &ofxBankAccount{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxBankAccountChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } - case xml.EndElement: - if ofxBankAccountChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxBankAccountChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readBankAccountElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readBankAccountElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxBankAccountBankIdName { - account.BankId = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxBankAccountBranchIdName { - account.BranchId = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxBankAccountAccountIdName { - account.AccountId = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxBankAccountAccountTypeName { - account.AccountType = ofxAccountType(r.getActualElementValue(name, value, elementNotHasEndElement, strictMode)) - } else if name == ofxBankAccountAccountKeyName { - account.AccountKey = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return account, nil -} - -func (r *ofxFileReader) readCreditAccountElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxCreditCardAccount, error) { - account := &ofxCreditCardAccount{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxCreditCardAccountChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } - case xml.EndElement: - if ofxCreditCardAccountChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxCreditCardAccountChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readCreditAccountElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readCreditAccountElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxCreditCardAccountAccountIdName { - account.AccountId = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxCreditCardAccountAccountKeyName { - account.AccountKey = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return account, nil -} - -func (r *ofxFileReader) readBankTransactionListElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxBankTransactionList, error) { - transactionList := &ofxBankTransactionList{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxBankTransactionListChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } else if token.Name.Local == ofxBankTransactionListStatementTransactionsName { - ofxBaseStatementTransaction, backAccountTo, _, err := r.readStatementTransactionElement(ctx, strictMode, "STMTTRN") - - if err != nil { - return nil, err - } - - transaction := &ofxBankStatementTransaction{ - ofxBaseStatementTransaction: *ofxBaseStatementTransaction, - AccountTo: backAccountTo, - } - - transactionList.StatementTransactions = append(transactionList.StatementTransactions, transaction) - } - case xml.EndElement: - if ofxBankTransactionListChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxBankTransactionListChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readBankTransactionListElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readBankTransactionListElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxBankTransactionListStartDateName { - transactionList.StartDate = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxBankTransactionListEndDateName { - transactionList.EndDate = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return transactionList, nil -} - -func (r *ofxFileReader) readCreditCardTransactionListElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxCreditCardTransactionList, error) { - transactionList := &ofxCreditCardTransactionList{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxCreditCardTransactionListChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } else if token.Name.Local == ofxCreditCardTransactionListStatementTransactionsName { - ofxBaseStatementTransaction, _, creditCardAccountTo, err := r.readStatementTransactionElement(ctx, strictMode, "STMTTRN") - - if err != nil { - return nil, err - } - - transaction := &ofxCreditCardStatementTransaction{ - ofxBaseStatementTransaction: *ofxBaseStatementTransaction, - AccountTo: creditCardAccountTo, - } - - transactionList.StatementTransactions = append(transactionList.StatementTransactions, transaction) - } - case xml.EndElement: - if ofxCreditCardTransactionListChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxCreditCardTransactionListChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readCreditCardTransactionListElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readCreditCardTransactionListElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxCreditCardTransactionListStartDateName { - transactionList.StartDate = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxCreditCardTransactionListEndDateName { - transactionList.EndDate = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return transactionList, nil -} - -func (r *ofxFileReader) readStatementTransactionElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxBaseStatementTransaction, *ofxBankAccount, *ofxCreditCardAccount, error) { - var bankAccountTo *ofxBankAccount - var creditCardAccountTo *ofxCreditCardAccount - transaction := &ofxBaseStatementTransaction{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxTransactionChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } else if token.Name.Local == ofxTransactionPayeeName { - element, err := r.readPayeeElement(ctx, strictMode, ofxTransactionPayeeName) - - if err != nil { - return nil, nil, nil, err - } - - transaction.Payee = element - } else if token.Name.Local == ofxTransactionBankAccountToName { - element, err := r.readBankAccountElement(ctx, strictMode, ofxTransactionBankAccountToName) - - if err != nil { - return nil, nil, nil, err - } - - bankAccountTo = element - } else if token.Name.Local == ofxTransactionCreditCardAccountToName { - element, err := r.readCreditAccountElement(ctx, strictMode, ofxTransactionCreditCardAccountToName) - - if err != nil { - return nil, nil, nil, err - } - - creditCardAccountTo = element - } - case xml.EndElement: - if ofxTransactionChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxTransactionChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readStatementTransactionElement] not found element", parentElementName) - return nil, nil, nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readStatementTransactionElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, nil, nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxTransactionTransactionIdName { - transaction.TransactionId = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxTransactionTransactionTypeName { - transaction.TransactionType = ofxTransactionType(r.getActualElementValue(name, value, elementNotHasEndElement, strictMode)) - } else if name == ofxTransactionPostedDateName { - transaction.PostedDate = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxTransactionAmountName { - transaction.Amount = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxTransactionNameName { - transaction.Name = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxTransactionMemoName { - transaction.Memo = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxTransactionCurrencyName { - transaction.Currency = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxTransactionOriginalCurrencyName { - transaction.OriginalCurrency = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return transaction, bankAccountTo, creditCardAccountTo, nil -} - -func (r *ofxFileReader) readPayeeElement(ctx core.Context, strictMode bool, parentElementName string) (*ofxPayee, error) { - payee := &ofxPayee{} - hasEndElement := false - elementNotHasEndElement := make(map[string]bool) - elementValues := make(map[string]string) - currentElementName := "" - - for { - token, err := r.xmlDecoder.RawToken() - - if err == io.EOF { - break - } - - switch token := token.(type) { - case xml.StartElement: - if ofxPayeeChildrenNames[token.Name.Local] { - currentElementName = token.Name.Local - elementNotHasEndElement[token.Name.Local] = true - } - case xml.EndElement: - if ofxPayeeChildrenNames[token.Name.Local] { - delete(elementNotHasEndElement, token.Name.Local) - } else if token.Name.Local == parentElementName { - hasEndElement = true - break - } - case xml.CharData: - if ofxPayeeChildrenNames[currentElementName] { - elementValues[currentElementName] = string(token) - } - - currentElementName = "" - } - - if hasEndElement { - break - } - } - - if strictMode && !hasEndElement { - log.Errorf(ctx, "[ofxFileReader.readPayeeElement] not found element", parentElementName) - return nil, errs.ErrInvalidOFXFile - } - - if strictMode && len(elementNotHasEndElement) > 0 { - log.Errorf(ctx, "[ofxFileReader.readPayeeElement] not found end element for %s", r.getNotHasEndElementNames(elementNotHasEndElement)) - return nil, errs.ErrInvalidOFXFile - } - - for name, value := range elementValues { - if name == ofxPayeeNameName { - payee.Name = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeeAddress1Name { - payee.Address1 = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeeAddress2Name { - payee.Address2 = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeeAddress3Name { - payee.Address3 = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeeCityName { - payee.City = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeeStateName { - payee.State = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeePostalCodeName { - payee.PostalCode = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeeCountryName { - payee.Country = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } else if name == ofxPayeePhoneName { - payee.Phone = r.getActualElementValue(name, value, elementNotHasEndElement, strictMode) - } - } - - return payee, nil -} - -func (r *ofxFileReader) getActualElementValue(name string, value string, elementNotHasEndElement map[string]bool, strictMode bool) string { - if strictMode { - return value - } - - _, notHasEndElement := elementNotHasEndElement[name] - - if !notHasEndElement { - return value - } - - for i := 0; i < len(value); i++ { - if value[i] == '\r' || value[i] == '\n' { - return value[0:i] - } - } - - return value -} - -func (r *ofxFileReader) getNotHasEndElementNames(elementNotHasEndElement map[string]bool) string { - builder := strings.Builder{} - - for name := range elementNotHasEndElement { - if builder.Len() > 0 { - builder.WriteRune(',') - } - - builder.WriteString(name) - } - - return builder.String() -} - -func createNewOFXFileReader(ctx core.Context, data []byte) (*ofxFileReader, error) { +func createNewOFXFileReader(ctx core.Context, data []byte) (ofxFileReader, error) { firstNonCrLfIndex := 0 for i := 0; i < len(data); i++ { @@ -1056,7 +98,7 @@ func createNewOFXFileReader(ctx core.Context, data []byte) (*ofxFileReader, erro return nil, errs.ErrInvalidOFXFile } -func createNewOFX1FileReader(ctx core.Context, data []byte) (*ofxFileReader, error) { +func createNewOFX1FileReader(ctx core.Context, data []byte) (ofxFileReader, error) { fileHeader, fileData, dataType, enc, err := readOFX1FileHeader(ctx, data) if err != nil { @@ -1090,19 +132,15 @@ func createNewOFX1FileReader(ctx core.Context, data []byte) (*ofxFileReader, err sgmlData := buffer.String() stringReader := strings.NewReader(sgmlData) + sgmlDecoder := sgml.NewDecoder(stringReader) - xmlDecoder := xml.NewDecoder(stringReader) - xmlDecoder.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) { - return input, nil - } - - return &ofxFileReader{ - fileHeader: fileHeader, - xmlDecoder: xmlDecoder, + return &ofxVersion1FileReader{ + fileHeader: fileHeader, + sgmlDecoder: sgmlDecoder, }, nil } -func createNewOFX2FileReader(ctx core.Context, data []byte, withHeader bool) (*ofxFileReader, error) { +func createNewOFX2FileReader(ctx core.Context, data []byte, withHeader bool) (ofxFileReader, error) { var fileHeader *ofxFileHeader = nil var err error @@ -1122,7 +160,7 @@ func createNewOFX2FileReader(ctx core.Context, data []byte, withHeader bool) (*o xmlDecoder := xml.NewDecoder(bytes.NewReader(data)) xmlDecoder.CharsetReader = charset.NewReaderLabel - return &ofxFileReader{ + return &ofxVersion2FileReader{ fileHeader: fileHeader, xmlDecoder: xmlDecoder, }, nil diff --git a/pkg/converters/ofx/ofx_data_reader_test.go b/pkg/converters/ofx/ofx_data_reader_test.go index af084215..71bb4244 100644 --- a/pkg/converters/ofx/ofx_data_reader_test.go +++ b/pkg/converters/ofx/ofx_data_reader_test.go @@ -131,7 +131,372 @@ func TestCreateNewOFXFileReader_OFX1WithoutBreakLine(t *testing.T) { assert.Equal(t, "123.45", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Amount) } -func TestCreateNewOFXFileReader_OFX1WithBlanklines(t *testing.T) { +func TestCreateNewOFXFileReader_OFX1ParseBankAccountFrom(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "1234567890\n"+ + "2345678901\n"+ + "3456789012\n"+ + "CHECKING\n"+ + "4567890123\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.BankMessageResponseV1) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom) + + account := ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom + assert.Equal(t, "1234567890", account.BankId) + assert.Equal(t, "2345678901", account.BranchId) + assert.Equal(t, "3456789012", account.AccountId) + assert.Equal(t, ofxCheckingAccount, account.AccountType) + assert.Equal(t, "4567890123", account.AccountKey) +} + +func TestCreateNewOFXFileReader_OFX1ParseCreditCardAccountFrom(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "3456789012\n"+ + "4567890123\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1) + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse.StatementResponse) + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom) + + account := ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom + assert.Equal(t, "3456789012", account.AccountId) + assert.Equal(t, "4567890123", account.AccountKey) +} + +func TestCreateNewOFXFileReader_OFX1ParseBankTransactionList(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "20240901012345.000[+8:CST]\n"+ + "20240901235959.000[+8:CST]\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.BankMessageResponseV1) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList) + + transactionList := ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList + assert.Equal(t, "20240901012345.000[+8:CST]", transactionList.StartDate) + assert.Equal(t, "20240901235959.000[+8:CST]", transactionList.EndDate) +} + +func TestCreateNewOFXFileReader_OFX1ParseCreditTransactionList(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "20240901012345.000[+8:CST]\n"+ + "20240901235959.000[+8:CST]\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1) + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse.StatementResponse) + assert.NotNil(t, ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList) + + transactionList := ofxFile.CreditCardMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList + assert.Equal(t, "20240901012345.000[+8:CST]", transactionList.StartDate) + assert.Equal(t, "20240901235959.000[+8:CST]", transactionList.EndDate) +} + +func TestCreateNewOFXFileReader_OFX1ParseTransaction(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "123\n"+ + "\n"+ + "\n"+ + "\n"+ + "1234567890\n"+ + "CASH\n"+ + "20240901012345.000[+8:CST]\n"+ + "123.45\n"+ + "Test Name\n"+ + "Some Text\n"+ + "CNY\n"+ + "USD\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.BankMessageResponseV1) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList) + assert.Equal(t, 1, len(ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions)) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0]) + + transaction := ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0] + assert.Equal(t, "1234567890", transaction.TransactionId) + assert.Equal(t, ofxCashWithdrawalTransaction, transaction.TransactionType) + assert.Equal(t, "20240901012345.000[+8:CST]", transaction.PostedDate) + assert.Equal(t, "123.45", transaction.Amount) + assert.Equal(t, "Test Name", transaction.Name) + assert.Equal(t, "Some Text", transaction.Memo) + assert.Equal(t, "CNY", transaction.Currency) + assert.Equal(t, "USD", transaction.OriginalCurrency) +} + +func TestCreateNewOFXFileReader_OFX1ParseTransactionPayee(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "123\n"+ + "\n"+ + "\n"+ + "\n"+ + "DEP\n"+ + "20240901012345.000[+8:CST]\n"+ + "123.45\n"+ + "\n"+ + "Test Name\n"+ + "Address 1\n"+ + "Address 2\n"+ + "Address 3\n"+ + "City Name\n"+ + "State Name\n"+ + "10000000\n"+ + "Country Name\n"+ + "11111111111\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.BankMessageResponseV1) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList) + assert.Equal(t, 1, len(ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions)) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0]) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Payee) + + payee := ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Payee + assert.Equal(t, "Test Name", payee.Name) + assert.Equal(t, "Address 1", payee.Address1) + assert.Equal(t, "Address 2", payee.Address2) + assert.Equal(t, "Address 3", payee.Address3) + assert.Equal(t, "City Name", payee.City) + assert.Equal(t, "State Name", payee.State) + assert.Equal(t, "10000000", payee.PostalCode) + assert.Equal(t, "Country Name", payee.Country) + assert.Equal(t, "11111111111", payee.Phone) +} + +func TestCreateNewOFXFileReader_OFX1WithEndElement(t *testing.T) { + context := core.NewNullContext() + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "CNY\n"+ + "\n"+ + "123\n"+ + "\n"+ + "\n"+ + "\n"+ + "DEP\n"+ + "20240901012345.000[+8:CST]\n"+ + "123.45\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "\n"+ + "")) + + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.Nil(t, err) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.FileHeader) + assert.Equal(t, ofxVersion1, ofxFile.FileHeader.OFXDeclarationVersion) + assert.Equal(t, "103", ofxFile.FileHeader.OFXDataVersion) + assert.Equal(t, "NONE", ofxFile.FileHeader.Security) + assert.Equal(t, "NONE", ofxFile.FileHeader.OldFileUid) + assert.Equal(t, "NONE", ofxFile.FileHeader.NewFileUid) + + assert.NotNil(t, ofxFile.BankMessageResponseV1) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse) + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse) + + assert.Equal(t, "CNY", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.DefaultCurrency) + + assert.NotNil(t, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom) + assert.Equal(t, "123", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.AccountFrom.AccountId) + + assert.Equal(t, 1, len(ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions)) + assert.Equal(t, ofxDepositTransaction, ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].TransactionType) + assert.Equal(t, "20240901012345.000[+8:CST]", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].PostedDate) + assert.Equal(t, "123.45", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Amount) +} + +func TestCreateNewOFXFileReader_OFX1WithBlanklinesInHeader(t *testing.T) { context := core.NewNullContext() reader, err := createNewOFXFileReader(context, []byte( "\n"+ @@ -194,6 +559,104 @@ func TestCreateNewOFXFileReader_OFX1WithBlanklines(t *testing.T) { assert.Equal(t, "123.45", ofxFile.BankMessageResponseV1.StatementTransactionResponse.StatementResponse.TransactionList.StatementTransactions[0].Amount) } +func TestCreateNewOFXFileReader_OFX1WithoutCharset(t *testing.T) { + context := core.NewNullContext() + + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "FOO:BAR\n"+ + "\n"+ + "\n"+ + "")) + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.FileHeader) + assert.Equal(t, ofxVersion1, ofxFile.FileHeader.OFXDeclarationVersion) + assert.Equal(t, "103", ofxFile.FileHeader.OFXDataVersion) + assert.Equal(t, "NONE", ofxFile.FileHeader.Security) + assert.Equal(t, "NONE", ofxFile.FileHeader.OldFileUid) + assert.Equal(t, "NONE", ofxFile.FileHeader.NewFileUid) +} + +func TestCreateNewOFXFileReader_OFX1WithInvalidHeaderVersion(t *testing.T) { + context := core.NewNullContext() + _, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:200\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "")) + + assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message) +} + +func TestCreateNewOFXFileReader_OFX1WithInvalidHeader(t *testing.T) { + context := core.NewNullContext() + _, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:XML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "\n"+ + "\n"+ + "")) + + assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message) +} + +func TestCreateNewOFXFileReader_OFX1WithUnknownHeader(t *testing.T) { + context := core.NewNullContext() + + reader, err := createNewOFXFileReader(context, []byte( + "OFXHEADER:100\n"+ + "DATA:OFXSGML\n"+ + "VERSION:103\n"+ + "SECURITY:NONE\n"+ + "ENCODING:USASCII\n"+ + "CHARSET:1252\n"+ + "COMPRESSION:NONE\n"+ + "OLDFILEUID:NONE\n"+ + "NEWFILEUID:NONE\n"+ + "FOO:BAR\n"+ + "\n"+ + "\n"+ + "")) + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.FileHeader) + assert.Equal(t, ofxVersion1, ofxFile.FileHeader.OFXDeclarationVersion) + assert.Equal(t, "103", ofxFile.FileHeader.OFXDataVersion) + assert.Equal(t, "NONE", ofxFile.FileHeader.Security) + assert.Equal(t, "NONE", ofxFile.FileHeader.OldFileUid) + assert.Equal(t, "NONE", ofxFile.FileHeader.NewFileUid) +} + func TestCreateNewOFXFileReader_OFX2(t *testing.T) { context := core.NewNullContext() reader, err := createNewOFXFileReader(context, []byte( @@ -328,14 +791,15 @@ func TestCreateNewOFXFileReader_OFX2WithInvalidHeader(t *testing.T) { ""+ ""+ "")) + assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message) _, err = createNewOFXFileReader(context, []byte( ""+ ""+ ""+ "")) - assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message) + _, err = createNewOFXFileReader(context, []byte( ""+ ""+ @@ -344,6 +808,27 @@ func TestCreateNewOFXFileReader_OFX2WithInvalidHeader(t *testing.T) { assert.EqualError(t, err, errs.ErrInvalidOFXFile.Message) } +func TestCreateNewOFXFileReader_OFX2WithUnknownHeader(t *testing.T) { + context := core.NewNullContext() + + reader, err := createNewOFXFileReader(context, []byte( + ""+ + ""+ + ""+ + "")) + assert.Nil(t, err) + + ofxFile, err := reader.read(context) + assert.NotNil(t, ofxFile) + + assert.NotNil(t, ofxFile.FileHeader) + assert.Equal(t, ofxVersion2, ofxFile.FileHeader.OFXDeclarationVersion) + assert.Equal(t, "211", ofxFile.FileHeader.OFXDataVersion) + assert.Equal(t, "NONE", ofxFile.FileHeader.Security) + assert.Equal(t, "NONE", ofxFile.FileHeader.OldFileUid) + assert.Equal(t, "NONE", ofxFile.FileHeader.NewFileUid) +} + func TestCreateNewOFXFileReader_OFX2WithSGML(t *testing.T) { context := core.NewNullContext() reader, err := createNewOFXFileReader(context, []byte( diff --git a/pkg/converters/sgml/sgml_decoder.go b/pkg/converters/sgml/sgml_decoder.go new file mode 100644 index 00000000..5fc0c2a9 --- /dev/null +++ b/pkg/converters/sgml/sgml_decoder.go @@ -0,0 +1,321 @@ +package sgml + +import ( + "encoding/xml" + "io" + "reflect" + "sync" + + "github.com/mayswind/ezbookkeeping/pkg/errs" +) + +const sgmlTagName = "sgml" +const sgmlNameFieldName = "SGMLName" +const xmlTagName = "xml" // reuse xml tag +const xmlNameFieldName = "XMLName" // reuse xml tag + +// sgmlFieldType represents SGML field type +type sgmlFieldType byte + +// Transaction template types +const ( + sgmlNotSupportedField sgmlFieldType = 0 + sgmlTextualField sgmlFieldType = 1 + sgmlStructField sgmlFieldType = 2 + sgmlStructSliceField sgmlFieldType = 3 +) + +// sgmlTypeInfo represents the struct of SGML type reflection info +type sgmlTypeInfo struct { + supportedFields map[string]*sgmlFieldInfo +} + +// sgmlFieldInfo represents the struct of SGML field info +type sgmlFieldInfo struct { + sgmlFieldName string + sgmlFieldType sgmlFieldType + structFieldName string +} + +type Decoder struct { + xmlDecoder *xml.Decoder +} + +var sgmlTypeInfoMap sync.Map // map[reflect.Type]*typeInfo + +// Decode unmarshal the specified struct instance and returns whether error occurs +func (d *Decoder) Decode(v any) error { + value := reflect.ValueOf(v).Elem() + finalValue := value + finalType := value.Type() + + for finalValue.Kind() == reflect.Pointer { + finalValue = value.Elem() + finalType = finalValue.Type() + } + + rootNameField, exists := finalType.FieldByName(sgmlNameFieldName) + + if !exists { + rootNameField, exists = finalType.FieldByName(xmlNameFieldName) + } + + if !exists { + return nil + } + + rootElementName := rootNameField.Tag.Get(sgmlTagName) + + if rootElementName == "" { + rootElementName = rootNameField.Tag.Get(xmlTagName) + } + + for { + token, err := d.xmlDecoder.RawToken() + + if err == io.EOF { + break + } + + switch token := token.(type) { + case xml.StartElement: + if token.Name.Local == rootElementName { + return d.unmarshal(value.Elem(), rootElementName) + } + } + } + + return nil +} + +func (d *Decoder) unmarshal(element reflect.Value, elementName string) error { + typeInfo, err := d.getStructTypeInfo(element.Type()) + + if err != nil { + return err + } + + if typeInfo == nil { + return errs.ErrInvalidSGMLFile + } + + textualFieldWithoutEndElementNames := make(map[string]bool) + textualFieldValues := make(map[string]string) + + hasEndElement := false + currentSGMLFieldName := "" + + for { + token, err := d.xmlDecoder.RawToken() + + if err == io.EOF { + break + } + + switch token := token.(type) { + case xml.StartElement: + if fieldInfo, exists := typeInfo.supportedFields[token.Name.Local]; exists { + if fieldInfo.sgmlFieldType == sgmlStructField || fieldInfo.sgmlFieldType == sgmlStructSliceField { + field := element.FieldByName(fieldInfo.structFieldName) + childElementType := field.Type() + childElementKind := field.Kind() + var childElement reflect.Value + + if fieldInfo.sgmlFieldType == sgmlStructSliceField { + childElementType = childElementType.Elem() + childElementKind = childElementType.Kind() + } + + if childElementKind == reflect.Pointer { + childElement = reflect.New(childElementType.Elem()) + } else if childElementKind == reflect.Struct { + childElement = reflect.New(childElementType) + } + + err := d.unmarshal(childElement.Elem(), fieldInfo.sgmlFieldName) + + if err != nil { + return err + } + + if childElementKind == reflect.Struct { + childElement = childElement.Elem() + } + + if fieldInfo.sgmlFieldType == sgmlStructField { + field.Set(childElement) + } else if fieldInfo.sgmlFieldType == sgmlStructSliceField { + if field.Len() == 0 { + slice := reflect.MakeSlice(reflect.SliceOf(childElement.Type()), 0, 0) + field.Set(reflect.Append(slice, childElement)) + } else { + field.Set(reflect.Append(field.Addr().Elem(), childElement)) + } + } + } else if fieldInfo.sgmlFieldType == sgmlTextualField { + currentSGMLFieldName = token.Name.Local + textualFieldWithoutEndElementNames[token.Name.Local] = true + } + } + case xml.EndElement: + if fieldInfo, exists := typeInfo.supportedFields[token.Name.Local]; exists { + if fieldInfo.sgmlFieldType == sgmlTextualField { + delete(textualFieldWithoutEndElementNames, token.Name.Local) + } + } else if token.Name.Local == elementName { + hasEndElement = true + break + } + case xml.CharData: + if currentSGMLFieldName != "" { + if fieldInfo, exists := typeInfo.supportedFields[currentSGMLFieldName]; exists { + if fieldInfo.sgmlFieldType == sgmlTextualField { + textualFieldValues[currentSGMLFieldName] = string(token) + } + } + } + + currentSGMLFieldName = "" + } + + if hasEndElement { + break + } + } + + if !hasEndElement { + return errs.ErrInvalidSGMLFile + } + + for sgmlFieldName, fieldValue := range textualFieldValues { + finalValue := d.getActualFieldValue(sgmlFieldName, fieldValue, textualFieldWithoutEndElementNames) + fieldInfo, exists := typeInfo.supportedFields[sgmlFieldName] + + if !exists { + continue + } + + field := element.FieldByName(fieldInfo.structFieldName) + field.SetString(finalValue) + } + + return nil +} + +func (d *Decoder) getStructTypeInfo(reflectType reflect.Type) (*sgmlTypeInfo, error) { + if reflectType.Kind() != reflect.Struct { + return nil, nil + } + + typeInfo, exists := sgmlTypeInfoMap.Load(reflectType) + + if exists { + return typeInfo.(*sgmlTypeInfo), nil + } + + newTypeInfo := &sgmlTypeInfo{ + supportedFields: make(map[string]*sgmlFieldInfo), + } + + for i := 0; i < reflectType.NumField(); i++ { + field := reflectType.Field(i) + + if field.Anonymous { + fieldType := field.Type + + if fieldType.Kind() == reflect.Struct { + fieldSgmlTypeInfo, err := d.getStructTypeInfo(fieldType) + + if err != nil { + return nil, err + } + + for sgmlFieldName, fieldInfo := range fieldSgmlTypeInfo.supportedFields { + newTypeInfo.supportedFields[sgmlFieldName] = fieldInfo + } + } + + continue + } else if !field.IsExported() { + continue + } + + sgmlFieldName := field.Tag.Get(sgmlTagName) + + if sgmlFieldName == "" { + sgmlFieldName = field.Tag.Get(xmlTagName) + } + + if sgmlFieldName == "" || field.Name == sgmlNameFieldName || field.Name == xmlNameFieldName { + continue + } + + sgmlFieldType := sgmlNotSupportedField + finalFieldType := field.Type + + for finalFieldType.Kind() == reflect.Pointer { + finalFieldType = finalFieldType.Elem() + } + + switch finalFieldType.Kind() { + case reflect.String: + sgmlFieldType = sgmlTextualField + case reflect.Struct: + sgmlFieldType = sgmlStructField + case reflect.Slice: + childFinalFieldType := finalFieldType.Elem() + + for childFinalFieldType.Kind() == reflect.Pointer { + childFinalFieldType = childFinalFieldType.Elem() + } + + if childFinalFieldType.Kind() == reflect.Struct { + sgmlFieldType = sgmlStructSliceField + } + default: + sgmlFieldType = sgmlNotSupportedField + } + + if sgmlFieldType == sgmlNotSupportedField { + return nil, errs.ErrInvalidSGMLFile + } + + newTypeInfo.supportedFields[sgmlFieldName] = &sgmlFieldInfo{ + sgmlFieldName: sgmlFieldName, + sgmlFieldType: sgmlFieldType, + structFieldName: field.Name, + } + } + + typeInfo, _ = sgmlTypeInfoMap.LoadOrStore(reflectType, newTypeInfo) + + return typeInfo.(*sgmlTypeInfo), nil +} + +func (d *Decoder) getActualFieldValue(fieldName string, fieldValue string, textualFieldWithoutEndElementNames map[string]bool) string { + _, notHasEndElement := textualFieldWithoutEndElementNames[fieldName] + + if !notHasEndElement { + return fieldValue + } + + for i := 0; i < len(fieldValue); i++ { + if fieldValue[i] == '\r' || fieldValue[i] == '\n' { + return fieldValue[0:i] + } + } + + return fieldValue +} + +// NewDecoder creates a new SGML parser reading from specified io reader +func NewDecoder(reader io.Reader) *Decoder { + xmlDecoder := xml.NewDecoder(reader) + xmlDecoder.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) { + return input, nil + } + + return &Decoder{ + xmlDecoder: xmlDecoder, + } +} diff --git a/pkg/converters/sgml/sgml_decoder_test.go b/pkg/converters/sgml/sgml_decoder_test.go new file mode 100644 index 00000000..202df814 --- /dev/null +++ b/pkg/converters/sgml/sgml_decoder_test.go @@ -0,0 +1,359 @@ +package sgml + +import ( + "encoding/xml" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mayswind/ezbookkeeping/pkg/errs" +) + +type TestSimpleStruct struct { + SGMLName string `sgml:"Root"` + Text1 string `sgml:"Text1"` + Text2 string `sgml:"Text2"` +} + +type TestNestedStruct1 struct { + SGMLName string `sgml:"Root"` + Child TestSimpleStruct `sgml:"Child"` + Text3 string `sgml:"Text3"` + Text4 string `sgml:"Text4"` +} + +type TestNestedStruct2 struct { + SGMLName string `sgml:"Root"` + Child *TestSimpleStruct `sgml:"Child"` + Text3 string `sgml:"Text3"` + Text4 string `sgml:"Text4"` +} + +type TestEmbeddedStruct struct { + TestSimpleStruct + Text5 string `sgml:"Text5"` + Text6 string `sgml:"Text6"` +} + +type TestSliceStruct1 struct { + SGMLName string `sgml:"Root"` + Children []TestSimpleStruct `sgml:"Child"` + Text7 string `sgml:"Text7"` +} + +type TestSliceStruct2 struct { + SGMLName string `sgml:"Root"` + Children []*TestSimpleStruct `sgml:"Child"` + Text7 string `sgml:"Text7"` +} + +type TestSimpleStructWithXMLTag struct { + XMLName xml.Name `xml:"Root"` + Text1 string `xml:"Text1"` + Text2 string `xml:"Text2"` +} + +type TestStructWithXMLTag struct { + XMLName xml.Name `xml:"Root"` + Child TestSimpleStructWithXMLTag `xml:"Child"` + Text3 string `xml:"Text3"` + Text4 string `xml:"Text4"` +} + +type TestNotExportedFieldStruct struct { + SGMLName string `sgml:"Root"` + Text1 string `sgml:"Text1"` + Text2 string + text3 string `sgml:"Text3"` +} + +type TestUnsupportedStruct struct { + SGMLName string `sgml:"Root"` + Number int `sgml:"Number"` +} + +type TestEmbeddedUnsupportedStruct struct { + TestUnsupportedStruct + Text1 string `sgml:"Text1"` +} + +func TestDecoderDecode(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct := &TestSimpleStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, "Foo", testStruct.Text1) + assert.Equal(t, "Bar", testStruct.Text2) +} + +func TestDecoderDecode_WithRedundantFields(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "Foo\n" + + "Bar\n" + + "Hello\n" + + "\n" + + "World\n" + + "\n" + + "\n")) + + testStruct := &TestSimpleStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, "Foo", testStruct.Text1) + assert.Equal(t, "Bar", testStruct.Text2) +} + +func TestDecoderDecode_WithEndElement(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct := &TestSimpleStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, "Foo", testStruct.Text1) + assert.Equal(t, "Bar", testStruct.Text2) +} + +func TestDecoderDecode_WithoutBreakLine(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "" + + "Foo" + + "Bar" + + "")) + + testStruct := &TestSimpleStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, "Foo", testStruct.Text1) + assert.Equal(t, "Bar", testStruct.Text2) +} + +func TestDecoderDecode_NestedStruct(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "\n" + + "Hello\n" + + "World\n" + + "\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct := &TestNestedStruct1{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.NotNil(t, testStruct.Child) + assert.Equal(t, "Hello", testStruct.Child.Text1) + assert.Equal(t, "World", testStruct.Child.Text2) + assert.Equal(t, "Foo", testStruct.Text3) + assert.Equal(t, "Bar", testStruct.Text4) +} + +func TestDecoderDecode_NestedStructUsingPointer(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "\n" + + "Hello\n" + + "World\n" + + "\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct := &TestNestedStruct2{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.NotNil(t, testStruct.Child) + assert.Equal(t, "Hello", testStruct.Child.Text1) + assert.Equal(t, "World", testStruct.Child.Text2) + assert.Equal(t, "Foo", testStruct.Text3) + assert.Equal(t, "Bar", testStruct.Text4) +} + +func TestDecoderDecode_EmbeddedStruct(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "Hello\n" + + "World\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct := &TestEmbeddedStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, "Hello", testStruct.Text1) + assert.Equal(t, "World", testStruct.Text2) + assert.Equal(t, "Foo", testStruct.Text5) + assert.Equal(t, "Bar", testStruct.Text6) +} + +func TestDecoderDecode_StructSlice(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "\n" + + "Hello\n" + + "World\n" + + "\n" + + "\n" + + "Hello2\n" + + "World2\n" + + "\n" + + "Foo\n" + + "\n")) + + testStruct := &TestSliceStruct1{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, 2, len(testStruct.Children)) + assert.Equal(t, "Hello", testStruct.Children[0].Text1) + assert.Equal(t, "World", testStruct.Children[0].Text2) + assert.Equal(t, "Hello2", testStruct.Children[1].Text1) + assert.Equal(t, "World2", testStruct.Children[1].Text2) + assert.Equal(t, "Foo", testStruct.Text7) +} + +func TestDecoderDecode_StructSliceUsingPointer(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "\n" + + "Hello\n" + + "World\n" + + "\n" + + "\n" + + "Hello2\n" + + "World2\n" + + "\n" + + "Foo\n" + + "\n")) + + testStruct := &TestSliceStruct2{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, 2, len(testStruct.Children)) + assert.Equal(t, "Hello", testStruct.Children[0].Text1) + assert.Equal(t, "World", testStruct.Children[0].Text2) + assert.Equal(t, "Hello2", testStruct.Children[1].Text1) + assert.Equal(t, "World2", testStruct.Children[1].Text2) + assert.Equal(t, "Foo", testStruct.Text7) +} + +func TestDecoderDecode_UsingXMLTag(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "\n" + + "Hello\n" + + "World\n" + + "\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct := &TestStructWithXMLTag{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.NotNil(t, testStruct.Child) + assert.Equal(t, "Hello", testStruct.Child.Text1) + assert.Equal(t, "World", testStruct.Child.Text2) + assert.Equal(t, "Foo", testStruct.Text3) + assert.Equal(t, "Bar", testStruct.Text4) +} + +func TestDecoderDecode_WithNotExportedFields(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "Foo\n" + + "Bar\n" + + "Hello World\n" + + "\n")) + + testStruct := &TestNotExportedFieldStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.Nil(t, err) + assert.NotNil(t, testStruct) + assert.Equal(t, "Foo", testStruct.Text1) + assert.Equal(t, "", testStruct.Text2) + assert.Equal(t, "", testStruct.text3) +} + +func TestDecoderDecode_StructWithoutEndElement(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "Foo\n" + + "Bar\n")) + + testStruct := &TestSimpleStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.EqualError(t, err, errs.ErrInvalidSGMLFile.Message) + + sgmlDecoder = NewDecoder(strings.NewReader( + "\n" + + "\n" + + "Hello\n" + + "World\n" + + "Foo\n" + + "Bar\n" + + "\n")) + + testStruct2 := &TestNestedStruct2{} + err = sgmlDecoder.Decode(&testStruct2) + + assert.EqualError(t, err, errs.ErrInvalidSGMLFile.Message) +} + +func TestDecoderDecode_WithNotSupportedField(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "1234\n" + + "\n")) + + testStruct := &TestUnsupportedStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.EqualError(t, err, errs.ErrInvalidSGMLFile.Message) +} + +func TestDecoderDecode_WithEmbeddedNotSupportedField(t *testing.T) { + sgmlDecoder := NewDecoder(strings.NewReader( + "\n" + + "1234\n" + + "Foo\n" + + "\n")) + + testStruct := &TestEmbeddedUnsupportedStruct{} + err := sgmlDecoder.Decode(&testStruct) + + assert.EqualError(t, err, errs.ErrInvalidSGMLFile.Message) +} diff --git a/pkg/errs/converter.go b/pkg/errs/converter.go index 7c26e2fc..1f61d9fc 100644 --- a/pkg/errs/converter.go +++ b/pkg/errs/converter.go @@ -24,4 +24,5 @@ var ( ErrThereAreNotSupportedTransactionType = NewNormalError(NormalSubcategoryConverter, 17, http.StatusBadRequest, "there are not supported transaction type") ErrInvalidIIFFile = NewNormalError(NormalSubcategoryConverter, 18, http.StatusBadRequest, "invalid iif file") ErrInvalidOFXFile = NewNormalError(NormalSubcategoryConverter, 19, http.StatusBadRequest, "invalid ofx file") + ErrInvalidSGMLFile = NewNormalError(NormalSubcategoryConverter, 20, http.StatusBadRequest, "invalid sgml file") ) diff --git a/src/locales/en.json b/src/locales/en.json index dc999d1d..4a1b0724 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1135,6 +1135,7 @@ "there are not supported transaction type": "There are not supported transaction type in import file", "invalid iif file": "Invalid IIF file", "invalid ofx file": "Invalid OFX file", + "invalid sgml file": "Invalid SGML file", "query items cannot be blank": "There are no query items", "query items too much": "There are too many query items", "query items have invalid item": "There is invalid item in query items", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index f1f80d27..3672651d 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -1135,6 +1135,7 @@ "there are not supported transaction type": "导入文件中有不支持的交易类型", "invalid iif file": "无效的 IIF 文件", "invalid ofx file": "无效的 OFX 文件", + "invalid sgml file": "无效的 SGML 文件", "query items cannot be blank": "请求项目不能为空", "query items too much": "请求项目过多", "query items have invalid item": "请求项目中有非法项目",