From e143c8f098e65ed7e0b6ec422c3226f36208aa00 Mon Sep 17 00:00:00 2001 From: MaysWind Date: Wed, 3 Dec 2025 23:56:13 +0800 Subject: [PATCH] automatically detect file encoding when importing delimiter-separated values (DSV) file --- package-lock.json | 33 ++------ package.json | 1 + ...stom_transaction_data_dsv_file_importer.go | 4 +- src/consts/file.ts | 35 +++++++- src/lib/file.ts | 58 ++++++++++++++ src/locales/de.json | 2 + src/locales/en.json | 2 + src/locales/es.json | 2 + src/locales/fr.json | 2 + src/locales/helpers.ts | 5 ++ src/locales/it.json | 2 + src/locales/ja.json | 2 + src/locales/ko.json | 2 + src/locales/nl.json | 2 + src/locales/pt_BR.json | 2 + src/locales/ru.json | 2 + src/locales/th.json | 2 + src/locales/uk.json | 2 + src/locales/vi.json | 2 + src/locales/zh_Hans.json | 2 + src/locales/zh_Hant.json | 2 + .../transactions/import/ImportDialog.vue | 80 +++++++++++++++++-- third-party-dependencies.json | 6 ++ 23 files changed, 215 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 03634d86..375a6741 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@vuepic/vue-datepicker": "^12.0.5", "axios": "^1.13.2", "cbor-js": "^0.1.0", + "chardet": "^2.1.1", "clipboard": "^2.0.11", "crypto-js": "^4.2.0", "dom7": "^4.0.6", @@ -99,7 +100,6 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -2005,7 +2005,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -2029,7 +2028,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -5534,7 +5532,6 @@ "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -5636,7 +5633,6 @@ "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.48.0", "@typescript-eslint/types": "8.48.0", @@ -6448,7 +6444,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "devOptional": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -6947,7 +6942,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", @@ -7115,6 +7109,12 @@ "node": ">=10" } }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, "node_modules/chokidar": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", @@ -7755,7 +7755,6 @@ "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.0.0.tgz", "integrity": "sha512-Tte/grDQRiETQP4xz3iZWSvoHrkCQtwqd6hs+mifXcjrCuo2iKWbajFObuLJVBlDIJlOzgQPd1hsaKt/3+OMkQ==", "license": "Apache-2.0", - "peer": true, "dependencies": { "tslib": "2.3.0", "zrender": "6.0.0" @@ -8029,7 +8028,6 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -8090,7 +8088,6 @@ "integrity": "sha512-nA5yUs/B1KmKzvC42fyD0+l9Yd+LtEpVhWRbXuDj0e+ZURcTtyRbMDWUeJmTAh2wC6jC83raS63anNM2YT3NPw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "natural-compare": "^1.4.0", @@ -9820,7 +9817,6 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -11408,7 +11404,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -12127,7 +12122,6 @@ "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -12496,7 +12490,6 @@ "integrity": "sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -12639,7 +12632,6 @@ "integrity": "sha512-N+7WK20/wOr7CzA2snJcUSSNTCzeCGUTFY3OgeQP3mZ1aj9NMQ0mSTXwlrnd89j33zzQJGqIN52GIOmYrfq46A==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.0.2", @@ -13581,7 +13573,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13704,7 +13695,6 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -13869,7 +13859,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14170,7 +14159,6 @@ "integrity": "sha512-NL8jTlbo0Tn4dUEXEsUg8KeyG/Lkmc4Fnzb8JXN/Ykm9G4HNImjtABMJgkQoVjOBN/j2WAwDTRytdqJbZsah7w==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -14385,7 +14373,6 @@ "integrity": "sha512-I/wd6QS+DO6lHmuGoi1UTyvvBTQ2KDzQZ9oowJQEJ6OcjWfJnscYXx2ptm6S7fJSASuZT8jGRBL3LV4oS3LpaA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@vuetify/loader-shared": "^2.1.1", "debug": "^4.3.3", @@ -14424,7 +14411,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -14444,7 +14430,6 @@ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.25.tgz", "integrity": "sha512-YLVdgv2K13WJ6n+kD5owehKtEXwdwXuj2TTyJMsO7pSeKw2bfRNZGjhB7YzrpbMYj5b5QsUebHpOqR3R3ziy/g==", "license": "MIT", - "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.25", "@vue/compiler-sfc": "3.5.25", @@ -14561,7 +14546,6 @@ "integrity": "sha512-L/G9IUjOWhBU0yun89rv8fKqmKC+T0HfhrFjlIml71WpfBv9eb4E9Bev8FMbyueBIU9vxQqbd+oOsVcDa5amGw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@volar/typescript": "2.4.23", "@vue/language-core": "3.1.5" @@ -14601,7 +14585,6 @@ "resolved": "https://registry.npmjs.org/vuetify/-/vuetify-3.11.0.tgz", "integrity": "sha512-ITGeT3uaTIwI2SdyTvtE45tY6FlS2oWklfLU47s2K0ZHnu1it35p9lz8oE15Id8ThtKyQojQGobMkN+korheEw==", "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/johnleider" @@ -14932,7 +14915,6 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -15070,7 +15052,6 @@ "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "rollup": "dist/bin/rollup" }, diff --git a/package.json b/package.json index 4ebdf2d0..25a80ba4 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@vuepic/vue-datepicker": "^12.0.5", "axios": "^1.13.2", "cbor-js": "^0.1.0", + "chardet": "^2.1.1", "clipboard": "^2.0.11", "crypto-js": "^4.2.0", "dom7": "^4.0.6", diff --git a/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go index 5f8f9494..a3c92b66 100644 --- a/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go +++ b/pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go @@ -33,8 +33,8 @@ var supportedFileTypeSeparators = map[string]rune{ var supportedFileEncodings = map[string]encoding.Encoding{ "utf-8": unicode.UTF8, // UTF-8 "utf-8-bom": unicode.UTF8BOM, // UTF-8 with BOM - "utf-16le": unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), // UTF-16 Little Endian - "utf-16be": unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM), // UTF-16 Big Endian + "utf-16le": unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), // UTF-16 Little Endian + "utf-16be": unicode.UTF16(unicode.BigEndian, unicode.UseBOM), // UTF-16 Big Endian "utf-16le-bom": unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM), // UTF-16 Little Endian with BOM "utf-16be-bom": unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM), // UTF-16 Big Endian with BOM "cp437": charmap.CodePage437, // OEM United States (CP-437) diff --git a/src/consts/file.ts b/src/consts/file.ts index 2b5205e0..5a1f53f0 100644 --- a/src/consts/file.ts +++ b/src/consts/file.ts @@ -9,8 +9,10 @@ export const SUPPORTED_DOCUMENT_LANGUAGES_FOR_IMPORT_FILE: Record = { + 'UTF-8': UTF_8, + 'UTF-16LE': 'utf-16le', + 'UTF-16BE': 'utf-16be', + // 'UTF-32 LE': '', // not supported + // 'UTF-32 BE': '', // not supported + 'ISO-2022-JP': 'iso-2022-jp', + // 'ISO-2022-KR': '', // not supported + // 'ISO-2022-CN': '', // not supported + 'Shift_JIS': 'shift_jis', + 'Big5': 'big5', + 'EUC-JP': 'euc-jp', + 'EUC-KR': 'euc-kr', + 'GB18030': 'gb18030', + 'ISO-8859-1': 'iso-8859-1', + 'ISO-8859-2': 'iso-8859-2', + 'ISO-8859-5': 'iso-8859-5', + 'ISO-8859-6': 'iso-8859-6', + 'ISO-8859-7': 'iso-8859-7', + 'ISO-8859-8': 'iso-8859-8', + 'ISO-8859-9': 'iso-8859-9', + 'windows-1250': 'windows-1250', + 'windows-1251': 'windows-1251', + 'windows-1252': 'windows-1252', + 'windows-1253': 'windows-1253', + 'windows-1254': 'windows-1254', + 'windows-1255': 'windows-1255', + 'windows-1256': 'windows-1256', + 'KOI8-R':'koi8r' +}; + export const SUPPORTED_IMPORT_FILE_CATEGORY_AND_TYPES: ImportFileCategoryAndTypes[] = [ { categoryName: 'ezBookkeeping File Format', diff --git a/src/lib/file.ts b/src/lib/file.ts index e680914d..d3a90399 100644 --- a/src/lib/file.ts +++ b/src/lib/file.ts @@ -1,5 +1,9 @@ +import chardet, { type Match } from 'chardet'; + import type { ImportFileTypeAndExtensions } from '@/core/file.ts'; +import { UTF_8, CHARDET_ENCODING_NAME_MAPPING } from '@/consts/file.ts'; + import { isString } from './common.ts'; export function getFileExtension(filename: string): string { @@ -41,3 +45,57 @@ export function isFileExtensionSupported(filename: string, supportedExtensions: return false; } + +export function detectFileEncoding(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + + reader.onload = () => { + const arrayBuffer = reader.result as ArrayBuffer; + const uint8Array = new Uint8Array(arrayBuffer); + const possibleEncodings: Match[] = chardet.analyse(uint8Array); + + if (!possibleEncodings || possibleEncodings.length < 1) { + reject(new Error('unable to detect file encoding')); + return; + } + + const mostPossibleEncoding: Match = possibleEncodings[0] as Match; + + if (!mostPossibleEncoding.name || mostPossibleEncoding.confidence < 50) { + // check whether all characters are ASCII + let isAllAscii = true; + + for (const byte of uint8Array) { + if (byte > 0x7F) { + isAllAscii = false; + break; + } + } + + if (isAllAscii) { + resolve(UTF_8); + return; + } + + reject(new Error('unable to detect file encoding')); + return; + } + + const encoding = CHARDET_ENCODING_NAME_MAPPING[mostPossibleEncoding.name]; + + if (!encoding) { + reject(new Error(`unsupported file encoding: ${mostPossibleEncoding.name}`)); + return; + } + + resolve(encoding); + }; + + reader.onerror = () => { + reject(new Error('failed to read file for encoding detection')); + }; + + reader.readAsArrayBuffer(file); + }); +} diff --git a/src/locales/de.json b/src/locales/de.json index f513c64c..96f1ca9f 100644 --- a/src/locales/de.json +++ b/src/locales/de.json @@ -1443,6 +1443,7 @@ "No results": "Keine Ergebnisse", "Unknown": "Unbekannt", "Auto detect": "Auto detect", + "Detecting...": "Detecting...", "Miscellaneous": "Verschiedenes", "Default": "Standard", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Datendatei", "Data to import": "Data to import", "Please select a file to import": "Bitte wählen Sie eine Datei zum Importieren aus", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Include Header Line", "Time Format": "Time Format", "Transaction Type Mapping": "Transaction Type Mapping", diff --git a/src/locales/en.json b/src/locales/en.json index 97748fda..9afc8048 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1443,6 +1443,7 @@ "No results": "No results", "Unknown": "Unknown", "Auto detect": "Auto detect", + "Detecting...": "Detecting...", "Miscellaneous": "Miscellaneous", "Default": "Default", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Data File", "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Include Header Line", "Time Format": "Time Format", "Transaction Type Mapping": "Transaction Type Mapping", diff --git a/src/locales/es.json b/src/locales/es.json index cf687158..51b96fc4 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -1443,6 +1443,7 @@ "No results": "Sin resultados", "Unknown": "Desconocido", "Auto detect": "Auto detect", + "Detecting...": "Detecting...", "Miscellaneous": "Misceláneas", "Default": "Por defecto", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Archivo de datos", "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Include Header Line", "Time Format": "Time Format", "Transaction Type Mapping": "Transaction Type Mapping", diff --git a/src/locales/fr.json b/src/locales/fr.json index c702b6ee..61455370 100644 --- a/src/locales/fr.json +++ b/src/locales/fr.json @@ -1443,6 +1443,7 @@ "No results": "Aucun résultat", "Unknown": "Inconnu", "Auto detect": "Détection automatique", + "Detecting...": "Detecting...", "Miscellaneous": "Divers", "Default": "Par défaut", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Fichier de données", "Data to import": "Données à importer", "Please select a file to import": "Veuillez sélectionner un fichier à importer", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Inclure la ligne d'en-tête", "Time Format": "Format d'heure", "Transaction Type Mapping": "Mappage du type de transaction", diff --git a/src/locales/helpers.ts b/src/locales/helpers.ts index a5d52e5b..78a67b2c 100644 --- a/src/locales/helpers.ts +++ b/src/locales/helpers.ts @@ -2140,6 +2140,10 @@ export function useI18n() { return ret; } + function getLocalizedFileEncodingName(encoding: string): string { + return t(`encoding.${encoding}`); + } + function getLocalizedOAuth2ProviderName(oauth2Provider: string, oidcDisplayNames: Record): string { if (oauth2Provider === 'oidc') { const providerDisplayName = getServerMultiLanguageConfigContent(oidcDisplayNames); @@ -2452,6 +2456,7 @@ export function useI18n() { getAmountPrependAndAppendText, getCategorizedAccountsWithDisplayBalance, // other format functions + getLocalizedFileEncodingName, getLocalizedOAuth2ProviderName, getLocalizedOAuth2LoginText, // localization setting functions diff --git a/src/locales/it.json b/src/locales/it.json index 2aef241f..0e7fca00 100644 --- a/src/locales/it.json +++ b/src/locales/it.json @@ -1443,6 +1443,7 @@ "No results": "Nessun risultato", "Unknown": "Sconosciuto", "Auto detect": "Rilevamento automatico", + "Detecting...": "Detecting...", "Miscellaneous": "Varie", "Default": "Predefinito", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "File dati", "Data to import": "Dati da importare", "Please select a file to import": "Seleziona un file da importare", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Includi riga di intestazione", "Time Format": "Formato ora", "Transaction Type Mapping": "Mappatura tipo transazione", diff --git a/src/locales/ja.json b/src/locales/ja.json index 3b0b33fa..02baf01c 100644 --- a/src/locales/ja.json +++ b/src/locales/ja.json @@ -1443,6 +1443,7 @@ "No results": "結果はありません", "Unknown": "不明", "Auto detect": "自動検出", + "Detecting...": "Detecting...", "Miscellaneous": "その他", "Default": "デフォルト", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "データファイル", "Data to import": "インポートするデータ", "Please select a file to import": "インポートするファイルを選択してください", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "ヘッダー行を含める", "Time Format": "時刻形式", "Transaction Type Mapping": "取引タイプのマッピング", diff --git a/src/locales/ko.json b/src/locales/ko.json index 0b35210f..aa1a30cd 100644 --- a/src/locales/ko.json +++ b/src/locales/ko.json @@ -1443,6 +1443,7 @@ "No results": "결과 없음", "Unknown": "알 수 없음", "Auto detect": "자동 감지", + "Detecting...": "Detecting...", "Miscellaneous": "기타", "Default": "기본값", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "데이터 파일", "Data to import": "가져올 데이터", "Please select a file to import": "가져올 파일을 선택하십시오", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "헤더 행 포함", "Time Format": "시간 형식", "Transaction Type Mapping": "거래 유형 매핑", diff --git a/src/locales/nl.json b/src/locales/nl.json index ed4ffca2..8902416e 100644 --- a/src/locales/nl.json +++ b/src/locales/nl.json @@ -1443,6 +1443,7 @@ "No results": "Geen resultaten", "Unknown": "Onbekend", "Auto detect": "Automatisch detecteren", + "Detecting...": "Detecting...", "Miscellaneous": "Diversen", "Default": "Standaard", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Gegevensbestand", "Data to import": "Te importeren gegevens", "Please select a file to import": "Selecteer een bestand om te importeren", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Kopregel opnemen", "Time Format": "Tijdsformaat", "Transaction Type Mapping": "Transactietypetoewijzing", diff --git a/src/locales/pt_BR.json b/src/locales/pt_BR.json index 60e4dc5b..c71b22e5 100644 --- a/src/locales/pt_BR.json +++ b/src/locales/pt_BR.json @@ -1443,6 +1443,7 @@ "No results": "Sem resultados", "Unknown": "Desconhecido", "Auto detect": "Detecção automática", + "Detecting...": "Detecting...", "Miscellaneous": "Diversos", "Default": "Padrão", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Arquivo de Dados", "Data to import": "Dados para importar", "Please select a file to import": "Por favor, selecione um arquivo para importar", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Incluir Linha de Cabeçalho", "Time Format": "Formato de Tempo", "Transaction Type Mapping": "Mapeamento de Tipo de Transação", diff --git a/src/locales/ru.json b/src/locales/ru.json index 9469b594..0c9388c4 100644 --- a/src/locales/ru.json +++ b/src/locales/ru.json @@ -1443,6 +1443,7 @@ "No results": "Нет результатов", "Unknown": "Неизвестно", "Auto detect": "Auto detect", + "Detecting...": "Detecting...", "Miscellaneous": "Разное", "Default": "По умолчанию", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Файл данных", "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Include Header Line", "Time Format": "Time Format", "Transaction Type Mapping": "Transaction Type Mapping", diff --git a/src/locales/th.json b/src/locales/th.json index 0dc0789c..4ec9c305 100644 --- a/src/locales/th.json +++ b/src/locales/th.json @@ -1443,6 +1443,7 @@ "No results": "ไม่มีผลลัพธ์", "Unknown": "ไม่ทราบ", "Auto detect": "ตรวจสอบอัตโนมัติ", + "Detecting...": "Detecting...", "Miscellaneous": "อื่น ๆ", "Default": "ค่าเริ่มต้น", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "ไฟล์ข้อมูล", "Data to import": "ข้อมูลที่จะนำเข้า", "Please select a file to import": "กรุณาเลือกไฟล์เพื่อนำเข้า", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "รวมแถวหัวตาราง", "Time Format": "รูปแบบเวลา", "Transaction Type Mapping": "การแมปประเภทรายการ", diff --git a/src/locales/uk.json b/src/locales/uk.json index 34018831..9bd4b4a0 100644 --- a/src/locales/uk.json +++ b/src/locales/uk.json @@ -1443,6 +1443,7 @@ "No results": "Немає результатів", "Unknown": "Невідомо", "Auto detect": "Автовизначення", + "Detecting...": "Detecting...", "Miscellaneous": "Різне", "Default": "По замовчуванню", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Файл даних", "Data to import": "Дані для імпорту", "Please select a file to import": "Будь ласка, виберіть файл для імпорту", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Включити рядок заголовка", "Time Format": "Формат часу", "Transaction Type Mapping": "Відповідність типів транзакцій", diff --git a/src/locales/vi.json b/src/locales/vi.json index 42e1c5c0..3240a89b 100644 --- a/src/locales/vi.json +++ b/src/locales/vi.json @@ -1443,6 +1443,7 @@ "No results": "Không có kết quả", "Unknown": "Không rõ", "Auto detect": "Auto detect", + "Detecting...": "Detecting...", "Miscellaneous": "Linh tinh", "Default": "Mặc định", "Included": "Included", @@ -1896,6 +1897,7 @@ "Data File": "Tệp dữ liệu", "Data to import": "Data to import", "Please select a file to import": "Please select a file to import", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.", "Include Header Line": "Include Header Line", "Time Format": "Time Format", "Transaction Type Mapping": "Transaction Type Mapping", diff --git a/src/locales/zh_Hans.json b/src/locales/zh_Hans.json index f742417e..5ec0f8a9 100644 --- a/src/locales/zh_Hans.json +++ b/src/locales/zh_Hans.json @@ -1443,6 +1443,7 @@ "No results": "无结果", "Unknown": "未知", "Auto detect": "自动检测", + "Detecting...": "正在检测...", "Miscellaneous": "杂项", "Default": "默认", "Included": "包含", @@ -1896,6 +1897,7 @@ "Data File": "数据文件", "Data to import": "要导入的数据", "Please select a file to import": "请选择要导入的文件", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "无法自动检测文件编码。请选择实际的编码。", "Include Header Line": "包含标题行", "Time Format": "时间格式", "Amount Format": "金额格式", diff --git a/src/locales/zh_Hant.json b/src/locales/zh_Hant.json index 2f3a5894..a19893d1 100644 --- a/src/locales/zh_Hant.json +++ b/src/locales/zh_Hant.json @@ -1443,6 +1443,7 @@ "No results": "無結果", "Unknown": "未知", "Auto detect": "自動偵測", + "Detecting...": "正在偵測...", "Miscellaneous": "雜項", "Default": "預設", "Included": "包含", @@ -1896,6 +1897,7 @@ "Data File": "資料檔案", "Data to import": "要匯入的資料", "Please select a file to import": "請選擇要匯入的檔案", + "Unable to detect the file encoding automatically. Please select the actual encoding.": "無法自動偵測檔案編碼。請選擇實際的編碼。", "Include Header Line": "包含標頭列", "Time Format": "時間格式", "Amount Format": "金額格式", diff --git a/src/views/desktop/transactions/import/ImportDialog.vue b/src/views/desktop/transactions/import/ImportDialog.vue index c0df42da..35cfd011 100644 --- a/src/views/desktop/transactions/import/ImportDialog.vue +++ b/src/views/desktop/transactions/import/ImportDialog.vue @@ -237,7 +237,7 @@ :prepend-icon="mdiClose" @click="close(false)" v-if="currentStep !== 'finalResult'">{{ tt('Cancel') }} {{ tt('Next') }} @@ -293,10 +293,12 @@ import { type LocalizedImportFileTypeSupportedEncodings, KnownFileType } from '@/core/file.ts'; +import { UTF_8 } from '@/consts/file.ts'; + import { ImportTransaction } from '@/models/imported_transaction.ts'; import { isDefined, isNumber } from '@/lib/common.ts'; -import { findExtensionByType, isFileExtensionSupported } from '@/lib/file.ts'; +import { findExtensionByType, isFileExtensionSupported, detectFileEncoding } from '@/lib/file.ts'; import { generateRandomUUID } from '@/lib/misc.ts'; import logger from '@/lib/logger.ts'; @@ -330,7 +332,8 @@ const { joinMultiText, getCurrentNumeralSystemType, getAllSupportedImportFileCagtegoryAndTypes, - formatNumberToLocalizedNumerals + formatNumberToLocalizedNumerals, + getLocalizedFileEncodingName } = useI18n(); const accountsStore = useAccountsStore(); @@ -377,7 +380,9 @@ const currentStep = ref('uploadFile'); const importProcess = ref(0); const fileType = ref('ezbookkeeping'); const fileSubType = ref('ezbookkeeping_csv'); -const fileEncoding = ref('utf-8'); +const fileEncoding = ref('auto'); +const detectingFileEncoding = ref(false); +const autoDetectedFileEncoding = ref(undefined); const processDSVMethod = ref(ImportDSVProcessMethod.ColumnMapping); const importFile = ref(null); const importData = ref(''); @@ -396,7 +401,39 @@ const numeralSystem = computed(() => getCurrentNumeralSystemType( const allSupportedImportFileCategoryAndTypes = computed(() => getAllSupportedImportFileCagtegoryAndTypes()); const allFileSubTypes = computed(() => allSupportedImportFileTypesMap.value[fileType.value]?.subTypes); -const allSupportedEncodings = computed(() => allSupportedImportFileTypesMap.value[fileType.value]?.supportedEncodings); +const allSupportedEncodings = computed(() => { + const supportedEncodings = allSupportedImportFileTypesMap.value[fileType.value]?.supportedEncodings; + + if (!supportedEncodings) { + return undefined; + } + + const ret: LocalizedImportFileTypeSupportedEncodings[] = []; + let autoDetectDisplayName = tt('Auto detect'); + + if (importFile.value) { + if (detectingFileEncoding.value) { + autoDetectDisplayName += ` [${tt('Detecting...')}]`; + } else if (autoDetectedFileEncoding.value) { + autoDetectDisplayName += ` [${getLocalizedFileEncodingName(autoDetectedFileEncoding.value)}]`; + } else { + autoDetectDisplayName += ` [${tt('Unknown')}]`; + } + } + + const autoDetectEncoding: LocalizedImportFileTypeSupportedEncodings = { + displayName: autoDetectDisplayName, + encoding: 'auto' + }; + + ret.push(autoDetectEncoding); + + if (supportedEncodings && supportedEncodings.length) { + ret.push(...supportedEncodings); + } + + return ret; +}); const isImportDataFromTextbox = computed(() => allSupportedImportFileTypesMap.value[fileType.value]?.dataFromTextbox ?? false); const supportedAdditionalOptions = computed(() => allSupportedImportFileTypesMap.value[fileType.value]?.supportedAdditionalOptions); @@ -508,7 +545,9 @@ function getDisplayCount(count: number): string { function open(): Promise { fileType.value = 'ezbookkeeping'; fileSubType.value = 'ezbookkeeping_csv'; - fileEncoding.value = 'utf-8'; + fileEncoding.value = 'auto'; + detectingFileEncoding.value = false; + autoDetectedFileEncoding.value = undefined; processDSVMethod.value = ImportDSVProcessMethod.ColumnMapping; currentStep.value = 'uploadFile'; importProcess.value = 0; @@ -570,7 +609,21 @@ function setImportFile(event: Event): void { } importFile.value = el.files[0] as File; + detectingFileEncoding.value = false; + autoDetectedFileEncoding.value = undefined; el.value = ''; + + if (allSupportedEncodings.value) { + detectingFileEncoding.value = true; + + detectFileEncoding(importFile.value).then(detectedEncoding => { + detectingFileEncoding.value = false; + autoDetectedFileEncoding.value = detectedEncoding; + }).catch(() => { + detectingFileEncoding.value = false; + autoDetectedFileEncoding.value = undefined; + }); + } } function parseData(): void { @@ -583,7 +636,11 @@ function parseData(): void { } if (allSupportedEncodings.value) { - encoding = fileEncoding.value; + if (fileEncoding.value === 'auto') { + encoding = autoDetectedFileEncoding.value; + } else { + encoding = fileEncoding.value; + } } if (!isImportDataFromTextbox.value) { @@ -592,6 +649,13 @@ function parseData(): void { return; } + if (allSupportedEncodings.value) { + if (fileEncoding.value === 'auto' && !autoDetectedFileEncoding.value) { + snackbar.value?.showError('Unable to detect the file encoding automatically. Please select the actual encoding.'); + return; + } + } + uploadFile = importFile.value; } else if (isImportDataFromTextbox.value) { if (!importData.value) { @@ -608,7 +672,7 @@ function parseData(): void { return; } - encoding = 'utf-8'; + encoding = UTF_8; } else { // should not happen, but ts would check whether uploadFile has been assigned a value snackbar.value?.showMessage('An error occurred'); return; diff --git a/third-party-dependencies.json b/third-party-dependencies.json index 563e2608..11f6dc77 100644 --- a/third-party-dependencies.json +++ b/third-party-dependencies.json @@ -289,6 +289,12 @@ "url": "https://leafletjs.com/", "licenseUrl": "https://github.com/Leaflet/Leaflet/blob/v1.9.4/LICENSE" }, + { + "name": "Chardet", + "copyright": "Copyright (C) 2024 Dmitry Shirokov", + "url": "https://github.com/runk/node-chardet", + "licenseUrl": "https://github.com/runk/node-chardet/blob/v2.1.1/LICENSE" + }, { "name": "crypto-js", "copyright": "Copyright (c) 2009-2013 Jeff Mott, Copyright (c) 2013-2016 Evan Vosberg",