automatically detect file encoding when importing delimiter-separated values (DSV) file

package-lock.json

@@ -13,6 +13,7 @@
         "@vuepic/vue-datepicker": "^12.0.5",
         "axios": "^1.13.2",
         "cbor-js": "^0.1.0",
+        "chardet": "^2.1.1",
         "clipboard": "^2.0.11",
         "crypto-js": "^4.2.0",
         "dom7": "^4.0.6",
@@ -99,7 +100,6 @@
       "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@babel/code-frame": "^7.27.1",
         "@babel/generator": "^7.28.5",
@@ -2005,7 +2005,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -2029,7 +2028,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       }
@@ -5534,7 +5532,6 @@
       "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "undici-types": "~7.16.0"
       }
@@ -5636,7 +5633,6 @@
       "integrity": "sha512-jCzKdm/QK0Kg4V4IK/oMlRZlY+QOcdjv89U2NgKHZk1CYTj82/RVSx1mV/0gqCVMJ/DA+Zf/S4NBWNF8GQ+eqQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@typescript-eslint/scope-manager": "8.48.0",
         "@typescript-eslint/types": "8.48.0",
@@ -6448,7 +6444,6 @@
       "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -6947,7 +6942,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "baseline-browser-mapping": "^2.8.25",
         "caniuse-lite": "^1.0.30001754",
@@ -7115,6 +7109,12 @@
         "node": ">=10"
       }
     },
+    "node_modules/chardet": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz",
+      "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==",
+      "license": "MIT"
+    },
     "node_modules/chokidar": {
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz",
@@ -7755,7 +7755,6 @@
       "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.0.0.tgz",
       "integrity": "sha512-Tte/grDQRiETQP4xz3iZWSvoHrkCQtwqd6hs+mifXcjrCuo2iKWbajFObuLJVBlDIJlOzgQPd1hsaKt/3+OMkQ==",
       "license": "Apache-2.0",
-      "peer": true,
       "dependencies": {
         "tslib": "2.3.0",
         "zrender": "6.0.0"
@@ -8029,7 +8028,6 @@
       "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.8.0",
         "@eslint-community/regexpp": "^4.12.1",
@@ -8090,7 +8088,6 @@
       "integrity": "sha512-nA5yUs/B1KmKzvC42fyD0+l9Yd+LtEpVhWRbXuDj0e+ZURcTtyRbMDWUeJmTAh2wC6jC83raS63anNM2YT3NPw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.4.0",
         "natural-compare": "^1.4.0",
@@ -9820,7 +9817,6 @@
       "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@jest/core": "30.2.0",
         "@jest/types": "30.2.0",
@@ -11408,7 +11404,6 @@
         }
       ],
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "nanoid": "^3.3.11",
         "picocolors": "^1.1.1",
@@ -12127,7 +12122,6 @@
       "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "cssesc": "^3.0.0",
         "util-deprecate": "^1.0.2"
@@ -12496,7 +12490,6 @@
       "integrity": "sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@types/estree": "1.0.8"
       },
@@ -12639,7 +12632,6 @@
       "integrity": "sha512-N+7WK20/wOr7CzA2snJcUSSNTCzeCGUTFY3OgeQP3mZ1aj9NMQ0mSTXwlrnd89j33zzQJGqIN52GIOmYrfq46A==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "chokidar": "^4.0.0",
         "immutable": "^5.0.2",
@@ -13581,7 +13573,6 @@
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -13704,7 +13695,6 @@
       "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@cspotcode/source-map-support": "^0.8.0",
         "@tsconfig/node10": "^1.0.7",
@@ -13869,7 +13859,6 @@
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "devOptional": true,
       "license": "Apache-2.0",
-      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -14170,7 +14159,6 @@
       "integrity": "sha512-NL8jTlbo0Tn4dUEXEsUg8KeyG/Lkmc4Fnzb8JXN/Ykm9G4HNImjtABMJgkQoVjOBN/j2WAwDTRytdqJbZsah7w==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "esbuild": "^0.25.0",
         "fdir": "^6.5.0",
@@ -14385,7 +14373,6 @@
       "integrity": "sha512-I/wd6QS+DO6lHmuGoi1UTyvvBTQ2KDzQZ9oowJQEJ6OcjWfJnscYXx2ptm6S7fJSASuZT8jGRBL3LV4oS3LpaA==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@vuetify/loader-shared": "^2.1.1",
         "debug": "^4.3.3",
@@ -14424,7 +14411,6 @@
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "devOptional": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -14444,7 +14430,6 @@
       "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.25.tgz",
       "integrity": "sha512-YLVdgv2K13WJ6n+kD5owehKtEXwdwXuj2TTyJMsO7pSeKw2bfRNZGjhB7YzrpbMYj5b5QsUebHpOqR3R3ziy/g==",
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@vue/compiler-dom": "3.5.25",
         "@vue/compiler-sfc": "3.5.25",
@@ -14561,7 +14546,6 @@
       "integrity": "sha512-L/G9IUjOWhBU0yun89rv8fKqmKC+T0HfhrFjlIml71WpfBv9eb4E9Bev8FMbyueBIU9vxQqbd+oOsVcDa5amGw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@volar/typescript": "2.4.23",
         "@vue/language-core": "3.1.5"
@@ -14601,7 +14585,6 @@
       "resolved": "https://registry.npmjs.org/vuetify/-/vuetify-3.11.0.tgz",
       "integrity": "sha512-ITGeT3uaTIwI2SdyTvtE45tY6FlS2oWklfLU47s2K0ZHnu1it35p9lz8oE15Id8ThtKyQojQGobMkN+korheEw==",
       "license": "MIT",
-      "peer": true,
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/johnleider"
@@ -14932,7 +14915,6 @@
       "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
         "fast-uri": "^3.0.1",
@@ -15070,7 +15052,6 @@
       "integrity": "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "rollup": "dist/bin/rollup"
       },

package.json

@@ -23,6 +23,7 @@
     "@vuepic/vue-datepicker": "^12.0.5",
     "axios": "^1.13.2",
     "cbor-js": "^0.1.0",
+    "chardet": "^2.1.1",
     "clipboard": "^2.0.11",
     "crypto-js": "^4.2.0",
     "dom7": "^4.0.6",

pkg/converters/dsv/custom_transaction_data_dsv_file_importer.go

@@ -33,8 +33,8 @@ var supportedFileTypeSeparators = map[string]rune{
 var supportedFileEncodings = map[string]encoding.Encoding{
 	"utf-8":        unicode.UTF8,                                           // UTF-8
 	"utf-8-bom":    unicode.UTF8BOM,                                        // UTF-8 with BOM
-	"utf-16le":     unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM), // UTF-16 Little Endian
+	"utf-16le":     unicode.UTF16(unicode.LittleEndian, unicode.UseBOM),    // UTF-16 Little Endian
-	"utf-16be":     unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),    // UTF-16 Big Endian
+	"utf-16be":     unicode.UTF16(unicode.BigEndian, unicode.UseBOM),       // UTF-16 Big Endian
 	"utf-16le-bom": unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM), // UTF-16 Little Endian with BOM
 	"utf-16be-bom": unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM),    // UTF-16 Big Endian with BOM
 	"cp437":        charmap.CodePage437,                                    // OEM United States (CP-437)

src/consts/file.ts

@@ -9,8 +9,10 @@ export const SUPPORTED_DOCUMENT_LANGUAGES_FOR_IMPORT_FILE: Record<string, string
     'zh-Hant': 'zh-Hans',
 };
 
+export const UTF_8 = 'utf-8';
+
 export const SUPPORTED_FILE_ENCODINGS: string[] = [
-    'utf-8', // UTF-8
+    UTF_8, // UTF-8
     'utf-8-bom', // UTF-8 with BOM
     'utf-16le', // UTF-16 Little Endian
     'utf-16be', // UTF-16 Big Endian
@@ -64,6 +66,37 @@ export const SUPPORTED_FILE_ENCODINGS: string[] = [
     'shift_jis', // Japanese (Shift_JIS)
 ];
 
+export const CHARDET_ENCODING_NAME_MAPPING: Record<string, string> = {
+    'UTF-8': UTF_8,
+    'UTF-16LE': 'utf-16le',
+    'UTF-16BE': 'utf-16be',
+    // 'UTF-32 LE': '', // not supported
+    // 'UTF-32 BE': '', // not supported
+    'ISO-2022-JP': 'iso-2022-jp',
+    // 'ISO-2022-KR': '', // not supported
+    // 'ISO-2022-CN': '', // not supported
+    'Shift_JIS': 'shift_jis',
+    'Big5': 'big5',
+    'EUC-JP': 'euc-jp',
+    'EUC-KR': 'euc-kr',
+    'GB18030': 'gb18030',
+    'ISO-8859-1': 'iso-8859-1',
+    'ISO-8859-2': 'iso-8859-2',
+    'ISO-8859-5': 'iso-8859-5',
+    'ISO-8859-6': 'iso-8859-6',
+    'ISO-8859-7': 'iso-8859-7',
+    'ISO-8859-8': 'iso-8859-8',
+    'ISO-8859-9': 'iso-8859-9',
+    'windows-1250': 'windows-1250',
+    'windows-1251': 'windows-1251',
+    'windows-1252': 'windows-1252',
+    'windows-1253': 'windows-1253',
+    'windows-1254': 'windows-1254',
+    'windows-1255': 'windows-1255',
+    'windows-1256': 'windows-1256',
+    'KOI8-R':'koi8r'
+};
+
 export const SUPPORTED_IMPORT_FILE_CATEGORY_AND_TYPES: ImportFileCategoryAndTypes[] = [
     {
         categoryName: 'ezBookkeeping File Format',

src/lib/file.ts

@@ -1,5 +1,9 @@
+import chardet, { type Match } from 'chardet';
+
 import type { ImportFileTypeAndExtensions } from '@/core/file.ts';
 
+import { UTF_8, CHARDET_ENCODING_NAME_MAPPING } from '@/consts/file.ts';
+
 import { isString } from './common.ts';
 
 export function getFileExtension(filename: string): string {
@@ -41,3 +45,57 @@ export function isFileExtensionSupported(filename: string, supportedExtensions:
 
     return false;
 }
+
+export function detectFileEncoding(file: File): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const reader = new FileReader();
+
+        reader.onload = () => {
+            const arrayBuffer = reader.result as ArrayBuffer;
+            const uint8Array = new Uint8Array(arrayBuffer);
+            const possibleEncodings: Match[] = chardet.analyse(uint8Array);
+
+            if (!possibleEncodings || possibleEncodings.length < 1) {
+                reject(new Error('unable to detect file encoding'));
+                return;
+            }
+
+            const mostPossibleEncoding: Match = possibleEncodings[0] as Match;
+
+            if (!mostPossibleEncoding.name || mostPossibleEncoding.confidence < 50) {
+                // check whether all characters are ASCII
+                let isAllAscii = true;
+
+                for (const byte of uint8Array) {
+                    if (byte > 0x7F) {
+                        isAllAscii = false;
+                        break;
+                    }
+                }
+
+                if (isAllAscii) {
+                    resolve(UTF_8);
+                    return;
+                }
+
+                reject(new Error('unable to detect file encoding'));
+                return;
+            }
+
+            const encoding = CHARDET_ENCODING_NAME_MAPPING[mostPossibleEncoding.name];
+
+            if (!encoding) {
+                reject(new Error(`unsupported file encoding: ${mostPossibleEncoding.name}`));
+                return;
+            }
+
+            resolve(encoding);
+        };
+
+        reader.onerror = () => {
+            reject(new Error('failed to read file for encoding detection'));
+        };
+
+        reader.readAsArrayBuffer(file);
+    });
+}

src/locales/de.json

@@ -1443,6 +1443,7 @@
     "No results": "Keine Ergebnisse",
     "Unknown": "Unbekannt",
     "Auto detect": "Auto detect",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Verschiedenes",
     "Default": "Standard",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Datendatei",
     "Data to import": "Data to import",
     "Please select a file to import": "Bitte wählen Sie eine Datei zum Importieren aus",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Include Header Line",
     "Time Format": "Time Format",
     "Transaction Type Mapping": "Transaction Type Mapping",

src/locales/en.json

@@ -1443,6 +1443,7 @@
     "No results": "No results",
     "Unknown": "Unknown",
     "Auto detect": "Auto detect",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Miscellaneous",
     "Default": "Default",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Data File",
     "Data to import": "Data to import",
     "Please select a file to import": "Please select a file to import",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Include Header Line",
     "Time Format": "Time Format",
     "Transaction Type Mapping": "Transaction Type Mapping",

src/locales/es.json

@@ -1443,6 +1443,7 @@
     "No results": "Sin resultados",
     "Unknown": "Desconocido",
     "Auto detect": "Auto detect",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Misceláneas",
     "Default": "Por defecto",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Archivo de datos",
     "Data to import": "Data to import",
     "Please select a file to import": "Please select a file to import",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Include Header Line",
     "Time Format": "Time Format",
     "Transaction Type Mapping": "Transaction Type Mapping",

src/locales/fr.json

@@ -1443,6 +1443,7 @@
     "No results": "Aucun résultat",
     "Unknown": "Inconnu",
     "Auto detect": "Détection automatique",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Divers",
     "Default": "Par défaut",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Fichier de données",
     "Data to import": "Données à importer",
     "Please select a file to import": "Veuillez sélectionner un fichier à importer",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Inclure la ligne d'en-tête",
     "Time Format": "Format d'heure",
     "Transaction Type Mapping": "Mappage du type de transaction",

src/locales/helpers.ts

@@ -2140,6 +2140,10 @@ export function useI18n() {
         return ret;
     }
 
+    function getLocalizedFileEncodingName(encoding: string): string {
+        return t(`encoding.${encoding}`);
+    }
+
     function getLocalizedOAuth2ProviderName(oauth2Provider: string, oidcDisplayNames: Record<string, string>): string {
         if (oauth2Provider === 'oidc') {
             const providerDisplayName = getServerMultiLanguageConfigContent(oidcDisplayNames);
@@ -2452,6 +2456,7 @@ export function useI18n() {
         getAmountPrependAndAppendText,
         getCategorizedAccountsWithDisplayBalance,
         // other format functions
+        getLocalizedFileEncodingName,
         getLocalizedOAuth2ProviderName,
         getLocalizedOAuth2LoginText,
         // localization setting functions

src/locales/it.json

@@ -1443,6 +1443,7 @@
     "No results": "Nessun risultato",
     "Unknown": "Sconosciuto",
     "Auto detect": "Rilevamento automatico",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Varie",
     "Default": "Predefinito",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "File dati",
     "Data to import": "Dati da importare",
     "Please select a file to import": "Seleziona un file da importare",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Includi riga di intestazione",
     "Time Format": "Formato ora",
     "Transaction Type Mapping": "Mappatura tipo transazione",

src/locales/ja.json

@@ -1443,6 +1443,7 @@
     "No results": "結果はありません",
     "Unknown": "不明",
     "Auto detect": "自動検出",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "その他",
     "Default": "デフォルト",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "データファイル",
     "Data to import": "インポートするデータ",
     "Please select a file to import": "インポートするファイルを選択してください",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "ヘッダー行を含める",
     "Time Format": "時刻形式",
     "Transaction Type Mapping": "取引タイプのマッピング",

src/locales/ko.json

@@ -1443,6 +1443,7 @@
     "No results": "결과 없음",
     "Unknown": "알 수 없음",
     "Auto detect": "자동 감지",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "기타",
     "Default": "기본값",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "데이터 파일",
     "Data to import": "가져올 데이터",
     "Please select a file to import": "가져올 파일을 선택하십시오",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "헤더 행 포함",
     "Time Format": "시간 형식",
     "Transaction Type Mapping": "거래 유형 매핑",

src/locales/nl.json

@@ -1443,6 +1443,7 @@
     "No results": "Geen resultaten",
     "Unknown": "Onbekend",
     "Auto detect": "Automatisch detecteren",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Diversen",
     "Default": "Standaard",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Gegevensbestand",
     "Data to import": "Te importeren gegevens",
     "Please select a file to import": "Selecteer een bestand om te importeren",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Kopregel opnemen",
     "Time Format": "Tijdsformaat",
     "Transaction Type Mapping": "Transactietypetoewijzing",

src/locales/pt_BR.json

@@ -1443,6 +1443,7 @@
     "No results": "Sem resultados",
     "Unknown": "Desconhecido",
     "Auto detect": "Detecção automática",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Diversos",
     "Default": "Padrão",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Arquivo de Dados",
     "Data to import": "Dados para importar",
     "Please select a file to import": "Por favor, selecione um arquivo para importar",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Incluir Linha de Cabeçalho",
     "Time Format": "Formato de Tempo",
     "Transaction Type Mapping": "Mapeamento de Tipo de Transação",

src/locales/ru.json

@@ -1443,6 +1443,7 @@
     "No results": "Нет результатов",
     "Unknown": "Неизвестно",
     "Auto detect": "Auto detect",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Разное",
     "Default": "По умолчанию",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Файл данных",
     "Data to import": "Data to import",
     "Please select a file to import": "Please select a file to import",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Include Header Line",
     "Time Format": "Time Format",
     "Transaction Type Mapping": "Transaction Type Mapping",

src/locales/th.json

@@ -1443,6 +1443,7 @@
     "No results": "ไม่มีผลลัพธ์",
     "Unknown": "ไม่ทราบ",
     "Auto detect": "ตรวจสอบอัตโนมัติ",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "อื่น ๆ",
     "Default": "ค่าเริ่มต้น",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "ไฟล์ข้อมูล",
     "Data to import": "ข้อมูลที่จะนำเข้า",
     "Please select a file to import": "กรุณาเลือกไฟล์เพื่อนำเข้า",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "รวมแถวหัวตาราง",
     "Time Format": "รูปแบบเวลา",
     "Transaction Type Mapping": "การแมปประเภทรายการ",

src/locales/uk.json

@@ -1443,6 +1443,7 @@
     "No results": "Немає результатів",
     "Unknown": "Невідомо",
     "Auto detect": "Автовизначення",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Різне",
     "Default": "По замовчуванню",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Файл даних",
     "Data to import": "Дані для імпорту",
     "Please select a file to import": "Будь ласка, виберіть файл для імпорту",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Включити рядок заголовка",
     "Time Format": "Формат часу",
     "Transaction Type Mapping": "Відповідність типів транзакцій",

src/locales/vi.json

@@ -1443,6 +1443,7 @@
     "No results": "Không có kết quả",
     "Unknown": "Không rõ",
     "Auto detect": "Auto detect",
+    "Detecting...": "Detecting...",
     "Miscellaneous": "Linh tinh",
     "Default": "Mặc định",
     "Included": "Included",
@@ -1896,6 +1897,7 @@
     "Data File": "Tệp dữ liệu",
     "Data to import": "Data to import",
     "Please select a file to import": "Please select a file to import",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "Unable to detect the file encoding automatically. Please select the actual encoding.",
     "Include Header Line": "Include Header Line",
     "Time Format": "Time Format",
     "Transaction Type Mapping": "Transaction Type Mapping",

src/locales/zh_Hans.json

@@ -1443,6 +1443,7 @@
     "No results": "无结果",
     "Unknown": "未知",
     "Auto detect": "自动检测",
+    "Detecting...": "正在检测...",
     "Miscellaneous": "杂项",
     "Default": "默认",
     "Included": "包含",
@@ -1896,6 +1897,7 @@
     "Data File": "数据文件",
     "Data to import": "要导入的数据",
     "Please select a file to import": "请选择要导入的文件",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "无法自动检测文件编码。请选择实际的编码。",
     "Include Header Line": "包含标题行",
     "Time Format": "时间格式",
     "Amount Format": "金额格式",

src/locales/zh_Hant.json

@@ -1443,6 +1443,7 @@
     "No results": "無結果",
     "Unknown": "未知",
     "Auto detect": "自動偵測",
+    "Detecting...": "正在偵測...",
     "Miscellaneous": "雜項",
     "Default": "預設",
     "Included": "包含",
@@ -1896,6 +1897,7 @@
     "Data File": "資料檔案",
     "Data to import": "要匯入的資料",
     "Please select a file to import": "請選擇要匯入的檔案",
+    "Unable to detect the file encoding automatically. Please select the actual encoding.": "無法自動偵測檔案編碼。請選擇實際的編碼。",
     "Include Header Line": "包含標頭列",
     "Time Format": "時間格式",
     "Amount Format": "金額格式",

src/views/desktop/transactions/import/ImportDialog.vue

@@ -237,7 +237,7 @@
                        :prepend-icon="mdiClose" @click="close(false)"
                        v-if="currentStep !== 'finalResult'">{{ tt('Cancel') }}</v-btn>
                 <v-btn class="button-icon-with-direction" color="primary"
-                       :disabled="loading || submitting || (!isImportDataFromTextbox && !importFile) || (isImportDataFromTextbox && !importData)"
+                       :disabled="loading || submitting || (!isImportDataFromTextbox && !importFile) || (isImportDataFromTextbox && !importData) || (!isImportDataFromTextbox && allSupportedEncodings && fileEncoding === 'auto' && !autoDetectedFileEncoding)"
                        :append-icon="!submitting ? mdiArrowRight : undefined" @click="parseData"
                        v-if="currentStep === 'defineColumn' || currentStep === 'executeCustomScript' || currentStep === 'uploadFile'">
                     {{ tt('Next') }}
@@ -293,10 +293,12 @@ import {
     type LocalizedImportFileTypeSupportedEncodings,
     KnownFileType
 } from '@/core/file.ts';
+import { UTF_8 } from '@/consts/file.ts';
+
 import { ImportTransaction } from '@/models/imported_transaction.ts';
 
 import { isDefined, isNumber } from '@/lib/common.ts';
-import { findExtensionByType, isFileExtensionSupported } from '@/lib/file.ts';
+import { findExtensionByType, isFileExtensionSupported, detectFileEncoding } from '@/lib/file.ts';
 import { generateRandomUUID } from '@/lib/misc.ts';
 import logger from '@/lib/logger.ts';
 
@@ -330,7 +332,8 @@ const {
     joinMultiText,
     getCurrentNumeralSystemType,
     getAllSupportedImportFileCagtegoryAndTypes,
-    formatNumberToLocalizedNumerals
+    formatNumberToLocalizedNumerals,
+    getLocalizedFileEncodingName
 } = useI18n();
 
 const accountsStore = useAccountsStore();
@@ -377,7 +380,9 @@ const currentStep = ref<ImportTransactionDialogStep>('uploadFile');
 const importProcess = ref<number>(0);
 const fileType = ref<string>('ezbookkeeping');
 const fileSubType = ref<string>('ezbookkeeping_csv');
-const fileEncoding = ref<string>('utf-8');
+const fileEncoding = ref<string>('auto');
+const detectingFileEncoding = ref<boolean>(false);
+const autoDetectedFileEncoding = ref<string | undefined>(undefined);
 const processDSVMethod = ref<ImportDSVProcessMethod>(ImportDSVProcessMethod.ColumnMapping);
 const importFile = ref<File | null>(null);
 const importData = ref<string>('');
@@ -396,7 +401,39 @@ const numeralSystem = computed<NumeralSystem>(() => getCurrentNumeralSystemType(
 
 const allSupportedImportFileCategoryAndTypes = computed<LocalizedImportFileCategoryAndTypes[]>(() => getAllSupportedImportFileCagtegoryAndTypes());
 const allFileSubTypes = computed<LocalizedImportFileTypeSubType[] | undefined>(() => allSupportedImportFileTypesMap.value[fileType.value]?.subTypes);
-const allSupportedEncodings = computed<LocalizedImportFileTypeSupportedEncodings[] | undefined>(() => allSupportedImportFileTypesMap.value[fileType.value]?.supportedEncodings);
+const allSupportedEncodings = computed<LocalizedImportFileTypeSupportedEncodings[] | undefined>(() => {
+    const supportedEncodings = allSupportedImportFileTypesMap.value[fileType.value]?.supportedEncodings;
+
+    if (!supportedEncodings) {
+        return undefined;
+    }
+
+    const ret: LocalizedImportFileTypeSupportedEncodings[] = [];
+    let autoDetectDisplayName = tt('Auto detect');
+
+    if (importFile.value) {
+        if (detectingFileEncoding.value) {
+            autoDetectDisplayName += ` [${tt('Detecting...')}]`;
+        } else if (autoDetectedFileEncoding.value) {
+            autoDetectDisplayName += ` [${getLocalizedFileEncodingName(autoDetectedFileEncoding.value)}]`;
+        } else {
+            autoDetectDisplayName += ` [${tt('Unknown')}]`;
+        }
+    }
+
+    const autoDetectEncoding: LocalizedImportFileTypeSupportedEncodings = {
+        displayName: autoDetectDisplayName,
+        encoding: 'auto'
+    };
+
+    ret.push(autoDetectEncoding);
+
+    if (supportedEncodings && supportedEncodings.length) {
+        ret.push(...supportedEncodings);
+    }
+
+    return ret;
+});
 const isImportDataFromTextbox = computed<boolean>(() => allSupportedImportFileTypesMap.value[fileType.value]?.dataFromTextbox ?? false);
 const supportedAdditionalOptions = computed<ImportFileTypeSupportedAdditionalOptions | undefined>(() => allSupportedImportFileTypesMap.value[fileType.value]?.supportedAdditionalOptions);
 
@@ -508,7 +545,9 @@ function getDisplayCount(count: number): string {
 function open(): Promise<void> {
     fileType.value = 'ezbookkeeping';
     fileSubType.value = 'ezbookkeeping_csv';
-    fileEncoding.value = 'utf-8';
+    fileEncoding.value = 'auto';
+    detectingFileEncoding.value = false;
+    autoDetectedFileEncoding.value = undefined;
     processDSVMethod.value = ImportDSVProcessMethod.ColumnMapping;
     currentStep.value = 'uploadFile';
     importProcess.value = 0;
@@ -570,7 +609,21 @@ function setImportFile(event: Event): void {
     }
 
     importFile.value = el.files[0] as File;
+    detectingFileEncoding.value = false;
+    autoDetectedFileEncoding.value = undefined;
     el.value = '';
+
+    if (allSupportedEncodings.value) {
+        detectingFileEncoding.value = true;
+
+        detectFileEncoding(importFile.value).then(detectedEncoding => {
+            detectingFileEncoding.value = false;
+            autoDetectedFileEncoding.value = detectedEncoding;
+        }).catch(() => {
+            detectingFileEncoding.value = false;
+            autoDetectedFileEncoding.value = undefined;
+        });
+    }
 }
 
 function parseData(): void {
@@ -583,7 +636,11 @@ function parseData(): void {
     }
 
     if (allSupportedEncodings.value) {
-        encoding = fileEncoding.value;
+        if (fileEncoding.value === 'auto') {
+            encoding = autoDetectedFileEncoding.value;
+        } else {
+            encoding = fileEncoding.value;
+        }
     }
 
     if (!isImportDataFromTextbox.value) {
@@ -592,6 +649,13 @@ function parseData(): void {
             return;
         }
 
+        if (allSupportedEncodings.value) {
+            if (fileEncoding.value === 'auto' && !autoDetectedFileEncoding.value) {
+                snackbar.value?.showError('Unable to detect the file encoding automatically. Please select the actual encoding.');
+                return;
+            }
+        }
+
         uploadFile = importFile.value;
     } else if (isImportDataFromTextbox.value) {
         if (!importData.value) {
@@ -608,7 +672,7 @@ function parseData(): void {
             return;
         }
 
-        encoding = 'utf-8';
+        encoding = UTF_8;
     } else { // should not happen, but ts would check whether uploadFile has been assigned a value
         snackbar.value?.showMessage('An error occurred');
         return;

third-party-dependencies.json

@@ -289,6 +289,12 @@
         "url": "https://leafletjs.com/",
         "licenseUrl": "https://github.com/Leaflet/Leaflet/blob/v1.9.4/LICENSE"
     },
+    {
+        "name": "Chardet",
+        "copyright": "Copyright (C) 2024 Dmitry Shirokov",
+        "url": "https://github.com/runk/node-chardet",
+        "licenseUrl": "https://github.com/runk/node-chardet/blob/v2.1.1/LICENSE"
+    },
     {
         "name": "crypto-js",
         "copyright": "Copyright (c) 2009-2013 Jeff Mott, Copyright (c) 2013-2016 Evan Vosberg",