support custom script to process delimiter-separated values (data) file / data

src/views/desktop/transactions/import/ImportDialog.vue

@@ -22,6 +22,21 @@
                             </v-list>
                         </v-menu>
                     </v-btn>
+                    <v-btn density="comfortable" color="default" variant="text" class="ms-2"
+                           :icon="true" :disabled="loading || submitting"
+                           v-if="currentStep === 'executeCustomScript' && importTransactionExecuteCustomScriptTab?.menus">
+                        <v-icon :icon="mdiDotsVertical" />
+                        <v-menu activator="parent" max-height="500">
+                            <v-list>
+                                <v-list-item :key="index"
+                                             :prepend-icon="menu.prependIcon"
+                                             :title="menu.title"
+                                             :disabled="menu.disabled"
+                                             @click="menu.onClick()"
+                                             v-for="(menu, index) in importTransactionExecuteCustomScriptTab.menus"/>
+                            </v-list>
+                        </v-menu>
+                    </v-btn>
                     <v-btn density="comfortable" color="default" variant="text" class="ms-2"
                            :icon="true" :disabled="loading || submitting"
                            v-if="currentStep === 'checkData' && importTransactionCheckDataTab?.filterMenus">
@@ -115,6 +130,21 @@
                             />
                         </v-col>
 
+                        <v-col cols="12" md="12" v-if="fileType === 'dsv' || fileType === 'dsv_data'">
+                            <v-select
+                                item-title="displayName"
+                                item-value="type"
+                                :disabled="submitting"
+                                :label="tt('Handling Method')"
+                                :placeholder="tt('Handling Method')"
+                                :items="[
+                                    { displayName: tt('Column Mapping'), type: ImportDSVProcessMethod.ColumnMapping },
+                                    { displayName: tt('Custom Script'), type: ImportDSVProcessMethod.CustomScript }
+                                 ]"
+                                v-model="processDSVMethod"
+                            />
+                        </v-col>
+
                         <v-col cols="12" md="12" v-if="!isImportDataFromTextbox">
                             <v-text-field
                                 readonly
@@ -157,6 +187,13 @@
                         :disabled="loading || submitting"
                     />
                 </v-window-item>
+                <v-window-item value="executeCustomScript">
+                    <import-transaction-execute-custom-script-tab
+                        ref="importTransactionExecuteCustomScriptTab"
+                        :parsed-file-data="parsedFileData"
+                        :disabled="loading || submitting"
+                    />
+                </v-window-item>
                 <v-window-item value="checkData">
                     <import-transaction-check-data-tab
                         ref="importTransactionCheckDataTab"
@@ -177,7 +214,7 @@
                 <v-btn class="button-icon-with-direction" color="primary"
                        :disabled="loading || submitting || (!isImportDataFromTextbox && !importFile) || (isImportDataFromTextbox && !importData)"
                        :append-icon="!submitting ? mdiArrowRight : undefined" @click="parseData"
-                       v-if="currentStep === 'defineColumn' || currentStep === 'uploadFile'">
+                       v-if="currentStep === 'defineColumn' || currentStep === 'executeCustomScript' || currentStep === 'uploadFile'">
                     {{ tt('Next') }}
                     <v-progress-circular indeterminate size="22" class="ms-2" v-if="submitting"></v-progress-circular>
                 </v-btn>
@@ -206,6 +243,7 @@ import type { StepBarItem } from '@/components/desktop/StepsBar.vue';
 import ConfirmDialog from '@/components/desktop/ConfirmDialog.vue';
 import SnackBar from '@/components/desktop/SnackBar.vue';
 import ImportTransactionDefineColumnTab from './tabs/ImportTransactionDefineColumnTab.vue';
+import ImportTransactionExecuteCustomScriptTab from './tabs/ImportTransactionExecuteCustomScriptTab.vue';
 import ImportTransactionCheckDataTab from './tabs/ImportTransactionCheckDataTab.vue';
 
 import { ref, computed, useTemplateRef, watch } from 'vue';
@@ -222,6 +260,7 @@ import { useStatisticsStore } from '@/stores/statistics.ts';
 import { itemAndIndex } from '@/core/base.ts';
 import { type NumeralSystem } from '@/core/numeral.ts';
 import { TransactionType } from '@/core/transaction.ts';
+import { KnownFileType } from '@/core/file.ts';
 
 import type { LocalizedImportFileCategoryAndTypes, LocalizedImportFileType, LocalizedImportFileTypeSubType, LocalizedImportFileTypeSupportedEncodings } from '@/core/file.ts';
 import { ImportTransaction } from '@/models/imported_transaction.ts';
@@ -243,9 +282,14 @@ import {
 type ConfirmDialogType = InstanceType<typeof ConfirmDialog>;
 type SnackBarType = InstanceType<typeof SnackBar>;
 type ImportTransactionDefineColumnTabType = InstanceType<typeof ImportTransactionDefineColumnTab>;
+type ImportTransactionExecuteCustomScriptTabType = InstanceType<typeof ImportTransactionExecuteCustomScriptTab>;
 type ImportTransactionCheckDataTabType = InstanceType<typeof ImportTransactionCheckDataTab>;
 
-type ImportTransactionDialogStep = 'uploadFile' | 'defineColumn' | 'checkData' | 'finalResult';
+type ImportTransactionDialogStep = 'uploadFile' | 'defineColumn' | 'executeCustomScript' | 'checkData' | 'finalResult';
+enum ImportDSVProcessMethod {
+    ColumnMapping,
+    CustomScript
+};
 
 defineProps<{
     persistent?: boolean;
@@ -268,6 +312,7 @@ const statisticsStore = useStatisticsStore();
 const confirmDialog = useTemplateRef<ConfirmDialogType>('confirmDialog');
 const snackbar = useTemplateRef<SnackBarType>('snackbar');
 const importTransactionDefineColumnTab = useTemplateRef<ImportTransactionDefineColumnTabType>('importTransactionDefineColumnTab');
+const importTransactionExecuteCustomScriptTab = useTemplateRef<ImportTransactionExecuteCustomScriptTabType>('importTransactionExecuteCustomScriptTab');
 const importTransactionCheckDataTab = useTemplateRef<ImportTransactionCheckDataTabType>('importTransactionCheckDataTab');
 const fileInput = useTemplateRef<HTMLInputElement>('fileInput');
 
@@ -278,6 +323,7 @@ const importProcess = ref<number>(0);
 const fileType = ref<string>('ezbookkeeping');
 const fileSubType = ref<string>('ezbookkeeping_csv');
 const fileEncoding = ref<string>('utf-8');
+const processDSVMethod = ref<ImportDSVProcessMethod>(ImportDSVProcessMethod.ColumnMapping);
 const importFile = ref<File | null>(null);
 const importData = ref<string>('');
 const parsedFileData = ref<string[][] | undefined>(undefined);
@@ -307,11 +353,19 @@ const allSteps = computed<StepBarItem[]>(() => {
     ];
 
     if (fileType.value === 'dsv' || fileType.value === 'dsv_data') {
-        steps.push({
-            name: 'defineColumn',
-            title: tt('Define Column'),
-            subTitle: tt('Define and Check Column Mapping')
-        });
+        if (processDSVMethod.value === ImportDSVProcessMethod.CustomScript) {
+            steps.push({
+                name: 'executeCustomScript',
+                title: tt('Execute Custom Script'),
+                subTitle: tt('Execute Custom Script to Parse Data')
+            });
+        } else {
+            steps.push({
+                name: 'defineColumn',
+                title: tt('Define Column'),
+                subTitle: tt('Define and Check Column Mapping')
+            });
+        }
     }
 
     steps.push(...[
@@ -378,12 +432,14 @@ function open(): Promise<void> {
     fileType.value = 'ezbookkeeping';
     fileSubType.value = 'ezbookkeeping_csv';
     fileEncoding.value = 'utf-8';
+    processDSVMethod.value = ImportDSVProcessMethod.ColumnMapping;
     currentStep.value = 'uploadFile';
     importProcess.value = 0;
     importFile.value = null;
     importData.value = '';
     parsedFileData.value = undefined;
     importTransactionDefineColumnTab.value?.reset();
+    importTransactionExecuteCustomScriptTab.value?.reset();
     importTransactions.value = undefined;
     importTransactionCheckDataTab.value?.reset();
     showState.value = true;
@@ -466,9 +522,9 @@ function parseData(): void {
         }
 
         if (type === 'custom_csv') {
-            uploadFile = new File([importData.value], 'import.csv', { type: 'text/csv' });
+            uploadFile = KnownFileType.CSV.createFile(importData.value, 'import');
         } else if (type === 'custom_tsv') {
-            uploadFile = new File([importData.value], 'import.tsv', { type: 'text/tab-separated-values' });
+            uploadFile = KnownFileType.TSV.createFile(importData.value, 'import');
         } else {
             snackbar.value?.showError('Parameter Invalid');
             return;
@@ -491,9 +547,15 @@ function parseData(): void {
             importFile: uploadFile
         }).then(response => {
             if (response && response.length) {
-                importTransactionDefineColumnTab.value?.reset();
-                parsedFileData.value = response;
-                currentStep.value = 'defineColumn';
+                if (processDSVMethod.value === ImportDSVProcessMethod.CustomScript) {
+                    importTransactionExecuteCustomScriptTab.value?.reset();
+                    parsedFileData.value = response;
+                    currentStep.value = 'executeCustomScript';
+                } else {
+                    importTransactionDefineColumnTab.value?.reset();
+                    parsedFileData.value = response;
+                    currentStep.value = 'defineColumn';
+                }
             } else {
                 parsedFileData.value = undefined;
                 snackbar.value?.showError('No data to import');
@@ -519,7 +581,7 @@ function parseData(): void {
         let geoLocationOrder: string | undefined = undefined;
         let tagSeparator: string | undefined = undefined;
 
-        if (isDsvFileType) {
+        if (isDsvFileType && processDSVMethod.value === ImportDSVProcessMethod.ColumnMapping) {
             const defineColumnResult = importTransactionDefineColumnTab.value?.generateResult();
 
             if (!defineColumnResult) {
@@ -536,6 +598,16 @@ function parseData(): void {
             geoLocationSeparator = defineColumnResult.geoLocationSeparator;
             geoLocationOrder = defineColumnResult.geoLocationOrder;
             tagSeparator = defineColumnResult.tagSeparator;
+        } else if (isDsvFileType && processDSVMethod.value === ImportDSVProcessMethod.CustomScript) {
+            const executeCustomScriptResult = importTransactionExecuteCustomScriptTab.value?.generateResult();
+
+            if (!executeCustomScriptResult) {
+                return;
+            }
+
+            type = 'ezbookkeeping_json';
+            encoding = undefined;
+            uploadFile = KnownFileType.JSON.createFile(executeCustomScriptResult, 'import');
         }
 
         submitting.value = true;

src/views/desktop/transactions/import/tabs/ImportTransactionExecuteCustomScriptTab.vue

@@ -0,0 +1,346 @@
+<template>
+    <v-row>
+        <v-col cols="12" md="6">
+            <div class="d-flex w-100 mb-2">
+                <v-btn density="compact" variant="tonal" :prepend-icon="mdiPlay"
+                       :disabled="disabled || !sandboxLoaded || executingScript" :loading="executingScript"
+                       @click="executeCustomScript()">
+                    <template #loader>
+                        <v-progress-circular indeterminate size="18" class="me-1"/>
+                        <span>{{ tt('Execute Custom Script') }}</span>
+                    </template>
+                    <span>{{ tt('Execute Custom Script') }}</span>
+                </v-btn>
+            </div>
+            <v-textarea class="w-100" style="height: 360px" :readonly="disabled"
+                        v-model="customScript"></v-textarea>
+        </v-col>
+        <v-col cols="12" md="6">
+            <div class="d-flex w-100 mb-2">
+                <v-btn density="compact" color="default" variant="text"
+                       :disabled="disabled || !sandboxLoaded || executingScript || !previewResult">
+                    <span>{{ tt('format.misc.previewCount', { count: previewCount > 0 ? getDisplayCount(previewCount) : tt('All') }) }}</span>
+                    <v-menu activator="parent">
+                        <v-list>
+                            <v-list-item :key="count.value" :title="count.name"
+                                         v-for="count in previewCounts"
+                                         @click="previewCount = count.value"></v-list-item>
+                        </v-list>
+                    </v-menu>
+                </v-btn>
+            </div>
+            <div class="w-100 code-container">
+                <v-textarea class="w-100 always-cursor-text" style="height: 360px" :readonly="true"
+                            :value="displayPreviewResult"></v-textarea>
+            </div>
+        </v-col>
+    </v-row>
+
+    <iframe id="sandbox" ref="sandbox" sandbox="allow-scripts" style="display: none;"></iframe>
+    <snack-bar ref="snackbar" />
+</template>
+
+<script setup lang="ts">
+import SnackBar from '@/components/desktop/SnackBar.vue';
+
+import { ref, computed, useTemplateRef, onMounted, onUnmounted } from 'vue';
+
+import { useI18n } from '@/locales/helpers.ts';
+
+import type { NameNumeralValue } from '@/core/base.ts';
+import type { NumeralSystem } from '@/core/numeral.ts';
+import { KnownFileType } from '@/core/file.ts';
+
+import type { ImportTransactionRequest, ImportTransactionRequestItem } from '@/models/imported_transaction.ts';
+
+import { isDefined } from '@/lib/common.ts';
+import {
+    openTextFileContent,
+    startDownloadFile
+} from '@/lib/ui/common.ts';
+import logger from '@/lib/logger.ts';
+
+import {
+    mdiPlay,
+    mdiFolderOpenOutline,
+    mdiContentSaveOutline
+} from '@mdi/js';
+
+type SnackBarType = InstanceType<typeof SnackBar>;
+
+type SandboxRequest = {
+    parsedFileData: string[][];
+    code: string;
+};
+
+type SandboxResponse = {
+    result?: string;
+    knownError?: string;
+    error?: string;
+};
+
+interface ImportTransactionDefineColumnMenu {
+    prependIcon: string;
+    title: string;
+    disabled?: boolean;
+    onClick: () => void;
+}
+
+const props = defineProps<{
+    parsedFileData?: string[][];
+    disabled?: boolean;
+}>();
+
+const { tt, getCurrentNumeralSystemType } = useI18n();
+
+const sandbox = useTemplateRef<HTMLIFrameElement>('sandbox');
+const snackbar = useTemplateRef<SnackBarType>('snackbar');
+
+const sandboxLoaded = ref<boolean>(false);
+const customScript = ref<string>('');
+const previewResult = ref<ImportTransactionRequestItem[] | undefined>(undefined);
+const executingScript = ref<boolean>(false);
+const previewCount = ref<number>(10);
+
+const numeralSystem = computed<NumeralSystem>(() => getCurrentNumeralSystemType());
+const previewCounts = computed<NameNumeralValue[]>(() => getTablePageOptions(previewResult.value?.length));
+
+const sampleScript = computed<string>(() => `// ${tt('sample.importTransactionCustomScript.headerComment')}
+/**
+ * ${tt('sample.importTransactionCustomScript.functionDescription')}
+ * @param {array} row - ${tt('sample.importTransactionCustomScript.functionParamRowDescription')}
+ * @param {number} index - ${tt('sample.importTransactionCustomScript.functionParamIndexDescription')}
+ * @returns {object|null} ${tt('sample.importTransactionCustomScript.functionReturnDescription')}
+ */
+function parse(row, index) {
+    if (index < 1) {
+        return null;
+    }
+
+    return {
+        time: row[0], // ${tt('sample.importTransactionCustomScript.fieldTimeDescription')}
+        utcOffset: '480', // ${tt('sample.importTransactionCustomScript.fieldUtcOffsetDescription')}
+        type: TransactionType.Expense, // ${tt('sample.importTransactionCustomScript.fieldTypeDescription')}
+        categoryName: row[4], // ${tt('sample.importTransactionCustomScript.fieldCategoryNameDescription')}
+        sourceAccountName: row[5], // ${tt('sample.importTransactionCustomScript.fieldSourceAccountNameDescription')}
+        destinationAccountName: row[8], // ${tt('sample.importTransactionCustomScript.fieldDestinationAccountNameDescription')}
+        sourceAmount: row[7], // ${tt('sample.importTransactionCustomScript.fieldSourceAmountDescription')}
+        destinationAmount: row[10], // ${tt('sample.importTransactionCustomScript.fieldDestinationAmountDescription')}
+        geoLocation: undefined, // ${tt('sample.importTransactionCustomScript.fieldGeoLocationDescription')}
+        tagNames: '', // ${tt('sample.importTransactionCustomScript.fieldTagNamesDescription')}
+        description: row[13] // ${tt('sample.importTransactionCustomScript.fieldCommentDescription')}
+    };
+}`);
+
+const displayPreviewResult = computed<string>(() => {
+    if (executingScript.value) {
+        return tt('Executing Script...');
+    } else if (previewResult.value) {
+        const rows = previewResult.value.slice(0, previewCount.value);
+        return JSON.stringify(rows, null, 2);
+    } else {
+        return tt('No Preview Result');
+    }
+});
+
+const menus = computed<ImportTransactionDefineColumnMenu[]>(() => [
+    {
+        prependIcon: mdiFolderOpenOutline,
+        title: tt('Load Script File'),
+        onClick: loadScriptFile
+    },
+    {
+        prependIcon: mdiContentSaveOutline,
+        title: tt('Save Script File'),
+        onClick: saveScriptFile
+    }
+]);
+
+function getDisplayCount(count: number): string {
+    return numeralSystem.value.formatNumber(count);
+}
+
+function getTablePageOptions(linesCount?: number): NameNumeralValue[] {
+    const pageOptions: NameNumeralValue[] = [];
+
+    if (!linesCount || linesCount < 1) {
+        pageOptions.push({ value: -1, name: tt('All') });
+        return pageOptions;
+    }
+
+    for (const count of [ 10, 50, 100 ]) {
+        if (linesCount < count) {
+            break;
+        }
+
+        pageOptions.push({ value: count, name: getDisplayCount(count) });
+    }
+
+    pageOptions.push({ value: -1, name: tt('All') });
+
+    return pageOptions;
+}
+
+function reloadSandbox(): void {
+    sandboxLoaded.value = false;
+
+    if (sandbox.value) {
+        sandbox.value.src = 'about:blank';
+        sandbox.value.srcdoc = `
+            <script>
+                window.TransactionType = {
+                    Income: 'Income',
+                    Expense: 'Expense',
+                    Transfer: 'Transfer'
+                };
+
+                window.addEventListener('message', function (event) {
+                    try {
+                        const data = JSON.parse(event.data);
+                        const parsedFileData = data.parsedFileData;
+                        eval(data.code);
+
+                        if (window.parse) {
+                            const result = [];
+
+                            for (let i = 0; i < parsedFileData.length; i++) {
+                                try {
+                                    const row = parsedFileData[i];
+                                    const transaction = window.parse(row, i);
+
+                                    if (transaction) {
+                                        result.push(transaction);
+                                    }
+                                } catch (error) {
+                                    window.parent.postMessage({ error: error.message }, '*');
+                                    return;
+                                }
+                            }
+
+                            window.parent.postMessage({ result: JSON.stringify(result) }, '*');
+                        } else {
+                            window.parent.postMessage({ knownError: 'No parse function defined' }, '*');
+                        }
+                    } catch (error) {
+                        window.parent.postMessage({ error: error.message }, '*');
+                    }
+                });
+            <\/script>
+        `;
+
+        sandbox.value.onload = () => {
+            sandboxLoaded.value = true;
+        };
+    }
+}
+
+function executeCustomScript(): void {
+    if (!sandbox.value || props.disabled || executingScript.value) {
+        return;
+    }
+
+    executingScript.value = true;
+
+    const sandboxRequest: SandboxRequest = {
+        parsedFileData: props.parsedFileData || [],
+        code: customScript.value + '\n\n;window.parse = parse;'
+    };
+
+    sandbox.value?.contentWindow?.postMessage(JSON.stringify(sandboxRequest), '*');
+}
+
+function generateResult(): string | undefined {
+    if (!previewResult.value) {
+        snackbar.value?.showError('Please execute the custom script first');
+        return undefined;
+    }
+
+    const result: ImportTransactionRequest = {
+        transactions: previewResult.value
+    };
+
+    return JSON.stringify(result);
+}
+
+function reset(): void {
+    customScript.value = sampleScript.value;
+    previewResult.value = undefined;
+    executingScript.value = false;
+    previewCount.value = 10;
+}
+
+function loadScriptFile(): void {
+    openTextFileContent({
+        allowedExtensions: KnownFileType.JS.contentType
+    }).then(content => {
+        customScript.value = content;
+    }).catch(error => {
+        logger.error('Failed to load script file', error);
+        snackbar.value?.showError('Cannot load script file');
+    });
+}
+
+function saveScriptFile(): void {
+    const fileName = KnownFileType.JS.formatFileName(tt('dataExport.defaultImportHandlingScript'));
+    startDownloadFile(fileName, KnownFileType.JS.createBlob(customScript.value));
+}
+
+function onMessage(event: MessageEvent<SandboxResponse>): void {
+    if (event.source !== sandbox.value?.contentWindow) {
+        return;
+    }
+
+    executingScript.value = false;
+
+    const data = event.data;
+
+    if (data.knownError) {
+        snackbar.value?.showError(data.knownError);
+        previewResult.value = undefined;
+    } else if (data.error) {
+        logger.error('Failed to execute custom script: ' + data.error);
+        snackbar.value?.showError('Failed to execute custom script');
+        previewResult.value = undefined;
+    } else if (data.result) {
+        const originalResult = JSON.parse(data.result) as Record<string, unknown>[];
+        const finalResult: ImportTransactionRequestItem[] = [];
+
+        for (const item of originalResult) {
+            const finalItem: ImportTransactionRequestItem = {
+                time: (isDefined(item['time'])) ? String(item['time']) : '',
+                utcOffset: (isDefined(item['utcOffset'])) ? String(item['utcOffset']) : '',
+                type: (isDefined(item['type'])) ? String(item['type']) : '',
+                categoryName: (isDefined(item['categoryName']) && item['categoryName'] !== '') ? String(item['categoryName']) : undefined,
+                sourceAccountName: (isDefined(item['sourceAccountName']) && item['sourceAccountName'] !== '') ? String(item['sourceAccountName']) : undefined,
+                destinationAccountName: (isDefined(item['destinationAccountName']) && item['destinationAccountName'] !== '') ? String(item['destinationAccountName']) : undefined,
+                sourceAmount: (isDefined(item['sourceAmount'])) ? String(item['sourceAmount']) : '',
+                destinationAmount: (isDefined(item['destinationAmount']) && item['destinationAmount'] !== '') ? String(item['destinationAmount']) : undefined,
+                geoLocation: (isDefined(item['geoLocation']) && item['geoLocation']) ? String(item['geoLocation']) : undefined,
+                tagNames: (isDefined(item['tagNames']) && item['tagNames']) ? String(item['tagNames']) : undefined,
+                comment: (isDefined(item['description']) && item['description']) ? String(item['description']) : undefined
+            };
+            finalResult.push(finalItem);
+        }
+
+        previewResult.value = finalResult;
+    }
+
+    reloadSandbox();
+}
+
+onMounted(() => {
+    customScript.value = sampleScript.value;
+    reloadSandbox();
+    window.addEventListener('message', onMessage);
+});
+
+onUnmounted(() => {
+    window.removeEventListener('message', onMessage);
+});
+
+defineExpose({
+    menus,
+    generateResult,
+    reset
+});
+</script>