create transactions from AI receipt image recognition results

src/views/desktop/transactions/ListPage.vue

@@ -63,11 +63,16 @@
                                             <v-btn class="ms-3" color="default" variant="outlined"
                                                    :disabled="loading || !canAddTransaction" @click="add()">
                                                 {{ tt('Add') }}
-                                                <v-menu activator="parent" :open-on-hover="true" v-if="allTransactionTemplates && allTransactionTemplates.length">
+                                                <v-menu activator="parent" :open-on-hover="true" v-if="isTransactionFromAIImageRecognitionEnabled() || (allTransactionTemplates && allTransactionTemplates.length)">
                                                     <v-list>
-                                                        <v-list-item :title="template.name"
+                                                        <v-list-item key="AIImageRecognition"
+                                                                     :title="tt('AI Image Recognition')"
+                                                                     :prepend-icon="mdiMagicStaff"
+                                                                     v-if="isTransactionFromAIImageRecognitionEnabled()"
+                                                                     @click="addByRecognizingImage"></v-list-item>
+                                                        <v-list-item :key="template.id"
+                                                                     :title="template.name"
                                                                      :prepend-icon="mdiTextBoxOutline"
-                                                                     :key="template.id"
                                                                      v-for="template in allTransactionTemplates"
                                                                      @click="add(template)"></v-list-item>
                                                     </v-list>
@@ -620,6 +625,7 @@
                             @error="onShowDateRangeError" />
 
     <edit-dialog ref="editDialog" :type="TransactionEditPageType.Transaction" />
+    <a-i-image-recognition-dialog ref="aiImageRecognitionDialog" />
     <import-dialog ref="importDialog" :persistent="true" />
 
     <v-dialog width="800" v-model="showFilterAccountDialog">
@@ -647,6 +653,7 @@ import PaginationButtons from '@/components/desktop/PaginationButtons.vue';
 import ConfirmDialog from '@/components/desktop/ConfirmDialog.vue';
 import SnackBar from '@/components/desktop/SnackBar.vue';
 import EditDialog from './list/dialogs/EditDialog.vue';
+import AIImageRecognitionDialog from './list/dialogs/AIImageRecognitionDialog.vue';
 import ImportDialog from './import/ImportDialog.vue';
 import AccountFilterSettingsCard from '@/views/desktop/common/cards/AccountFilterSettingsCard.vue';
 import CategoryFilterSettingsCard from '@/views/desktop/common/cards/CategoryFilterSettingsCard.vue';
@@ -716,7 +723,7 @@ import {
     categoryTypeToTransactionType,
     transactionTypeToCategoryType
 } from '@/lib/category.ts';
-import { isDataExportingEnabled, isDataImportingEnabled } from '@/lib/server_settings.ts';
+import { isDataExportingEnabled, isDataImportingEnabled, isTransactionFromAIImageRecognitionEnabled } from '@/lib/server_settings.ts';
 import { startDownloadFile } from '@/lib/ui/common.ts';
 import { scrollToSelectedItem } from '@/lib/ui/desktop.ts';
 import logger from '@/lib/logger.ts';
@@ -738,6 +745,7 @@ import {
     mdiMinusBoxMultipleOutline,
     mdiCloseBoxMultipleOutline,
     mdiPound,
+    mdiMagicStaff,
     mdiTextBoxOutline
 } from '@mdi/js';
 
@@ -760,6 +768,7 @@ const props = defineProps<TransactionListProps>();
 type ConfirmDialogType = InstanceType<typeof ConfirmDialog>;
 type SnackBarType = InstanceType<typeof SnackBar>;
 type EditDialogType = InstanceType<typeof EditDialog>;
+type AIImageRecognitionDialogType = InstanceType<typeof AIImageRecognitionDialog>;
 type ImportDialogType = InstanceType<typeof ImportDialog>;
 
 interface TransactionTemplateWithIcon {
@@ -859,6 +868,7 @@ const tagFilterMenu = useTemplateRef<VMenu>('tagFilterMenu');
 const confirmDialog = useTemplateRef<ConfirmDialogType>('confirmDialog');
 const snackbar = useTemplateRef<SnackBarType>('snackbar');
 const editDialog = useTemplateRef<EditDialogType>('editDialog');
+const aiImageRecognitionDialog = useTemplateRef<AIImageRecognitionDialogType>('aiImageRecognitionDialog');
 const importDialog = useTemplateRef<ImportDialogType>('importDialog');
 
 const activeTab = ref<string>('transactionPage');
@@ -1597,6 +1607,33 @@ function add(template?: TransactionTemplate): void {
     });
 }
 
+function addByRecognizingImage(): void {
+    aiImageRecognitionDialog.value?.open().then(result => {
+        editDialog.value?.open({
+            time: result.time,
+            type: result.type,
+            categoryId: result.categoryId,
+            accountId: result.sourceAccountId,
+            destinationAccountId: result.destinationAccountId,
+            amount: result.sourceAmount,
+            destinationAmount: result.destinationAmount,
+            tagIds: result.tagIds ? result.tagIds.join(',') : undefined,
+            comment: result.comment,
+            noTransactionDraft: true
+        }).then(result => {
+            if (result && result.message) {
+                snackbar.value?.showMessage(result.message);
+            }
+
+            reload(false, false);
+        }).catch(error => {
+            if (error) {
+                snackbar.value?.showError(error);
+            }
+        });
+    });
+}
+
 function importTransaction(): void {
     importDialog.value?.open().then(() => {
         reload(false, false);

src/views/desktop/transactions/list/dialogs/AIImageRecognitionDialog.vue

@@ -0,0 +1,208 @@
+<template>
+    <v-dialog width="800" :persistent="loading || recognizing || !!imageFile" v-model="showState">
+        <v-card class="pa-2 pa-sm-4 pa-md-4">
+            <template #title>
+                <div class="d-flex align-center justify-center">
+                    <h4 class="text-h4">{{ tt('AI Image Recognition') }}</h4>
+                </div>
+            </template>
+
+            <v-card-text class="d-flex justify-center w-100 my-md-4 pt-0">
+                <div class="w-100 border position-relative"
+                     @dragenter.prevent="onDragEnter"
+                     @dragover.prevent
+                     @dragleave.prevent="onDragLeave"
+                     @drop.prevent="onDrop">
+                    <div class="d-flex w-100 fill-height justify-center align-center justify-content-center"
+                         :class="{ 'dropzone': true, 'dropzone-dragover': isDragOver }" style="height: 480px">
+                        <h3 v-if="!imageFile && !isDragOver">{{ tt('Drag and drop a receipt or transaction image here, or click to select one') }}</h3>
+                        <h3 v-if="isDragOver">{{ tt('Release to load image') }}</h3>
+                    </div>
+                    <v-img height="480px" :class="{ 'cursor-pointer': !loading || !recognizing || !isDragOver }"
+                           :src="imageSrc" @click="showOpenImageDialog">
+                        <template #placeholder>
+                            <div class="w-100 fill-height bg-grey-200"></div>
+                        </template>
+                    </v-img>
+                </div>
+            </v-card-text>
+
+            <v-card-text class="overflow-y-visible">
+                <div ref="buttonContainer" class="w-100 d-flex justify-center gap-4">
+                    <v-btn :disabled="loading || recognizing || !imageFile" @click="recognize">
+                        {{ tt('Recognize') }}
+                        <v-progress-circular indeterminate size="22" class="ms-2" v-if="recognizing"></v-progress-circular>
+                    </v-btn>
+                    <v-btn color="secondary" variant="tonal" :disabled="loading || recognizing"
+                           @click="cancel">{{ tt('Cancel') }}</v-btn>
+                </div>
+            </v-card-text>
+        </v-card>
+    </v-dialog>
+
+    <snack-bar ref="snackbar" />
+    <input ref="imageInput" type="file" style="display: none" :accept="SUPPORTED_IMAGE_EXTENSIONS" @change="openImage($event)" />
+</template>
+
+<script setup lang="ts">
+import SnackBar from '@/components/desktop/SnackBar.vue';
+
+import { ref, useTemplateRef } from 'vue';
+
+import { useI18n } from '@/locales/helpers.ts';
+
+import { useTransactionsStore } from '@/stores/transaction.ts';
+
+import { KnownFileType } from '@/core/file.ts';
+import { SUPPORTED_IMAGE_EXTENSIONS } from '@/consts/file.ts';
+
+import type { RecognizedReceiptImageResponse } from '@/models/large_language_model.ts';
+
+import { compressJpgImage } from '@/lib/ui/common.ts';
+import logger from '@/lib/logger.ts';
+
+type SnackBarType = InstanceType<typeof SnackBar>;
+
+const { tt } = useI18n();
+
+const transactionsStore = useTransactionsStore();
+
+const snackbar = useTemplateRef<SnackBarType>('snackbar');
+const imageInput = useTemplateRef<HTMLInputElement>('imageInput');
+
+let resolveFunc: ((response: RecognizedReceiptImageResponse) => void) | null = null;
+let rejectFunc: ((reason?: unknown) => void) | null = null;
+
+const showState = ref<boolean>(false);
+const loading = ref<boolean>(false);
+const recognizing = ref<boolean>(false);
+const imageFile = ref<File | null>(null);
+const imageSrc = ref<string | undefined>(undefined);
+const isDragOver = ref<boolean>(false);
+
+function loadImage(file: File): void {
+    compressJpgImage(file, 1280, 1280, 0.8).then(blob => {
+        imageFile.value = KnownFileType.JPG.createFileFromBlob(blob, "image");
+        imageSrc.value = URL.createObjectURL(blob);
+    }).catch(error => {
+        imageFile.value = null;
+        imageSrc.value = undefined;
+        logger.error('failed to compress image', error);
+        snackbar.value?.showError('Unable to load image');
+    });
+}
+
+function open(): Promise<RecognizedReceiptImageResponse> {
+    showState.value = true;
+    loading.value = false;
+    recognizing.value = false;
+    imageFile.value = null;
+    imageSrc.value = undefined;
+
+    return new Promise((resolve, reject) => {
+        resolveFunc = resolve;
+        rejectFunc = reject;
+    });
+}
+
+function showOpenImageDialog(): void {
+    if (loading.value || recognizing.value || isDragOver.value) {
+        return;
+    }
+
+    imageInput.value?.click();
+}
+
+function openImage(event: Event): void {
+    if (!event || !event.target) {
+        return;
+    }
+
+    const el = event.target as HTMLInputElement;
+
+    if (!el.files || !el.files.length || !el.files[0]) {
+        return;
+    }
+
+    const image = el.files[0] as File;
+
+    el.value = '';
+
+    loadImage(image);
+}
+
+function recognize(): void {
+    if (loading.value || recognizing.value || !imageFile.value) {
+        return;
+    }
+
+    recognizing.value = true;
+
+    transactionsStore.recognizeReceiptImage({
+        imageFile: imageFile.value
+    }).then(response => {
+        resolveFunc?.(response);
+        showState.value = false;
+        recognizing.value = false;
+    }).catch(error => {
+        recognizing.value = false;
+
+        if (!error.processed) {
+            snackbar.value?.showError(error);
+        }
+    });
+}
+
+function cancel(): void {
+    rejectFunc?.();
+    showState.value = false;
+    loading.value = false;
+    recognizing.value = false;
+    imageFile.value = null;
+    imageSrc.value = undefined;
+}
+
+function onDragEnter(): void {
+    if (loading.value || recognizing.value) {
+        return;
+    }
+
+    isDragOver.value = true;
+}
+
+function onDragLeave(): void {
+    isDragOver.value = false;
+}
+
+function onDrop(event: DragEvent): void {
+    if (loading.value || recognizing.value) {
+        return;
+    }
+
+    isDragOver.value = false;
+
+    if (event.dataTransfer && event.dataTransfer.files && event.dataTransfer.files.length && event.dataTransfer.files[0]) {
+        loadImage(event.dataTransfer.files[0] as File);
+    }
+}
+
+defineExpose({
+    open
+});
+</script>
+
+<style>
+.dropzone {
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    pointer-events: none;
+    border-radius: 8px;
+    z-index: 10;
+}
+
+.dropzone-dragover {
+    border: 6px dashed rgba(var(--v-border-color),var(--v-border-opacity));
+}
+</style>

src/views/mobile/HomePage.vue

@@ -188,7 +188,14 @@
         <f7-popover class="template-popover-menu" target-el="#homepage-add-button"
                     v-model:opened="showTransactionTemplatePopover">
             <f7-list dividers v-if="allTransactionTemplates">
-                <f7-list-item :title="template.name" :key="template.id"
+                <f7-list-item key="AIImageRecognition" :title="tt('AI Image Recognition')"
+                              @click="showAIReceiptImageRecognitionSheet = true; showTransactionTemplatePopover = false"
+                              v-if="isTransactionFromAIImageRecognitionEnabled()">
+                    <template #media>
+                        <f7-icon f7="wand_stars"></f7-icon>
+                    </template>
+                </f7-list-item>
+                <f7-list-item :key="template.id" :title="template.name"
                               :link="'/transaction/add?templateId=' + template.id"
                               v-for="template in allTransactionTemplates">
                     <template #media>
@@ -197,11 +204,15 @@
                 </f7-list-item>
             </f7-list>
         </f7-popover>
+
+        <a-i-image-recognition-sheet v-model:show="showAIReceiptImageRecognitionSheet"
+                                     @recognition:change="onReceiptRecognitionChanged"/>
     </f7-page>
 </template>
 
 <script setup lang="ts">
 import { ref, computed } from 'vue';
+import type { Router } from 'framework7/types';
 
 import { useI18n } from '@/locales/helpers.ts';
 import { useI18nUIComponents } from '@/lib/ui/mobile.ts';
@@ -215,8 +226,14 @@ import { useOverviewStore } from '@/stores/overview.ts';
 import { DateRange } from '@/core/datetime.ts';
 import { TemplateType } from '@/core/template.ts';
 import { TransactionTemplate } from '@/models/transaction_template.ts';
+import type { RecognizedReceiptImageResponse } from '@/models/large_language_model.ts';
 
 import { isUserLogined, isUserUnlocked } from '@/lib/userstate.ts';
+import { isTransactionFromAIImageRecognitionEnabled } from '@/lib/server_settings.ts';
+
+const props = defineProps<{
+    f7router: Router.Router;
+}>();
 
 const { tt } = useI18n();
 const { showToast } = useI18nUIComponents();
@@ -236,6 +253,7 @@ const overviewStore = useOverviewStore();
 
 const loading = ref<boolean>(true);
 const showTransactionTemplatePopover = ref<boolean>(false);
+const showAIReceiptImageRecognitionSheet = ref<boolean>(false);
 
 const allTransactionTemplates = computed<TransactionTemplate[]>(() => {
     const allTemplates = transactionTemplatesStore.allVisibleTemplates;
@@ -243,7 +261,7 @@ const allTransactionTemplates = computed<TransactionTemplate[]>(() => {
 });
 
 function openTransactionTemplatePopover(): void {
-    if (allTransactionTemplates.value && allTransactionTemplates.value.length) {
+    if (isTransactionFromAIImageRecognitionEnabled() || (allTransactionTemplates.value && allTransactionTemplates.value.length)) {
         showTransactionTemplatePopover.value = true;
     }
 }
@@ -291,6 +309,48 @@ function reload(done?: () => void): void {
     });
 }
 
+function onReceiptRecognitionChanged(result: RecognizedReceiptImageResponse): void {
+    const params: string[] = [];
+
+    if (result.type) {
+        params.push(`type=${result.type}`);
+    }
+
+    if (result.time) {
+        params.push(`time=${result.time}`);
+    }
+
+    if (result.categoryId) {
+        params.push(`categoryId=${result.categoryId}`);
+    }
+
+    if (result.sourceAccountId) {
+        params.push(`accountId=${result.sourceAccountId}`);
+    }
+
+    if (result.destinationAccountId) {
+        params.push(`destinationAccountId=${result.destinationAccountId}`);
+    }
+
+    if (result.sourceAmount) {
+        params.push(`amount=${result.sourceAmount}`);
+    }
+
+    if (result.destinationAmount) {
+        params.push(`destinationAmount=${result.destinationAmount}`);
+    }
+
+    if (result.tagIds) {
+        params.push(`tagIds=${result.tagIds.join(',')}`);
+    }
+
+    if (result.comment) {
+        params.push(`comment=${encodeURIComponent(result.comment)}`);
+    }
+
+    props.f7router.navigate(`/transaction/add?${params.join('&')}`);
+}
+
 function onPageAfterIn(): void {
     if (!loading.value) {
         reload();