support receiving images from the Web Share Target API Level 2 and directly opening AI image recognition on mobile version

src/components/mobile/AIImageRecognitionSheet.vue

@@ -70,12 +70,12 @@ const cancelRecognizingUuid = ref<string | undefined>(undefined);
 const imageFile = ref<File | null>(null);
 const imageSrc = ref<string | undefined>(undefined);
 
-function loadImage(file: File): void {
+function loadImage(image: Blob): void {
     loading.value = true;
     imageFile.value = null;
     imageSrc.value = undefined;
 
-    compressJpgImage(file, 1280, 1280, 0.8).then(blob => {
+    compressJpgImage(image, 1280, 1280, 0.8).then(blob => {
         imageFile.value = KnownFileType.JPG.createFileFromBlob(blob, "image");
         imageSrc.value = URL.createObjectURL(blob);
         loading.value = false;
@@ -184,6 +184,10 @@ function onSheetOpen(): void {
 function onSheetClosed(): void {
     close();
 }
+
+defineExpose({
+    loadImage
+});
 </script>
 
 <style>

src/consts/cache.ts

@@ -3,6 +3,7 @@ export const SW_RUNTIME_CACHE_NAME_PREFIX: string = 'workbox-runtime-';
 export const SW_ASSETS_CACHE_NAME: string = 'ezbookkeeping-assets-cache';
 export const SW_CODE_CACHE_NAME: string = 'ezbookkeeping-code-cache';
 export const SW_MAP_CACHE_NAME: string = 'ezbookkeeping-map-cache';
+export const SW_SHARE_CACHE_NAME: string = 'ezbookkeeping-share-cache';
 
 export const SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG: string = 'UPDATE_MAP_CACHE_CONFIG';
 export const SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG_RESPONSE: string = 'UPDATE_MAP_CACHE_CONFIG_RESPONSE';

src/lib/cache.ts

@@ -9,6 +9,7 @@ import {
     SW_ASSETS_CACHE_NAME,
     SW_CODE_CACHE_NAME,
     SW_MAP_CACHE_NAME,
+    SW_SHARE_CACHE_NAME,
     SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG,
     SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG_RESPONSE,
     MAP_CACHE_MAX_ENTRIES
@@ -103,6 +104,42 @@ async function getCacheTotalSize(cacheName: string): Promise<number> {
     return totalSize;
 }
 
+export function getShareCacheImageBlob(): Promise<Blob | undefined> {
+    if (!window.caches) {
+        logger.error('caches API is not supported in this browser');
+        return Promise.resolve(undefined);
+    }
+
+    return new Promise((resolve) => {
+        window.caches.open(SW_SHARE_CACHE_NAME).then(cache => {
+            cache.match(SW_SHARE_CACHE_NAME).then(response => {
+                if (!response) {
+                    resolve(undefined);
+                    return;
+                }
+
+                response.blob().then(blob => {
+                    cache.delete(SW_SHARE_CACHE_NAME).then(() => {
+                        resolve(blob);
+                    }).catch(error => {
+                        logger.warn('failed to delete share cache image blob', error);
+                        resolve(blob);
+                    });
+                }).catch(error => {
+                    logger.error('failed to read share cache image blob', error);
+                    resolve(undefined);
+                });
+            }).catch(error => {
+                logger.error('failed to match share cache image blob', error);
+                resolve(undefined);
+            });
+        }).catch(error => {
+            logger.error('failed to open share cache', error);
+            resolve(undefined);
+        });
+    });
+}
+
 export function loadBrowserCacheStatistics(): Promise<BrowserCacheStatistics> {
     return new Promise((resolve, reject) => {
         const caches = window.caches;

src/lib/ui/common.ts

@@ -188,7 +188,7 @@ export function startDownloadFile(fileName: string, fileData: Blob): void {
     dataLink.click();
 }
 
-export function compressJpgImage(file: File, maxWidth: number, maxHeight: number, quality: number): Promise<Blob> {
+export function compressJpgImage(blob: Blob, maxWidth: number, maxHeight: number, quality: number): Promise<Blob> {
     return new Promise((resolve, reject) => {
         const reader = new FileReader();
 
@@ -242,6 +242,6 @@ export function compressJpgImage(file: File, maxWidth: number, maxHeight: number
             reject(error);
         };
 
-        reader.readAsDataURL(file);
+        reader.readAsDataURL(blob);
     });
 }

src/sw.ts

@@ -177,6 +177,9 @@ declare const self: ServiceWorkerGlobalScope;
 const SW_ASSETS_CACHE_NAME: string = 'ezbookkeeping-assets-cache';
 const SW_CODE_CACHE_NAME: string = 'ezbookkeeping-code-cache';
 const SW_MAP_CACHE_NAME: string = 'ezbookkeeping-map-cache';
+const SW_SHARE_CACHE_NAME: string = 'ezbookkeeping-share-cache';
+const SW_SHARE_IMAGE_URL_PATHNAME: string = '__share__image__';
+const SW_SHARE_IMAGE_PARAM_NAME: string = 'image';
 
 const SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG: string = 'UPDATE_MAP_CACHE_CONFIG';
 const SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG_RESPONSE: string = 'UPDATE_MAP_CACHE_CONFIG_RESPONSE';
@@ -275,6 +278,43 @@ registerRoute(
     })
 );
 
+self.addEventListener('fetch', (event: FetchEvent) => {
+    const request: Request = event.request;
+
+    if (request.method !== 'POST' || !request.url.endsWith(SW_SHARE_IMAGE_URL_PATHNAME)) {
+        return;
+    }
+
+    event.respondWith((async (): Promise<Response> => {
+        let redirectUrl = request.url;
+        let lastShareIndex = redirectUrl.lastIndexOf(SW_SHARE_IMAGE_URL_PATHNAME);
+        redirectUrl = redirectUrl.substring(0, lastShareIndex);
+
+        try {
+            const formData = await request.formData();
+            const image = formData.get(SW_SHARE_IMAGE_PARAM_NAME);
+
+            if (image instanceof Blob) {
+                const cache: Cache = await caches.open(SW_SHARE_CACHE_NAME);
+                const response = new Response(image, {
+                    headers: {
+                        'Content-Type': image.type
+                    }
+                });
+
+                const putPromise = cache.put(SW_SHARE_CACHE_NAME, response.clone());
+                event.waitUntil(putPromise);
+                await putPromise;
+            }
+
+            return Response.redirect(redirectUrl, 303);
+        } catch (ex) {
+            console.error('failed to handle share image upload in service worker', ex);
+            return Response.redirect(redirectUrl, 303);
+        }
+    })());
+});
+
 self.addEventListener('message', (event: ExtendableMessageEvent) => {
     try {
         if (event.data && event.data.type === SW_MESSAGE_TYPE_UPDATE_MAP_CACHE_CONFIG && 'payload' in event.data) {

src/views/desktop/MainLayout.vue

@@ -220,8 +220,11 @@ import { useDesktopPageStore } from '@/stores/desktopPage.ts';
 
 import { APPLICATION_LOGO_PATH } from '@/consts/asset.ts';
 import { ThemeType } from '@/core/theme.ts';
+
+import { getShareCacheImageBlob } from '@/lib/cache.ts';
 import { isUserScheduledTransactionEnabled } from '@/lib/server_settings.ts';
 import { getSystemTheme, setExpenseAndIncomeAmountColor } from '@/lib/ui/common.ts';
+import logger from '@/lib/logger.ts';
 
 import {
     mdiMenu,
@@ -300,6 +303,14 @@ function handleNavScroll(e: Event): void {
     isVerticalNavScrolled.value = (e.target as HTMLElement).scrollTop > 0;
 }
 
+function clearShareImageCache(): void {
+    getShareCacheImageBlob().then(blob => {
+        if (blob) {
+            logger.warn('desktop version does not support receving shared image, the share image cache has been cleared');
+        }
+    });
+}
+
 function lock(): void {
     rootStore.lock();
     router.replace('/unlock');
@@ -334,6 +345,8 @@ function logout(): void {
 function showAddDialogInTransactionListPage(): void {
     desktopPageStore.setShowAddTransactionDialogInTransactionList();
 }
+
+clearShareImageCache();
 </script>
 
 <style>

src/views/mobile/HomePage.vue

@@ -206,13 +206,16 @@
             </f7-list>
         </f7-popover>
 
-        <a-i-image-recognition-sheet v-model:show="showAIReceiptImageRecognitionSheet"
+        <a-i-image-recognition-sheet ref="aiImageRecognitionSheet"
+                                     v-model:show="showAIReceiptImageRecognitionSheet"
                                      @recognition:change="onReceiptRecognitionChanged"/>
     </f7-page>
 </template>
 
 <script setup lang="ts">
-import { ref, computed } from 'vue';
+import AIImageRecognitionSheet from '@/components/mobile/AIImageRecognitionSheet.vue';
+
+import { ref, computed, useTemplateRef } from 'vue';
 import type { Router } from 'framework7/types';
 
 import { useI18n } from '@/locales/helpers.ts';
@@ -230,8 +233,11 @@ import { TransactionTemplate } from '@/models/transaction_template.ts';
 import type { RecognizedReceiptImageResponse } from '@/models/large_language_model.ts';
 
 import { isUserLogined, isUserUnlocked } from '@/lib/userstate.ts';
+import { getShareCacheImageBlob } from '@/lib/cache.ts';
 import { isTransactionFromAIImageRecognitionEnabled } from '@/lib/server_settings.ts';
 
+type AIImageRecognitionSheetType = InstanceType<typeof AIImageRecognitionSheet>;
+
 const props = defineProps<{
     f7router: Router.Router;
 }>();
@@ -252,6 +258,8 @@ const transactionCategoriesStore = useTransactionCategoriesStore();
 const transactionTemplatesStore = useTransactionTemplatesStore();
 const overviewStore = useOverviewStore();
 
+const aiImageRecognitionSheet = useTemplateRef<AIImageRecognitionSheetType>('aiImageRecognitionSheet');
+
 const loading = ref<boolean>(true);
 const showTransactionTemplatePopover = ref<boolean>(false);
 const showAIReceiptImageRecognitionSheet = ref<boolean>(false);
@@ -272,13 +280,19 @@ function init(): void {
         loading.value = true;
 
         const promises = [
+            getShareCacheImageBlob(),
             accountsStore.loadAllAccounts({ force: false }),
             transactionCategoriesStore.loadAllCategories({ force: false }),
             transactionTemplatesStore.loadAllTemplates({ templateType: TemplateType.Normal.type,  force: false }),
             overviewStore.loadTransactionOverview({ force: false })
         ];
 
-        Promise.all(promises).then(() => {
+        Promise.all(promises).then(responses => {
+            if (responses[0] && responses[0] instanceof Blob) {
+                aiImageRecognitionSheet.value?.loadImage(responses[0]);
+                showAIReceiptImageRecognitionSheet.value = true;
+            }
+
             loading.value = false;
         }).catch(error => {
             loading.value = false;

vite.config.ts

@@ -139,7 +139,20 @@ export default defineConfig(() => {
                             sizes: '512x512',
                             type: 'image/png'
                         }
-                    ]
+                    ],
+                    share_target: {
+                        action: './__share__image__',
+                        method: 'POST',
+                        enctype: 'multipart/form-data',
+                        params: {
+                            files: [
+                                {
+                                    'name': 'image',
+                                    'accept': ['image/*']
+                                }
+                            ]
+                        }
+                    }
                 },
                 injectManifest: {
                     globDirectory: 'dist/',