diff options
34 files changed, 545 insertions, 101 deletions
diff --git a/apps/mobile/components/bookmarks/BookmarkCard.tsx b/apps/mobile/components/bookmarks/BookmarkCard.tsx index ce294a6f..3cbd064e 100644 --- a/apps/mobile/components/bookmarks/BookmarkCard.tsx +++ b/apps/mobile/components/bookmarks/BookmarkCard.tsx @@ -1,3 +1,4 @@ +import React from "react"; import { ActivityIndicator, Alert, @@ -300,11 +301,15 @@ function AssetCard({ } const title = bookmark.title ?? bookmark.content.fileName; + const assetImage = + bookmark.assets.find((r) => r.assetType == "assetScreenshot")?.id ?? + bookmark.content.assetId; + return ( <View className="flex gap-2"> <Pressable onPress={onOpenBookmark}> <BookmarkAssetImage - assetId={bookmark.content.assetId} + assetId={assetImage} className="h-56 min-h-56 w-full object-cover" /> </Pressable> diff --git a/apps/web/components/admin/AdminActions.tsx b/apps/web/components/admin/AdminActions.tsx index 34b3d63a..fb151ac8 100644 --- a/apps/web/components/admin/AdminActions.tsx +++ b/apps/web/components/admin/AdminActions.tsx @@ -37,6 +37,21 @@ export default function AdminActions() { }, }); + const { mutate: reprocessAssetsFixMode, isPending: isReprocessingPending } = + api.admin.reprocessAssetsFixMode.useMutation({ + onSuccess: () => { + toast({ + description: "Reprocessing enqueued", + }); + }, + onError: (e) => { + toast({ + variant: "destructive", + description: e.message, + }); + }, + }); + const { mutate: reRunInferenceOnAllBookmarks, isPending: isInferencePending, @@ -126,6 +141,13 @@ export default function AdminActions() { </ActionButton> <ActionButton variant="destructive" + loading={isReprocessingPending} + onClick={() => reprocessAssetsFixMode()} + > + {t("admin.actions.reprocess_assets_fix_mode")} + </ActionButton> + <ActionButton + variant="destructive" loading={isTidyAssetsPending} onClick={() => tidyAssets()} > diff --git a/apps/web/components/dashboard/bookmarks/AssetCard.tsx b/apps/web/components/dashboard/bookmarks/AssetCard.tsx index 61b3bc8d..0cb75b3f 100644 --- a/apps/web/components/dashboard/bookmarks/AssetCard.tsx +++ b/apps/web/components/dashboard/bookmarks/AssetCard.tsx @@ -2,6 +2,8 @@ import Image from "next/image"; import Link from "next/link"; +import { cn } from "@/lib/utils"; +import { FileText } from "lucide-react"; import type { ZBookmarkTypeAsset } from "@hoarder/shared/types/bookmarks"; import { getAssetUrl } from "@hoarder/shared-react/utils/assetUtils"; @@ -32,12 +34,28 @@ function AssetImage({ ); } case "pdf": { + const screenshotAssetId = bookmark.assets.find( + (r) => r.assetType === "assetScreenshot", + )?.id; + if (!screenshotAssetId) { + return ( + <div + className={cn(className, "flex items-center justify-center")} + title="PDF screenshot not available. Run asset preprocessing job to generate one screenshot" + > + <FileText size={80} /> + </div> + ); + } return ( - <iframe - title={bookmarkedAsset.assetId} - className={className} - src={getAssetUrl(bookmarkedAsset.assetId)} - /> + <Link href={`/dashboard/preview/${bookmark.id}`}> + <Image + alt="asset" + src={getAssetUrl(screenshotAssetId)} + fill={true} + className={className} + /> + </Link> ); } default: { diff --git a/apps/web/components/dashboard/preview/AssetContentSection.tsx b/apps/web/components/dashboard/preview/AssetContentSection.tsx index 03ab8a43..8590d2ad 100644 --- a/apps/web/components/dashboard/preview/AssetContentSection.tsx +++ b/apps/web/components/dashboard/preview/AssetContentSection.tsx @@ -1,42 +1,117 @@ +import { useMemo, useState } from "react"; import Image from "next/image"; import Link from "next/link"; +import { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { useTranslation } from "@/lib/i18n/client"; +import { getAssetUrl } from "@hoarder/shared-react/utils/assetUtils"; import { BookmarkTypes, ZBookmark } from "@hoarder/shared/types/bookmarks"; -export function AssetContentSection({ bookmark }: { bookmark: ZBookmark }) { +// 20 MB +const BIG_FILE_SIZE = 20 * 1024 * 1024; + +function PDFContentSection({ bookmark }: { bookmark: ZBookmark }) { if (bookmark.content.type != BookmarkTypes.ASSET) { throw new Error("Invalid content type"); } + const { t } = useTranslation(); - switch (bookmark.content.assetType) { - case "image": { - return ( - <div className="relative h-full min-w-full"> - <Link - href={`/api/assets/${bookmark.content.assetId}`} - target="_blank" - > - <Image - alt="asset" - fill={true} - className="object-contain" - src={`/api/assets/${bookmark.content.assetId}`} - /> - </Link> - </div> - ); + const initialSection = useMemo(() => { + if (bookmark.content.type != BookmarkTypes.ASSET) { + throw new Error("Invalid content type"); } - case "pdf": { - return ( - <iframe - title={bookmark.content.assetId} - className="h-full w-full" - src={`/api/assets/${bookmark.content.assetId}`} - /> - ); + + const screenshot = bookmark.assets.find( + (item) => item.assetType === "assetScreenshot", + ); + const bigSize = + bookmark.content.size && bookmark.content.size > BIG_FILE_SIZE; + if (bigSize && screenshot) { + return "screenshot"; } - default: { + return "pdf"; + }, [bookmark]); + const [section, setSection] = useState(initialSection); + + const screenshot = bookmark.assets.find( + (r) => r.assetType === "assetScreenshot", + )?.id; + + const content = + section === "screenshot" && screenshot ? ( + <div className="relative h-full min-w-full"> + <Image + alt="screenshot" + src={getAssetUrl(screenshot)} + fill={true} + className="object-contain" + /> + </div> + ) : ( + <iframe + title={bookmark.content.assetId} + className="h-full w-full" + src={getAssetUrl(bookmark.content.assetId)} + /> + ); + + return ( + <div className="flex h-full flex-col items-center gap-2"> + <div className="flex w-full items-center justify-center gap-4"> + <Select onValueChange={setSection} value={section}> + <SelectTrigger className="w-fit"> + <SelectValue /> + </SelectTrigger> + <SelectContent> + <SelectGroup> + <SelectItem value="screenshot" disabled={!screenshot}> + {t("common.screenshot")} + </SelectItem> + <SelectItem value="pdf">PDF</SelectItem> + </SelectGroup> + </SelectContent> + </Select> + </div> + {content} + </div> + ); +} + +function ImageContentSection({ bookmark }: { bookmark: ZBookmark }) { + if (bookmark.content.type != BookmarkTypes.ASSET) { + throw new Error("Invalid content type"); + } + return ( + <div className="relative h-full min-w-full"> + <Link href={getAssetUrl(bookmark.content.assetId)} target="_blank"> + <Image + alt="asset" + fill={true} + className="object-contain" + src={getAssetUrl(bookmark.content.assetId)} + /> + </Link> + </div> + ); +} + +export function AssetContentSection({ bookmark }: { bookmark: ZBookmark }) { + if (bookmark.content.type != BookmarkTypes.ASSET) { + throw new Error("Invalid content type"); + } + switch (bookmark.content.assetType) { + case "image": + return <ImageContentSection bookmark={bookmark} />; + case "pdf": + return <PDFContentSection bookmark={bookmark} />; + default: return <div>Unsupported asset type</div>; - } } } diff --git a/apps/web/components/dashboard/preview/AttachmentBox.tsx b/apps/web/components/dashboard/preview/AttachmentBox.tsx index 6547ae51..32939cb0 100644 --- a/apps/web/components/dashboard/preview/AttachmentBox.tsx +++ b/apps/web/components/dashboard/preview/AttachmentBox.tsx @@ -45,6 +45,7 @@ export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) { const { t } = useTranslation(); const typeToIcon: Record<ZAssetType, React.ReactNode> = { screenshot: <Camera className="size-4" />, + assetScreenshot: <Camera className="size-4" />, fullPageArchive: <Archive className="size-4" />, precrawledArchive: <Archive className="size-4" />, bannerImage: <Image className="size-4" />, diff --git a/apps/web/lib/i18n/locales/ar/translation.json b/apps/web/lib/i18n/locales/ar/translation.json index 7bd0bcad..e9239e70 100644 --- a/apps/web/lib/i18n/locales/ar/translation.json +++ b/apps/web/lib/i18n/locales/ar/translation.json @@ -153,7 +153,7 @@ } }, "admin": { - "admin_settings": "إعدادات المدير", + "admin_settings": "إعدادات المشرف", "server_stats": { "server_stats": "إحصائيات الخادم", "total_users": "إجمالي المستخدمين", @@ -161,15 +161,36 @@ "server_version": "إصدار الخادم" }, "background_jobs": { - "background_jobs": "المهام التلقائية", + "background_jobs": "المهام الخلفية", "crawler_jobs": "مهام الاستكشاف", "indexing_jobs": "مهام الفهرسة", - "inference_jobs": "مهام التحليل الذكي", - "tidy_assets_jobs": "مهام تنظيم الملفات", + "inference_jobs": "مهام الاستدلال", + "tidy_assets_jobs": "مهام تنظيم الوسائط", "job": "مهمة", "queued": "في قائمة الانتظار", - "pending": "معلق", - "failed": "فشل" + "pending": "قيد الانتظار", + "failed": "فشلت" + }, + "actions": { + "recrawl_failed_links_only": "إعادة استكشاف الروابط الفاشلة فقط", + "recrawl_all_links": "إعادة استكشاف جميع الروابط", + "without_inference": "بدون استدلال", + "regenerate_ai_tags_for_failed_bookmarks_only": "إعادة إنشاء علامات الذكاء الاصطناعي للإشارات المرجعية الفاشلة فقط", + "regenerate_ai_tags_for_all_bookmarks": "إعادة إنشاء علامات الذكاء الاصطناعي لجميع الإشارات المرجعية", + "reindex_all_bookmarks": "إعادة فهرسة جميع الإشارات المرجعية", + "compact_assets": "ضغط الوسائط", + "reprocess_assets_fix_mode": "إعادة معالجة الوسائط (وضع الإصلاح)" + }, + "users_list": { + "users_list": "قائمة المستخدمين", + "create_user": "إنشاء مستخدم", + "change_role": "تغيير الدور", + "reset_password": "إعادة تعيين كلمة المرور", + "delete_user": "حذف المستخدم", + "num_bookmarks": "عدد الإشارات المرجعية", + "asset_sizes": "أحجام الوسائط", + "local_user": "مستخدم محلي", + "confirm_password": "تأكيد كلمة المرور" } }, "options": { diff --git a/apps/web/lib/i18n/locales/da/translation.json b/apps/web/lib/i18n/locales/da/translation.json index 3822d5c6..4fe69650 100644 --- a/apps/web/lib/i18n/locales/da/translation.json +++ b/apps/web/lib/i18n/locales/da/translation.json @@ -94,7 +94,8 @@ "recrawl_all_links": "Gennemsøg alle links", "without_inference": "Uden inferens", "regenerate_ai_tags_for_all_bookmarks": "Genopret AI-tags for alle bogmærker", - "reindex_all_bookmarks": "Genindeksér alle bogmærker" + "reindex_all_bookmarks": "Genindeksér alle bogmærker", + "reprocess_assets_fix_mode": "Genbehandling af aktiver (Fix Mode)" }, "background_jobs": { "inference_jobs": "Inferensopgaver", diff --git a/apps/web/lib/i18n/locales/de/translation.json b/apps/web/lib/i18n/locales/de/translation.json index c20a2273..ccebf1f1 100644 --- a/apps/web/lib/i18n/locales/de/translation.json +++ b/apps/web/lib/i18n/locales/de/translation.json @@ -175,7 +175,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "KI-Tags nur für fehlgeschlagene Lesezeichen neu generieren", "regenerate_ai_tags_for_all_bookmarks": "KI-Tags für alle Lesezeichen neu generieren", "reindex_all_bookmarks": "Alle Lesezeichen neu indizieren", - "compact_assets": "Assets komprimieren" + "compact_assets": "Assets komprimieren", + "reprocess_assets_fix_mode": "Assets neu verarbeiten (Fix-Modus)" }, "users_list": { "users_list": "Benutzerliste", diff --git a/apps/web/lib/i18n/locales/en/translation.json b/apps/web/lib/i18n/locales/en/translation.json index 2e80f2f4..81ef942f 100644 --- a/apps/web/lib/i18n/locales/en/translation.json +++ b/apps/web/lib/i18n/locales/en/translation.json @@ -178,7 +178,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "Regenerate AI Tags for Failed Bookmarks Only", "regenerate_ai_tags_for_all_bookmarks": "Regenerate AI Tags for All Bookmarks", "reindex_all_bookmarks": "Reindex All Bookmarks", - "compact_assets": "Compact Assets" + "compact_assets": "Compact Assets", + "reprocess_assets_fix_mode": "Reprocess Assets (Fix Mode)" }, "users_list": { "users_list": "Users List", diff --git a/apps/web/lib/i18n/locales/es/translation.json b/apps/web/lib/i18n/locales/es/translation.json index 40c6cb01..3a1a7e3c 100644 --- a/apps/web/lib/i18n/locales/es/translation.json +++ b/apps/web/lib/i18n/locales/es/translation.json @@ -146,7 +146,8 @@ "compact_assets": "Optimizar multimedia", "without_inference": "Sin inferencia", "recrawl_failed_links_only": "Recrawlear solo los enlaces fallidos", - "recrawl_all_links": "Recrawlear todos los enlaces" + "recrawl_all_links": "Recrawlear todos los enlaces", + "reprocess_assets_fix_mode": "Reprocesar assets (modo fijo)" }, "users_list": { "users_list": "Lista de usuarios", diff --git a/apps/web/lib/i18n/locales/fr/translation.json b/apps/web/lib/i18n/locales/fr/translation.json index b7834a7b..1772c2ff 100644 --- a/apps/web/lib/i18n/locales/fr/translation.json +++ b/apps/web/lib/i18n/locales/fr/translation.json @@ -146,7 +146,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "Régénérer les tags AI uniquement pour les favoris échoués", "regenerate_ai_tags_for_all_bookmarks": "Régénérer les tags AI pour tous les favoris", "reindex_all_bookmarks": "Réindexer tous les favoris", - "compact_assets": "Compacter les assets" + "compact_assets": "Compacter les assets", + "reprocess_assets_fix_mode": "Reprocesser les assets (mode fix)" }, "users_list": { "users_list": "Liste des utilisateurs", diff --git a/apps/web/lib/i18n/locales/gl/translation.json b/apps/web/lib/i18n/locales/gl/translation.json index eb65ca64..363ffac8 100644 --- a/apps/web/lib/i18n/locales/gl/translation.json +++ b/apps/web/lib/i18n/locales/gl/translation.json @@ -178,7 +178,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "Rexenerar etiquetas IA so en marcadores errados", "regenerate_ai_tags_for_all_bookmarks": "Rexenerar etiquetas IA para todos os marcadores", "reindex_all_bookmarks": "Reindexar marcadores", - "compact_assets": "Optimizar multimedia" + "compact_assets": "Optimizar multimedia", + "reprocess_assets_fix_mode": "Reprocesar assets (modo fixo)" }, "users_list": { "users_list": "Listado de usuarios", diff --git a/apps/web/lib/i18n/locales/hr/translation.json b/apps/web/lib/i18n/locales/hr/translation.json index 6e250924..7a72d295 100644 --- a/apps/web/lib/i18n/locales/hr/translation.json +++ b/apps/web/lib/i18n/locales/hr/translation.json @@ -36,7 +36,8 @@ "recrawl_all_links": "Ponovno pregledavanje svih veza", "regenerate_ai_tags_for_all_bookmarks": "Ponovno generiranje AI oznaka za sve oznake", "without_inference": "Bez zaključivanja", - "compact_assets": "Kompaktiranje resursa" + "compact_assets": "Kompaktiranje resursa", + "reprocess_assets_fix_mode": "Ponovno postupanje s resursima (fiksni mod)" } }, "layouts": { diff --git a/apps/web/lib/i18n/locales/hu/translation.json b/apps/web/lib/i18n/locales/hu/translation.json index 38ef96b4..439212f4 100644 --- a/apps/web/lib/i18n/locales/hu/translation.json +++ b/apps/web/lib/i18n/locales/hu/translation.json @@ -258,7 +258,8 @@ "regenerate_ai_tags_for_all_bookmarks": "Minden könyvjelző MI címkéjének lecserélése", "regenerate_ai_tags_for_failed_bookmarks_only": "Hibás könyvjelzők MI címkéjének lecserélése", "reindex_all_bookmarks": "Minden könyvjelző újraindexelése", - "compact_assets": "Kompakt tulajdonok" + "compact_assets": "Kompakt tulajdonok", + "reprocess_assets_fix_mode": "Tulajdonok függvényezése (Fix Mod)" }, "users_list": { "asset_sizes": "Tulajdon méretek", diff --git a/apps/web/lib/i18n/locales/it/translation.json b/apps/web/lib/i18n/locales/it/translation.json index e24b6b7f..4b093b72 100644 --- a/apps/web/lib/i18n/locales/it/translation.json +++ b/apps/web/lib/i18n/locales/it/translation.json @@ -201,7 +201,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "Rigenera tag AI solo per i segnalibri falliti", "regenerate_ai_tags_for_all_bookmarks": "Rigenera tag AI per tutti i segnalibri", "compact_assets": "Compatta asset", - "reindex_all_bookmarks": "Reindicizza tutti i segnalibri" + "reindex_all_bookmarks": "Reindicizza tutti i segnalibri", + "reprocess_assets_fix_mode": "Riprocessa asset (modalità fissa)" }, "users_list": { "users_list": "Lista utenti", diff --git a/apps/web/lib/i18n/locales/pl/translation.json b/apps/web/lib/i18n/locales/pl/translation.json index 0d026542..66921560 100644 --- a/apps/web/lib/i18n/locales/pl/translation.json +++ b/apps/web/lib/i18n/locales/pl/translation.json @@ -148,7 +148,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "Regeneruj tagi AI tylko dla nieudanych zakładek", "regenerate_ai_tags_for_all_bookmarks": "Regeneruj tagi AI dla wszystkich zakładek", "reindex_all_bookmarks": "Ponowne indeksowanie wszystkich zakładek", - "compact_assets": "Kompaktuj zasoby" + "compact_assets": "Kompaktuj zasoby", + "reprocess_assets_fix_mode": "Ponowne przetwarzanie zasobów (tryb fiksny)" } }, "tags": { diff --git a/apps/web/lib/i18n/locales/ru/translation.json b/apps/web/lib/i18n/locales/ru/translation.json index 4a8cdd52..1d4c50bd 100644 --- a/apps/web/lib/i18n/locales/ru/translation.json +++ b/apps/web/lib/i18n/locales/ru/translation.json @@ -211,7 +211,8 @@ "compact_assets": "Сжать ресурсы", "regenerate_ai_tags_for_failed_bookmarks_only": "Перегенерировать ИИ метки только для неудачных закладок", "reindex_all_bookmarks": "Переиндексировать все закладки", - "recrawl_all_links": "Пересканировать все ссылки" + "recrawl_all_links": "Пересканировать все ссылки", + "reprocess_assets_fix_mode": "Перепроцессировать ресурсы (фиксный режим)" }, "admin_settings": "Настройки администратора" }, diff --git a/apps/web/lib/i18n/locales/tr/translation.json b/apps/web/lib/i18n/locales/tr/translation.json index 9840c6f0..227f6dac 100644 --- a/apps/web/lib/i18n/locales/tr/translation.json +++ b/apps/web/lib/i18n/locales/tr/translation.json @@ -148,7 +148,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "Yalnızca Başarısız Yer İşaretleri için Yapay Zeka Etiketlerini Yeniden Oluştur", "regenerate_ai_tags_for_all_bookmarks": "Tüm Yer İşaretleri için Yapay Zeka Etiketlerini Yeniden Oluştur", "reindex_all_bookmarks": "Tüm Yer İşaretlerini Yeniden Dizine Al", - "compact_assets": "Varlıkları Sıkıştır" + "compact_assets": "Varlıkları Sıkıştır", + "reprocess_assets_fix_mode": "Varlıkları Yeniden İşle (Fix Mod)" }, "users_list": { "users_list": "Kullanıcı Listesi", diff --git a/apps/web/lib/i18n/locales/zh/translation.json b/apps/web/lib/i18n/locales/zh/translation.json index 84a9e17a..d798b716 100644 --- a/apps/web/lib/i18n/locales/zh/translation.json +++ b/apps/web/lib/i18n/locales/zh/translation.json @@ -175,7 +175,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "仅为失败书签重新生成AI标签", "regenerate_ai_tags_for_all_bookmarks": "为所有书签重新生成AI标签", "reindex_all_bookmarks": "重新索引所有书签", - "compact_assets": "压缩资产" + "compact_assets": "压缩资产", + "reprocess_assets_fix_mode": "重新处理资产(固定模式)" }, "users_list": { "users_list": "用户列表", diff --git a/apps/web/lib/i18n/locales/zhtw/translation.json b/apps/web/lib/i18n/locales/zhtw/translation.json index aada5492..284b5de2 100644 --- a/apps/web/lib/i18n/locales/zhtw/translation.json +++ b/apps/web/lib/i18n/locales/zhtw/translation.json @@ -156,7 +156,8 @@ "regenerate_ai_tags_for_failed_bookmarks_only": "僅重新產生失敗書籤的 AI 標籤", "regenerate_ai_tags_for_all_bookmarks": "重新產生所有書籤的 AI 標籤", "reindex_all_bookmarks": "重新索引所有書籤", - "compact_assets": "壓縮資源" + "compact_assets": "壓縮資源", + "reprocess_assets_fix_mode": "重新處理資源(固定模式)" }, "users_list": { "users_list": "使用者清單", diff --git a/apps/workers/assetPreprocessingWorker.ts b/apps/workers/assetPreprocessingWorker.ts index 5c4937e5..f94eeb9e 100644 --- a/apps/workers/assetPreprocessingWorker.ts +++ b/apps/workers/assetPreprocessingWorker.ts @@ -2,12 +2,18 @@ import os from "os"; import { eq } from "drizzle-orm"; import { DequeuedJob, Runner } from "liteque"; import PDFParser from "pdf2json"; +import { fromBuffer } from "pdf2pic"; import { createWorker } from "tesseract.js"; import type { AssetPreprocessingRequest } from "@hoarder/shared/queues"; import { db } from "@hoarder/db"; -import { bookmarkAssets, bookmarks } from "@hoarder/db/schema"; -import { readAsset } from "@hoarder/shared/assetdb"; +import { + assets, + AssetTypes, + bookmarkAssets, + bookmarks, +} from "@hoarder/db/schema"; +import { newAssetId, readAsset, saveAsset } from "@hoarder/shared/assetdb"; import serverConfig from "@hoarder/shared/config"; import logger from "@hoarder/shared/logger"; import { @@ -67,17 +73,14 @@ async function readImageText(buffer: Buffer) { async function readPDFText(buffer: Buffer): Promise<{ text: string; - metadata: Record<string, string>; + metadata: Record<string, object>; }> { return new Promise((resolve, reject) => { - // Need raw text flag represents as number (1), reference : https://github.com/modesty/pdf2json/issues/76#issuecomment-236569265 - const pdfParser = new PDFParser(null, 1); + const pdfParser = new PDFParser(null, true); pdfParser.on("pdfParser_dataError", reject); pdfParser.on("pdfParser_dataReady", (pdfData) => { resolve({ - // The type isn't set correctly, reference : https://github.com/modesty/pdf2json/issues/327 - // eslint-disable-next-line - text: (pdfParser as any).getRawTextContent(), + text: pdfParser.getRawTextContent(), metadata: pdfData.Meta, }); }); @@ -85,11 +88,102 @@ async function readPDFText(buffer: Buffer): Promise<{ }); } -async function preprocessImage( +export async function extractAndSavePDFScreenshot( jobId: string, asset: Buffer, -): Promise<{ content: string; metadata: string | null } | undefined> { + bookmark: NonNullable<Awaited<ReturnType<typeof getBookmark>>>, + isFixMode: boolean, +): Promise<boolean> { + { + const alreadyHasScreenshot = + bookmark.assets.find( + (r) => r.assetType === AssetTypes.ASSET_SCREENSHOT, + ) !== undefined; + if (alreadyHasScreenshot && isFixMode) { + logger.info( + `[assetPreprocessing][${jobId}] Skipping PDF screenshot generation as it's already been generated.`, + ); + return false; + } + } + logger.info( + `[assetPreprocessing][${jobId}] Attempting to generate PDF screenshot for bookmarkId: ${bookmark.id}`, + ); + try { + /** + * If you encountered any issues with this library, make sure you have ghostscript and graphicsmagick installed following this URL + * https://github.com/yakovmeister/pdf2image/blob/HEAD/docs/gm-installation.md + */ + const screenshot = await fromBuffer(asset, { + density: 100, + quality: 100, + format: "png", + preserveAspectRatio: true, + })(1, { responseType: "buffer" }); + + if (!screenshot.buffer) { + logger.error( + `[assetPreprocessing][${jobId}] Failed to generate PDF screenshot`, + ); + return false; + } + + // Store the screenshot + const assetId = newAssetId(); + const fileName = "screenshot.png"; + const contentType = "image/png"; + await saveAsset({ + userId: bookmark.userId, + assetId, + asset: screenshot.buffer, + metadata: { + contentType, + fileName, + }, + }); + + // Insert into database + await db.insert(assets).values({ + id: assetId, + bookmarkId: bookmark.id, + userId: bookmark.userId, + assetType: AssetTypes.ASSET_SCREENSHOT, + contentType, + size: screenshot.buffer.byteLength, + fileName, + }); + + logger.info( + `[assetPreprocessing][${jobId}] Successfully saved PDF screenshot to database`, + ); + return true; + } catch (error) { + logger.error( + `[assetPreprocessing][${jobId}] Failed to process PDF screenshot: ${error}`, + ); + return false; + } +} + +async function extractAndSaveImageText( + jobId: string, + asset: Buffer, + bookmark: NonNullable<Awaited<ReturnType<typeof getBookmark>>>, + isFixMode: boolean, +): Promise<boolean> { + { + const alreadyHasText = !!bookmark.asset.content; + if (alreadyHasText && isFixMode) { + logger.info( + `[assetPreprocessing][${jobId}] Skipping image text extraction as it's already been extracted.`, + ); + return false; + } + } let imageText = null; + logger.info( + `[assetPreprocessing][${jobId}] Attempting to extract text from image.`, + ); try { imageText = await readImageText(asset); } catch (e) { @@ -98,19 +192,40 @@ async function preprocessImage( ); } if (!imageText) { - return undefined; + return false; } logger.info( `[assetPreprocessing][${jobId}] Extracted ${imageText.length} characters from image.`, ); - return { content: imageText, metadata: null }; + await db + .update(bookmarkAssets) + .set({ + content: imageText, + metadata: null, + }) + .where(eq(bookmarkAssets.id, bookmark.id)); + return true; } -async function preProcessPDF( +async function extractAndSavePDFText( jobId: string, asset: Buffer, -): Promise<{ content: string; metadata: string | null } | undefined> { + bookmark: NonNullable<Awaited<ReturnType<typeof getBookmark>>>, + isFixMode: boolean, +): Promise<boolean> { + { + const alreadyHasText = !!bookmark.asset.content; + if (alreadyHasText && isFixMode) { + logger.info( + `[assetPreprocessing][${jobId}] Skipping PDF text extraction as it's already been extracted.`, + ); + return false; + } + } + logger.info( + `[assetPreprocessing][${jobId}] Attempting to extract text from pdf.`, + ); const pdfParse = await readPDFText(asset); if (!pdfParse?.text) { throw new Error( @@ -120,13 +235,28 @@ async function preProcessPDF( logger.info( `[assetPreprocessing][${jobId}] Extracted ${pdfParse.text.length} characters from pdf.`, ); - return { - content: pdfParse.text, - metadata: pdfParse.metadata ? JSON.stringify(pdfParse.metadata) : null, - }; + await db + .update(bookmarkAssets) + .set({ + content: pdfParse.text, + metadata: pdfParse.metadata ? JSON.stringify(pdfParse.metadata) : null, + }) + .where(eq(bookmarkAssets.id, bookmark.id)); + return true; +} + +async function getBookmark(bookmarkId: string) { + return db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, bookmarkId), + with: { + asset: true, + assets: true, + }, + }); } async function run(req: DequeuedJob<AssetPreprocessingRequest>) { + const isFixMode = req.data.fixMode; const jobId = req.id; const bookmarkId = req.data.bookmarkId; @@ -134,6 +264,7 @@ async function run(req: DequeuedJob<AssetPreprocessingRequest>) { where: eq(bookmarks.id, bookmarkId), with: { asset: true, + assets: true, }, }); @@ -162,15 +293,29 @@ async function run(req: DequeuedJob<AssetPreprocessingRequest>) { ); } - let result: { content: string; metadata: string | null } | undefined = - undefined; - + let anythingChanged = false; switch (bookmark.asset.assetType) { case "image": - result = await preprocessImage(jobId, asset); + anythingChanged ||= await extractAndSaveImageText( + jobId, + asset, + bookmark, + isFixMode, + ); break; case "pdf": - result = await preProcessPDF(jobId, asset); + anythingChanged ||= await extractAndSavePDFText( + jobId, + asset, + bookmark, + isFixMode, + ); + anythingChanged ||= await extractAndSavePDFScreenshot( + jobId, + asset, + bookmark, + isFixMode, + ); break; default: throw new Error( @@ -178,20 +323,12 @@ async function run(req: DequeuedJob<AssetPreprocessingRequest>) { ); } - if (result) { - await db - .update(bookmarkAssets) - .set({ - content: result.content, - metadata: result.metadata, - }) - .where(eq(bookmarkAssets.id, bookmarkId)); - } - - await OpenAIQueue.enqueue({ - bookmarkId, - }); + if (anythingChanged) { + await OpenAIQueue.enqueue({ + bookmarkId, + }); - // Update the search index - await triggerSearchReindex(bookmarkId); + // Update the search index + await triggerSearchReindex(bookmarkId); + } } diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 7611494e..17dba443 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -592,6 +592,7 @@ async function handleAsAssetBookmark( }); await AssetPreprocessingQueue.enqueue({ bookmarkId, + fixMode: false, }); } diff --git a/apps/workers/package.json b/apps/workers/package.json index ebcae757..122c7cb1 100644 --- a/apps/workers/package.json +++ b/apps/workers/package.json @@ -30,7 +30,8 @@ "metascraper-url": "^5.45.22", "node-cron": "^3.0.3", "node-fetch": "^3.3.2", - "pdf2json": "^3.0.5", + "pdf2json": "^3.1.5", + "pdf2pic": "^3.1.3", "pdfjs-dist": "^4.0.379", "puppeteer": "^22.0.0", "puppeteer-extra": "^3.3.6", @@ -65,4 +66,4 @@ ] }, "prettier": "@hoarder/prettier-config" -} +}
\ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index f4b05f79..2eba6fb8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -72,7 +72,7 @@ COPY --chmod=755 ./docker/root/etc/s6-overlay /etc/s6-overlay ###################### # Install runtime deps ###################### -RUN apk add --no-cache monolith yt-dlp +RUN apk add --no-cache monolith yt-dlp graphicsmagick ghostscript ###################### # Prepare the web app diff --git a/hoarder-linux.sh b/hoarder-linux.sh index e536caec..1ad80578 100644 --- a/hoarder-linux.sh +++ b/hoarder-linux.sh @@ -36,6 +36,8 @@ install() { sudo \ unzip \ gnupg \ + graphicsmagick \ + ghostscript \ ca-certificates if [[ "$OS" == "noble" ]]; then apt-get install -y software-properties-common diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 6bd67448..111081b8 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -164,6 +164,7 @@ export const bookmarkLinks = sqliteTable( export const enum AssetTypes { LINK_BANNER_IMAGE = "linkBannerImage", LINK_SCREENSHOT = "linkScreenshot", + ASSET_SCREENSHOT = "assetScreenshot", LINK_FULL_PAGE_ARCHIVE = "linkFullPageArchive", LINK_PRECRAWLED_ARCHIVE = "linkPrecrawledArchive", LINK_VIDEO = "linkVideo", @@ -180,6 +181,7 @@ export const assets = sqliteTable( enum: [ AssetTypes.LINK_BANNER_IMAGE, AssetTypes.LINK_SCREENSHOT, + AssetTypes.ASSET_SCREENSHOT, AssetTypes.LINK_FULL_PAGE_ARCHIVE, AssetTypes.LINK_PRECRAWLED_ARCHIVE, AssetTypes.LINK_VIDEO, diff --git a/packages/open-api/hoarder-openapi-spec.json b/packages/open-api/hoarder-openapi-spec.json index 182cf3b0..3af444b8 100644 --- a/packages/open-api/hoarder-openapi-spec.json +++ b/packages/open-api/hoarder-openapi-spec.json @@ -224,6 +224,10 @@ "sourceUrl": { "type": "string", "nullable": true + }, + "size": { + "type": "number", + "nullable": true } }, "required": [ @@ -260,6 +264,7 @@ "type": "string", "enum": [ "screenshot", + "assetScreenshot", "bannerImage", "fullPageArchive", "video", @@ -1121,6 +1126,7 @@ "type": "string", "enum": [ "screenshot", + "assetScreenshot", "bannerImage", "fullPageArchive", "video", @@ -1153,6 +1159,7 @@ "type": "string", "enum": [ "screenshot", + "assetScreenshot", "bannerImage", "fullPageArchive", "video", diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts index 89738fcf..974f7893 100644 --- a/packages/shared/assetdb.ts +++ b/packages/shared/assetdb.ts @@ -4,6 +4,7 @@ import { Glob } from "glob"; import { z } from "zod"; import serverConfig from "./config"; +import logger from "./logger"; const ROOT_PATH = path.join(serverConfig.dataDir, "assets"); @@ -241,3 +242,35 @@ export async function* getAllAssets() { }; } } + +export async function storeScreenshot( + screenshot: Buffer | undefined, + userId: string, + jobId: string, +) { + if (!serverConfig.crawler.storeScreenshot) { + logger.info( + `[Crawler][${jobId}] Skipping storing the screenshot as per the config.`, + ); + return null; + } + if (!screenshot) { + logger.info( + `[Crawler][${jobId}] Skipping storing the screenshot as it's empty.`, + ); + return null; + } + const assetId = newAssetId(); + const contentType = "image/png"; + const fileName = "screenshot.png"; + await saveAsset({ + userId, + assetId, + metadata: { contentType, fileName }, + asset: screenshot, + }); + logger.info( + `[Crawler][${jobId}] Stored the screenshot as assetId: ${assetId}`, + ); + return { assetId, contentType, fileName, size: screenshot.byteLength }; +} diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index cbe58f8d..5484ffb2 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -98,6 +98,13 @@ export async function triggerSearchDeletion(bookmarkId: string) { }); } +export async function triggerReprocessingFixMode(bookmarkId: string) { + await AssetPreprocessingQueue.enqueue({ + bookmarkId, + fixMode: true, + }); +} + export const zvideoRequestSchema = z.object({ bookmarkId: z.string(), url: z.string(), @@ -143,6 +150,7 @@ export const FeedQueue = new SqliteQueue<ZFeedRequestSchema>( // Preprocess Assets export const zAssetPreprocessingRequestSchema = z.object({ bookmarkId: z.string(), + fixMode: z.boolean().optional().default(false), }); export type AssetPreprocessingRequest = z.infer< typeof zAssetPreprocessingRequestSchema diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index b6a74474..9644095c 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -17,6 +17,7 @@ export type ZSortOrder = z.infer<typeof zSortOrder>; export const zAssetTypesSchema = z.enum([ "screenshot", + "assetScreenshot", "bannerImage", "fullPageArchive", "video", @@ -61,6 +62,7 @@ export const zBookmarkedAssetSchema = z.object({ assetId: z.string(), fileName: z.string().nullish(), sourceUrl: z.string().nullish(), + size: z.number().nullish(), }); export type ZBookmarkedAsset = z.infer<typeof zBookmarkedAssetSchema>; diff --git a/packages/trpc/lib/attachments.ts b/packages/trpc/lib/attachments.ts index f4fda9cd..3ad79a5a 100644 --- a/packages/trpc/lib/attachments.ts +++ b/packages/trpc/lib/attachments.ts @@ -6,6 +6,7 @@ import { ZAssetType, zAssetTypesSchema } from "@hoarder/shared/types/bookmarks"; export function mapDBAssetTypeToUserType(assetType: AssetTypes): ZAssetType { const map: Record<AssetTypes, z.infer<typeof zAssetTypesSchema>> = { [AssetTypes.LINK_SCREENSHOT]: "screenshot", + [AssetTypes.ASSET_SCREENSHOT]: "assetScreenshot", [AssetTypes.LINK_FULL_PAGE_ARCHIVE]: "fullPageArchive", [AssetTypes.LINK_PRECRAWLED_ARCHIVE]: "precrawledArchive", [AssetTypes.LINK_BANNER_IMAGE]: "bannerImage", @@ -21,6 +22,7 @@ export function mapSchemaAssetTypeToDB( ): AssetTypes { const map: Record<ZAssetType, AssetTypes> = { screenshot: AssetTypes.LINK_SCREENSHOT, + assetScreenshot: AssetTypes.ASSET_SCREENSHOT, fullPageArchive: AssetTypes.LINK_FULL_PAGE_ARCHIVE, precrawledArchive: AssetTypes.LINK_PRECRAWLED_ARCHIVE, bannerImage: AssetTypes.LINK_BANNER_IMAGE, @@ -34,6 +36,7 @@ export function mapSchemaAssetTypeToDB( export function humanFriendlyNameForAssertType(type: ZAssetType) { const map: Record<ZAssetType, string> = { screenshot: "Screenshot", + assetScreenshot: "Asset Screenshot", fullPageArchive: "Full Page Archive", precrawledArchive: "Precrawled Archive", bannerImage: "Banner Image", @@ -47,6 +50,7 @@ export function humanFriendlyNameForAssertType(type: ZAssetType) { export function isAllowedToAttachAsset(type: ZAssetType) { const map: Record<ZAssetType, boolean> = { screenshot: true, + assetScreenshot: true, fullPageArchive: false, precrawledArchive: false, bannerImage: true, @@ -60,6 +64,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) { export function isAllowedToDetachAsset(type: ZAssetType) { const map: Record<ZAssetType, boolean> = { screenshot: true, + assetScreenshot: true, fullPageArchive: true, precrawledArchive: false, bannerImage: true, diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts index c7dd7575..6393c950 100644 --- a/packages/trpc/routers/admin.ts +++ b/packages/trpc/routers/admin.ts @@ -9,6 +9,7 @@ import { OpenAIQueue, SearchIndexingQueue, TidyAssetsQueue, + triggerReprocessingFixMode, triggerSearchReindex, } from "@hoarder/shared/queues"; import { @@ -154,6 +155,15 @@ export const adminAppRouter = router({ await Promise.all(bookmarkIds.map((b) => triggerSearchReindex(b.id))); }), + reprocessAssetsFixMode: adminProcedure.mutation(async ({ ctx }) => { + const bookmarkIds = await ctx.db.query.bookmarkAssets.findMany({ + columns: { + id: true, + }, + }); + + await Promise.all(bookmarkIds.map((b) => triggerReprocessingFixMode(b.id))); + }), reRunInferenceOnAllBookmarks: adminProcedure .input( z.object({ diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 12ec9ccb..6ab863fb 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -259,6 +259,7 @@ function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark { assetId: asset.assetId, fileName: asset.fileName, sourceUrl: asset.sourceUrl, + size: assets.find((a) => a.id == asset.assetId)?.size, }; break; } @@ -441,6 +442,7 @@ export const bookmarksAppRouter = router({ case BookmarkTypes.ASSET: { await AssetPreprocessingQueue.enqueue({ bookmarkId: bookmark.id, + fixMode: false, }); break; } @@ -830,6 +832,7 @@ export const bookmarksAppRouter = router({ assetType: bookmarkAssets.assetType, fileName: bookmarkAssets.fileName, sourceUrl: bookmarkAssets.sourceUrl ?? null, + size: null, // This will get filled in the asset loop }; break; } @@ -881,6 +884,13 @@ export const bookmarksAppRouter = router({ } acc[bookmarkId].content = content; } + if (acc[bookmarkId].content.type == BookmarkTypes.ASSET) { + const content = acc[bookmarkId].content; + if (row.assets.id == content.assetId) { + // If this is the bookmark's main aset, caputure its size. + content.size = row.assets.size; + } + } acc[bookmarkId].assets.push({ id: row.assets.id, assetType: mapDBAssetTypeToUserType(row.assets.assetType), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2b5f8ac..6c313683 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -804,8 +804,11 @@ importers: specifier: ^3.3.2 version: 3.3.2 pdf2json: - specifier: ^3.0.5 - version: 3.0.5 + specifier: ^3.1.5 + version: 3.1.5 + pdf2pic: + specifier: ^3.1.3 + version: 3.1.3 pdfjs-dist: specifier: ^4.0.379 version: 4.0.379 @@ -5432,6 +5435,12 @@ packages: resolution: {integrity: sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==} engines: {node: '>= 0.4'} + array-parallel@0.1.3: + resolution: {integrity: sha512-TDPTwSWW5E4oiFiKmz6RGJ/a80Y91GuLgUYuLd49+XBS75tYo8PNgaT2K/OxuQYqkoI852MDGBorg9OcUSTQ8w==} + + array-series@0.1.5: + resolution: {integrity: sha512-L0XlBwfx9QetHOsbLDrE/vh2t018w9462HM3iaFfxRiK83aJjAt/Ja3NMkOW7FICwWTlQBa3ZbL5FKhuQWkDrg==} + array-timsort@1.0.3: resolution: {integrity: sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ==} @@ -6335,6 +6344,9 @@ packages: cross-fetch@4.0.0: resolution: {integrity: sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==} + cross-spawn@4.0.2: + resolution: {integrity: sha512-yAXz/pA1tD8Gtg2S98Ekf/sewp3Lcp3YoFKJ4Hkp5h5yLWnKVTDU0kwjKJ8NDCYcfTLfyGkzTikst+jWypT1iA==} + cross-spawn@6.0.5: resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==} engines: {node: '>=4.8'} @@ -8112,6 +8124,10 @@ packages: globrex@0.1.2: resolution: {integrity: sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==} + gm@1.25.0: + resolution: {integrity: sha512-4kKdWXTtgQ4biIo7hZA396HT062nDVVHPjQcurNZ3o/voYN+o5FUC5kOwuORbpExp3XbTJ3SU7iRipiIhQtovw==} + engines: {node: '>=14'} + gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} @@ -9336,6 +9352,9 @@ packages: resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} engines: {node: 20 || >=22} + lru-cache@4.1.5: + resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} + lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} @@ -10559,13 +10578,17 @@ packages: pathval@1.1.1: resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} - pdf2json@3.0.5: - resolution: {integrity: sha512-Un1yLbSlk/zfwrltgguskExIioXZlFSFwsyXU0cnBorLywbTbcdzmJJEebh+U2cFCtR7y8nDs5lPHAe7ldxjZg==} - engines: {node: '>=18.12.1', npm: '>=8.19.2'} + pdf2json@3.1.5: + resolution: {integrity: sha512-djZPInDLNuJU+o6GaJNvcoUh6MtUAx3IYTQCTxywHzeg1jC5YWgz/XzlgmduxxBblpMTqY2fjcWwvyRdGPTyrQ==} + engines: {node: '>=20.18.0', npm: '>=10.8.2'} hasBin: true bundledDependencies: - '@xmldom/xmldom' + pdf2pic@3.1.3: + resolution: {integrity: sha512-KbW4Qb7iHw2fBRWtA9FTc4pZg9cokiFIzc6cE7dzelTrhXWolfQuG1fYVC0E2BRmK/w7xfBjQ+OEsPZPO3QEew==} + engines: {node: '>=14'} + pdfjs-dist@4.0.379: resolution: {integrity: sha512-6H0Gv1nna+wmrr3CakaKlZ4rbrL8hvGIFAgg4YcoFuGC0HC4B2DVjXEGTFjJEjLlf8nYi3C3/MYRcM5bNx0elA==} engines: {node: '>=18'} @@ -11123,6 +11146,9 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + pseudomap@1.0.2: + resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} + psl@1.9.0: resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} @@ -13617,6 +13643,9 @@ packages: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} + yallist@2.1.2: + resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} + yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -19803,6 +19832,12 @@ snapshots: get-intrinsic: 1.2.4 is-string: 1.0.7 + array-parallel@0.1.3: + dev: false + + array-series@0.1.5: + dev: false + array-timsort@1.0.3: dev: false @@ -21045,6 +21080,12 @@ snapshots: - encoding dev: false + cross-spawn@4.0.2: + dependencies: + lru-cache: 4.1.5 + which: 1.3.1 + dev: false + cross-spawn@6.0.5: dependencies: nice-try: 1.0.5 @@ -23366,6 +23407,16 @@ snapshots: globrex@0.1.2: dev: true + gm@1.25.0: + dependencies: + array-parallel: 0.1.3 + array-series: 0.1.5 + cross-spawn: 4.0.2 + debug: 3.2.7 + transitivePeerDependencies: + - supports-color + dev: false + gopd@1.0.1: dependencies: get-intrinsic: 1.2.4 @@ -24941,6 +24992,12 @@ snapshots: lru-cache@11.0.1: dev: false + lru-cache@4.1.5: + dependencies: + pseudomap: 1.0.2 + yallist: 2.1.2 + dev: false + lru-cache@5.1.1: dependencies: yallist: 3.1.1 @@ -26931,7 +26988,14 @@ snapshots: pathval@1.1.1: dev: true - pdf2json@3.0.5: + pdf2json@3.1.5: + dev: false + + pdf2pic@3.1.3: + dependencies: + gm: 1.25.0 + transitivePeerDependencies: + - supports-color dev: false pdfjs-dist@4.0.379: @@ -27549,6 +27613,9 @@ snapshots: proxy-from-env@1.1.0: dev: false + pseudomap@1.0.2: + dev: false + psl@1.9.0: dev: false @@ -30985,6 +31052,9 @@ snapshots: y18n@5.0.8: dev: false + yallist@2.1.2: + dev: false + yallist@3.1.1: {} yallist@4.0.0: {} |