From 68e27adb029cb7bb7b51b8ea594163931a495c61 Mon Sep 17 00:00:00 2001
From: Mohamed Bassem <me@mbassem.com>
Date: Sun, 2 Feb 2025 15:44:48 +0000
Subject: fix: Dont rearchive singlefile uploads and consider them as archives

---
 apps/web/components/dashboard/preview/LinkContentSection.tsx | 9 +++++++--
 apps/workers/crawlerWorker.ts                                | 8 ++++++--
 packages/shared/types/bookmarks.ts                           | 1 +
 packages/trpc/routers/bookmarks.ts                           | 6 ++++++
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/apps/web/components/dashboard/preview/LinkContentSection.tsx b/apps/web/components/dashboard/preview/LinkContentSection.tsx
index f09cc31f..f1fe3820 100644
--- a/apps/web/components/dashboard/preview/LinkContentSection.tsx
+++ b/apps/web/components/dashboard/preview/LinkContentSection.tsx
@@ -26,10 +26,12 @@ import {
 } from "@hoarder/shared/types/bookmarks";
 
 function FullPageArchiveSection({ link }: { link: ZBookmarkedLink }) {
+  const archiveAssetId =
+    link.fullPageArchiveAssetId ?? link.precrawledArchiveAssetId;
   return (
     <iframe
       title={link.url}
-      src={`/api/assets/${link.fullPageArchiveAssetId}`}
+      src={`/api/assets/${archiveAssetId}`}
       className="relative h-full min-w-full"
     />
   );
@@ -199,7 +201,10 @@ export default function LinkContentSection({
             </SelectItem>
             <SelectItem
               value="archive"
-              disabled={!bookmark.content.fullPageArchiveAssetId}
+              disabled={
+                !bookmark.content.fullPageArchiveAssetId &&
+                !bookmark.content.precrawledArchiveAssetId
+              }
             >
               {t("common.archive")}
             </SelectItem>
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index 91adb185..7611494e 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -669,7 +669,8 @@ async function crawlAndParseUrl(
       .set({
         title: meta.title,
         description: meta.description,
-        imageUrl: meta.image,
+        // Don't store data URIs as they're not valid URLs and are usually quite large
+        imageUrl: meta.image?.startsWith("data:") ? null : meta.image,
         favicon: meta.logo,
         content: readableContent?.textContent,
         htmlContent: readableContent?.content,
@@ -705,7 +706,10 @@ async function crawlAndParseUrl(
   ]);
 
   return async () => {
-    if (serverConfig.crawler.fullPageArchive || archiveFullPage) {
+    if (
+      !precrawledArchiveAssetId &&
+      (serverConfig.crawler.fullPageArchive || archiveFullPage)
+    ) {
       const {
         assetId: fullPageArchiveAssetId,
         size,
diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts
index 0bb98f82..b6a74474 100644
--- a/packages/shared/types/bookmarks.ts
+++ b/packages/shared/types/bookmarks.ts
@@ -40,6 +40,7 @@ export const zBookmarkedLinkSchema = z.object({
   imageAssetId: z.string().nullish(),
   screenshotAssetId: z.string().nullish(),
   fullPageArchiveAssetId: z.string().nullish(),
+  precrawledArchiveAssetId: z.string().nullish(),
   videoAssetId: z.string().nullish(),
   favicon: z.string().nullish(),
   htmlContent: z.string().nullish(),
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 817ceef3..12ec9ccb 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -234,6 +234,9 @@ function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark {
         fullPageArchiveAssetId: assets.find(
           (a) => a.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE,
         )?.id,
+        precrawledArchiveAssetId: assets.find(
+          (a) => a.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE,
+        )?.id,
         imageAssetId: assets.find(
           (a) => a.assetType == AssetTypes.LINK_BANNER_IMAGE,
         )?.id,
@@ -873,6 +876,9 @@ export const bookmarksAppRouter = router({
               if (row.assets.assetType == AssetTypes.LINK_VIDEO) {
                 content.videoAssetId = row.assets.id;
               }
+              if (row.assets.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE) {
+                content.precrawledArchiveAssetId = row.assets.id;
+              }
               acc[bookmarkId].content = content;
             }
             acc[bookmarkId].assets.push({
-- 
cgit v1.2.3-70-g09d2