diff options
| -rw-r--r-- | apps/web/components/dashboard/preview/LinkContentSection.tsx | 9 | ||||
| -rw-r--r-- | apps/workers/crawlerWorker.ts | 8 | ||||
| -rw-r--r-- | packages/shared/types/bookmarks.ts | 1 | ||||
| -rw-r--r-- | packages/trpc/routers/bookmarks.ts | 6 |
4 files changed, 20 insertions, 4 deletions
diff --git a/apps/web/components/dashboard/preview/LinkContentSection.tsx b/apps/web/components/dashboard/preview/LinkContentSection.tsx index f09cc31f..f1fe3820 100644 --- a/apps/web/components/dashboard/preview/LinkContentSection.tsx +++ b/apps/web/components/dashboard/preview/LinkContentSection.tsx @@ -26,10 +26,12 @@ import { } from "@hoarder/shared/types/bookmarks"; function FullPageArchiveSection({ link }: { link: ZBookmarkedLink }) { + const archiveAssetId = + link.fullPageArchiveAssetId ?? link.precrawledArchiveAssetId; return ( <iframe title={link.url} - src={`/api/assets/${link.fullPageArchiveAssetId}`} + src={`/api/assets/${archiveAssetId}`} className="relative h-full min-w-full" /> ); @@ -199,7 +201,10 @@ export default function LinkContentSection({ </SelectItem> <SelectItem value="archive" - disabled={!bookmark.content.fullPageArchiveAssetId} + disabled={ + !bookmark.content.fullPageArchiveAssetId && + !bookmark.content.precrawledArchiveAssetId + } > {t("common.archive")} </SelectItem> diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 91adb185..7611494e 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -669,7 +669,8 @@ async function crawlAndParseUrl( .set({ title: meta.title, description: meta.description, - imageUrl: meta.image, + // Don't store data URIs as they're not valid URLs and are usually quite large + imageUrl: meta.image?.startsWith("data:") ? null : meta.image, favicon: meta.logo, content: readableContent?.textContent, htmlContent: readableContent?.content, @@ -705,7 +706,10 @@ async function crawlAndParseUrl( ]); return async () => { - if (serverConfig.crawler.fullPageArchive || archiveFullPage) { + if ( + !precrawledArchiveAssetId && + (serverConfig.crawler.fullPageArchive || archiveFullPage) + ) { const { assetId: fullPageArchiveAssetId, size, diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index 0bb98f82..b6a74474 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -40,6 +40,7 @@ export const zBookmarkedLinkSchema = z.object({ imageAssetId: z.string().nullish(), screenshotAssetId: z.string().nullish(), fullPageArchiveAssetId: z.string().nullish(), + precrawledArchiveAssetId: z.string().nullish(), videoAssetId: z.string().nullish(), favicon: z.string().nullish(), htmlContent: z.string().nullish(), diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 817ceef3..12ec9ccb 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -234,6 +234,9 @@ function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark { fullPageArchiveAssetId: assets.find( (a) => a.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE, )?.id, + precrawledArchiveAssetId: assets.find( + (a) => a.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE, + )?.id, imageAssetId: assets.find( (a) => a.assetType == AssetTypes.LINK_BANNER_IMAGE, )?.id, @@ -873,6 +876,9 @@ export const bookmarksAppRouter = router({ if (row.assets.assetType == AssetTypes.LINK_VIDEO) { content.videoAssetId = row.assets.id; } + if (row.assets.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE) { + content.precrawledArchiveAssetId = row.assets.id; + } acc[bookmarkId].content = content; } acc[bookmarkId].assets.push({ |