diff options
21 files changed, 1352 insertions, 114 deletions
diff --git a/apps/mobile/components/bookmarks/BookmarkCard.tsx b/apps/mobile/components/bookmarks/BookmarkCard.tsx index 6662e76a..c995d593 100644 --- a/apps/mobile/components/bookmarks/BookmarkCard.tsx +++ b/apps/mobile/components/bookmarks/BookmarkCard.tsx @@ -21,33 +21,17 @@ import { useDeleteBookmark, useUpdateBookmark, } from "@hoarder/shared-react/hooks/bookmarks"; +import { + getBookmarkLinkImageUrl, + isBookmarkStillLoading, + isBookmarkStillTagging, +} from "@hoarder/shared-react/utils/bookmarkUtils"; import { TailwindResolver } from "../TailwindResolver"; import { Divider } from "../ui/Divider"; import { Skeleton } from "../ui/Skeleton"; import { useToast } from "../ui/Toast"; -const MAX_LOADING_MSEC = 30 * 1000; - -export function isBookmarkStillCrawling(bookmark: ZBookmark) { - return ( - bookmark.content.type === "link" && - !bookmark.content.crawledAt && - Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC - ); -} - -export function isBookmarkStillTagging(bookmark: ZBookmark) { - return ( - bookmark.taggingStatus === "pending" && - Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC - ); -} - -export function isBookmarkStillLoading(bookmark: ZBookmark) { - return isBookmarkStillTagging(bookmark) || isBookmarkStillCrawling(bookmark); -} - function ActionBar({ bookmark }: { bookmark: ZBookmark }) { const { toast } = useToast(); @@ -176,6 +160,7 @@ function TagList({ bookmark }: { bookmark: ZBookmark }) { } function LinkCard({ bookmark }: { bookmark: ZBookmark }) { + const { settings } = useAppSettings(); if (bookmark.content.type !== "link") { throw new Error("Wrong content type rendered"); } @@ -183,18 +168,36 @@ function LinkCard({ bookmark }: { bookmark: ZBookmark }) { const url = bookmark.content.url; const parsedUrl = new URL(url); - const imageComp = bookmark.content.imageUrl ? ( - <Image - source={{ uri: bookmark.content.imageUrl }} - className="h-56 min-h-56 w-full object-cover" - /> - ) : ( - <Image - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - source={require("@/assets/blur.jpeg")} - className="h-56 w-full rounded-t-lg" - /> - ); + const imageUrl = getBookmarkLinkImageUrl(bookmark.content); + + let imageComp; + if (imageUrl) { + imageComp = ( + <Image + source={ + imageUrl.localAsset + ? { + uri: `${settings.address}${imageUrl.url}`, + headers: { + Authorization: `Bearer ${settings.apiKey}`, + }, + } + : { + uri: imageUrl.url, + } + } + className="h-56 min-h-56 w-full object-cover" + /> + ); + } else { + imageComp = ( + <Image + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + source={require("@/assets/blur.jpeg")} + className="h-56 w-full rounded-t-lg" + /> + ); + } return ( <View className="flex gap-2"> diff --git a/apps/web/app/api/assets/route.ts b/apps/web/app/api/assets/route.ts index c77751d3..a1ebea0f 100644 --- a/apps/web/app/api/assets/route.ts +++ b/apps/web/app/api/assets/route.ts @@ -2,7 +2,7 @@ import { createContextFromRequest } from "@/server/api/client"; import { TRPCError } from "@trpc/server"; import type { ZUploadResponse } from "@hoarder/shared/types/uploads"; -import { saveAsset } from "@hoarder/shared/assetdb"; +import { newAssetId, saveAsset } from "@hoarder/shared/assetdb"; import serverConfig from "@hoarder/shared/config"; const SUPPORTED_ASSET_TYPES = new Set([ @@ -46,7 +46,7 @@ export async function POST(request: Request) { return Response.json({ error: "Bad request" }, { status: 400 }); } - const assetId = crypto.randomUUID(); + const assetId = newAssetId(); const fileName = data.name; await saveAsset({ diff --git a/apps/web/components/dashboard/bookmarks/AssetCard.tsx b/apps/web/components/dashboard/bookmarks/AssetCard.tsx index c9a43575..40f435de 100644 --- a/apps/web/components/dashboard/bookmarks/AssetCard.tsx +++ b/apps/web/components/dashboard/bookmarks/AssetCard.tsx @@ -1,13 +1,14 @@ "use client"; import Image from "next/image"; -import { isBookmarkStillTagging } from "@/lib/bookmarkUtils"; import { api } from "@/lib/trpc"; import type { ZBookmark, ZBookmarkTypeAsset, } from "@hoarder/shared/types/bookmarks"; +import { getAssetUrl } from "@hoarder/shared-react/utils/assetUtils"; +import { isBookmarkStillTagging } from "@hoarder/shared-react/utils/bookmarkUtils"; import { BookmarkLayoutAdaptingCard } from "./BookmarkLayoutAdaptingCard"; @@ -24,7 +25,7 @@ function AssetImage({ return ( <Image alt="asset" - src={`/api/assets/${bookmarkedAsset.assetId}`} + src={getAssetUrl(bookmarkedAsset.assetId)} fill={true} className={className} /> @@ -35,7 +36,7 @@ function AssetImage({ <iframe title={bookmarkedAsset.assetId} className={className} - src={`/api/assets/${bookmarkedAsset.assetId}`} + src={getAssetUrl(bookmarkedAsset.assetId)} /> ); } diff --git a/apps/web/components/dashboard/bookmarks/BookmarkLayoutAdaptingCard.tsx b/apps/web/components/dashboard/bookmarks/BookmarkLayoutAdaptingCard.tsx index 42c4db21..d282c3f4 100644 --- a/apps/web/components/dashboard/bookmarks/BookmarkLayoutAdaptingCard.tsx +++ b/apps/web/components/dashboard/bookmarks/BookmarkLayoutAdaptingCard.tsx @@ -1,7 +1,6 @@ import type { BookmarksLayoutTypes } from "@/lib/userLocalSettings/types"; import React from "react"; import Link from "next/link"; -import { isBookmarkStillTagging } from "@/lib/bookmarkUtils"; import { bookmarkLayoutSwitch, useBookmarkLayout, @@ -10,6 +9,7 @@ import { cn } from "@/lib/utils"; import dayjs from "dayjs"; import type { ZBookmark } from "@hoarder/shared/types/bookmarks"; +import { isBookmarkStillTagging } from "@hoarder/shared-react/utils/bookmarkUtils"; import BookmarkActionBar from "./BookmarkActionBar"; import TagList from "./TagList"; diff --git a/apps/web/components/dashboard/bookmarks/LinkCard.tsx b/apps/web/components/dashboard/bookmarks/LinkCard.tsx index ef0ae6f2..3bb1698f 100644 --- a/apps/web/components/dashboard/bookmarks/LinkCard.tsx +++ b/apps/web/components/dashboard/bookmarks/LinkCard.tsx @@ -1,13 +1,14 @@ "use client"; import Link from "next/link"; -import { - isBookmarkStillCrawling, - isBookmarkStillLoading, -} from "@/lib/bookmarkUtils"; import { api } from "@/lib/trpc"; import type { ZBookmarkTypeLink } from "@hoarder/shared/types/bookmarks"; +import { + getBookmarkLinkImageUrl, + isBookmarkStillCrawling, + isBookmarkStillLoading, +} from "@hoarder/shared-react/utils/bookmarkUtils"; import { BookmarkLayoutAdaptingCard } from "./BookmarkLayoutAdaptingCard"; @@ -33,7 +34,7 @@ function LinkImage({ // A dummy white pixel for when there's no image. // TODO: Better handling for cards with no images const image = - link.imageUrl ?? + getBookmarkLinkImageUrl(link)?.url ?? "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAA1JREFUGFdj+P///38ACfsD/QVDRcoAAAAASUVORK5CYII="; return ( <Link href={link.url} target="_blank"> diff --git a/apps/web/components/dashboard/bookmarks/TextCard.tsx b/apps/web/components/dashboard/bookmarks/TextCard.tsx index 9d5c8d8b..74b3e8e5 100644 --- a/apps/web/components/dashboard/bookmarks/TextCard.tsx +++ b/apps/web/components/dashboard/bookmarks/TextCard.tsx @@ -1,13 +1,13 @@ "use client"; import { useState } from "react"; -import { isBookmarkStillTagging } from "@/lib/bookmarkUtils"; import { api } from "@/lib/trpc"; import { bookmarkLayoutSwitch } from "@/lib/userLocalSettings/bookmarksLayout"; import { cn } from "@/lib/utils"; import Markdown from "react-markdown"; import type { ZBookmark } from "@hoarder/shared/types/bookmarks"; +import { isBookmarkStillTagging } from "@hoarder/shared-react/utils/bookmarkUtils"; import { BookmarkedTextViewer } from "./BookmarkedTextViewer"; import { BookmarkLayoutAdaptingCard } from "./BookmarkLayoutAdaptingCard"; diff --git a/apps/web/components/dashboard/preview/BookmarkPreview.tsx b/apps/web/components/dashboard/preview/BookmarkPreview.tsx index 29e8e39a..581ec4bd 100644 --- a/apps/web/components/dashboard/preview/BookmarkPreview.tsx +++ b/apps/web/components/dashboard/preview/BookmarkPreview.tsx @@ -11,20 +11,21 @@ import { TooltipPortal, TooltipTrigger, } from "@/components/ui/tooltip"; -import { - isBookmarkStillCrawling, - isBookmarkStillLoading, -} from "@/lib/bookmarkUtils"; import { api } from "@/lib/trpc"; import dayjs from "dayjs"; import relativeTime from "dayjs/plugin/relativeTime"; import { CalendarDays, ExternalLink } from "lucide-react"; import type { ZBookmark } from "@hoarder/shared/types/bookmarks"; +import { + isBookmarkStillCrawling, + isBookmarkStillLoading, +} from "@hoarder/shared-react/utils/bookmarkUtils"; import ActionBar from "./ActionBar"; import { AssetContentSection } from "./AssetContentSection"; import { EditableTitle } from "./EditableTitle"; +import LinkContentSection from "./LinkContentSection"; import { NoteEditor } from "./NoteEditor"; import { TextContentSection } from "./TextContentSection"; @@ -90,7 +91,10 @@ export default function BookmarkPreview({ let content; switch (bookmark.content.type) { - case "link": + case "link": { + content = <LinkContentSection bookmark={bookmark} />; + break; + } case "text": { content = <TextContentSection bookmark={bookmark} />; break; diff --git a/apps/web/components/dashboard/preview/LinkContentSection.tsx b/apps/web/components/dashboard/preview/LinkContentSection.tsx new file mode 100644 index 00000000..ff08c661 --- /dev/null +++ b/apps/web/components/dashboard/preview/LinkContentSection.tsx @@ -0,0 +1,77 @@ +import { useState } from "react"; +import { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { ScrollArea } from "@radix-ui/react-scroll-area"; + +import { ZBookmark, ZBookmarkedLink } from "@hoarder/shared/types/bookmarks"; + +function ScreenshotSection({ link }: { link: ZBookmarkedLink }) { + // eslint-disable-next-line @next/next/no-img-element + return <img alt="screenshot" src={`/api/assets/${link.screenshotAssetId}`} />; +} + +function CachedContentSection({ link }: { link: ZBookmarkedLink }) { + let content; + if (!link.htmlContent) { + content = ( + <div className="text-destructive">Failed to fetch link content ...</div> + ); + } else { + content = ( + <div + dangerouslySetInnerHTML={{ + __html: link.htmlContent || "", + }} + className="prose mx-auto dark:prose-invert" + /> + ); + } + return content; +} + +export default function LinkContentSection({ + bookmark, +}: { + bookmark: ZBookmark; +}) { + const [section, setSection] = useState<string>("cached"); + + if (bookmark.content.type != "link") { + throw new Error("Invalid content type"); + } + + let content; + if (section === "cached") { + content = <CachedContentSection link={bookmark.content} />; + } else { + content = <ScreenshotSection link={bookmark.content} />; + } + + return ( + <div className="flex flex-col items-center gap-2"> + <Select onValueChange={setSection} value={section}> + <SelectTrigger className="w-fit"> + <SelectValue /> + </SelectTrigger> + <SelectContent> + <SelectGroup> + <SelectItem value="cached">Cached Content</SelectItem> + <SelectItem + value="screenshot" + disabled={!bookmark.content.screenshotAssetId} + > + Screenshot + </SelectItem> + </SelectGroup> + </SelectContent> + </Select> + <ScrollArea className="h-full">{content}</ScrollArea> + </div> + ); +} diff --git a/apps/web/components/dashboard/preview/TextContentSection.tsx b/apps/web/components/dashboard/preview/TextContentSection.tsx index a73ad722..eba7d28b 100644 --- a/apps/web/components/dashboard/preview/TextContentSection.tsx +++ b/apps/web/components/dashboard/preview/TextContentSection.tsx @@ -4,36 +4,14 @@ import Markdown from "react-markdown"; import type { ZBookmark } from "@hoarder/shared/types/bookmarks"; export function TextContentSection({ bookmark }: { bookmark: ZBookmark }) { - let content; - switch (bookmark.content.type) { - case "link": { - if (!bookmark.content.htmlContent) { - content = ( - <div className="text-destructive"> - Failed to fetch link content ... - </div> - ); - } else { - content = ( - <div - dangerouslySetInnerHTML={{ - __html: bookmark.content.htmlContent || "", - }} - className="prose mx-auto dark:prose-invert" - /> - ); - } - break; - } - case "text": { - content = ( - <Markdown className="prose mx-auto dark:prose-invert"> - {bookmark.content.text} - </Markdown> - ); - break; - } + if (bookmark.content.type != "text") { + throw new Error("Invalid content type"); } - - return <ScrollArea className="h-full">{content}</ScrollArea>; + return ( + <ScrollArea className="h-full"> + <Markdown className="prose mx-auto dark:prose-invert"> + {bookmark.content.text} + </Markdown> + </ScrollArea> + ); } diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 91b0a03f..27e9e14c 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -24,7 +24,8 @@ import { withTimeout } from "utils"; import type { ZCrawlLinkRequest } from "@hoarder/shared/queues"; import { db } from "@hoarder/db"; -import { bookmarkLinks } from "@hoarder/db/schema"; +import { bookmarkLinks, bookmarks } from "@hoarder/db/schema"; +import { newAssetId, saveAsset } from "@hoarder/shared/assetdb"; import serverConfig from "@hoarder/shared/config"; import logger from "@hoarder/shared/logger"; import { @@ -155,15 +156,16 @@ async function changeBookmarkStatus( .where(eq(bookmarkLinks.id, bookmarkId)); } -async function getBookmarkUrl(bookmarkId: string) { - const bookmark = await db.query.bookmarkLinks.findFirst({ - where: eq(bookmarkLinks.id, bookmarkId), +async function getBookmarkDetails(bookmarkId: string) { + const bookmark = await db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, bookmarkId), + with: { link: true }, }); - if (!bookmark) { + if (!bookmark || !bookmark.link) { throw new Error("The bookmark either doesn't exist or not a link"); } - return bookmark.url; + return { url: bookmark.link.url, userId: bookmark.userId }; } /** @@ -208,13 +210,116 @@ async function crawlPage(jobId: string, url: string) { logger.info(`[Crawler][${jobId}] Finished waiting for the page to load.`); - const htmlContent = await page.content(); - return htmlContent; + const [htmlContent, screenshot] = await Promise.all([ + page.content(), + page.screenshot({ + // If you change this, you need to change the asset type in the store function. + type: "png", + encoding: "binary", + }), + ]); + logger.info( + `[Crawler][${jobId}] Finished capturing page content and a screenshot.`, + ); + return { htmlContent, screenshot, url: page.url() }; } finally { await context.close(); } } +async function extractMetadata( + htmlContent: string, + url: string, + jobId: string, +) { + logger.info( + `[Crawler][${jobId}] Will attempt to extract metadata from page ...`, + ); + const meta = await metascraperParser({ + url, + html: htmlContent, + // We don't want to validate the URL again as we've already done it by visiting the page. + // This was added because URL validation fails if the URL ends with a question mark (e.g. empty query params). + validateUrl: false, + }); + logger.info(`[Crawler][${jobId}] Done extracting metadata from the page.`); + return meta; +} + +function extractReadableContent( + htmlContent: string, + url: string, + jobId: string, +) { + logger.info( + `[Crawler][${jobId}] Will attempt to extract readable content ...`, + ); + const window = new JSDOM("").window; + const purify = DOMPurify(window); + const purifiedHTML = purify.sanitize(htmlContent); + const purifiedDOM = new JSDOM(purifiedHTML, { url }); + const readableContent = new Readability(purifiedDOM.window.document).parse(); + logger.info(`[Crawler][${jobId}] Done extracting readable content.`); + return readableContent; +} + +async function storeScreenshot( + screenshot: Buffer, + userId: string, + jobId: string, +) { + const assetId = newAssetId(); + await saveAsset({ + userId, + assetId, + metadata: { contentType: "image/png", fileName: "screenshot.png" }, + asset: screenshot, + }); + logger.info( + `[Crawler][${jobId}] Stored the screenshot as assetId: ${assetId}`, + ); + return assetId; +} + +async function downloadAndStoreImage( + url: string, + userId: string, + jobId: string, +) { + try { + logger.info(`[Crawler][${jobId}] Downloading image from "${url}"`); + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to download image: ${response.status}`); + } + const buffer = await response.arrayBuffer(); + const assetId = newAssetId(); + + const contentType = response.headers.get("content-type"); + if (!contentType) { + throw new Error("No content type in the response"); + } + + await saveAsset({ + userId, + assetId, + metadata: { contentType }, + asset: Buffer.from(buffer), + }); + + logger.info( + `[Crawler][${jobId}] Downloaded the image as assetId: ${assetId}`, + ); + + return assetId; + } catch (e) { + logger.error( + `[Crawler][${jobId}] Failed to download and store image: ${e}`, + ); + return null; + } +} + async function runCrawler(job: Job<ZCrawlLinkRequest, void>) { const jobId = job.id ?? "unknown"; @@ -227,35 +332,30 @@ async function runCrawler(job: Job<ZCrawlLinkRequest, void>) { } const { bookmarkId } = request.data; - const url = await getBookmarkUrl(bookmarkId); + const { url, userId } = await getBookmarkDetails(bookmarkId); logger.info( `[Crawler][${jobId}] Will crawl "${url}" for link with id "${bookmarkId}"`, ); validateUrl(url); - const htmlContent = await crawlPage(jobId, url); - - logger.info( - `[Crawler][${jobId}] Will attempt to parse the content of the page ...`, - ); - const meta = await metascraperParser({ - url, - html: htmlContent, - // We don't want to validate the URL again as we've already done it by visiting the page. - // This was added because URL validation fails if the URL ends with a question mark (e.g. empty query params). - validateUrl: false, - }); - logger.info(`[Crawler][${jobId}] Done parsing the content of the page.`); + const { + htmlContent, + screenshot, + url: browserUrl, + } = await crawlPage(jobId, url); - const window = new JSDOM("").window; - const purify = DOMPurify(window); - const purifiedHTML = purify.sanitize(htmlContent); - const purifiedDOM = new JSDOM(purifiedHTML, { url }); - const readableContent = new Readability(purifiedDOM.window.document).parse(); + const [meta, readableContent, screenshotAssetId] = await Promise.all([ + extractMetadata(htmlContent, browserUrl, jobId), + extractReadableContent(htmlContent, browserUrl, jobId), + storeScreenshot(screenshot, userId, jobId), + ]); + let imageAssetId: string | null = null; + if (meta.image) { + imageAssetId = await downloadAndStoreImage(meta.image, userId, jobId); + } // TODO(important): Restrict the size of content to store - await db .update(bookmarkLinks) .set({ @@ -265,6 +365,8 @@ async function runCrawler(job: Job<ZCrawlLinkRequest, void>) { favicon: meta.logo, content: readableContent?.textContent, htmlContent: readableContent?.content, + screenshotAssetId, + imageAssetId, crawledAt: new Date(), }) .where(eq(bookmarkLinks.id, bookmarkId)); diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a21dcb92..be21dfa5 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,6 +26,7 @@ services: - --disable-gpu - --remote-debugging-address=0.0.0.0 - --remote-debugging-port=9222 + - --hide-scrollbars meilisearch: image: getmeili/meilisearch:v1.6 restart: unless-stopped diff --git a/packages/db/drizzle/0020_sudden_dagger.sql b/packages/db/drizzle/0020_sudden_dagger.sql new file mode 100644 index 00000000..ef6615be --- /dev/null +++ b/packages/db/drizzle/0020_sudden_dagger.sql @@ -0,0 +1,2 @@ +ALTER TABLE bookmarkLinks ADD `screenshotAssetId` text;--> statement-breakpoint +ALTER TABLE bookmarkLinks ADD `imageAssetId` text;
\ No newline at end of file diff --git a/packages/db/drizzle/meta/0020_snapshot.json b/packages/db/drizzle/meta/0020_snapshot.json new file mode 100644 index 00000000..a64a7229 --- /dev/null +++ b/packages/db/drizzle/meta/0020_snapshot.json @@ -0,0 +1,1000 @@ +{ + "version": "5", + "dialect": "sqlite", + "id": "cdb3562b-0d7a-4f1d-bbc9-71b1119d8d88", + "prevId": "3e975d6c-1289-487f-8e78-6f4eda5243d2", + "tables": { + "account": { + "name": "account", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "columns": [ + "provider", + "providerAccountId" + ], + "name": "account_provider_providerAccountId_pk" + } + }, + "uniqueConstraints": {} + }, + "apiKey": { + "name": "apiKey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyId": { + "name": "keyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apiKey_keyId_unique": { + "name": "apiKey_keyId_unique", + "columns": [ + "keyId" + ], + "isUnique": true + }, + "apiKey_name_userId_unique": { + "name": "apiKey_name_userId_unique", + "columns": [ + "name", + "userId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "apiKey_userId_user_id_fk": { + "name": "apiKey_userId_user_id_fk", + "tableFrom": "apiKey", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkAssets": { + "name": "bookmarkAssets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assetId": { + "name": "assetId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkAssets_id_bookmarks_id_fk": { + "name": "bookmarkAssets_id_bookmarks_id_fk", + "tableFrom": "bookmarkAssets", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkLinks": { + "name": "bookmarkLinks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imageUrl": { + "name": "imageUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favicon": { + "name": "favicon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "htmlContent": { + "name": "htmlContent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "screenshotAssetId": { + "name": "screenshotAssetId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imageAssetId": { + "name": "imageAssetId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawledAt": { + "name": "crawledAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawlStatus": { + "name": "crawlStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkLinks_id_bookmarks_id_fk": { + "name": "bookmarkLinks_id_bookmarks_id_fk", + "tableFrom": "bookmarkLinks", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkLists": { + "name": "bookmarkLists", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarkLists_userId_idx": { + "name": "bookmarkLists_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarkLists_userId_user_id_fk": { + "name": "bookmarkLists_userId_user_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarkLists_parentId_bookmarkLists_id_fk": { + "name": "bookmarkLists_parentId_bookmarkLists_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "parentId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkTags": { + "name": "bookmarkTags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarkTags_name_idx": { + "name": "bookmarkTags_name_idx", + "columns": [ + "name" + ], + "isUnique": false + }, + "bookmarkTags_userId_idx": { + "name": "bookmarkTags_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarkTags_userId_name_unique": { + "name": "bookmarkTags_userId_name_unique", + "columns": [ + "userId", + "name" + ], + "isUnique": true + } + }, + "foreignKeys": { + "bookmarkTags_userId_user_id_fk": { + "name": "bookmarkTags_userId_user_id_fk", + "tableFrom": "bookmarkTags", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarkTexts": { + "name": "bookmarkTexts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkTexts_id_bookmarks_id_fk": { + "name": "bookmarkTexts_id_bookmarks_id_fk", + "tableFrom": "bookmarkTexts", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarks": { + "name": "bookmarks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "favourited": { + "name": "favourited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "taggingStatus": { + "name": "taggingStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarks_userId_idx": { + "name": "bookmarks_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarks_archived_idx": { + "name": "bookmarks_archived_idx", + "columns": [ + "archived" + ], + "isUnique": false + }, + "bookmarks_favourited_idx": { + "name": "bookmarks_favourited_idx", + "columns": [ + "favourited" + ], + "isUnique": false + }, + "bookmarks_createdAt_idx": { + "name": "bookmarks_createdAt_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarks_userId_user_id_fk": { + "name": "bookmarks_userId_user_id_fk", + "tableFrom": "bookmarks", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "bookmarksInLists": { + "name": "bookmarksInLists", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarksInLists_bookmarkId_idx": { + "name": "bookmarksInLists_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "bookmarksInLists_listId_idx": { + "name": "bookmarksInLists_listId_idx", + "columns": [ + "listId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarksInLists_bookmarkId_bookmarks_id_fk": { + "name": "bookmarksInLists_bookmarkId_bookmarks_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarksInLists_listId_bookmarkLists_id_fk": { + "name": "bookmarksInLists_listId_bookmarkLists_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "listId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bookmarksInLists_bookmarkId_listId_pk": { + "columns": [ + "bookmarkId", + "listId" + ], + "name": "bookmarksInLists_bookmarkId_listId_pk" + } + }, + "uniqueConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "tagsOnBookmarks": { + "name": "tagsOnBookmarks", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachedAt": { + "name": "attachedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attachedBy": { + "name": "attachedBy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tagsOnBookmarks_tagId_idx": { + "name": "tagsOnBookmarks_tagId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "tagsOnBookmarks_bookmarkId_idx": { + "name": "tagsOnBookmarks_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tagsOnBookmarks_bookmarkId_bookmarks_id_fk": { + "name": "tagsOnBookmarks_bookmarkId_bookmarks_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tagsOnBookmarks_tagId_bookmarkTags_id_fk": { + "name": "tagsOnBookmarks_tagId_bookmarkTags_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "tagId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tagsOnBookmarks_bookmarkId_tagId_pk": { + "columns": [ + "bookmarkId", + "tagId" + ], + "name": "tagsOnBookmarks_bookmarkId_tagId_pk" + } + }, + "uniqueConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'user'" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "verificationToken": { + "name": "verificationToken", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verificationToken_identifier_token_pk": { + "columns": [ + "identifier", + "token" + ], + "name": "verificationToken_identifier_token_pk" + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +}
\ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 67a48b68..735a3b73 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -141,6 +141,13 @@ "when": 1713432890859, "tag": "0019_many_vertigo", "breakpoints": true + }, + { + "idx": 20, + "version": "5", + "when": 1713539346326, + "tag": "0020_sudden_dagger", + "breakpoints": true } ] }
\ No newline at end of file diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 037be814..e323c25f 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -141,6 +141,8 @@ export const bookmarkLinks = sqliteTable("bookmarkLinks", { favicon: text("favicon"), content: text("content"), htmlContent: text("htmlContent"), + screenshotAssetId: text("screenshotAssetId"), + imageAssetId: text("imageAssetId"), crawledAt: integer("crawledAt", { mode: "timestamp" }), crawlStatus: text("crawlStatus", { enum: ["pending", "failure", "success"], diff --git a/packages/shared-react/utils/assetUtils.ts b/packages/shared-react/utils/assetUtils.ts new file mode 100644 index 00000000..119451d9 --- /dev/null +++ b/packages/shared-react/utils/assetUtils.ts @@ -0,0 +1,3 @@ +export function getAssetUrl(assetId: string) { + return `/api/assets/${assetId}`; +} diff --git a/apps/web/lib/bookmarkUtils.tsx b/packages/shared-react/utils/bookmarkUtils.ts index 475ba383..da199a40 100644 --- a/apps/web/lib/bookmarkUtils.tsx +++ b/packages/shared-react/utils/bookmarkUtils.ts @@ -1,7 +1,24 @@ -import type { ZBookmark } from "@hoarder/shared/types/bookmarks"; +import type { + ZBookmark, + ZBookmarkedLink, +} from "@hoarder/shared/types/bookmarks"; + +import { getAssetUrl } from "./assetUtils"; const MAX_LOADING_MSEC = 30 * 1000; +export function getBookmarkLinkImageUrl(bookmark: ZBookmarkedLink) { + if (bookmark.imageAssetId) { + return { url: getAssetUrl(bookmark.imageAssetId), localAsset: true }; + } + if (bookmark.screenshotAssetId) { + return { url: getAssetUrl(bookmark.screenshotAssetId), localAsset: true }; + } + return bookmark.imageUrl + ? { url: bookmark.imageUrl, localAsset: false } + : null; +} + export function isBookmarkStillCrawling(bookmark: ZBookmark) { return ( bookmark.content.type == "link" && diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts index 90fc7182..1033c594 100644 --- a/packages/shared/assetdb.ts +++ b/packages/shared/assetdb.ts @@ -15,6 +15,10 @@ export const zAssetMetadataSchema = z.object({ fileName: z.string().nullish(), }); +export function newAssetId() { + return crypto.randomUUID(); +} + export async function saveAsset({ userId, assetId, diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index 2cf8152b..f58473b4 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -10,6 +10,8 @@ export const zBookmarkedLinkSchema = z.object({ title: z.string().nullish(), description: z.string().nullish(), imageUrl: z.string().url().nullish(), + imageAssetId: z.string().nullish(), + screenshotAssetId: z.string().nullish(), favicon: z.string().url().nullish(), htmlContent: z.string().nullish(), crawledAt: z.date().nullish(), diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 0383d3f2..1e154e7b 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -93,6 +93,28 @@ type BookmarkQueryReturnType = Awaited< ReturnType<typeof dummyDrizzleReturnType> >; +async function cleanupAssetForBookmark( + bookmark: Pick<BookmarkQueryReturnType, "asset" | "link" | "userId">, +) { + const assetIds = []; + if (bookmark.asset) { + assetIds.push(bookmark.asset.assetId); + } + if (bookmark.link) { + if (bookmark.link.screenshotAssetId) { + assetIds.push(bookmark.link.screenshotAssetId); + } + if (bookmark.link.imageAssetId) { + assetIds.push(bookmark.link.imageAssetId); + } + } + await Promise.all( + assetIds.map((assetId) => + deleteAsset({ userId: bookmark.userId, assetId }), + ), + ); +} + function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark { const { tagsOnBookmarks, link, text, asset, ...rest } = bookmark; @@ -291,8 +313,15 @@ export const bookmarksAppRouter = router({ .input(z.object({ bookmarkId: z.string() })) .use(ensureBookmarkOwnership) .mutation(async ({ input, ctx }) => { - const asset = await ctx.db.query.bookmarkAssets.findFirst({ - where: and(eq(bookmarkAssets.id, input.bookmarkId)), + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: and( + eq(bookmarks.id, input.bookmarkId), + eq(bookmarks.userId, ctx.user.id), + ), + with: { + asset: true, + link: true, + }, }); const deleted = await ctx.db .delete(bookmarks) @@ -306,8 +335,12 @@ export const bookmarksAppRouter = router({ bookmarkId: input.bookmarkId, type: "delete", }); - if (deleted.changes > 0 && asset) { - await deleteAsset({ userId: ctx.user.id, assetId: asset.assetId }); + if (deleted.changes > 0 && bookmark) { + await cleanupAssetForBookmark({ + asset: bookmark.asset, + link: bookmark.link, + userId: ctx.user.id, + }); } }), recrawlBookmark: authedProcedure diff --git a/tooling/eslint/base.js b/tooling/eslint/base.js index 123b25fb..4aa34798 100644 --- a/tooling/eslint/base.js +++ b/tooling/eslint/base.js @@ -28,6 +28,7 @@ const config = { "@typescript-eslint/no-floating-promises": "off", "@typescript-eslint/unbound-method": "off", "@typescript-eslint/restrict-template-expressions": "off", + "@typescript-eslint/prefer-optional-chain": "off", }, ignorePatterns: [ "**/*.config.js", |