From 10506173cd5309e7c63d83055243abc67cecad4f Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 11 Jan 2025 18:09:51 +0000 Subject: feat: Add support for singlefile extension uploads. #172 --- apps/web/app/api/assets/route.ts | 75 ++++++++++++++-------- apps/web/app/api/v1/bookmarks/singlefile/route.ts | 54 ++++++++++++++++ .../components/dashboard/preview/AttachmentBox.tsx | 1 + apps/workers/crawlerWorker.ts | 36 +++++++++-- apps/workers/workerUtils.ts | 3 + docs/docs/14-Guides/03-singlefile.md | 20 ++++++ packages/db/schema.ts | 2 + packages/e2e_tests/tests/api/bookmarks.test.ts | 51 +++++++++++++++ packages/open-api/hoarder-openapi-spec.json | 6 ++ packages/sdk/src/hoarder-api.d.ts | 4 ++ packages/shared/assetdb.ts | 7 ++ packages/shared/types/bookmarks.ts | 7 +- packages/trpc/lib/attachments.ts | 5 ++ packages/trpc/routers/bookmarks.ts | 41 ++++++++++-- 14 files changed, 275 insertions(+), 37 deletions(-) create mode 100644 apps/web/app/api/v1/bookmarks/singlefile/route.ts create mode 100644 docs/docs/14-Guides/03-singlefile.md diff --git a/apps/web/app/api/assets/route.ts b/apps/web/app/api/assets/route.ts index 0e52ff93..81ee454e 100644 --- a/apps/web/app/api/assets/route.ts +++ b/apps/web/app/api/assets/route.ts @@ -9,43 +9,43 @@ import { SUPPORTED_UPLOAD_ASSET_TYPES, } from "@hoarder/shared/assetdb"; import serverConfig from "@hoarder/shared/config"; +import { AuthedContext } from "@hoarder/trpc"; const MAX_UPLOAD_SIZE_BYTES = serverConfig.maxAssetSizeMb * 1024 * 1024; export const dynamic = "force-dynamic"; -export async function POST(request: Request) { - const ctx = await createContextFromRequest(request); - if (!ctx.user) { - return Response.json({ error: "Unauthorized" }, { status: 401 }); - } - if (serverConfig.demoMode) { - throw new TRPCError({ - message: "Mutations are not allowed in demo mode", - code: "FORBIDDEN", - }); - } - const formData = await request.formData(); + +export async function uploadFromPostData( + user: AuthedContext["user"], + db: AuthedContext["db"], + formData: FormData, +): Promise< + | { error: string; status: number } + | { + assetId: string; + contentType: string; + fileName: string; + size: number; + } +> { const data = formData.get("file") ?? formData.get("image"); let buffer; let contentType; if (data instanceof File) { contentType = data.type; if (!SUPPORTED_UPLOAD_ASSET_TYPES.has(contentType)) { - return Response.json( - { error: "Unsupported asset type" }, - { status: 400 }, - ); + return { error: "Unsupported asset type", status: 400 }; } if (data.size > MAX_UPLOAD_SIZE_BYTES) { - return Response.json({ error: "Asset is too big" }, { status: 413 }); + return { error: "Asset is too big", status: 413 }; } buffer = Buffer.from(await data.arrayBuffer()); } else { - return Response.json({ error: "Bad request" }, { status: 400 }); + return { error: "Bad request", status: 400 }; } const fileName = data.name; - const [assetDb] = await ctx.db + const [assetDb] = await db .insert(assets) .values({ id: newAssetId(), @@ -53,25 +53,50 @@ export async function POST(request: Request) { // And without an attached bookmark. assetType: AssetTypes.UNKNOWN, bookmarkId: null, - userId: ctx.user.id, + userId: user.id, contentType, size: data.size, fileName, }) .returning(); - const assetId = assetDb.id; await saveAsset({ - userId: ctx.user.id, - assetId, + userId: user.id, + assetId: assetDb.id, metadata: { contentType, fileName }, asset: buffer, }); - return Response.json({ - assetId, + return { + assetId: assetDb.id, contentType, size: buffer.byteLength, fileName, + }; +} + +export async function POST(request: Request) { + const ctx = await createContextFromRequest(request); + if (ctx.user === null) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + if (serverConfig.demoMode) { + throw new TRPCError({ + message: "Mutations are not allowed in demo mode", + code: "FORBIDDEN", + }); + } + const formData = await request.formData(); + + const resp = await uploadFromPostData(ctx.user, ctx.db, formData); + if ("error" in resp) { + return Response.json({ error: resp.error }, { status: resp.status }); + } + + return Response.json({ + assetId: resp.assetId, + contentType: resp.contentType, + size: resp.size, + fileName: resp.fileName, } satisfies ZUploadResponse); } diff --git a/apps/web/app/api/v1/bookmarks/singlefile/route.ts b/apps/web/app/api/v1/bookmarks/singlefile/route.ts new file mode 100644 index 00000000..3f8ac2f7 --- /dev/null +++ b/apps/web/app/api/v1/bookmarks/singlefile/route.ts @@ -0,0 +1,54 @@ +import { createContextFromRequest } from "@/server/api/client"; +import { TRPCError } from "@trpc/server"; + +import serverConfig from "@hoarder/shared/config"; +import { BookmarkTypes } from "@hoarder/shared/types/bookmarks"; +import { createCallerFactory } from "@hoarder/trpc"; +import { appRouter } from "@hoarder/trpc/routers/_app"; + +import { uploadFromPostData } from "../../../assets/route"; + +export const dynamic = "force-dynamic"; + +export async function POST(req: Request) { + const ctx = await createContextFromRequest(req); + if (!ctx.user) { + return Response.json({ error: "Unauthorized" }, { status: 401 }); + } + if (serverConfig.demoMode) { + throw new TRPCError({ + message: "Mutations are not allowed in demo mode", + code: "FORBIDDEN", + }); + } + const formData = await req.formData(); + const up = await uploadFromPostData(ctx.user, ctx.db, formData); + + if ("error" in up) { + return Response.json({ error: up.error }, { status: up.status }); + } + + const url = formData.get("url"); + if (!url) { + throw new TRPCError({ + message: "URL is required", + code: "BAD_REQUEST", + }); + } + if (typeof url !== "string") { + throw new TRPCError({ + message: "URL must be a string", + code: "BAD_REQUEST", + }); + } + + const createCaller = createCallerFactory(appRouter); + const api = createCaller(ctx); + + const bookmark = await api.bookmarks.createBookmark({ + type: BookmarkTypes.LINK, + url, + precrawledArchiveId: up.assetId, + }); + return Response.json(bookmark, { status: 201 }); +} diff --git a/apps/web/components/dashboard/preview/AttachmentBox.tsx b/apps/web/components/dashboard/preview/AttachmentBox.tsx index 32184c30..6547ae51 100644 --- a/apps/web/components/dashboard/preview/AttachmentBox.tsx +++ b/apps/web/components/dashboard/preview/AttachmentBox.tsx @@ -46,6 +46,7 @@ export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) { const typeToIcon: Record = { screenshot: , fullPageArchive: , + precrawledArchive: , bannerImage: , video: