aboutsummaryrefslogtreecommitdiffstats
path: root/apps
diff options
context:
space:
mode:
Diffstat (limited to 'apps')
-rw-r--r--apps/web/app/api/assets/route.ts75
-rw-r--r--apps/web/app/api/v1/bookmarks/singlefile/route.ts54
-rw-r--r--apps/web/components/dashboard/preview/AttachmentBox.tsx1
-rw-r--r--apps/workers/crawlerWorker.ts36
-rw-r--r--apps/workers/workerUtils.ts3
5 files changed, 138 insertions, 31 deletions
diff --git a/apps/web/app/api/assets/route.ts b/apps/web/app/api/assets/route.ts
index 0e52ff93..81ee454e 100644
--- a/apps/web/app/api/assets/route.ts
+++ b/apps/web/app/api/assets/route.ts
@@ -9,43 +9,43 @@ import {
SUPPORTED_UPLOAD_ASSET_TYPES,
} from "@hoarder/shared/assetdb";
import serverConfig from "@hoarder/shared/config";
+import { AuthedContext } from "@hoarder/trpc";
const MAX_UPLOAD_SIZE_BYTES = serverConfig.maxAssetSizeMb * 1024 * 1024;
export const dynamic = "force-dynamic";
-export async function POST(request: Request) {
- const ctx = await createContextFromRequest(request);
- if (!ctx.user) {
- return Response.json({ error: "Unauthorized" }, { status: 401 });
- }
- if (serverConfig.demoMode) {
- throw new TRPCError({
- message: "Mutations are not allowed in demo mode",
- code: "FORBIDDEN",
- });
- }
- const formData = await request.formData();
+
+export async function uploadFromPostData(
+ user: AuthedContext["user"],
+ db: AuthedContext["db"],
+ formData: FormData,
+): Promise<
+ | { error: string; status: number }
+ | {
+ assetId: string;
+ contentType: string;
+ fileName: string;
+ size: number;
+ }
+> {
const data = formData.get("file") ?? formData.get("image");
let buffer;
let contentType;
if (data instanceof File) {
contentType = data.type;
if (!SUPPORTED_UPLOAD_ASSET_TYPES.has(contentType)) {
- return Response.json(
- { error: "Unsupported asset type" },
- { status: 400 },
- );
+ return { error: "Unsupported asset type", status: 400 };
}
if (data.size > MAX_UPLOAD_SIZE_BYTES) {
- return Response.json({ error: "Asset is too big" }, { status: 413 });
+ return { error: "Asset is too big", status: 413 };
}
buffer = Buffer.from(await data.arrayBuffer());
} else {
- return Response.json({ error: "Bad request" }, { status: 400 });
+ return { error: "Bad request", status: 400 };
}
const fileName = data.name;
- const [assetDb] = await ctx.db
+ const [assetDb] = await db
.insert(assets)
.values({
id: newAssetId(),
@@ -53,25 +53,50 @@ export async function POST(request: Request) {
// And without an attached bookmark.
assetType: AssetTypes.UNKNOWN,
bookmarkId: null,
- userId: ctx.user.id,
+ userId: user.id,
contentType,
size: data.size,
fileName,
})
.returning();
- const assetId = assetDb.id;
await saveAsset({
- userId: ctx.user.id,
- assetId,
+ userId: user.id,
+ assetId: assetDb.id,
metadata: { contentType, fileName },
asset: buffer,
});
- return Response.json({
- assetId,
+ return {
+ assetId: assetDb.id,
contentType,
size: buffer.byteLength,
fileName,
+ };
+}
+
+export async function POST(request: Request) {
+ const ctx = await createContextFromRequest(request);
+ if (ctx.user === null) {
+ return Response.json({ error: "Unauthorized" }, { status: 401 });
+ }
+ if (serverConfig.demoMode) {
+ throw new TRPCError({
+ message: "Mutations are not allowed in demo mode",
+ code: "FORBIDDEN",
+ });
+ }
+ const formData = await request.formData();
+
+ const resp = await uploadFromPostData(ctx.user, ctx.db, formData);
+ if ("error" in resp) {
+ return Response.json({ error: resp.error }, { status: resp.status });
+ }
+
+ return Response.json({
+ assetId: resp.assetId,
+ contentType: resp.contentType,
+ size: resp.size,
+ fileName: resp.fileName,
} satisfies ZUploadResponse);
}
diff --git a/apps/web/app/api/v1/bookmarks/singlefile/route.ts b/apps/web/app/api/v1/bookmarks/singlefile/route.ts
new file mode 100644
index 00000000..3f8ac2f7
--- /dev/null
+++ b/apps/web/app/api/v1/bookmarks/singlefile/route.ts
@@ -0,0 +1,54 @@
+import { createContextFromRequest } from "@/server/api/client";
+import { TRPCError } from "@trpc/server";
+
+import serverConfig from "@hoarder/shared/config";
+import { BookmarkTypes } from "@hoarder/shared/types/bookmarks";
+import { createCallerFactory } from "@hoarder/trpc";
+import { appRouter } from "@hoarder/trpc/routers/_app";
+
+import { uploadFromPostData } from "../../../assets/route";
+
+export const dynamic = "force-dynamic";
+
+export async function POST(req: Request) {
+ const ctx = await createContextFromRequest(req);
+ if (!ctx.user) {
+ return Response.json({ error: "Unauthorized" }, { status: 401 });
+ }
+ if (serverConfig.demoMode) {
+ throw new TRPCError({
+ message: "Mutations are not allowed in demo mode",
+ code: "FORBIDDEN",
+ });
+ }
+ const formData = await req.formData();
+ const up = await uploadFromPostData(ctx.user, ctx.db, formData);
+
+ if ("error" in up) {
+ return Response.json({ error: up.error }, { status: up.status });
+ }
+
+ const url = formData.get("url");
+ if (!url) {
+ throw new TRPCError({
+ message: "URL is required",
+ code: "BAD_REQUEST",
+ });
+ }
+ if (typeof url !== "string") {
+ throw new TRPCError({
+ message: "URL must be a string",
+ code: "BAD_REQUEST",
+ });
+ }
+
+ const createCaller = createCallerFactory(appRouter);
+ const api = createCaller(ctx);
+
+ const bookmark = await api.bookmarks.createBookmark({
+ type: BookmarkTypes.LINK,
+ url,
+ precrawledArchiveId: up.assetId,
+ });
+ return Response.json(bookmark, { status: 201 });
+}
diff --git a/apps/web/components/dashboard/preview/AttachmentBox.tsx b/apps/web/components/dashboard/preview/AttachmentBox.tsx
index 32184c30..6547ae51 100644
--- a/apps/web/components/dashboard/preview/AttachmentBox.tsx
+++ b/apps/web/components/dashboard/preview/AttachmentBox.tsx
@@ -46,6 +46,7 @@ export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) {
const typeToIcon: Record<ZAssetType, React.ReactNode> = {
screenshot: <Camera className="size-4" />,
fullPageArchive: <Archive className="size-4" />,
+ precrawledArchive: <Archive className="size-4" />,
bannerImage: <Image className="size-4" />,
video: <Video className="size-4" />,
bookmarkAsset: <Paperclip className="size-4" />,
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index 252da3b2..16b1f4ae 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -41,6 +41,7 @@ import {
getAssetSize,
IMAGE_ASSET_TYPES,
newAssetId,
+ readAsset,
saveAsset,
saveAssetFromFile,
silentDeleteAsset,
@@ -582,14 +583,35 @@ async function crawlAndParseUrl(
oldScreenshotAssetId: string | undefined,
oldImageAssetId: string | undefined,
oldFullPageArchiveAssetId: string | undefined,
+ precrawledArchiveAssetId: string | undefined,
archiveFullPage: boolean,
) {
- const {
- htmlContent,
- screenshot,
- statusCode,
- url: browserUrl,
- } = await crawlPage(jobId, url);
+ let result: {
+ htmlContent: string;
+ screenshot: Buffer | undefined;
+ statusCode: number | null;
+ url: string;
+ };
+
+ if (precrawledArchiveAssetId) {
+ logger.info(
+ `[Crawler][${jobId}] The page has been precrawled. Will use the precrawled archive instead.`,
+ );
+ const asset = await readAsset({
+ userId,
+ assetId: precrawledArchiveAssetId,
+ });
+ result = {
+ htmlContent: asset.asset.toString(),
+ screenshot: undefined,
+ statusCode: 200,
+ url,
+ };
+ } else {
+ result = await crawlPage(jobId, url);
+ }
+
+ const { htmlContent, screenshot, statusCode, url: browserUrl } = result;
const [meta, readableContent, screenshotAssetInfo] = await Promise.all([
extractMetadata(htmlContent, browserUrl, jobId),
@@ -701,6 +723,7 @@ async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) {
screenshotAssetId: oldScreenshotAssetId,
imageAssetId: oldImageAssetId,
fullPageArchiveAssetId: oldFullPageArchiveAssetId,
+ precrawledArchiveAssetId,
} = await getBookmarkDetails(bookmarkId);
logger.info(
@@ -730,6 +753,7 @@ async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) {
oldScreenshotAssetId,
oldImageAssetId,
oldFullPageArchiveAssetId,
+ precrawledArchiveAssetId,
archiveFullPage,
);
diff --git a/apps/workers/workerUtils.ts b/apps/workers/workerUtils.ts
index e93d241b..2b365c73 100644
--- a/apps/workers/workerUtils.ts
+++ b/apps/workers/workerUtils.ts
@@ -44,5 +44,8 @@ export async function getBookmarkDetails(bookmarkId: string) {
videoAssetId: bookmark.assets.find(
(a) => a.assetType == AssetTypes.LINK_VIDEO,
)?.id,
+ precrawledArchiveAssetId: bookmark.assets.find(
+ (a) => a.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE,
+ )?.id,
};
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-29 13:21:38 +0000