feat: Allow specifying the overwrite mode for singlefile archives. Fixes #1125

author: Mohamed Bassem <me@mbassem.com> 2025-06-01 20:53:12 +0000
committer: Mohamed Bassem <me@mbassem.com> 2025-06-01 20:53:12 +0000
commit: e59be245d5e3005b5b5dadf78ad7115cc800c663 (patch)
tree: 67e973a0d2c54cc3c5c64829d811999c219b6fca /packages
parent: ea1d0023bfee55358ebb1a96f3d06e783a219c0d (diff)
download: karakeep-e59be245d5e3005b5b5dadf78ad7115cc800c663.tar.zst
3 files changed, 308 insertions, 37 deletions
diff --git a/packages/api/routes/bookmarks.ts b/packages/api/routes/bookmarks.ts
index fbc46d2f..abf0daae 100644
--- a/packages/api/routes/bookmarks.ts
+++ b/packages/api/routes/bookmarks.ts
@@ -90,6 +90,21 @@ const app = new Hono()
   .post(
     "/singlefile",
     zValidator(
+      "query",
+      z.object({
+        ifexists: z
+          .enum([
+            "skip",
+            "overwrite",
+            "overwrite-recrawl",
+            "append",
+            "append-recrawl",
+          ])
+          .optional()
+          .default("skip"),
+      }),
+    ),
+    zValidator(
       "form",
       z.object({
         url: z.string(),
@@ -107,7 +122,59 @@ const app = new Hono()
         url: form.url,
         precrawledArchiveId: up.assetId,
       });
-      return c.json(bookmark, 201);
+      if (bookmark.alreadyExists) {
+        const ifexists = c.req.valid("query").ifexists;
+        switch (ifexists) {
+          case "skip":
+            break;
+          case "overwrite-recrawl":
+          case "overwrite": {
+            const existingPrecrawledArchiveId = bookmark.assets
+              .filter((a) => a.assetType == "precrawledArchive")
+              .at(-1)?.id;
+            if (existingPrecrawledArchiveId) {
+              await c.var.api.assets.replaceAsset({
+                bookmarkId: bookmark.id,
+                oldAssetId: existingPrecrawledArchiveId,
+                newAssetId: up.assetId,
+              });
+            } else {
+              await c.var.api.assets.attachAsset({
+                bookmarkId: bookmark.id,
+                asset: {
+                  id: up.assetId,
+                  assetType: "precrawledArchive",
+                },
+              });
+            }
+            if (ifexists == "overwrite-recrawl") {
+              await c.var.api.bookmarks.recrawlBookmark({
+                bookmarkId: bookmark.id,
+              });
+            }
+            break;
+          }
+          case "append-recrawl":
+          case "append": {
+            await c.var.api.assets.attachAsset({
+              bookmarkId: bookmark.id,
+              asset: {
+                id: up.assetId,
+                assetType: "precrawledArchive",
+              },
+            });
+            if (ifexists == "append-recrawl") {
+              await c.var.api.bookmarks.recrawlBookmark({
+                bookmarkId: bookmark.id,
+              });
+            }
+            break;
+          }
+        }
+        return c.json(bookmark, 200);
+      } else {
+        return c.json(bookmark, 201);
+      }
     },
   )
 
diff --git a/packages/e2e_tests/tests/api/bookmarks.test.ts b/packages/e2e_tests/tests/api/bookmarks.test.ts
index 6c56f689..d40c1add 100644
--- a/packages/e2e_tests/tests/api/bookmarks.test.ts
+++ b/packages/e2e_tests/tests/api/bookmarks.test.ts
@@ -397,54 +397,258 @@ describe("Bookmarks API", () => {
     expect(finalPage!.nextCursor).toBeNull();
   });
 
-  it("should support precrawling via singlefile", async () => {
-    const file = new File(["<html>HELLO WORLD</html>"], "test.html", {
-      type: "text/html",
-    });
+  describe("singlefile", () => {
+    async function uploadSinglefileAsset(ifexists?: string) {
+      const file = new File(["<html>HELLO WORLD</html>"], "test.html", {
+        type: "text/html",
+      });
 
-    const formData = new FormData();
-    formData.append("url", "https://example.com");
-    formData.append("file", file);
+      const formData = new FormData();
+      formData.append("url", "https://example.com");
+      formData.append("file", file);
 
-    // OpenAPI typescript doesn't support multipart/form-data
-    // Upload the singlefile archive
-    const response = await fetch(
-      `http://localhost:${port}/api/v1/bookmarks/singlefile`,
-      {
+      const url = new URL(
+        `http://localhost:${port}/api/v1/bookmarks/singlefile`,
+      );
+      if (ifexists) {
+        url.searchParams.append("ifexists", ifexists);
+      }
+
+      const response = await fetch(url.toString(), {
         method: "POST",
         headers: {
           authorization: `Bearer ${apiKey}`,
         },
         body: formData,
-      },
-    );
+      });
 
-    if (!response.ok) {
-      throw new Error(`Failed to upload asset: ${response.statusText}`);
+      if (!response.ok) {
+        return [null, response] as const;
+      }
+
+      const data = (await response.json()) as { id: string };
+      return [data, response] as const;
     }
 
-    expect(response.status).toBe(201);
+    it("should support precrawling via singlefile with ifexists=skip", async () => {
+      // First upload: create a bookmark
+      const [data, response] = await uploadSinglefileAsset();
+      expect(response?.status).toBe(201);
+      const bookmarkId = data?.id;
+      if (!bookmarkId) throw new Error("Bookmark ID not found");
+
+      // Get the bookmark and record the precrawled asset id
+      const { data: bookmark, response: getResponse1 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse1.status).toBe(200);
+      const assetIds = bookmark!.assets
+        .filter((a) => a.assetType === "precrawledArchive")
+        .map((a) => a.id);
+      expect(assetIds.length).toBe(1);
+      const firstAssetId = assetIds[0];
+
+      // Second upload with skip
+      const [data2, response2] = await uploadSinglefileAsset("skip");
+      expect(response2?.status).toBe(200);
+      expect(data2?.id).toBe(bookmarkId);
+
+      // Get the bookmark again
+      const { data: bookmark2, response: getResponse2 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse2.status).toBe(200);
+      const assetIds2 = bookmark2!.assets
+        .filter((a) => a.assetType === "precrawledArchive")
+        .map((a) => a.id);
+      expect(assetIds2).toEqual([firstAssetId]); // same asset
+    });
 
-    const { id: bookmarkId } = (await response.json()) as {
-      id: string;
-    };
+    it("should support precrawling via singlefile with ifexists=overwrite", async () => {
+      // First upload
+      const [data, response] = await uploadSinglefileAsset("overwrite");
+      expect(response?.status).toBe(201);
+      const bookmarkId = data?.id;
+      if (!bookmarkId) throw new Error("Bookmark ID not found");
+
+      // Record the asset
+      const { data: bookmark, response: getResponse1 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse1.status).toBe(200);
+      const firstAssetId = bookmark!.assets.find(
+        (a) => a.assetType === "precrawledArchive",
+      )?.id;
+      expect(firstAssetId).toBeDefined();
+
+      // Second upload with overwrite
+      const [data2, response2] = await uploadSinglefileAsset("overwrite");
+      expect(response2?.status).toBe(200);
+      expect(data2?.id).toBe(bookmarkId);
+
+      // Get the bookmark again
+      const { data: bookmark2, response: getResponse2 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse2.status).toBe(200);
+      const secondAssetId = bookmark2!.assets.find(
+        (a) => a.assetType === "precrawledArchive",
+      )?.id;
+      expect(secondAssetId).toBeDefined();
+      expect(secondAssetId).not.toBe(firstAssetId);
+      // There should be only one precrawledArchive asset
+      const precrawledAssets = bookmark2!.assets.filter(
+        (a) => a.assetType === "precrawledArchive",
+      );
+      expect(precrawledAssets.length).toBe(1);
+    });
 
-    // Get the created bookmark
-    const { data: retrievedBookmark, response: getResponse } = await client.GET(
-      "/bookmarks/{bookmarkId}",
-      {
-        params: {
-          path: {
-            bookmarkId: bookmarkId,
-          },
+    it("should support precrawling via singlefile with ifexists=overwrite-recrawl", async () => {
+      // First upload
+      const [data, response] = await uploadSinglefileAsset("overwrite-recrawl");
+      expect(response?.status).toBe(201);
+      const bookmarkId = data?.id;
+      if (!bookmarkId) throw new Error("Bookmark ID not found");
+
+      // Record the asset
+      const { data: bookmark, response: getResponse1 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
         },
-      },
-    );
+      );
+      expect(getResponse1.status).toBe(200);
+      const firstAssetId = bookmark!.assets.find(
+        (a) => a.assetType === "precrawledArchive",
+      )?.id;
+      expect(firstAssetId).toBeDefined();
+
+      // Second upload with overwrite-recrawl
+      const [data2, response2] =
+        await uploadSinglefileAsset("overwrite-recrawl");
+      expect(response2?.status).toBe(200);
+      expect(data2?.id).toBe(bookmarkId);
+
+      // Get the bookmark again
+      const { data: bookmark2, response: getResponse2 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse2.status).toBe(200);
+      const secondAssetId = bookmark2!.assets.find(
+        (a) => a.assetType === "precrawledArchive",
+      )?.id;
+      expect(secondAssetId).toBeDefined();
+      expect(secondAssetId).not.toBe(firstAssetId);
+      // There should be only one precrawledArchive asset
+      const precrawledAssets = bookmark2!.assets.filter(
+        (a) => a.assetType === "precrawledArchive",
+      );
+      expect(precrawledAssets.length).toBe(1);
+    });
 
-    expect(getResponse.status).toBe(200);
-    assert(retrievedBookmark!.content.type === "link");
-    expect(retrievedBookmark!.assets.map((a) => a.assetType)).toContain(
-      "precrawledArchive",
-    );
+    it("should support precrawling via singlefile with ifexists=append", async () => {
+      // First upload
+      const [data, response] = await uploadSinglefileAsset("append");
+      expect(response?.status).toBe(201);
+      const bookmarkId = data?.id;
+      if (!bookmarkId) throw new Error("Bookmark ID not found");
+
+      // Record the first asset
+      const { data: bookmark, response: getResponse1 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse1.status).toBe(200);
+      const firstAssetId = bookmark!.assets.find(
+        (a) => a.assetType === "precrawledArchive",
+      )?.id;
+      expect(firstAssetId).toBeDefined();
+
+      // Second upload with append
+      const [data2, response2] = await uploadSinglefileAsset("append");
+      expect(response2?.status).toBe(200);
+      expect(data2?.id).toBe(bookmarkId);
+
+      // Get the bookmark again
+      const { data: bookmark2, response: getResponse2 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse2.status).toBe(200);
+      const precrawledAssets = bookmark2!.assets.filter(
+        (a) => a.assetType === "precrawledArchive",
+      );
+      expect(precrawledAssets.length).toBe(2);
+      expect(precrawledAssets.map((a) => a.id)).toContain(firstAssetId);
+      // The second asset id should be different
+      const secondAssetId = precrawledAssets.find(
+        (asset) => asset.id !== firstAssetId,
+      )?.id;
+      expect(secondAssetId).toBeDefined();
+    });
+
+    it("should support precrawling via singlefile with ifexists=append-recrawl", async () => {
+      // First upload
+      const [data, response] = await uploadSinglefileAsset("append-recrawl");
+      expect(response?.status).toBe(201);
+      const bookmarkId = data?.id;
+      if (!bookmarkId) throw new Error("Bookmark ID not found");
+
+      // Record the first asset
+      const { data: bookmark, response: getResponse1 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse1.status).toBe(200);
+      const firstAssetId = bookmark!.assets.find(
+        (a) => a.assetType === "precrawledArchive",
+      )?.id;
+      expect(firstAssetId).toBeDefined();
+
+      // Second upload with append-recrawl
+      const [data2, response2] = await uploadSinglefileAsset("append-recrawl");
+      expect(response2?.status).toBe(200);
+      expect(data2?.id).toBe(bookmarkId);
+
+      // Get the bookmark again
+      const { data: bookmark2, response: getResponse2 } = await client.GET(
+        "/bookmarks/{bookmarkId}",
+        {
+          params: { path: { bookmarkId } },
+        },
+      );
+      expect(getResponse2.status).toBe(200);
+      const precrawledAssets = bookmark2!.assets.filter(
+        (a) => a.assetType === "precrawledArchive",
+      );
+      expect(precrawledAssets.length).toBe(2);
+      expect(precrawledAssets.map((a) => a.id)).toContain(firstAssetId);
+      // The second asset id should be different
+      const secondAssetId = precrawledAssets.find(
+        (asset) => asset.id !== firstAssetId,
+      )?.id;
+      expect(secondAssetId).toBeDefined();
+    });
   });
 });
diff --git a/packages/trpc/lib/attachments.ts b/packages/trpc/lib/attachments.ts
index 15cbba74..739aa8f5 100644
--- a/packages/trpc/lib/attachments.ts
+++ b/packages/trpc/lib/attachments.ts
@@ -55,7 +55,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) {
     screenshot: true,
     assetScreenshot: true,
     fullPageArchive: false,
-    precrawledArchive: false,
+    precrawledArchive: true,
     bannerImage: true,
     video: false,
     bookmarkAsset: false,
author	Mohamed Bassem <me@mbassem.com>	2025-06-01 20:53:12 +0000
committer	Mohamed Bassem <me@mbassem.com>	2025-06-01 20:53:12 +0000
commit	e59be245d5e3005b5b5dadf78ad7115cc800c663 (patch)
tree	67e973a0d2c54cc3c5c64829d811999c219b6fca /packages
parent	ea1d0023bfee55358ebb1a96f3d06e783a219c0d (diff)
download	karakeep-e59be245d5e3005b5b5dadf78ad7115cc800c663.tar.zst