rcgit

Commit c4bee9fe

patch snapshot

* tests: Attempt to fix flaky tests

* fix internal address

* fix assets tests

.github/workflows/ci.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 37136a5d..13500f20 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,6 +67,14 @@ jobs:
       - name: E2E Tests
         working-directory: packages/e2e_tests
         run: pnpm test
+
+      - name: Upload Docker Logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-docker-logs-${{ github.sha }}-${{ github.run_attempt }}
+          path: packages/e2e_tests/setup/docker-logs/
+          retention-days: 7
   open-api-spec:
     runs-on: ubuntu-latest
     steps:

packages/e2e_tests/.gitignore

diff --git a/packages/e2e_tests/.gitignore b/packages/e2e_tests/.gitignore
new file mode 100644
index 00000000..0cb27f19
--- /dev/null
+++ b/packages/e2e_tests/.gitignore
@@ -0,0 +1,2 @@
+# Docker logs captured during test runs
+setup/docker-logs/

packages/e2e_tests/docker-compose.yml

diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.yml
index 8cb143b3..c74b45d6 100644
--- a/packages/e2e_tests/docker-compose.yml
+++ b/packages/e2e_tests/docker-compose.yml
@@ -15,6 +15,7 @@ services:
       MEILI_ADDR: http://meilisearch:7700
       BROWSER_WEB_URL: http://chrome:9222
       CRAWLER_NUM_WORKERS: 6
+      CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx
   meilisearch:
     image: getmeili/meilisearch:v1.13.3
     restart: unless-stopped

packages/e2e_tests/setup/startContainers.ts

diff --git a/packages/e2e_tests/setup/startContainers.ts b/packages/e2e_tests/setup/startContainers.ts
index 87e812a2..4d2a89cf 100644
--- a/packages/e2e_tests/setup/startContainers.ts
+++ b/packages/e2e_tests/setup/startContainers.ts
@@ -56,6 +56,29 @@ export default async function ({ provide }: GlobalSetupContext) {
   process.env.KARAKEEP_PORT = port.toString();
 
   return async () => {
+    console.log("Capturing docker logs...");
+    try {
+      const logsDir = path.join(__dirname, "docker-logs");
+      execSync(`mkdir -p "${logsDir}"`, { cwd: __dirname });
+
+      const services = ["web", "meilisearch", "chrome", "nginx", "minio"];
+      for (const service of services) {
+        try {
+          execSync(
+            `/bin/sh -c 'docker compose logs ${service} > "${logsDir}/${service}.log" 2>&1'`,
+            {
+              cwd: __dirname,
+            },
+          );
+          console.log(`Captured logs for ${service}`);
+        } catch (error) {
+          console.error(`Failed to capture logs for ${service}:`, error);
+        }
+      }
+    } catch (error) {
+      console.error("Failed to capture docker logs:", error);
+    }
+
     console.log("Stopping docker compose...");
     execSync("docker compose down", {
       cwd: __dirname,

packages/e2e_tests/tests/api/assets.test.ts

diff --git a/packages/e2e_tests/tests/api/assets.test.ts b/packages/e2e_tests/tests/api/assets.test.ts
index 78a5c7fe..accc5cf0 100644
--- a/packages/e2e_tests/tests/api/assets.test.ts
+++ b/packages/e2e_tests/tests/api/assets.test.ts
@@ -183,6 +183,7 @@ describe("Assets API", () => {
     expect(firstAsset).toEqual({
       id: uploadResponse1.assetId,
       assetType: "bannerImage",
+      fileName: "test.pdf",
     });
 
     // Attach second asset
@@ -205,6 +206,7 @@ describe("Assets API", () => {
     expect(secondAsset).toEqual({
       id: uploadResponse2.assetId,
       assetType: "bannerImage",
+      fileName: "test.pdf",
     });
 
     // Get bookmark and verify assets
@@ -221,8 +223,16 @@ describe("Assets API", () => {
 
     expect(bookmarkWithAssets?.assets).toEqual(
       expect.arrayContaining([
-        { id: uploadResponse1.assetId, assetType: "bannerImage" },
-        { id: uploadResponse2.assetId, assetType: "bannerImage" },
+        {
+          id: uploadResponse1.assetId,
+          assetType: "bannerImage",
+          fileName: "test.pdf",
+        },
+        {
+          id: uploadResponse2.assetId,
+          assetType: "bannerImage",
+          fileName: "test.pdf",
+        },
       ]),
     );
 
@@ -258,8 +268,16 @@ describe("Assets API", () => {
 
     expect(bookmarkAfterReplace?.assets).toEqual(
       expect.arrayContaining([
-        { id: uploadResponse3.assetId, assetType: "bannerImage" },
-        { id: uploadResponse2.assetId, assetType: "bannerImage" },
+        {
+          id: uploadResponse3.assetId,
+          assetType: "bannerImage",
+          fileName: "test.pdf",
+        },
+        {
+          id: uploadResponse2.assetId,
+          assetType: "bannerImage",
+          fileName: "test.pdf",
+        },
       ]),
     );
 
@@ -291,7 +309,11 @@ describe("Assets API", () => {
     );
 
     expect(bookmarkAfterDetach?.assets).toEqual([
-      { id: uploadResponse3.assetId, assetType: "bannerImage" },
+      {
+        id: uploadResponse3.assetId,
+        assetType: "bannerImage",
+        fileName: "test.pdf",
+      },
     ]);
   });
 });

packages/e2e_tests/tests/api/bookmarks.test.ts

diff --git a/packages/e2e_tests/tests/api/bookmarks.test.ts b/packages/e2e_tests/tests/api/bookmarks.test.ts
index e1cfc8d5..a6bc85e3 100644
--- a/packages/e2e_tests/tests/api/bookmarks.test.ts
+++ b/packages/e2e_tests/tests/api/bookmarks.test.ts
@@ -3,6 +3,7 @@ import { assert, beforeEach, describe, expect, inject, it } from "vitest";
 import { createKarakeepClient } from "@karakeep/sdk";
 
 import { createTestUser } from "../../utils/api";
+import { waitUntil } from "../../utils/general";
 
 describe("Bookmarks API", () => {
   const port = inject("karakeepPort");
@@ -353,9 +354,22 @@ describe("Bookmarks API", () => {
       },
     });
 
-    // Wait 3 seconds for the search index to be updated
-    // TODO: Replace with a check that all queues are empty
-    await new Promise((f) => setTimeout(f, 3000));
+    await waitUntil(async () => {
+      const { data, response, error } = await client.GET("/bookmarks/search", {
+        params: {
+          query: {
+            q: "test bookmark",
+          },
+        },
+      });
+      if (error) {
+        throw error;
+      }
+      if (response.status !== 200) {
+        throw new Error(`Search request failed with status ${response.status}`);
+      }
+      return (data?.bookmarks.length ?? 0) >= 2;
+    }, 'Search index contains the new bookmarks for query "test bookmark"');
 
     // Search for bookmarks
     const { data: searchResults, response: searchResponse } = await client.GET(
@@ -387,9 +401,23 @@ describe("Bookmarks API", () => {
 
     await Promise.all(bookmarkPromises);
 
-    // Wait 3 seconds for the search index to be updated
-    // TODO: Replace with a check that all queues are empty
-    await new Promise((f) => setTimeout(f, 3000));
+    await waitUntil(async () => {
+      const { data, response, error } = await client.GET("/bookmarks/search", {
+        params: {
+          query: {
+            q: "pagination",
+            limit: 5,
+          },
+        },
+      });
+      if (error) {
+        throw error;
+      }
+      if (response.status !== 200) {
+        throw new Error(`Search request failed with status ${response.status}`);
+      }
+      return (data?.bookmarks.length ?? 0) >= 5;
+    }, "Search index contains the pagination test bookmarks");
 
     // Get first page
     const { data: firstPage, response: firstResponse } = await client.GET(

packages/e2e_tests/tests/api/public.test.ts

diff --git a/packages/e2e_tests/tests/api/public.test.ts b/packages/e2e_tests/tests/api/public.test.ts
index 4ad37036..59f50495 100644
--- a/packages/e2e_tests/tests/api/public.test.ts
+++ b/packages/e2e_tests/tests/api/public.test.ts
@@ -90,18 +90,14 @@ describe("Public API", () => {
 
     const trpcClient = getTrpcClient(apiKey);
     // Wait for link bookmark to be crawled and have a banner image (screenshot)
-    await waitUntil(
-      async () => {
-        const res = await trpcClient.bookmarks.getBookmark.query({
-          bookmarkId: createBookmark1.id,
-        });
-        assert(res.content.type === BookmarkTypes.LINK);
-        // Check for screenshotAssetId as bannerImageUrl might be derived from it or original imageUrl
-        return !!res.content.screenshotAssetId || !!res.content.imageUrl;
-      },
-      "Bookmark is crawled and has banner info",
-      20000, // Increased timeout as crawling can take time
-    );
+    await waitUntil(async () => {
+      const res = await trpcClient.bookmarks.getBookmark.query({
+        bookmarkId: createBookmark1.id,
+      });
+      assert(res.content.type === BookmarkTypes.LINK);
+      // Check for screenshotAssetId as bannerImageUrl might be derived from it or original imageUrl
+      return !!res.content.screenshotAssetId || !!res.content.imageUrl;
+    }, "Bookmark is crawled and has banner info");
 
     const res = await trpcClient.publicBookmarks.getPublicBookmarksInList.query(
       {

packages/e2e_tests/tests/workers/crawler.test.ts

diff --git a/packages/e2e_tests/tests/workers/crawler.test.ts b/packages/e2e_tests/tests/workers/crawler.test.ts
index bd01a22e..534f9f2a 100644
--- a/packages/e2e_tests/tests/workers/crawler.test.ts
+++ b/packages/e2e_tests/tests/workers/crawler.test.ts
@@ -49,16 +49,12 @@ describe("Crawler Tests", () => {
     });
     assert(bookmark);
 
-    await waitUntil(
-      async () => {
-        const data = await getBookmark(bookmark!.id);
-        assert(data);
-        assert(data.content.type === "link");
-        return data.content.crawledAt !== null;
-      },
-      "Bookmark is crawled",
-      20000,
-    );
+    await waitUntil(async () => {
+      const data = await getBookmark(bookmark!.id);
+      assert(data);
+      assert(data.content.type === "link");
+      return data.content.crawledAt !== null;
+    }, "Bookmark is crawled");
 
     bookmark = await getBookmark(bookmark.id);
     assert(bookmark && bookmark.content.type === "link");
@@ -80,15 +76,11 @@ describe("Crawler Tests", () => {
     });
     assert(bookmark);
 
-    await waitUntil(
-      async () => {
-        const data = await getBookmark(bookmark!.id);
-        assert(data);
-        return data.content.type === "asset";
-      },
-      "Bookmark is crawled and converted to an image",
-      20000,
-    );
+    await waitUntil(async () => {
+      const data = await getBookmark(bookmark!.id);
+      assert(data);
+      return data.content.type === "asset";
+    }, "Bookmark is crawled and converted to an image");
 
     bookmark = await getBookmark(bookmark.id);
     assert(bookmark && bookmark.content.type === "asset");