diff options
| -rw-r--r-- | packages/e2e_tests/docker-compose.yml | 16 | ||||
| -rw-r--r-- | packages/e2e_tests/setup/html/hello.html | 12 | ||||
| -rw-r--r-- | packages/e2e_tests/setup/html/image.png | bin | 0 -> 3195946 bytes | |||
| -rw-r--r-- | packages/e2e_tests/setup/startContainers.ts | 23 | ||||
| -rw-r--r-- | packages/e2e_tests/tests/workers/crawler.test.ts | 100 | ||||
| -rw-r--r-- | packages/e2e_tests/utils/general.ts | 24 |
6 files changed, 161 insertions, 14 deletions
diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.yml index e1f30f8b..c48d4d63 100644 --- a/packages/e2e_tests/docker-compose.yml +++ b/packages/e2e_tests/docker-compose.yml @@ -12,9 +12,25 @@ services: NEXTAUTH_SECRET: secret MEILI_MASTER_KEY: dummy MEILI_ADDR: http://meilisearch:7700 + BROWSER_WEB_URL: http://chrome:9222 meilisearch: image: getmeili/meilisearch:v1.13.3 restart: unless-stopped environment: MEILI_NO_ANALYTICS: "true" MEILI_MASTER_KEY: dummy + chrome: + image: gcr.io/zenika-hub/alpine-chrome:123 + restart: unless-stopped + command: + - --no-sandbox + - --disable-gpu + - --disable-dev-shm-usage + - --remote-debugging-address=0.0.0.0 + - --remote-debugging-port=9222 + - --hide-scrollbars + nginx: + image: nginx:alpine + restart: unless-stopped + volumes: + - ./setup/html:/usr/share/nginx/html diff --git a/packages/e2e_tests/setup/html/hello.html b/packages/e2e_tests/setup/html/hello.html new file mode 100644 index 00000000..da551251 --- /dev/null +++ b/packages/e2e_tests/setup/html/hello.html @@ -0,0 +1,12 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="UTF-8" /> + <meta http-equiv="X-UA-Compatible" content="IE=edge" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>My test title</title> + </head> + <body> + <h1>Hello World</h1> + </body> +</html> diff --git a/packages/e2e_tests/setup/html/image.png b/packages/e2e_tests/setup/html/image.png Binary files differnew file mode 100644 index 00000000..b0dce115 --- /dev/null +++ b/packages/e2e_tests/setup/html/image.png diff --git a/packages/e2e_tests/setup/startContainers.ts b/packages/e2e_tests/setup/startContainers.ts index 10b1b9d8..df07cc58 100644 --- a/packages/e2e_tests/setup/startContainers.ts +++ b/packages/e2e_tests/setup/startContainers.ts @@ -4,6 +4,8 @@ import path from "path"; import { fileURLToPath } from "url"; import type { GlobalSetupContext } from "vitest/node"; +import { waitUntil } from "../utils/general"; + async function getRandomPort(): Promise<number> { const server = net.createServer(); return new Promise<number>((resolve, reject) => { @@ -17,21 +19,14 @@ async function getRandomPort(): Promise<number> { } async function waitForHealthy(port: number, timeout = 60000): Promise<void> { - const startTime = Date.now(); - - while (Date.now() - startTime < timeout) { - try { + return waitUntil( + async () => { const response = await fetch(`http://localhost:${port}/api/health`); - if (response.status === 200) { - return; - } - } catch (error) { - // Ignore errors and retry - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - - throw new Error(`Health check failed after ${timeout}ms`); + return response.status === 200; + }, + "Container are healthy", + timeout, + ); } export default async function ({ provide }: GlobalSetupContext) { diff --git a/packages/e2e_tests/tests/workers/crawler.test.ts b/packages/e2e_tests/tests/workers/crawler.test.ts new file mode 100644 index 00000000..df276cae --- /dev/null +++ b/packages/e2e_tests/tests/workers/crawler.test.ts @@ -0,0 +1,100 @@ +import { assert, beforeEach, describe, expect, inject, it } from "vitest"; + +import { createKarakeepClient } from "@karakeep/sdk"; + +import { createTestUser } from "../../utils/api"; +import { waitUntil } from "../../utils/general"; + +describe("Crawler Tests", () => { + const port = inject("hoarderPort"); + + if (!port) { + throw new Error("Missing required environment variables"); + } + + let client: ReturnType<typeof createKarakeepClient>; + let apiKey: string; + + async function getBookmark(bookmarkId: string) { + const { data } = await client.GET(`/bookmarks/{bookmarkId}`, { + params: { + path: { + bookmarkId, + }, + query: { + includeContent: true, + }, + }, + }); + return data; + } + + beforeEach(async () => { + apiKey = await createTestUser(); + client = createKarakeepClient({ + baseUrl: `http://localhost:${port}/api/v1/`, + headers: { + "Content-Type": "application/json", + authorization: `Bearer ${apiKey}`, + }, + }); + }); + + it("should crawl a website", async () => { + let { data: bookmark } = await client.POST("/bookmarks", { + body: { + type: "link", + url: "http://nginx:80/hello.html", + }, + }); + assert(bookmark); + + await waitUntil( + async () => { + const data = await getBookmark(bookmark!.id); + assert(data); + assert(data.content.type === "link"); + return data.content.crawledAt !== null; + }, + "Bookmark is crawled", + 10000, + ); + + bookmark = await getBookmark(bookmark.id); + assert(bookmark && bookmark.content.type === "link"); + expect(bookmark.content.crawledAt).toBeDefined(); + expect(bookmark.content.htmlContent).toContain("Hello World"); + expect(bookmark.content.title).toContain("My test title"); + expect(bookmark.content.url).toBe("http://nginx:80/hello.html"); + expect( + bookmark.assets.find((a) => a.assetType === "screenshot"), + ).toBeDefined(); + }); + + it("image lings jobs be converted into images", async () => { + let { data: bookmark } = await client.POST("/bookmarks", { + body: { + type: "link", + url: "http://nginx:80/image.png", + }, + }); + assert(bookmark); + + await waitUntil( + async () => { + const data = await getBookmark(bookmark!.id); + assert(data); + return data.content.type === "asset"; + }, + "Bookmark is crawled and converted to an image", + 10000, + ); + + bookmark = await getBookmark(bookmark.id); + assert(bookmark && bookmark.content.type === "asset"); + expect(bookmark.content.assetType).toBe("image"); + expect(bookmark.content.assetId).toBeDefined(); + expect(bookmark.content.fileName).toBe("image.png"); + expect(bookmark.content.sourceUrl).toBe("http://nginx:80/image.png"); + }); +}); diff --git a/packages/e2e_tests/utils/general.ts b/packages/e2e_tests/utils/general.ts new file mode 100644 index 00000000..417a7cd5 --- /dev/null +++ b/packages/e2e_tests/utils/general.ts @@ -0,0 +1,24 @@ +export async function waitUntil( + f: () => Promise<boolean>, + description: string, + timeoutMs = 60000, +): Promise<void> { + const startTime = Date.now(); + + while (Date.now() - startTime < timeoutMs) { + console.log(`Waiting for ${description}...`); + try { + const res = await f(); + if (res) { + console.log(`${description}: success`); + return; + } + } catch (error) { + // Ignore errors and retry + console.log(`${description}: error, retrying...: ${error}`); + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + + throw new Error(`${description}: timeout after ${timeoutMs}ms`); +} |
