1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
import { assert, beforeEach, describe, expect, inject, it } from "vitest";
import { createKarakeepClient } from "@karakeep/sdk";
import { createTestUser } from "../../utils/api";
import { waitUntil } from "../../utils/general";
describe("Crawler Tests", () => {
const port = inject("karakeepPort");
if (!port) {
throw new Error("Missing required environment variables");
}
let client: ReturnType<typeof createKarakeepClient>;
let apiKey: string;
async function getBookmark(bookmarkId: string) {
const { data } = await client.GET(`/bookmarks/{bookmarkId}`, {
params: {
path: {
bookmarkId,
},
query: {
includeContent: true,
},
},
});
return data;
}
beforeEach(async () => {
apiKey = await createTestUser();
client = createKarakeepClient({
baseUrl: `http://localhost:${port}/api/v1/`,
headers: {
"Content-Type": "application/json",
authorization: `Bearer ${apiKey}`,
},
});
});
it("should crawl a website", async () => {
let { data: bookmark } = await client.POST("/bookmarks", {
body: {
type: "link",
url: "http://nginx:80/hello.html",
},
});
assert(bookmark);
await waitUntil(
async () => {
const data = await getBookmark(bookmark!.id);
assert(data);
assert(data.content.type === "link");
return data.content.crawledAt !== null;
},
"Bookmark is crawled",
20000,
);
bookmark = await getBookmark(bookmark.id);
assert(bookmark && bookmark.content.type === "link");
expect(bookmark.content.crawledAt).toBeDefined();
expect(bookmark.content.htmlContent).toContain("Hello World");
expect(bookmark.content.title).toContain("My test title");
expect(bookmark.content.url).toBe("http://nginx:80/hello.html");
expect(
bookmark.assets.find((a) => a.assetType === "screenshot"),
).toBeDefined();
});
it("image lings jobs be converted into images", async () => {
let { data: bookmark } = await client.POST("/bookmarks", {
body: {
type: "link",
url: "http://nginx:80/image.png",
},
});
assert(bookmark);
await waitUntil(
async () => {
const data = await getBookmark(bookmark!.id);
assert(data);
return data.content.type === "asset";
},
"Bookmark is crawled and converted to an image",
20000,
);
bookmark = await getBookmark(bookmark.id);
assert(bookmark && bookmark.content.type === "asset");
expect(bookmark.content.assetType).toBe("image");
expect(bookmark.content.assetId).toBeDefined();
expect(bookmark.content.fileName).toBe("image.png");
expect(bookmark.content.sourceUrl).toBe("http://nginx:80/image.png");
});
});
|
