aboutsummaryrefslogtreecommitdiffstats
path: root/packages
diff options
context:
space:
mode:
authorMohamed Bassem <me@mbassem.com>2025-01-11 18:09:51 +0000
committerMohamed Bassem <me@mbassem.com>2025-01-11 18:09:51 +0000
commit10506173cd5309e7c63d83055243abc67cecad4f (patch)
treef37f7dd704c63e34a1e5b0bffdda442b03179d9c /packages
parent107d923b3abd60329463957ca4604107b3427b2c (diff)
downloadkarakeep-10506173cd5309e7c63d83055243abc67cecad4f.tar.zst
feat: Add support for singlefile extension uploads. #172
Diffstat (limited to 'packages')
-rw-r--r--packages/db/schema.ts2
-rw-r--r--packages/e2e_tests/tests/api/bookmarks.test.ts51
-rw-r--r--packages/open-api/hoarder-openapi-spec.json6
-rw-r--r--packages/sdk/src/hoarder-api.d.ts4
-rw-r--r--packages/shared/assetdb.ts7
-rw-r--r--packages/shared/types/bookmarks.ts7
-rw-r--r--packages/trpc/lib/attachments.ts5
-rw-r--r--packages/trpc/routers/bookmarks.ts41
8 files changed, 117 insertions, 6 deletions
diff --git a/packages/db/schema.ts b/packages/db/schema.ts
index 19bf6db5..6498545a 100644
--- a/packages/db/schema.ts
+++ b/packages/db/schema.ts
@@ -158,6 +158,7 @@ export const enum AssetTypes {
LINK_BANNER_IMAGE = "linkBannerImage",
LINK_SCREENSHOT = "linkScreenshot",
LINK_FULL_PAGE_ARCHIVE = "linkFullPageArchive",
+ LINK_PRECRAWLED_ARCHIVE = "linkPrecrawledArchive",
LINK_VIDEO = "linkVideo",
BOOKMARK_ASSET = "bookmarkAsset",
UNKNOWN = "unknown",
@@ -173,6 +174,7 @@ export const assets = sqliteTable(
AssetTypes.LINK_BANNER_IMAGE,
AssetTypes.LINK_SCREENSHOT,
AssetTypes.LINK_FULL_PAGE_ARCHIVE,
+ AssetTypes.LINK_PRECRAWLED_ARCHIVE,
AssetTypes.LINK_VIDEO,
AssetTypes.BOOKMARK_ASSET,
AssetTypes.UNKNOWN,
diff --git a/packages/e2e_tests/tests/api/bookmarks.test.ts b/packages/e2e_tests/tests/api/bookmarks.test.ts
index 7c605aab..df3cefe2 100644
--- a/packages/e2e_tests/tests/api/bookmarks.test.ts
+++ b/packages/e2e_tests/tests/api/bookmarks.test.ts
@@ -394,4 +394,55 @@ describe("Bookmarks API", () => {
expect(finalPage!.bookmarks.length).toBe(1);
expect(finalPage!.nextCursor).toBeNull();
});
+
+ it("should support precrawling via singlefile", async () => {
+ const file = new File(["<html>HELLO WORLD</html>"], "test.html", {
+ type: "text/html",
+ });
+
+ const formData = new FormData();
+ formData.append("url", "https://example.com");
+ formData.append("file", file);
+
+ // OpenAPI typescript doesn't support multipart/form-data
+ // Upload the singlefile archive
+ const response = await fetch(
+ `http://localhost:${port}/api/v1/bookmarks/singlefile`,
+ {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${apiKey}`,
+ },
+ body: formData,
+ },
+ );
+
+ if (!response.ok) {
+ throw new Error(`Failed to upload asset: ${response.statusText}`);
+ }
+
+ expect(response.status).toBe(201);
+
+ const { id: bookmarkId } = (await response.json()) as {
+ id: string;
+ };
+
+ // Get the created bookmark
+ const { data: retrievedBookmark, response: getResponse } = await client.GET(
+ "/bookmarks/{bookmarkId}",
+ {
+ params: {
+ path: {
+ bookmarkId: bookmarkId,
+ },
+ },
+ },
+ );
+
+ expect(getResponse.status).toBe(200);
+ assert(retrievedBookmark!.content.type === "link");
+ expect(retrievedBookmark!.assets.map((a) => a.assetType)).toContain(
+ "precrawledArchive",
+ );
+ });
});
diff --git a/packages/open-api/hoarder-openapi-spec.json b/packages/open-api/hoarder-openapi-spec.json
index 7b2b9436..382733e0 100644
--- a/packages/open-api/hoarder-openapi-spec.json
+++ b/packages/open-api/hoarder-openapi-spec.json
@@ -256,6 +256,7 @@
"fullPageArchive",
"video",
"bookmarkAsset",
+ "precrawledArchive",
"unknown"
]
}
@@ -598,6 +599,9 @@
"url": {
"type": "string",
"format": "uri"
+ },
+ "precrawledArchiveId": {
+ "type": "string"
}
},
"required": [
@@ -1107,6 +1111,7 @@
"fullPageArchive",
"video",
"bookmarkAsset",
+ "precrawledArchive",
"unknown"
]
}
@@ -1138,6 +1143,7 @@
"fullPageArchive",
"video",
"bookmarkAsset",
+ "precrawledArchive",
"unknown"
]
}
diff --git a/packages/sdk/src/hoarder-api.d.ts b/packages/sdk/src/hoarder-api.d.ts
index f4d76a8a..482f6c3c 100644
--- a/packages/sdk/src/hoarder-api.d.ts
+++ b/packages/sdk/src/hoarder-api.d.ts
@@ -68,6 +68,7 @@ export interface paths {
type: "link";
/** Format: uri */
url: string;
+ precrawledArchiveId?: string;
}
| {
/** @enum {string} */
@@ -426,6 +427,7 @@ export interface paths {
| "fullPageArchive"
| "video"
| "bookmarkAsset"
+ | "precrawledArchive"
| "unknown";
};
};
@@ -446,6 +448,7 @@ export interface paths {
| "fullPageArchive"
| "video"
| "bookmarkAsset"
+ | "precrawledArchive"
| "unknown";
};
};
@@ -1250,6 +1253,7 @@ export interface components {
| "fullPageArchive"
| "video"
| "bookmarkAsset"
+ | "precrawledArchive"
| "unknown";
}[];
};
diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts
index 2ef69279..d2b9eebf 100644
--- a/packages/shared/assetdb.ts
+++ b/packages/shared/assetdb.ts
@@ -25,6 +25,13 @@ export const IMAGE_ASSET_TYPES: Set<string> = new Set<string>([
// The assets that we allow the users to upload
export const SUPPORTED_UPLOAD_ASSET_TYPES: Set<string> = new Set<string>([
...IMAGE_ASSET_TYPES,
+ ASSET_TYPES.TEXT_HTML,
+ ASSET_TYPES.APPLICATION_PDF,
+]);
+
+// The assets that we allow as a bookmark of type asset
+export const SUPPORTED_BOOKMARK_ASSET_TYPES: Set<string> = new Set<string>([
+ ...IMAGE_ASSET_TYPES,
ASSET_TYPES.APPLICATION_PDF,
]);
diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts
index a1e39280..0a414ff9 100644
--- a/packages/shared/types/bookmarks.ts
+++ b/packages/shared/types/bookmarks.ts
@@ -18,6 +18,7 @@ export const zAssetTypesSchema = z.enum([
"fullPageArchive",
"video",
"bookmarkAsset",
+ "precrawledArchive",
"unknown",
]);
export type ZAssetType = z.infer<typeof zAssetTypesSchema>;
@@ -126,7 +127,11 @@ export const zNewBookmarkRequestSchema = z
})
.and(
z.discriminatedUnion("type", [
- z.object({ type: z.literal(BookmarkTypes.LINK), url: z.string().url() }),
+ z.object({
+ type: z.literal(BookmarkTypes.LINK),
+ url: z.string().url(),
+ precrawledArchiveId: z.string().optional(),
+ }),
z.object({
type: z.literal(BookmarkTypes.TEXT),
text: z.string(),
diff --git a/packages/trpc/lib/attachments.ts b/packages/trpc/lib/attachments.ts
index 0fd41d1b..f4fda9cd 100644
--- a/packages/trpc/lib/attachments.ts
+++ b/packages/trpc/lib/attachments.ts
@@ -7,6 +7,7 @@ export function mapDBAssetTypeToUserType(assetType: AssetTypes): ZAssetType {
const map: Record<AssetTypes, z.infer<typeof zAssetTypesSchema>> = {
[AssetTypes.LINK_SCREENSHOT]: "screenshot",
[AssetTypes.LINK_FULL_PAGE_ARCHIVE]: "fullPageArchive",
+ [AssetTypes.LINK_PRECRAWLED_ARCHIVE]: "precrawledArchive",
[AssetTypes.LINK_BANNER_IMAGE]: "bannerImage",
[AssetTypes.LINK_VIDEO]: "video",
[AssetTypes.BOOKMARK_ASSET]: "bookmarkAsset",
@@ -21,6 +22,7 @@ export function mapSchemaAssetTypeToDB(
const map: Record<ZAssetType, AssetTypes> = {
screenshot: AssetTypes.LINK_SCREENSHOT,
fullPageArchive: AssetTypes.LINK_FULL_PAGE_ARCHIVE,
+ precrawledArchive: AssetTypes.LINK_PRECRAWLED_ARCHIVE,
bannerImage: AssetTypes.LINK_BANNER_IMAGE,
video: AssetTypes.LINK_VIDEO,
bookmarkAsset: AssetTypes.BOOKMARK_ASSET,
@@ -33,6 +35,7 @@ export function humanFriendlyNameForAssertType(type: ZAssetType) {
const map: Record<ZAssetType, string> = {
screenshot: "Screenshot",
fullPageArchive: "Full Page Archive",
+ precrawledArchive: "Precrawled Archive",
bannerImage: "Banner Image",
video: "Video",
bookmarkAsset: "Bookmark Asset",
@@ -45,6 +48,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) {
const map: Record<ZAssetType, boolean> = {
screenshot: true,
fullPageArchive: false,
+ precrawledArchive: false,
bannerImage: true,
video: false,
bookmarkAsset: false,
@@ -57,6 +61,7 @@ export function isAllowedToDetachAsset(type: ZAssetType) {
const map: Record<ZAssetType, boolean> = {
screenshot: true,
fullPageArchive: true,
+ precrawledArchive: false,
bannerImage: true,
video: true,
bookmarkAsset: false,
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 15e4cb7c..026bd322 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -22,7 +22,10 @@ import {
rssFeedImportsTable,
tagsOnBookmarks,
} from "@hoarder/db/schema";
-import { deleteAsset } from "@hoarder/shared/assetdb";
+import {
+ deleteAsset,
+ SUPPORTED_BOOKMARK_ASSET_TYPES,
+} from "@hoarder/shared/assetdb";
import serverConfig from "@hoarder/shared/config";
import { InferenceClientFactory } from "@hoarder/shared/inference";
import { buildSummaryPrompt } from "@hoarder/shared/prompts";
@@ -98,9 +101,6 @@ export const ensureAssetOwnership = async (opts: {
}) => {
const asset = await opts.ctx.db.query.assets.findFirst({
where: eq(bookmarks.id, opts.assetId),
- columns: {
- userId: true,
- },
});
if (!opts.ctx.user) {
throw new TRPCError({
@@ -120,6 +120,7 @@ export const ensureAssetOwnership = async (opts: {
message: "User is not allowed to access resource",
});
}
+ return asset;
};
async function getBookmark(ctx: AuthedContext, bookmarkId: string) {
@@ -307,6 +308,24 @@ export const bookmarksAppRouter = router({
})
.returning()
)[0];
+ if (input.precrawledArchiveId) {
+ await ensureAssetOwnership({
+ ctx,
+ assetId: input.precrawledArchiveId,
+ });
+ await tx
+ .update(assets)
+ .set({
+ bookmarkId: bookmark.id,
+ assetType: AssetTypes.LINK_PRECRAWLED_ARCHIVE,
+ })
+ .where(
+ and(
+ eq(assets.id, input.precrawledArchiveId),
+ eq(assets.userId, ctx.user.id),
+ ),
+ );
+ }
content = {
type: BookmarkTypes.LINK,
...link,
@@ -344,7 +363,19 @@ export const bookmarksAppRouter = router({
sourceUrl: null,
})
.returning();
- await ensureAssetOwnership({ ctx, assetId: input.assetId });
+ const uploadedAsset = await ensureAssetOwnership({
+ ctx,
+ assetId: input.assetId,
+ });
+ if (
+ !uploadedAsset.contentType ||
+ !SUPPORTED_BOOKMARK_ASSET_TYPES.has(uploadedAsset.contentType)
+ ) {
+ throw new TRPCError({
+ code: "BAD_REQUEST",
+ message: "Unsupported asset type",
+ });
+ }
await tx
.update(assets)
.set({
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-29 17:05:52 +0000