aboutsummaryrefslogtreecommitdiffstats
path: root/apps
diff options
context:
space:
mode:
Diffstat (limited to 'apps')
-rw-r--r--apps/mobile/lib/upload.ts6
-rw-r--r--apps/mobile/package.json2
-rw-r--r--apps/web/app/api/assets/route.ts7
-rw-r--r--apps/web/components/dashboard/UploadDropzone.tsx7
-rw-r--r--apps/web/components/dashboard/bookmarks/AssetCard.tsx7
-rw-r--r--apps/web/components/dashboard/preview/AssetContentSection.tsx39
-rw-r--r--apps/workers/openaiWorker.ts69
-rw-r--r--apps/workers/package.json2
-rw-r--r--apps/workers/searchWorker.ts7
-rw-r--r--apps/workers/utils.ts32
10 files changed, 141 insertions, 37 deletions
diff --git a/apps/mobile/lib/upload.ts b/apps/mobile/lib/upload.ts
index 56b2c7a5..f9d05967 100644
--- a/apps/mobile/lib/upload.ts
+++ b/apps/mobile/lib/upload.ts
@@ -38,7 +38,7 @@ export function useUploadAsset(
mutationFn: async (file: { type: string; name: string; uri: string }) => {
const formData = new FormData();
// @ts-expect-error This is a valid api in react native
- formData.append("image", {
+ formData.append("file", {
uri: file.uri,
name: file.name,
type: file.type,
@@ -57,7 +57,9 @@ export function useUploadAsset(
},
onSuccess: (resp) => {
const assetId = resp.assetId;
- createBookmark({ type: "asset", assetId, assetType: "image" });
+ const assetType =
+ resp.contentType === "application/pdf" ? "pdf" : "image";
+ createBookmark({ type: "asset", assetId, assetType });
},
onError: (e) => {
if (options.onError) {
diff --git a/apps/mobile/package.json b/apps/mobile/package.json
index 9f170040..248f1f53 100644
--- a/apps/mobile/package.json
+++ b/apps/mobile/package.json
@@ -31,7 +31,7 @@
"expo-navigation-bar": "~2.8.1",
"expo-router": "~3.4.8",
"expo-secure-store": "^12.8.1",
- "expo-share-intent": "^1.1.0",
+ "expo-share-intent": "1.1.0",
"expo-status-bar": "~1.11.1",
"expo-system-ui": "^2.9.3",
"expo-web-browser": "^12.8.2",
diff --git a/apps/web/app/api/assets/route.ts b/apps/web/app/api/assets/route.ts
index 0bb2f778..5b72033a 100644
--- a/apps/web/app/api/assets/route.ts
+++ b/apps/web/app/api/assets/route.ts
@@ -9,6 +9,7 @@ const SUPPORTED_ASSET_TYPES = new Set([
"image/jpeg",
"image/png",
"image/webp",
+ "application/pdf",
]);
const MAX_UPLOAD_SIZE_BYTES = serverConfig.maxAssetSizeMb * 1024 * 1024;
@@ -26,7 +27,7 @@ export async function POST(request: Request) {
});
}
const formData = await request.formData();
- const data = formData.get("image");
+ const data = formData.get("file") ?? formData.get("image");
let buffer;
let contentType;
if (data instanceof File) {
@@ -46,11 +47,12 @@ export async function POST(request: Request) {
}
const assetId = crypto.randomUUID();
+ const fileName = data.name;
await saveAsset({
userId: ctx.user.id,
assetId,
- metadata: { contentType },
+ metadata: { contentType, fileName },
asset: buffer,
});
@@ -58,5 +60,6 @@ export async function POST(request: Request) {
assetId,
contentType,
size: buffer.byteLength,
+ fileName,
} satisfies ZUploadResponse);
}
diff --git a/apps/web/components/dashboard/UploadDropzone.tsx b/apps/web/components/dashboard/UploadDropzone.tsx
index bd08d2cf..f6243885 100644
--- a/apps/web/components/dashboard/UploadDropzone.tsx
+++ b/apps/web/components/dashboard/UploadDropzone.tsx
@@ -29,7 +29,7 @@ function useUploadAsset({ onComplete }: { onComplete: () => void }) {
const { mutateAsync: runUpload } = useMutation({
mutationFn: async (file: File) => {
const formData = new FormData();
- formData.append("image", file);
+ formData.append("file", file);
const resp = await fetch("/api/assets", {
method: "POST",
body: formData,
@@ -40,8 +40,9 @@ function useUploadAsset({ onComplete }: { onComplete: () => void }) {
return zUploadResponseSchema.parse(await resp.json());
},
onSuccess: async (resp) => {
- const assetId = resp.assetId;
- return createBookmark({ type: "asset", assetId, assetType: "image" });
+ const assetType =
+ resp.contentType === "application/pdf" ? "pdf" : "image";
+ return createBookmark({ ...resp, type: "asset", assetType });
},
onError: (error, req) => {
const err = zUploadErrorSchema.parse(JSON.parse(error.message));
diff --git a/apps/web/components/dashboard/bookmarks/AssetCard.tsx b/apps/web/components/dashboard/bookmarks/AssetCard.tsx
index 460dbe98..8997a7e2 100644
--- a/apps/web/components/dashboard/bookmarks/AssetCard.tsx
+++ b/apps/web/components/dashboard/bookmarks/AssetCard.tsx
@@ -59,6 +59,13 @@ export default function AssetCard({
/>
</div>
)}
+ {bookmarkedAsset.assetType == "pdf" && (
+ <iframe
+ title={bookmarkedAsset.assetId}
+ className="h-56 max-h-56 w-full"
+ src={`/api/assets/${bookmarkedAsset.assetId}`}
+ />
+ )}
<div className="flex flex-col gap-y-1 overflow-hidden p-2">
<div className="flex h-full flex-wrap gap-1 overflow-hidden">
<TagList
diff --git a/apps/web/components/dashboard/preview/AssetContentSection.tsx b/apps/web/components/dashboard/preview/AssetContentSection.tsx
index 3fbbc519..4a025f9d 100644
--- a/apps/web/components/dashboard/preview/AssetContentSection.tsx
+++ b/apps/web/components/dashboard/preview/AssetContentSection.tsx
@@ -7,25 +7,30 @@ export function AssetContentSection({ bookmark }: { bookmark: ZBookmark }) {
throw new Error("Invalid content type");
}
- let content;
switch (bookmark.content.assetType) {
case "image": {
- switch (bookmark.content.assetType) {
- case "image": {
- content = (
- <div className="relative h-full min-w-full">
- <Image
- alt="asset"
- fill={true}
- className="object-contain"
- src={`/api/assets/${bookmark.content.assetId}`}
- />
- </div>
- );
- }
- }
- break;
+ return (
+ <div className="relative h-full min-w-full">
+ <Image
+ alt="asset"
+ fill={true}
+ className="object-contain"
+ src={`/api/assets/${bookmark.content.assetId}`}
+ />
+ </div>
+ );
+ }
+ case "pdf": {
+ return (
+ <iframe
+ title={bookmark.content.assetId}
+ className="h-full w-full"
+ src={`/api/assets/${bookmark.content.assetId}`}
+ />
+ );
+ }
+ default: {
+ return <div>Unsupported asset type</div>;
}
}
- return content;
}
diff --git a/apps/workers/openaiWorker.ts b/apps/workers/openaiWorker.ts
index c7b519e2..b07e02fe 100644
--- a/apps/workers/openaiWorker.ts
+++ b/apps/workers/openaiWorker.ts
@@ -5,7 +5,12 @@ import { z } from "zod";
import type { ZOpenAIRequest } from "@hoarder/shared/queues";
import { db } from "@hoarder/db";
-import { bookmarks, bookmarkTags, tagsOnBookmarks } from "@hoarder/db/schema";
+import {
+ bookmarkAssets,
+ bookmarks,
+ bookmarkTags,
+ tagsOnBookmarks,
+} from "@hoarder/db/schema";
import { readAsset } from "@hoarder/shared/assetdb";
import serverConfig from "@hoarder/shared/config";
import logger from "@hoarder/shared/logger";
@@ -18,6 +23,7 @@ import {
import type { InferenceClient } from "./inference";
import { InferenceClientFactory } from "./inference";
+import { readPDFText, truncateContent } from "./utils";
const openAIResponseSchema = z.object({
tags: z.array(z.string()),
@@ -91,14 +97,6 @@ CONTENT START HERE:
function buildPrompt(
bookmark: NonNullable<Awaited<ReturnType<typeof fetchBookmark>>>,
) {
- const truncateContent = (content: string) => {
- let words = content.split(" ");
- if (words.length > 1500) {
- words = words.slice(1500);
- content = words.join(" ");
- }
- return content;
- };
if (bookmark.link) {
if (!bookmark.link.description && !bookmark.link.content) {
throw new Error(
@@ -158,14 +156,48 @@ async function inferTagsFromImage(
);
}
const base64 = asset.toString("base64");
-
- return await inferenceClient.inferFromImage(
+ return inferenceClient.inferFromImage(
IMAGE_PROMPT_BASE,
metadata.contentType,
base64,
);
}
+async function inferTagsFromPDF(
+ jobId: string,
+ bookmark: NonNullable<Awaited<ReturnType<typeof fetchBookmark>>>,
+ inferenceClient: InferenceClient,
+) {
+ const { asset } = await readAsset({
+ userId: bookmark.userId,
+ assetId: bookmark.asset.assetId,
+ });
+ if (!asset) {
+ throw new Error(
+ `[inference][${jobId}] AssetId ${bookmark.asset.assetId} for bookmark ${bookmark.id} not found`,
+ );
+ }
+ const pdfParse = await readPDFText(asset);
+ if (!pdfParse?.text) {
+ throw new Error(
+ `[inference][${jobId}] PDF text is empty. Please make sure that the PDF includes text and not just images.`,
+ );
+ }
+
+ await db
+ .update(bookmarkAssets)
+ .set({
+ content: pdfParse.text,
+ metadata: pdfParse.metadata ? JSON.stringify(pdfParse.metadata) : null,
+ })
+ .where(eq(bookmarkAssets.id, bookmark.id));
+
+ const prompt = `${TEXT_PROMPT_BASE}
+Content: ${truncateContent(pdfParse.text)}
+`;
+ return inferenceClient.inferFromText(prompt);
+}
+
async function inferTagsFromText(
bookmark: NonNullable<Awaited<ReturnType<typeof fetchBookmark>>>,
inferenceClient: InferenceClient,
@@ -182,11 +214,24 @@ async function inferTags(
if (bookmark.link || bookmark.text) {
response = await inferTagsFromText(bookmark, inferenceClient);
} else if (bookmark.asset) {
- response = await inferTagsFromImage(jobId, bookmark, inferenceClient);
+ switch (bookmark.asset.assetType) {
+ case "image":
+ response = await inferTagsFromImage(jobId, bookmark, inferenceClient);
+ break;
+ case "pdf":
+ response = await inferTagsFromPDF(jobId, bookmark, inferenceClient);
+ break;
+ default:
+ throw new Error(`[inference][${jobId}] Unsupported bookmark type`);
+ }
} else {
throw new Error(`[inference][${jobId}] Unsupported bookmark type`);
}
+ if (!response) {
+ throw new Error(`[inference][${jobId}] Inference response is empty`);
+ }
+
try {
let tags = openAIResponseSchema.parse(JSON.parse(response.response)).tags;
logger.info(
diff --git a/apps/workers/package.json b/apps/workers/package.json
index c9de43a4..e14c576b 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -26,6 +26,8 @@
"metascraper-url": "^5.43.4",
"ollama": "^0.5.0",
"openai": "^4.29.0",
+ "pdf2json": "^3.0.5",
+ "pdfjs-dist": "^4.0.379",
"puppeteer": "^22.0.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-adblocker": "^2.13.6",
diff --git a/apps/workers/searchWorker.ts b/apps/workers/searchWorker.ts
index 79b0c8c1..fcef7a1b 100644
--- a/apps/workers/searchWorker.ts
+++ b/apps/workers/searchWorker.ts
@@ -48,6 +48,7 @@ async function runIndex(
with: {
link: true,
text: true,
+ asset: true,
tagsOnBookmarks: {
with: {
tag: true,
@@ -72,6 +73,12 @@ async function runIndex(
content: bookmark.link.content,
}
: undefined),
+ ...(bookmark.asset
+ ? {
+ content: bookmark.asset.content,
+ metadata: bookmark.asset.metadata,
+ }
+ : undefined),
...(bookmark.text ? { content: bookmark.text.text } : undefined),
note: bookmark.note,
createdAt: bookmark.createdAt.toISOString(),
diff --git a/apps/workers/utils.ts b/apps/workers/utils.ts
index 2f56d3f0..f8c48408 100644
--- a/apps/workers/utils.ts
+++ b/apps/workers/utils.ts
@@ -1,3 +1,5 @@
+import PDFParser from "pdf2json";
+
export function withTimeout<T, Ret>(
func: (param: T) => Promise<Ret>,
timeoutSec: number,
@@ -14,3 +16,33 @@ export function withTimeout<T, Ret>(
]);
};
}
+
+export async function readPDFText(buffer: Buffer): Promise<{
+ text: string;
+ metadata: Record<string, string>;
+}> {
+ return new Promise((resolve, reject) => {
+ // Need raw text flag represents as number (1), reference : https://github.com/modesty/pdf2json/issues/76#issuecomment-236569265
+ const pdfParser = new PDFParser(null, 1);
+ pdfParser.on("pdfParser_dataError", reject);
+ pdfParser.on("pdfParser_dataReady", (pdfData) => {
+ // eslint-disable-next-line
+ resolve({
+ // The type isn't set correctly, reference : https://github.com/modesty/pdf2json/issues/327
+ // eslint-disable-next-line
+ text: (pdfParser as any).getRawTextContent(),
+ metadata: pdfData.Meta,
+ });
+ });
+ pdfParser.parseBuffer(buffer);
+ });
+}
+
+export function truncateContent(content: string, length = 1500) {
+ let words = content.split(" ");
+ if (words.length > length) {
+ words = words.slice(length);
+ content = words.join(" ");
+ }
+ return content;
+}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-29 17:03:20 +0000