From aa7d68a00cbe9d7d3733f4cd02061d4586af061c Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Tue, 19 Mar 2024 02:21:54 +0000
Subject: refactor: Change asset storage to be the filesystem instead of sqlite

---
 apps/web/app/api/assets/[assetId]/route.ts | 16 ++++++----------
 apps/web/app/api/assets/route.ts           | 24 +++++++++++-------------
 apps/workers/openaiWorker.ts               | 19 +++++++++----------
 apps/workers/package.json                  |  1 -
 4 files changed, 26 insertions(+), 34 deletions(-)

(limited to 'apps')

diff --git a/apps/web/app/api/assets/[assetId]/route.ts b/apps/web/app/api/assets/[assetId]/route.ts
index 6b583e51..f3cf1ab4 100644
--- a/apps/web/app/api/assets/[assetId]/route.ts
+++ b/apps/web/app/api/assets/[assetId]/route.ts
@@ -1,8 +1,6 @@
 import { createContextFromRequest } from "@/server/api/client";
-import { and, eq } from "drizzle-orm";
 
-import { db } from "@hoarder/db";
-import { assets } from "@hoarder/db/schema";
+import { readAsset } from "@hoarder/shared/assetdb";
 
 export const dynamic = "force-dynamic";
 export async function GET(
@@ -13,17 +11,15 @@ export async function GET(
   if (!ctx.user) {
     return Response.json({ error: "Unauthorized" }, { status: 401 });
   }
-  const asset = await db.query.assets.findFirst({
-    where: and(eq(assets.id, params.assetId), eq(assets.userId, ctx.user.id)),
+  const { asset, metadata } = await readAsset({
+    userId: ctx.user.id,
+    assetId: params.assetId,
   });
 
-  if (!asset) {
-    return Response.json({ error: "Asset not found" }, { status: 404 });
-  }
-  return new Response(asset.blob as string, {
+  return new Response(asset, {
     status: 200,
     headers: {
-      "Content-type": asset.contentType,
+      "Content-type": metadata.contentType,
     },
   });
 }
diff --git a/apps/web/app/api/assets/route.ts b/apps/web/app/api/assets/route.ts
index 2caa4d4c..4e1746b3 100644
--- a/apps/web/app/api/assets/route.ts
+++ b/apps/web/app/api/assets/route.ts
@@ -1,8 +1,7 @@
 import { createContextFromRequest } from "@/server/api/client";
 
 import type { ZUploadResponse } from "@hoarder/trpc/types/uploads";
-import { db } from "@hoarder/db";
-import { assets } from "@hoarder/db/schema";
+import { saveAsset } from "@hoarder/shared/assetdb";
 
 const SUPPORTED_ASSET_TYPES = new Set(["image/jpeg", "image/png"]);
 
@@ -34,19 +33,18 @@ export async function POST(request: Request) {
     return Response.json({ error: "Bad request" }, { status: 400 });
   }
 
-  const [dbRes] = await db
-    .insert(assets)
-    .values({
-      encoding: "binary",
-      contentType: contentType,
-      blob: buffer,
-      userId: ctx.user.id,
-    })
-    .returning();
+  const assetId = crypto.randomUUID();
+
+  await saveAsset({
+    userId: ctx.user.id,
+    assetId,
+    metadata: { contentType },
+    asset: buffer,
+  });
 
   return Response.json({
-    assetId: dbRes.id,
-    contentType: dbRes.contentType,
+    assetId,
+    contentType,
     size: buffer.byteLength,
   } satisfies ZUploadResponse);
 }
diff --git a/apps/workers/openaiWorker.ts b/apps/workers/openaiWorker.ts
index 428f6027..f1ec0b10 100644
--- a/apps/workers/openaiWorker.ts
+++ b/apps/workers/openaiWorker.ts
@@ -3,12 +3,11 @@ import { and, eq, inArray } from "drizzle-orm";
 import OpenAI from "openai";
 import { z } from "zod";
 
-import Base64 from "js-base64";
-
 import { db } from "@hoarder/db";
-import { assets, bookmarks, bookmarkTags, tagsOnBookmarks } from "@hoarder/db/schema";
+import { bookmarks, bookmarkTags, tagsOnBookmarks } from "@hoarder/db/schema";
 import serverConfig from "@hoarder/shared/config";
 import logger from "@hoarder/shared/logger";
+import { readAsset } from "@hoarder/shared/assetdb";
 import {
   OpenAIQueue,
   queueConnectionDetails,
@@ -142,15 +141,15 @@ async function inferTagsFromImage(
   openai: OpenAI,
 ) {
 
-  const asset = await db.query.assets.findFirst({
-    where: eq(assets.id, bookmark.asset.assetId),
+  const { asset, metadata } = await readAsset({
+    userId: bookmark.userId,
+    assetId: bookmark.asset.assetId,
   });
 
   if (!asset) {
     throw new Error(`[openai][${jobId}] AssetId ${bookmark.asset.assetId} for bookmark ${bookmark.id} not found`);
   }
-
-  const base64 = Base64.encode(asset.blob as string);
+  const base64 = asset.toString('base64');
 
   const chatCompletion = await openai.chat.completions.create({
     model: "gpt-4-vision-preview",
@@ -162,7 +161,7 @@ async function inferTagsFromImage(
           {
             type: "image_url",
             image_url: {
-              url: `data:image/jpeg;base64,${base64}`,
+              url: `data:${metadata.contentType};base64,${base64}`,
               detail: "low",
             },
           },
@@ -176,7 +175,7 @@ async function inferTagsFromImage(
   if (!response) {
     throw new Error(`[openai][${jobId}] Got no message content from OpenAI`);
   }
-  return {response, totalTokens: chatCompletion.usage?.total_tokens};
+  return { response, totalTokens: chatCompletion.usage?.total_tokens };
 }
 
 async function inferTagsFromText(
@@ -194,7 +193,7 @@ async function inferTagsFromText(
   if (!response) {
     throw new Error(`[openai][${jobId}] Got no message content from OpenAI`);
   }
-  return {response, totalTokens: chatCompletion.usage?.total_tokens};
+  return { response, totalTokens: chatCompletion.usage?.total_tokens };
 }
 
 async function inferTags(
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 1c3813b8..f6d58eb4 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -14,7 +14,6 @@
     "dompurify": "^3.0.9",
     "dotenv": "^16.4.1",
     "drizzle-orm": "^0.29.4",
-    "js-base64": "^3.7.7",
     "jsdom": "^24.0.0",
     "metascraper": "^5.43.4",
     "metascraper-description": "^5.43.4",
-- 
cgit v1.2.3-70-g09d2