From c16173ea0fdbf6cc47b13756c0a77e8399669055 Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Sat, 12 Oct 2024 16:47:22 +0000
Subject: feature: Introduce a mechanism to cleanup dangling assets

---
 packages/shared/assetdb.ts   | 42 ++++++++++++++++++++++++++++++++++++++++++
 packages/shared/package.json |  1 +
 packages/shared/queues.ts    | 16 ++++++++++++++++
 3 files changed, 59 insertions(+)

(limited to 'packages/shared')

diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts
index 4edfa1ec..64413e9f 100644
--- a/packages/shared/assetdb.ts
+++ b/packages/shared/assetdb.ts
@@ -1,5 +1,6 @@
 import * as fs from "fs";
 import * as path from "path";
+import { Glob } from "glob";
 import { z } from "zod";
 
 import serverConfig from "./config";
@@ -120,6 +121,25 @@ export async function readAsset({
   return { asset, metadata };
 }
 
+export async function readAssetMetadata({
+  userId,
+  assetId,
+}: {
+  userId: string;
+  assetId: string;
+}) {
+  const assetDir = getAssetDir(userId, assetId);
+
+  const metadataStr = await fs.promises.readFile(
+    path.join(assetDir, "metadata.json"),
+    {
+      encoding: "utf8",
+    },
+  );
+
+  return zAssetMetadataSchema.parse(JSON.parse(metadataStr));
+}
+
 export async function getAssetSize({
   userId,
   assetId,
@@ -154,3 +174,25 @@ export async function deleteUserAssets({ userId }: { userId: string }) {
   }
   await fs.promises.rm(userDir, { recursive: true });
 }
+
+export async function* getAllAssets() {
+  const g = new Glob(`/**/**/asset.bin`, {
+    maxDepth: 3,
+    root: ROOT_PATH,
+    cwd: ROOT_PATH,
+    absolute: false,
+  });
+  for await (const file of g) {
+    const [userId, assetId] = file.split("/").slice(0, 2);
+    const [size, metadata] = await Promise.all([
+      getAssetSize({ userId, assetId }),
+      readAssetMetadata({ userId, assetId }),
+    ]);
+    yield {
+      userId,
+      assetId,
+      ...metadata,
+      size,
+    };
+  }
+}
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 2b1ae973..69d93075 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -6,6 +6,7 @@
   "type": "module",
   "dependencies": {
     "@hoarder/queue": "workspace:^0.1.0",
+    "glob": "^11.0.0",
     "meilisearch": "^0.37.0",
     "winston": "^3.11.0",
     "zod": "^3.22.4"
diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index cadeefd0..6b04b988 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -65,6 +65,22 @@ export const SearchIndexingQueue = new SqliteQueue<ZSearchIndexingRequest>(
   },
 );
 
+// Tidy Assets Worker
+export const zTidyAssetsRequestSchema = z.object({
+  cleanDanglingAssets: z.boolean().optional().default(false),
+  syncAssetMetadata: z.boolean().optional().default(false),
+});
+export type ZTidyAssetsRequest = z.infer<typeof zTidyAssetsRequestSchema>;
+export const TidyAssetsQueue = new SqliteQueue<ZTidyAssetsRequest>(
+  "tidy_assets_queue",
+  queueDB,
+  {
+    defaultJobArgs: {
+      numRetries: 1,
+    },
+  },
+);
+
 export async function triggerSearchReindex(bookmarkId: string) {
   await SearchIndexingQueue.enqueue({
     bookmarkId,
-- 
cgit v1.2.3-70-g09d2