feature: Full page archival with monolith. Fixes #132

3 files changed, 37 insertions, 1 deletions

diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts
index c070ad54..4cea06b0 100644
--- a/packages/shared/assetdb.ts
+++ b/packages/shared/assetdb.ts
@@ -6,13 +6,20 @@ import serverConfig from "./config";
 
 const ROOT_PATH = path.join(serverConfig.dataDir, "assets");
 
-export const SUPPORTED_ASSET_TYPES = new Set([
+// The assets that we allow the users to upload
+export const SUPPORTED_UPLOAD_ASSET_TYPES = new Set([
   "image/jpeg",
   "image/png",
   "image/webp",
   "application/pdf",
 ]);
 
+// The assets that we support saving in the asset db
+export const SUPPORTED_ASSET_TYPES = new Set([
+  ...SUPPORTED_UPLOAD_ASSET_TYPES,
+  "text/html",
+]);
+
 function getAssetDir(userId: string, assetId: string) {
   return path.join(ROOT_PATH, userId, assetId);
 }
@@ -52,6 +59,32 @@ export async function saveAsset({
   ]);
 }
 
+export async function saveAssetFromFile({
+  userId,
+  assetId,
+  assetPath,
+  metadata,
+}: {
+  userId: string;
+  assetId: string;
+  assetPath: string;
+  metadata: z.infer<typeof zAssetMetadataSchema>;
+}) {
+  if (!SUPPORTED_ASSET_TYPES.has(metadata.contentType)) {
+    throw new Error("Unsupported asset type");
+  }
+  const assetDir = getAssetDir(userId, assetId);
+  await fs.promises.mkdir(assetDir, { recursive: true });
+
+  await Promise.all([
+    fs.promises.rename(assetPath, path.join(assetDir, "asset.bin")),
+    fs.promises.writeFile(
+      path.join(assetDir, "metadata.json"),
+      JSON.stringify(metadata),
+    ),
+  ]);
+}
+
 export async function readAsset({
   userId,
   assetId,
diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 8bb4e830..2c739a0c 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -29,6 +29,7 @@ const allEnv = z.object({
   CRAWLER_DOWNLOAD_BANNER_IMAGE: stringBool("true"),
   CRAWLER_STORE_SCREENSHOT: stringBool("true"),
   CRAWLER_FULL_PAGE_SCREENSHOT: stringBool("false"),
+  CRAWLER_FULL_PAGE_ARCHIVE: stringBool("false"),
   MEILI_ADDR: z.string().optional(),
   MEILI_MASTER_KEY: z.string().default(""),
   LOG_LEVEL: z.string().default("debug"),
@@ -74,6 +75,7 @@ const serverConfigSchema = allEnv.transform((val) => {
       downloadBannerImage: val.CRAWLER_DOWNLOAD_BANNER_IMAGE,
       storeScreenshot: val.CRAWLER_STORE_SCREENSHOT,
       fullPageScreenshot: val.CRAWLER_FULL_PAGE_SCREENSHOT,
+      fullPageArchive: val.CRAWLER_FULL_PAGE_ARCHIVE,
     },
     meilisearch: val.MEILI_ADDR
       ? {
diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts
index 10dba0c8..06cd632e 100644
--- a/packages/shared/types/bookmarks.ts
+++ b/packages/shared/types/bookmarks.ts
@@ -12,6 +12,7 @@ export const zBookmarkedLinkSchema = z.object({
   imageUrl: z.string().url().nullish(),
   imageAssetId: z.string().nullish(),
   screenshotAssetId: z.string().nullish(),
+  fullPageArchiveAssetId: z.string().nullish(),
   favicon: z.string().url().nullish(),
   htmlContent: z.string().nullish(),
   crawledAt: z.date().nullish(),