feature: add support for PDF links. Fixes #28 (#216)

* feature request: pdf support #28
Added a new sourceUrl column to the asset bookmarks
Added transforming a link bookmark pointing at a pdf to an asset bookmark
made sure the "View Original" link is also shown for asset bookmarks that have a sourceURL
updated gitignore for IDEA

* remove pdf parsing from the crawler

* extract the http logic into its own function to avoid duplicating the post-processing actions (openai/index)

* Add 5s timeout to the content type fetch

---------

Co-authored-by: MohamedBassem <me@mbassem.com>

1 files changed, 14 insertions, 6 deletions

diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts
index 4cea06b0..fb625af8 100644
--- a/packages/shared/assetdb.ts
+++ b/packages/shared/assetdb.ts
@@ -6,18 +6,26 @@ import serverConfig from "./config";
 
 const ROOT_PATH = path.join(serverConfig.dataDir, "assets");
 
+export const enum ASSET_TYPES {
+  IMAGE_JPEG = "image/jpeg",
+  IMAGE_PNG = "image/png",
+  IMAGE_WEBP = "image/webp",
+  APPLICATION_PDF = "application/pdf",
+  TEXT_HTML = "text/html",
+}
+
 // The assets that we allow the users to upload
-export const SUPPORTED_UPLOAD_ASSET_TYPES = new Set([
-  "image/jpeg",
-  "image/png",
-  "image/webp",
-  "application/pdf",
+export const SUPPORTED_UPLOAD_ASSET_TYPES: Set<string> = new Set<string>([
+  ASSET_TYPES.IMAGE_JPEG,
+  ASSET_TYPES.IMAGE_PNG,
+  ASSET_TYPES.IMAGE_WEBP,
+  ASSET_TYPES.APPLICATION_PDF,
 ]);
 
 // The assets that we support saving in the asset db
 export const SUPPORTED_ASSET_TYPES = new Set([
   ...SUPPORTED_UPLOAD_ASSET_TYPES,
-  "text/html",
+  ASSET_TYPES.TEXT_HTML,
 ]);
 
 function getAssetDir(userId: string, assetId: string) {