From be1b7f7e1c0cb3d905e13aa1a95e295b816cbdeb Mon Sep 17 00:00:00 2001
From: kamtschatka <simon.schatka@gmx.at>
Date: Sat, 22 Jun 2024 18:52:40 +0200
Subject: feature: add support for PDF links. Fixes #28 (#216)

* feature request: pdf support #28
Added a new sourceUrl column to the asset bookmarks
Added transforming a link bookmark pointing at a pdf to an asset bookmark
made sure the "View Original" link is also shown for asset bookmarks that have a sourceURL
updated gitignore for IDEA

* remove pdf parsing from the crawler

* extract the http logic into its own function to avoid duplicating the post-processing actions (openai/index)

* Add 5s timeout to the content type fetch

---------

Co-authored-by: MohamedBassem <me@mbassem.com>
---
 packages/db/schema.ts | 63 ++++++++++++++++++++++++++++-----------------------
 1 file changed, 35 insertions(+), 28 deletions(-)

(limited to 'packages/db/schema.ts')

diff --git a/packages/db/schema.ts b/packages/db/schema.ts
index 3fd7897f..07f1686e 100644
--- a/packages/db/schema.ts
+++ b/packages/db/schema.ts
@@ -126,33 +126,37 @@ export const bookmarks = sqliteTable(
   }),
 );
 
-export const bookmarkLinks = sqliteTable("bookmarkLinks", {
-  id: text("id")
-    .notNull()
-    .primaryKey()
-    .$defaultFn(() => createId())
-    .references(() => bookmarks.id, { onDelete: "cascade" }),
-  url: text("url").notNull(),
+export const bookmarkLinks = sqliteTable(
+  "bookmarkLinks",
+  {
+    id: text("id")
+      .notNull()
+      .primaryKey()
+      .$defaultFn(() => createId())
+      .references(() => bookmarks.id, { onDelete: "cascade" }),
+    url: text("url").notNull(),
 
-  // Crawled info
-  title: text("title"),
-  description: text("description"),
-  imageUrl: text("imageUrl"),
-  favicon: text("favicon"),
-  content: text("content"),
-  htmlContent: text("htmlContent"),
-  screenshotAssetId: text("screenshotAssetId"),
-  fullPageArchiveAssetId: text("fullPageArchiveAssetId"),
-  imageAssetId: text("imageAssetId"),
-  crawledAt: integer("crawledAt", { mode: "timestamp" }),
-  crawlStatus: text("crawlStatus", {
-    enum: ["pending", "failure", "success"],
-  }).default("pending"),
-}, (bl) => {
-  return {
-    urlIdx: index("bookmarkLinks_url_idx").on(bl.url),
-  };
-});
+    // Crawled info
+    title: text("title"),
+    description: text("description"),
+    imageUrl: text("imageUrl"),
+    favicon: text("favicon"),
+    content: text("content"),
+    htmlContent: text("htmlContent"),
+    screenshotAssetId: text("screenshotAssetId"),
+    fullPageArchiveAssetId: text("fullPageArchiveAssetId"),
+    imageAssetId: text("imageAssetId"),
+    crawledAt: integer("crawledAt", { mode: "timestamp" }),
+    crawlStatus: text("crawlStatus", {
+      enum: ["pending", "failure", "success"],
+    }).default("pending"),
+  },
+  (bl) => {
+    return {
+      urlIdx: index("bookmarkLinks_url_idx").on(bl.url),
+    };
+  },
+);
 
 export const bookmarkTexts = sqliteTable("bookmarkTexts", {
   id: text("id")
@@ -174,6 +178,7 @@ export const bookmarkAssets = sqliteTable("bookmarkAssets", {
   content: text("content"),
   metadata: text("metadata"),
   fileName: text("fileName"),
+  sourceUrl: text("sourceUrl"),
 });
 
 export const bookmarkTags = sqliteTable(
@@ -231,8 +236,10 @@ export const bookmarkLists = sqliteTable(
     userId: text("userId")
       .notNull()
       .references(() => users.id, { onDelete: "cascade" }),
-    parentId: text("parentId")
-      .references((): AnySQLiteColumn => bookmarkLists.id, { onDelete: "set null" }),
+    parentId: text("parentId").references(
+      (): AnySQLiteColumn => bookmarkLists.id,
+      { onDelete: "set null" },
+    ),
   },
   (bl) => ({
     userIdIdx: index("bookmarkLists_userId_idx").on(bl.userId),
-- 
cgit v1.2.3-70-g09d2