diff options
| author | Mohamed Bassem <me@mbassem.com> | 2025-07-06 21:50:23 +0000 |
|---|---|---|
| committer | Mohamed Bassem <me@mbassem.com> | 2025-07-06 22:04:56 +0000 |
| commit | dee3a4d44ddb1999e7dec383889246e87f202d92 (patch) | |
| tree | 1984234f17eed886bc834543e1505ddbfb43228f /packages | |
| parent | 362be3008aa8b036c4c448a86e459044af8784c2 (diff) | |
| download | karakeep-dee3a4d44ddb1999e7dec383889246e87f202d92.tar.zst | |
feat: Store large html content in the asset db
Diffstat (limited to 'packages')
| -rw-r--r-- | packages/db/drizzle/0055_content_asset_id.sql | 2 | ||||
| -rw-r--r-- | packages/db/drizzle/meta/0055_snapshot.json | 2051 | ||||
| -rw-r--r-- | packages/db/drizzle/meta/_journal.json | 7 | ||||
| -rw-r--r-- | packages/db/schema.ts | 4 | ||||
| -rw-r--r-- | packages/shared/package.json | 2 | ||||
| -rw-r--r-- | packages/shared/types/bookmarks.ts | 2 | ||||
| -rw-r--r-- | packages/shared/utils/htmlUtils.ts | 17 | ||||
| -rw-r--r-- | packages/trpc/lib/attachments.ts | 5 | ||||
| -rw-r--r-- | packages/trpc/models/bookmarks.ts | 80 | ||||
| -rw-r--r-- | packages/trpc/routers/bookmarks.ts | 21 |
10 files changed, 2183 insertions, 8 deletions
diff --git a/packages/db/drizzle/0055_content_asset_id.sql b/packages/db/drizzle/0055_content_asset_id.sql new file mode 100644 index 00000000..533792d3 --- /dev/null +++ b/packages/db/drizzle/0055_content_asset_id.sql @@ -0,0 +1,2 @@ +ALTER TABLE `bookmarkLinks` ADD `contentAssetId` text;--> statement-breakpoint +ALTER TABLE `bookmarkLinks` DROP COLUMN `content`;
\ No newline at end of file diff --git a/packages/db/drizzle/meta/0055_snapshot.json b/packages/db/drizzle/meta/0055_snapshot.json new file mode 100644 index 00000000..024aba47 --- /dev/null +++ b/packages/db/drizzle/meta/0055_snapshot.json @@ -0,0 +1,2051 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "a7674152-1484-4144-9faa-2f4597ba619e", + "prevId": "ac5b67e5-b9a8-413b-bba7-80280d8ebfc8", + "tables": { + "account": { + "name": "account", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "columns": [ + "provider", + "providerAccountId" + ], + "name": "account_provider_providerAccountId_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apiKey": { + "name": "apiKey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyId": { + "name": "keyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apiKey_keyId_unique": { + "name": "apiKey_keyId_unique", + "columns": [ + "keyId" + ], + "isUnique": true + }, + "apiKey_name_userId_unique": { + "name": "apiKey_name_userId_unique", + "columns": [ + "name", + "userId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "apiKey_userId_user_id_fk": { + "name": "apiKey_userId_user_id_fk", + "tableFrom": "apiKey", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assets": { + "name": "assets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "contentType": { + "name": "contentType", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "assets_bookmarkId_idx": { + "name": "assets_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "assets_assetType_idx": { + "name": "assets_assetType_idx", + "columns": [ + "assetType" + ], + "isUnique": false + }, + "assets_userId_idx": { + "name": "assets_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "assets_bookmarkId_bookmarks_id_fk": { + "name": "assets_bookmarkId_bookmarks_id_fk", + "tableFrom": "assets", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assets_userId_user_id_fk": { + "name": "assets_userId_user_id_fk", + "tableFrom": "assets", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkAssets": { + "name": "bookmarkAssets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assetId": { + "name": "assetId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sourceUrl": { + "name": "sourceUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkAssets_id_bookmarks_id_fk": { + "name": "bookmarkAssets_id_bookmarks_id_fk", + "tableFrom": "bookmarkAssets", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkLinks": { + "name": "bookmarkLinks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "publisher": { + "name": "publisher", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "datePublished": { + "name": "datePublished", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dateModified": { + "name": "dateModified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imageUrl": { + "name": "imageUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favicon": { + "name": "favicon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "htmlContent": { + "name": "htmlContent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentAssetId": { + "name": "contentAssetId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawledAt": { + "name": "crawledAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawlStatus": { + "name": "crawlStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "crawlStatusCode": { + "name": "crawlStatusCode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 200 + } + }, + "indexes": { + "bookmarkLinks_url_idx": { + "name": "bookmarkLinks_url_idx", + "columns": [ + "url" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarkLinks_id_bookmarks_id_fk": { + "name": "bookmarkLinks_id_bookmarks_id_fk", + "tableFrom": "bookmarkLinks", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkLists": { + "name": "bookmarkLists", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "query": { + "name": "query", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rssToken": { + "name": "rssToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "bookmarkLists_userId_idx": { + "name": "bookmarkLists_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarkLists_userId_id_idx": { + "name": "bookmarkLists_userId_id_idx", + "columns": [ + "userId", + "id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "bookmarkLists_userId_user_id_fk": { + "name": "bookmarkLists_userId_user_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarkLists_parentId_bookmarkLists_id_fk": { + "name": "bookmarkLists_parentId_bookmarkLists_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "parentId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkTags": { + "name": "bookmarkTags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarkTags_name_idx": { + "name": "bookmarkTags_name_idx", + "columns": [ + "name" + ], + "isUnique": false + }, + "bookmarkTags_userId_idx": { + "name": "bookmarkTags_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarkTags_userId_name_unique": { + "name": "bookmarkTags_userId_name_unique", + "columns": [ + "userId", + "name" + ], + "isUnique": true + }, + "bookmarkTags_userId_id_idx": { + "name": "bookmarkTags_userId_id_idx", + "columns": [ + "userId", + "id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "bookmarkTags_userId_user_id_fk": { + "name": "bookmarkTags_userId_user_id_fk", + "tableFrom": "bookmarkTags", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkTexts": { + "name": "bookmarkTexts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sourceUrl": { + "name": "sourceUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkTexts_id_bookmarks_id_fk": { + "name": "bookmarkTexts_id_bookmarks_id_fk", + "tableFrom": "bookmarkTexts", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarks": { + "name": "bookmarks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "modifiedAt": { + "name": "modifiedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "favourited": { + "name": "favourited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "taggingStatus": { + "name": "taggingStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "summarizationStatus": { + "name": "summarizationStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarks_userId_idx": { + "name": "bookmarks_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarks_archived_idx": { + "name": "bookmarks_archived_idx", + "columns": [ + "archived" + ], + "isUnique": false + }, + "bookmarks_favourited_idx": { + "name": "bookmarks_favourited_idx", + "columns": [ + "favourited" + ], + "isUnique": false + }, + "bookmarks_createdAt_idx": { + "name": "bookmarks_createdAt_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarks_userId_user_id_fk": { + "name": "bookmarks_userId_user_id_fk", + "tableFrom": "bookmarks", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarksInLists": { + "name": "bookmarksInLists", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarksInLists_bookmarkId_idx": { + "name": "bookmarksInLists_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "bookmarksInLists_listId_idx": { + "name": "bookmarksInLists_listId_idx", + "columns": [ + "listId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarksInLists_bookmarkId_bookmarks_id_fk": { + "name": "bookmarksInLists_bookmarkId_bookmarks_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarksInLists_listId_bookmarkLists_id_fk": { + "name": "bookmarksInLists_listId_bookmarkLists_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "listId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bookmarksInLists_bookmarkId_listId_pk": { + "columns": [ + "bookmarkId", + "listId" + ], + "name": "bookmarksInLists_bookmarkId_listId_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config": { + "name": "config", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "customPrompts": { + "name": "customPrompts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appliesTo": { + "name": "appliesTo", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "customPrompts_userId_idx": { + "name": "customPrompts_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "customPrompts_userId_user_id_fk": { + "name": "customPrompts_userId_user_id_fk", + "tableFrom": "customPrompts", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "highlights": { + "name": "highlights", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "startOffset": { + "name": "startOffset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endOffset": { + "name": "endOffset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'yellow'" + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "highlights_bookmarkId_idx": { + "name": "highlights_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "highlights_userId_idx": { + "name": "highlights_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "highlights_bookmarkId_bookmarks_id_fk": { + "name": "highlights_bookmarkId_bookmarks_id_fk", + "tableFrom": "highlights", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "highlights_userId_user_id_fk": { + "name": "highlights_userId_user_id_fk", + "tableFrom": "highlights", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rssFeedImports": { + "name": "rssFeedImports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entryId": { + "name": "entryId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rssFeedId": { + "name": "rssFeedId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "rssFeedImports_feedIdIdx_idx": { + "name": "rssFeedImports_feedIdIdx_idx", + "columns": [ + "rssFeedId" + ], + "isUnique": false + }, + "rssFeedImports_entryIdIdx_idx": { + "name": "rssFeedImports_entryIdIdx_idx", + "columns": [ + "entryId" + ], + "isUnique": false + }, + "rssFeedImports_rssFeedId_entryId_unique": { + "name": "rssFeedImports_rssFeedId_entryId_unique", + "columns": [ + "rssFeedId", + "entryId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "rssFeedImports_rssFeedId_rssFeeds_id_fk": { + "name": "rssFeedImports_rssFeedId_rssFeeds_id_fk", + "tableFrom": "rssFeedImports", + "tableTo": "rssFeeds", + "columnsFrom": [ + "rssFeedId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "rssFeedImports_bookmarkId_bookmarks_id_fk": { + "name": "rssFeedImports_bookmarkId_bookmarks_id_fk", + "tableFrom": "rssFeedImports", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rssFeeds": { + "name": "rssFeeds", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastFetchedAt": { + "name": "lastFetchedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastFetchedStatus": { + "name": "lastFetchedStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "rssFeeds_userId_idx": { + "name": "rssFeeds_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "rssFeeds_userId_user_id_fk": { + "name": "rssFeeds_userId_user_id_fk", + "tableFrom": "rssFeeds", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ruleEngineActions": { + "name": "ruleEngineActions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ruleId": { + "name": "ruleId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "ruleEngineActions_userId_idx": { + "name": "ruleEngineActions_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "ruleEngineActions_ruleId_idx": { + "name": "ruleEngineActions_ruleId_idx", + "columns": [ + "ruleId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "ruleEngineActions_userId_user_id_fk": { + "name": "ruleEngineActions_userId_user_id_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineActions_ruleId_ruleEngineRules_id_fk": { + "name": "ruleEngineActions_ruleId_ruleEngineRules_id_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "ruleEngineRules", + "columnsFrom": [ + "ruleId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineActions_userId_tagId_fk": { + "name": "ruleEngineActions_userId_tagId_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "userId", + "tagId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineActions_userId_listId_fk": { + "name": "ruleEngineActions_userId_listId_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "userId", + "listId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ruleEngineRules": { + "name": "ruleEngineRules", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "condition": { + "name": "condition", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "ruleEngine_userId_idx": { + "name": "ruleEngine_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "ruleEngineRules_userId_user_id_fk": { + "name": "ruleEngineRules_userId_user_id_fk", + "tableFrom": "ruleEngineRules", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineRules_userId_tagId_fk": { + "name": "ruleEngineRules_userId_tagId_fk", + "tableFrom": "ruleEngineRules", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "userId", + "tagId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineRules_userId_listId_fk": { + "name": "ruleEngineRules_userId_listId_fk", + "tableFrom": "ruleEngineRules", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "userId", + "listId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tagsOnBookmarks": { + "name": "tagsOnBookmarks", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachedAt": { + "name": "attachedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attachedBy": { + "name": "attachedBy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tagsOnBookmarks_tagId_idx": { + "name": "tagsOnBookmarks_tagId_idx", + "columns": [ + "tagId" + ], + "isUnique": false + }, + "tagsOnBookmarks_bookmarkId_idx": { + "name": "tagsOnBookmarks_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tagsOnBookmarks_bookmarkId_bookmarks_id_fk": { + "name": "tagsOnBookmarks_bookmarkId_bookmarks_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tagsOnBookmarks_tagId_bookmarkTags_id_fk": { + "name": "tagsOnBookmarks_tagId_bookmarkTags_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "tagId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tagsOnBookmarks_bookmarkId_tagId_pk": { + "columns": [ + "bookmarkId", + "tagId" + ], + "name": "tagsOnBookmarks_bookmarkId_tagId_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "userSettings": { + "name": "userSettings", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bookmarkClickAction": { + "name": "bookmarkClickAction", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open_original_link'" + }, + "archiveDisplayBehaviour": { + "name": "archiveDisplayBehaviour", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'show'" + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'UTC'" + } + }, + "indexes": {}, + "foreignKeys": { + "userSettings_userId_user_id_fk": { + "name": "userSettings_userId_user_id_fk", + "tableFrom": "userSettings", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "salt": { + "name": "salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'user'" + }, + "bookmarkQuota": { + "name": "bookmarkQuota", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageQuota": { + "name": "storageQuota", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verificationToken": { + "name": "verificationToken", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verificationToken_identifier_token_pk": { + "columns": [ + "identifier", + "token" + ], + "name": "verificationToken_identifier_token_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhooks": { + "name": "webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhooks_userId_idx": { + "name": "webhooks_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "webhooks_userId_user_id_fk": { + "name": "webhooks_userId_user_id_fk", + "tableFrom": "webhooks", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +}
\ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index ea4e30b1..6c509b15 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -386,6 +386,13 @@ "when": 1751826417328, "tag": "0054_add_timezone", "breakpoints": true + }, + { + "idx": 55, + "version": "6", + "when": 1751839469055, + "tag": "0055_content_asset_id", + "breakpoints": true } ] }
\ No newline at end of file diff --git a/packages/db/schema.ts b/packages/db/schema.ts index ed9d586c..4375b201 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -161,8 +161,8 @@ export const bookmarkLinks = sqliteTable( dateModified: integer("dateModified", { mode: "timestamp" }), imageUrl: text("imageUrl"), favicon: text("favicon"), - content: text("content"), htmlContent: text("htmlContent"), + contentAssetId: text("contentAssetId"), crawledAt: integer("crawledAt", { mode: "timestamp" }), crawlStatus: text("crawlStatus", { enum: ["pending", "failure", "success"], @@ -179,6 +179,7 @@ export const enum AssetTypes { LINK_FULL_PAGE_ARCHIVE = "linkFullPageArchive", LINK_PRECRAWLED_ARCHIVE = "linkPrecrawledArchive", LINK_VIDEO = "linkVideo", + LINK_HTML_CONTENT = "linkHtmlContent", BOOKMARK_ASSET = "bookmarkAsset", UNKNOWN = "unknown", } @@ -196,6 +197,7 @@ export const assets = sqliteTable( AssetTypes.LINK_FULL_PAGE_ARCHIVE, AssetTypes.LINK_PRECRAWLED_ARCHIVE, AssetTypes.LINK_VIDEO, + AssetTypes.LINK_HTML_CONTENT, AssetTypes.BOOKMARK_ASSET, AssetTypes.UNKNOWN, ], diff --git a/packages/shared/package.json b/packages/shared/package.json index f4e521b6..6f22865f 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -7,6 +7,7 @@ "dependencies": { "@aws-sdk/client-s3": "^3.842.0", "glob": "^11.0.0", + "html-to-text": "^9.0.5", "js-tiktoken": "^1.0.20", "liteque": "^0.3.2", "meilisearch": "^0.37.0", @@ -20,6 +21,7 @@ "devDependencies": { "@karakeep/prettier-config": "workspace:^0.1.0", "@karakeep/tsconfig": "workspace:^0.1.0", + "@types/html-to-text": "^9.0.4", "vitest": "^1.6.1" }, "scripts": { diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index ea1ab717..f648bce5 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -16,6 +16,7 @@ export const zSortOrder = z.enum(["asc", "desc", "relevance"]); export type ZSortOrder = z.infer<typeof zSortOrder>; export const zAssetTypesSchema = z.enum([ + "linkHtmlContent", "screenshot", "assetScreenshot", "bannerImage", @@ -45,6 +46,7 @@ export const zBookmarkedLinkSchema = z.object({ videoAssetId: z.string().nullish(), favicon: z.string().nullish(), htmlContent: z.string().nullish(), + contentAssetId: z.string().nullish(), crawledAt: z.date().nullish(), author: z.string().nullish(), publisher: z.string().nullish(), diff --git a/packages/shared/utils/htmlUtils.ts b/packages/shared/utils/htmlUtils.ts new file mode 100644 index 00000000..60272899 --- /dev/null +++ b/packages/shared/utils/htmlUtils.ts @@ -0,0 +1,17 @@ +import { compile } from "html-to-text"; + +const compiledConvert = compile({ + selectors: [{ selector: "img", format: "skip" }], +}); + +/** + * Converts HTML content to plain text + */ +export function htmlToPlainText(htmlContent: string): string { + if (!htmlContent) { + return ""; + } + + // TODO, we probably should also remove singlefile inline images from the content + return compiledConvert(htmlContent); +} diff --git a/packages/trpc/lib/attachments.ts b/packages/trpc/lib/attachments.ts index 739aa8f5..e886b821 100644 --- a/packages/trpc/lib/attachments.ts +++ b/packages/trpc/lib/attachments.ts @@ -14,6 +14,7 @@ export function mapDBAssetTypeToUserType(assetType: AssetTypes): ZAssetType { [AssetTypes.LINK_PRECRAWLED_ARCHIVE]: "precrawledArchive", [AssetTypes.LINK_BANNER_IMAGE]: "bannerImage", [AssetTypes.LINK_VIDEO]: "video", + [AssetTypes.LINK_HTML_CONTENT]: "linkHtmlContent", [AssetTypes.BOOKMARK_ASSET]: "bookmarkAsset", [AssetTypes.UNKNOWN]: "bannerImage", }; @@ -31,6 +32,7 @@ export function mapSchemaAssetTypeToDB( bannerImage: AssetTypes.LINK_BANNER_IMAGE, video: AssetTypes.LINK_VIDEO, bookmarkAsset: AssetTypes.BOOKMARK_ASSET, + linkHtmlContent: AssetTypes.LINK_HTML_CONTENT, unknown: AssetTypes.UNKNOWN, }; return map[assetType]; @@ -45,6 +47,7 @@ export function humanFriendlyNameForAssertType(type: ZAssetType) { bannerImage: "Banner Image", video: "Video", bookmarkAsset: "Bookmark Asset", + linkHtmlContent: "HTML Content", unknown: "Unknown", }; return map[type]; @@ -59,6 +62,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) { bannerImage: true, video: false, bookmarkAsset: false, + linkHtmlContent: false, unknown: false, }; return map[type]; @@ -73,6 +77,7 @@ export function isAllowedToDetachAsset(type: ZAssetType) { bannerImage: true, video: true, bookmarkAsset: false, + linkHtmlContent: false, unknown: false, }; return map[type]; diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.ts index 986fca58..07b3832d 100644 --- a/packages/trpc/models/bookmarks.ts +++ b/packages/trpc/models/bookmarks.ts @@ -27,6 +27,7 @@ import { rssFeedImportsTable, tagsOnBookmarks, } from "@karakeep/db/schema"; +import { readAsset } from "@karakeep/shared/assetdb"; import serverConfig from "@karakeep/shared/config"; import { createSignedToken, @@ -46,6 +47,7 @@ import { getBookmarkLinkAssetIdOrUrl, getBookmarkTitle, } from "@karakeep/shared/utils/bookmarkUtils"; +import { htmlToPlainText } from "@karakeep/shared/utils/htmlUtils"; import { AuthedContext } from ".."; import { mapDBAssetTypeToUserType } from "../lib/attachments"; @@ -202,8 +204,11 @@ export class Bookmark implements PrivacyAware { imageUrl: row.bookmarkLinks.imageUrl, favicon: row.bookmarkLinks.favicon, htmlContent: input.includeContent - ? row.bookmarkLinks.htmlContent + ? row.bookmarkLinks.contentAssetId + ? null // Will be populated later from asset + : row.bookmarkLinks.htmlContent : null, + contentAssetId: row.bookmarkLinks.contentAssetId, crawledAt: row.bookmarkLinks.crawledAt, author: row.bookmarkLinks.author, publisher: row.bookmarkLinks.publisher, @@ -300,6 +305,33 @@ export class Bookmark implements PrivacyAware { const bookmarksArr = Object.values(bookmarksRes); + // Fetch HTML content from assets for bookmarks that have contentAssetId (large content) + if (input.includeContent) { + await Promise.all( + bookmarksArr.map(async (bookmark) => { + if ( + bookmark.content.type === BookmarkTypes.LINK && + bookmark.content.contentAssetId && + !bookmark.content.htmlContent // Only fetch if not already inline + ) { + try { + const asset = await readAsset({ + userId: ctx.user.id, + assetId: bookmark.content.contentAssetId, + }); + bookmark.content.htmlContent = asset.asset.toString("utf8"); + } catch (error) { + // If asset reading fails, keep htmlContent as null + console.warn( + `Failed to read HTML content asset ${bookmark.content.contentAssetId}:`, + error, + ); + } + } + }), + ); + } + bookmarksArr.sort((a, b) => { if (a.createdAt != b.createdAt) { return input.sortOrder === "asc" @@ -427,4 +459,50 @@ export class Bookmark implements PrivacyAware { bannerImageUrl: getBannerImageUrl(this.bookmark.content), }; } + + static async getBookmarkHtmlContent( + { + contentAssetId, + htmlContent, + }: { + contentAssetId: string | null; + htmlContent: string | null; + }, + userId: string, + ): Promise<string | null> { + if (contentAssetId) { + // Read large HTML content from asset + const asset = await readAsset({ + userId, + assetId: contentAssetId, + }); + return asset.asset.toString("utf8"); + } else if (htmlContent) { + return htmlContent; + } + return null; + } + + static async getBookmarkPlainTextContent( + { + contentAssetId, + htmlContent, + }: { + contentAssetId: string | null; + htmlContent: string | null; + }, + userId: string, + ): Promise<string | null> { + const content = await this.getBookmarkHtmlContent( + { + contentAssetId, + htmlContent, + }, + userId, + ); + if (!content) { + return null; + } + return htmlToPlainText(content); + } } diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index f1fe10d7..77f40878 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -118,7 +118,7 @@ async function getBookmark( }); } - return toZodSchema(bookmark, includeContent); + return await toZodSchema(bookmark, includeContent); } async function attemptToDedupLink(ctx: AuthedContext, url: string) { @@ -177,10 +177,10 @@ async function cleanupAssetForBookmark( ); } -function toZodSchema( +async function toZodSchema( bookmark: BookmarkQueryReturnType, includeContent: boolean, -): ZBookmark { +): Promise<ZBookmark> { const { tagsOnBookmarks, link, text, asset, assets, ...rest } = bookmark; let content: ZBookmarkContent = { @@ -208,7 +208,9 @@ function toZodSchema( description: link.description, imageUrl: link.imageUrl, favicon: link.favicon, - htmlContent: includeContent ? link.htmlContent : null, + htmlContent: includeContent + ? await Bookmark.getBookmarkHtmlContent(link, bookmark.userId) + : null, crawledAt: link.crawledAt, author: link.author, publisher: link.publisher, @@ -806,7 +808,9 @@ export const bookmarksAppRouter = router({ } return { - bookmarks: results.map((b) => toZodSchema(b, input.includeContent)), + bookmarks: await Promise.all( + results.map((b) => toZodSchema(b, input.includeContent)), + ), nextCursor: resp.hits.length + resp.offset >= resp.estimatedTotalHits ? null @@ -1052,10 +1056,15 @@ export const bookmarksAppRouter = router({ }); } + const content = await Bookmark.getBookmarkPlainTextContent( + bookmark, + ctx.user.id, + ); + const bookmarkDetails = ` Title: ${bookmark.title ?? ""} Description: ${bookmark.description ?? ""} -Content: ${bookmark.content ?? ""} +Content: ${content} Publisher: ${bookmark.publisher ?? ""} Author: ${bookmark.author ?? ""} `; |
