diff options
Diffstat (limited to 'packages')
| -rw-r--r-- | packages/db/schema.ts | 2 | ||||
| -rw-r--r-- | packages/open-api/hoarder-openapi-spec.json | 9 | ||||
| -rw-r--r-- | packages/shared/assetdb.ts | 16 | ||||
| -rw-r--r-- | packages/shared/config.ts | 6 | ||||
| -rw-r--r-- | packages/shared/queues.ts | 23 | ||||
| -rw-r--r-- | packages/shared/types/bookmarks.ts | 2 | ||||
| -rw-r--r-- | packages/trpc/lib/attachments.ts | 5 | ||||
| -rw-r--r-- | packages/trpc/routers/bookmarks.ts | 5 |
8 files changed, 68 insertions, 0 deletions
diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 033295bf..10c69d9d 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -165,6 +165,7 @@ export const enum AssetTypes { LINK_BANNER_IMAGE = "linkBannerImage", LINK_SCREENSHOT = "linkScreenshot", LINK_FULL_PAGE_ARCHIVE = "linkFullPageArchive", + LINK_VIDEO = "linkVideo", BOOKMARK_ASSET = "bookmarkAsset", UNKNOWN = "unknown", } @@ -179,6 +180,7 @@ export const assets = sqliteTable( AssetTypes.LINK_BANNER_IMAGE, AssetTypes.LINK_SCREENSHOT, AssetTypes.LINK_FULL_PAGE_ARCHIVE, + AssetTypes.LINK_VIDEO, AssetTypes.BOOKMARK_ASSET, AssetTypes.UNKNOWN, ], diff --git a/packages/open-api/hoarder-openapi-spec.json b/packages/open-api/hoarder-openapi-spec.json index a93a0b9c..eac98326 100644 --- a/packages/open-api/hoarder-openapi-spec.json +++ b/packages/open-api/hoarder-openapi-spec.json @@ -140,6 +140,10 @@ "type": "string", "nullable": true }, + "videoAssetId": { + "type": "string", + "nullable": true + }, "favicon": { "type": "string", "nullable": true, @@ -245,6 +249,7 @@ "screenshot", "bannerImage", "fullPageArchive", + "video", "bookmarkAsset", "unknown" ] @@ -488,6 +493,10 @@ "type": "string", "nullable": true }, + "videoAssetId": { + "type": "string", + "nullable": true + }, "favicon": { "type": "string", "nullable": true, diff --git a/packages/shared/assetdb.ts b/packages/shared/assetdb.ts index 64413e9f..fb7d2461 100644 --- a/packages/shared/assetdb.ts +++ b/packages/shared/assetdb.ts @@ -13,6 +13,7 @@ export const enum ASSET_TYPES { IMAGE_WEBP = "image/webp", APPLICATION_PDF = "application/pdf", TEXT_HTML = "text/html", + VIDEO_MP4 = "video/mp4", } export const IMAGE_ASSET_TYPES: Set<string> = new Set<string>([ @@ -31,6 +32,7 @@ export const SUPPORTED_UPLOAD_ASSET_TYPES: Set<string> = new Set<string>([ export const SUPPORTED_ASSET_TYPES: Set<string> = new Set<string>([ ...SUPPORTED_UPLOAD_ASSET_TYPES, ASSET_TYPES.TEXT_HTML, + ASSET_TYPES.VIDEO_MP4, ]); function getAssetDir(userId: string, assetId: string) { @@ -152,6 +154,20 @@ export async function getAssetSize({ return stat.size; } +/** + * Deletes the passed in asset if it exists and ignores any errors + * @param userId the id of the user the asset belongs to + * @param assetId the id of the asset to delete + */ +export async function silentDeleteAsset( + userId: string, + assetId: string | undefined, +) { + if (assetId) { + await deleteAsset({ userId, assetId }).catch(() => ({})); + } +} + export async function deleteAsset({ userId, assetId, diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 4b51d15d..35d3df54 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -42,6 +42,9 @@ const allEnv = z.object({ CRAWLER_STORE_SCREENSHOT: stringBool("true"), CRAWLER_FULL_PAGE_SCREENSHOT: stringBool("false"), CRAWLER_FULL_PAGE_ARCHIVE: stringBool("false"), + CRAWLER_VIDEO_DOWNLOAD: stringBool("false"), + CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE: z.coerce.number().default(50), + CRAWLER_VIDEO_DOWNLOAD_TIMEOUT_SEC: z.coerce.number().default(10 * 60), MEILI_ADDR: z.string().optional(), MEILI_MASTER_KEY: z.string().default(""), LOG_LEVEL: z.string().default("debug"), @@ -98,6 +101,9 @@ const serverConfigSchema = allEnv.transform((val) => { storeScreenshot: val.CRAWLER_STORE_SCREENSHOT, fullPageScreenshot: val.CRAWLER_FULL_PAGE_SCREENSHOT, fullPageArchive: val.CRAWLER_FULL_PAGE_ARCHIVE, + downloadVideo: val.CRAWLER_VIDEO_DOWNLOAD, + maxVideoDownloadSize: val.CRAWLER_VIDEO_DOWNLOAD_MAX_SIZE, + downloadVideoTimeout: val.CRAWLER_VIDEO_DOWNLOAD_TIMEOUT_SEC, }, ocr: { langs: val.OCR_LANGS, diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index 0cb30aae..6189a633 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -93,3 +93,26 @@ export async function triggerSearchDeletion(bookmarkId: string) { type: "delete", }); } + +export const zvideoRequestSchema = z.object({ + bookmarkId: z.string(), + url: z.string(), +}); +export type ZVideoRequest = z.infer<typeof zvideoRequestSchema>; + +export const VideoWorkerQueue = new SqliteQueue<ZVideoRequest>( + "video_queue", + queueDB, + { + defaultJobArgs: { + numRetries: 5, + }, + }, +); + +export async function triggerVideoWorker(bookmarkId: string, url: string) { + await VideoWorkerQueue.enqueue({ + bookmarkId, + url, + }); +} diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index c731cb32..2d46684e 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -15,6 +15,7 @@ export const zAssetTypesSchema = z.enum([ "screenshot", "bannerImage", "fullPageArchive", + "video", "bookmarkAsset", "unknown", ]); @@ -34,6 +35,7 @@ export const zBookmarkedLinkSchema = z.object({ imageAssetId: z.string().nullish(), screenshotAssetId: z.string().nullish(), fullPageArchiveAssetId: z.string().nullish(), + videoAssetId: z.string().nullish(), favicon: z.string().url().nullish(), htmlContent: z.string().nullish(), crawledAt: z.date().nullish(), diff --git a/packages/trpc/lib/attachments.ts b/packages/trpc/lib/attachments.ts index 175947f8..0fd41d1b 100644 --- a/packages/trpc/lib/attachments.ts +++ b/packages/trpc/lib/attachments.ts @@ -8,6 +8,7 @@ export function mapDBAssetTypeToUserType(assetType: AssetTypes): ZAssetType { [AssetTypes.LINK_SCREENSHOT]: "screenshot", [AssetTypes.LINK_FULL_PAGE_ARCHIVE]: "fullPageArchive", [AssetTypes.LINK_BANNER_IMAGE]: "bannerImage", + [AssetTypes.LINK_VIDEO]: "video", [AssetTypes.BOOKMARK_ASSET]: "bookmarkAsset", [AssetTypes.UNKNOWN]: "bannerImage", }; @@ -21,6 +22,7 @@ export function mapSchemaAssetTypeToDB( screenshot: AssetTypes.LINK_SCREENSHOT, fullPageArchive: AssetTypes.LINK_FULL_PAGE_ARCHIVE, bannerImage: AssetTypes.LINK_BANNER_IMAGE, + video: AssetTypes.LINK_VIDEO, bookmarkAsset: AssetTypes.BOOKMARK_ASSET, unknown: AssetTypes.UNKNOWN, }; @@ -32,6 +34,7 @@ export function humanFriendlyNameForAssertType(type: ZAssetType) { screenshot: "Screenshot", fullPageArchive: "Full Page Archive", bannerImage: "Banner Image", + video: "Video", bookmarkAsset: "Bookmark Asset", unknown: "Unknown", }; @@ -43,6 +46,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) { screenshot: true, fullPageArchive: false, bannerImage: true, + video: false, bookmarkAsset: false, unknown: false, }; @@ -54,6 +58,7 @@ export function isAllowedToDetachAsset(type: ZAssetType) { screenshot: true, fullPageArchive: true, bannerImage: true, + video: true, bookmarkAsset: false, unknown: false, }; diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 80dd4bec..9a27c25a 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -215,6 +215,8 @@ function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark { imageAssetId: assets.find( (a) => a.assetType == AssetTypes.LINK_BANNER_IMAGE, )?.id, + videoAssetId: assets.find((a) => a.assetType == AssetTypes.LINK_VIDEO) + ?.id, ...link, }; break; @@ -698,6 +700,9 @@ export const bookmarksAppRouter = router({ if (row.assets.assetType == AssetTypes.LINK_BANNER_IMAGE) { content.imageAssetId = row.assets.id; } + if (row.assets.assetType == AssetTypes.LINK_VIDEO) { + content.videoAssetId = row.assets.id; + } acc[bookmarkId].content = content; } acc[bookmarkId].assets.push({ |