aboutsummaryrefslogtreecommitdiffstats
path: root/apps
diff options
context:
space:
mode:
Diffstat (limited to 'apps')
-rw-r--r--apps/web/app/api/assets/[assetId]/route.ts11
-rw-r--r--apps/web/app/api/assets/route.ts17
-rw-r--r--apps/web/components/dashboard/preview/AttachmentBox.tsx112
-rw-r--r--apps/workers/crawlerWorker.ts143
4 files changed, 167 insertions, 116 deletions
diff --git a/apps/web/app/api/assets/[assetId]/route.ts b/apps/web/app/api/assets/[assetId]/route.ts
index f3cf1ab4..73237d8d 100644
--- a/apps/web/app/api/assets/[assetId]/route.ts
+++ b/apps/web/app/api/assets/[assetId]/route.ts
@@ -1,5 +1,7 @@
import { createContextFromRequest } from "@/server/api/client";
+import { and, eq } from "drizzle-orm";
+import { assets } from "@hoarder/db/schema";
import { readAsset } from "@hoarder/shared/assetdb";
export const dynamic = "force-dynamic";
@@ -11,6 +13,15 @@ export async function GET(
if (!ctx.user) {
return Response.json({ error: "Unauthorized" }, { status: 401 });
}
+
+ const assetDb = await ctx.db.query.assets.findFirst({
+ where: and(eq(assets.id, params.assetId), eq(assets.userId, ctx.user.id)),
+ });
+
+ if (!assetDb) {
+ return Response.json({ error: "Asset not found" }, { status: 404 });
+ }
+
const { asset, metadata } = await readAsset({
userId: ctx.user.id,
assetId: params.assetId,
diff --git a/apps/web/app/api/assets/route.ts b/apps/web/app/api/assets/route.ts
index 9028f556..0e52ff93 100644
--- a/apps/web/app/api/assets/route.ts
+++ b/apps/web/app/api/assets/route.ts
@@ -2,6 +2,7 @@ import { createContextFromRequest } from "@/server/api/client";
import { TRPCError } from "@trpc/server";
import type { ZUploadResponse } from "@hoarder/shared/types/uploads";
+import { assets, AssetTypes } from "@hoarder/db/schema";
import {
newAssetId,
saveAsset,
@@ -43,8 +44,22 @@ export async function POST(request: Request) {
return Response.json({ error: "Bad request" }, { status: 400 });
}
- const assetId = newAssetId();
const fileName = data.name;
+ const [assetDb] = await ctx.db
+ .insert(assets)
+ .values({
+ id: newAssetId(),
+ // Initially, uploads are uploaded for unknown purpose
+ // And without an attached bookmark.
+ assetType: AssetTypes.UNKNOWN,
+ bookmarkId: null,
+ userId: ctx.user.id,
+ contentType,
+ size: data.size,
+ fileName,
+ })
+ .returning();
+ const assetId = assetDb.id;
await saveAsset({
userId: ctx.user.id,
diff --git a/apps/web/components/dashboard/preview/AttachmentBox.tsx b/apps/web/components/dashboard/preview/AttachmentBox.tsx
index a8eaf0f4..436f1026 100644
--- a/apps/web/components/dashboard/preview/AttachmentBox.tsx
+++ b/apps/web/components/dashboard/preview/AttachmentBox.tsx
@@ -16,6 +16,7 @@ import {
ChevronsDownUp,
Download,
Image,
+ Paperclip,
Pencil,
Plus,
Trash2,
@@ -35,6 +36,7 @@ import {
import {
humanFriendlyNameForAssertType,
isAllowedToAttachAsset,
+ isAllowedToDetachAsset,
} from "@hoarder/trpc/lib/attachments";
export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) {
@@ -42,6 +44,8 @@ export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) {
screenshot: <Camera className="size-4" />,
fullPageArchive: <Archive className="size-4" />,
bannerImage: <Image className="size-4" />,
+ bookmarkAsset: <Paperclip className="size-4" />,
+ unknown: <Paperclip className="size-4" />,
};
const { mutate: attachAsset, isPending: isAttaching } =
@@ -100,11 +104,6 @@ export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) {
bookmark.assets.sort((a, b) => a.assetType.localeCompare(b.assetType));
- if (bookmark.content.type == BookmarkTypes.ASSET) {
- // Currently, we don't allow attaching assets to assets types.
- return null;
- }
-
return (
<Collapsible>
<CollapsibleTrigger className="flex w-full items-center justify-between gap-2 text-sm text-gray-400">
@@ -156,59 +155,62 @@ export default function AttachmentBox({ bookmark }: { bookmark: ZBookmark }) {
<Pencil className="size-4" />
</FilePickerButton>
)}
- <ActionConfirmingDialog
- title="Delete Attachment?"
- description={`Are you sure you want to delete the attachment of the bookmark?`}
- actionButton={(setDialogOpen) => (
- <ActionButton
- loading={isDetaching}
- variant="destructive"
- onClick={() =>
- detachAsset(
- { bookmarkId: bookmark.id, assetId: asset.id },
- { onSettled: () => setDialogOpen(false) },
- )
- }
- >
- <Trash2 className="mr-2 size-4" />
- Delete
- </ActionButton>
- )}
- >
- <Button variant="none" size="none" title="Delete">
- <Trash2 className="size-4" />
- </Button>
- </ActionConfirmingDialog>
+ {isAllowedToDetachAsset(asset.assetType) && (
+ <ActionConfirmingDialog
+ title="Delete Attachment?"
+ description={`Are you sure you want to delete the attachment of the bookmark?`}
+ actionButton={(setDialogOpen) => (
+ <ActionButton
+ loading={isDetaching}
+ variant="destructive"
+ onClick={() =>
+ detachAsset(
+ { bookmarkId: bookmark.id, assetId: asset.id },
+ { onSettled: () => setDialogOpen(false) },
+ )
+ }
+ >
+ <Trash2 className="mr-2 size-4" />
+ Delete
+ </ActionButton>
+ )}
+ >
+ <Button variant="none" size="none" title="Delete">
+ <Trash2 className="size-4" />
+ </Button>
+ </ActionConfirmingDialog>
+ )}
</div>
</div>
))}
- {!bookmark.assets.some((asset) => asset.assetType == "bannerImage") && (
- <FilePickerButton
- title="Attach a Banner"
- loading={isAttaching}
- accept=".jgp,.JPG,.jpeg,.png,.webp"
- multiple={false}
- variant="ghost"
- size="none"
- className="flex w-full items-center justify-center gap-2"
- onFileSelect={(file) =>
- uploadAsset(file, {
- onSuccess: (resp) => {
- attachAsset({
- bookmarkId: bookmark.id,
- asset: {
- id: resp.assetId,
- assetType: "bannerImage",
- },
- });
- },
- })
- }
- >
- <Plus className="size-4" />
- Attach a Banner
- </FilePickerButton>
- )}
+ {!bookmark.assets.some((asset) => asset.assetType == "bannerImage") &&
+ bookmark.content.type != BookmarkTypes.ASSET && (
+ <FilePickerButton
+ title="Attach a Banner"
+ loading={isAttaching}
+ accept=".jgp,.JPG,.jpeg,.png,.webp"
+ multiple={false}
+ variant="ghost"
+ size="none"
+ className="flex w-full items-center justify-center gap-2"
+ onFileSelect={(file) =>
+ uploadAsset(file, {
+ onSuccess: (resp) => {
+ attachAsset({
+ bookmarkId: bookmark.id,
+ asset: {
+ id: resp.assetId,
+ assetType: "bannerImage",
+ },
+ });
+ },
+ })
+ }
+ >
+ <Plus className="size-4" />
+ Attach a Banner
+ </FilePickerButton>
+ )}
</CollapsibleContent>
</Collapsible>
);
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index f830c500..74413c63 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -36,6 +36,7 @@ import { DequeuedJob, Runner } from "@hoarder/queue";
import {
ASSET_TYPES,
deleteAsset,
+ getAssetSize,
IMAGE_ASSET_TYPES,
newAssetId,
saveAsset,
@@ -192,6 +193,8 @@ export class CrawlerWorker {
}
}
+type DBAssetType = typeof assets.$inferInsert;
+
async function changeBookmarkStatus(
bookmarkId: string,
crawlStatus: "success" | "failure",
@@ -353,16 +356,18 @@ async function storeScreenshot(
return null;
}
const assetId = newAssetId();
+ const contentType = "image/png";
+ const fileName = "screenshot.png";
await saveAsset({
userId,
assetId,
- metadata: { contentType: "image/png", fileName: "screenshot.png" },
+ metadata: { contentType, fileName },
asset: screenshot,
});
logger.info(
`[Crawler][${jobId}] Stored the screenshot as assetId: ${assetId}`,
);
- return assetId;
+ return { assetId, contentType, fileName, size: screenshot.byteLength };
}
async function downloadAndStoreFile(
@@ -396,7 +401,7 @@ async function downloadAndStoreFile(
`[Crawler][${jobId}] Downloaded ${fileType} as assetId: ${assetId}`,
);
- return assetId;
+ return { assetId, userId, contentType, size: buffer.byteLength };
} catch (e) {
logger.error(
`[Crawler][${jobId}] Failed to download and store ${fileType}: ${e}`,
@@ -433,12 +438,14 @@ async function archiveWebpage(
input: html,
})`monolith - -Ije -t 5 -b ${url} -o ${assetPath}`;
+ const contentType = "text/html";
+
await saveAssetFromFile({
userId,
assetId,
assetPath,
metadata: {
- contentType: "text/html",
+ contentType,
},
});
@@ -446,7 +453,11 @@ async function archiveWebpage(
`[Crawler][${jobId}] Done archiving the page as assetId: ${assetId}`,
);
- return assetId;
+ return {
+ assetId,
+ contentType,
+ size: await getAssetSize({ userId, assetId }),
+ };
}
async function getContentType(
@@ -489,17 +500,31 @@ async function handleAsAssetBookmark(
jobId: string,
bookmarkId: string,
) {
- const assetId = await downloadAndStoreFile(url, userId, jobId, assetType);
- if (!assetId) {
+ const downloaded = await downloadAndStoreFile(url, userId, jobId, assetType);
+ if (!downloaded) {
return;
}
+ const fileName = path.basename(new URL(url).pathname);
await db.transaction(async (trx) => {
+ await updateAsset(
+ undefined,
+ {
+ id: downloaded.assetId,
+ bookmarkId,
+ userId,
+ assetType: AssetTypes.BOOKMARK_ASSET,
+ contentType: downloaded.contentType,
+ size: downloaded.size,
+ fileName,
+ },
+ trx,
+ );
await trx.insert(bookmarkAssets).values({
id: bookmarkId,
assetType,
- assetId,
+ assetId: downloaded.assetId,
content: null,
- fileName: path.basename(new URL(url).pathname),
+ fileName,
sourceUrl: url,
});
// Switch the type of the bookmark from LINK to ASSET
@@ -527,14 +552,24 @@ async function crawlAndParseUrl(
url: browserUrl,
} = await crawlPage(jobId, url);
- const [meta, readableContent, screenshotAssetId] = await Promise.all([
+ const [meta, readableContent, screenshotAssetInfo] = await Promise.all([
extractMetadata(htmlContent, browserUrl, jobId),
extractReadableContent(htmlContent, browserUrl, jobId),
storeScreenshot(screenshot, userId, jobId),
]);
- let imageAssetId: string | null = null;
+ let imageAssetInfo: DBAssetType | null = null;
if (meta.image) {
- imageAssetId = await downloadAndStoreImage(meta.image, userId, jobId);
+ const downloaded = await downloadAndStoreImage(meta.image, userId, jobId);
+ if (downloaded) {
+ imageAssetInfo = {
+ id: downloaded.assetId,
+ bookmarkId,
+ userId,
+ assetType: AssetTypes.LINK_BANNER_IMAGE,
+ contentType: downloaded.contentType,
+ size: downloaded.size,
+ };
+ }
}
// TODO(important): Restrict the size of content to store
@@ -552,22 +587,24 @@ async function crawlAndParseUrl(
})
.where(eq(bookmarkLinks.id, bookmarkId));
- await updateAsset(
- screenshotAssetId,
- oldScreenshotAssetId,
- bookmarkId,
- userId,
- AssetTypes.LINK_SCREENSHOT,
- txn,
- );
- await updateAsset(
- imageAssetId,
- oldImageAssetId,
- bookmarkId,
- userId,
- AssetTypes.LINK_BANNER_IMAGE,
- txn,
- );
+ if (screenshotAssetInfo) {
+ await updateAsset(
+ oldScreenshotAssetId,
+ {
+ id: screenshotAssetInfo.assetId,
+ bookmarkId,
+ userId,
+ assetType: AssetTypes.LINK_SCREENSHOT,
+ contentType: screenshotAssetInfo.contentType,
+ size: screenshotAssetInfo.size,
+ fileName: screenshotAssetInfo.fileName,
+ },
+ txn,
+ );
+ }
+ if (imageAssetInfo) {
+ await updateAsset(oldImageAssetId, imageAssetInfo, txn);
+ }
});
// Delete the old assets if any
@@ -582,20 +619,24 @@ async function crawlAndParseUrl(
return async () => {
if (serverConfig.crawler.fullPageArchive || archiveFullPage) {
- const fullPageArchiveAssetId = await archiveWebpage(
- htmlContent,
- browserUrl,
- userId,
- jobId,
- );
+ const {
+ assetId: fullPageArchiveAssetId,
+ size,
+ contentType,
+ } = await archiveWebpage(htmlContent, browserUrl, userId, jobId);
await db.transaction(async (txn) => {
await updateAsset(
- fullPageArchiveAssetId,
oldFullPageArchiveAssetId,
- bookmarkId,
- userId,
- AssetTypes.LINK_FULL_PAGE_ARCHIVE,
+ {
+ id: fullPageArchiveAssetId,
+ bookmarkId,
+ userId,
+ assetType: AssetTypes.LINK_FULL_PAGE_ARCHIVE,
+ contentType,
+ size,
+ fileName: null,
+ },
txn,
);
});
@@ -676,31 +717,13 @@ async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) {
await archivalLogic();
}
-/**
- * Removes the old asset and adds a new one instead
- * @param newAssetId the new assetId to add
- * @param oldAssetId the old assetId to remove (if it exists)
- * @param bookmarkId the id of the bookmark the asset belongs to
- * @param assetType the type of the asset
- * @param txn the transaction where this update should happen in
- */
async function updateAsset(
- newAssetId: string | null,
oldAssetId: string | undefined,
- bookmarkId: string,
- userId: string,
- assetType: AssetTypes,
+ newAsset: DBAssetType,
txn: HoarderDBTransaction,
) {
- if (newAssetId) {
- if (oldAssetId) {
- await txn.delete(assets).where(eq(assets.id, oldAssetId));
- }
- await txn.insert(assets).values({
- id: newAssetId,
- assetType,
- bookmarkId,
- userId,
- });
+ if (oldAssetId) {
+ await txn.delete(assets).where(eq(assets.id, oldAssetId));
}
+ await txn.insert(assets).values(newAsset);
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-30 14:57:11 +0000