From 8b69cddfb92b3b7548d3f90dbec1038c728ea5d9 Mon Sep 17 00:00:00 2001
From: kamtschatka <simon.schatka@gmx.at>
Date: Mon, 30 Sep 2024 02:02:48 +0200
Subject: feature(web): Add ability to manually trigger full page archives.
 Fixes #398 (#418)

* [Feature Request] Ability to select what to "crawl full page archive" #398
Added the ability to start a full page crawl for links and also in bulk operations
added the ability to refresh links as a bulk operation as well

* minor icon and wording changes

---------

Co-authored-by: MohamedBassem <me@mbassem.com>
---
 apps/workers/crawlerWorker.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'apps/workers/crawlerWorker.ts')

diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index 49e9495d..246e9050 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -214,7 +214,7 @@ async function getBookmarkDetails(bookmarkId: string) {
   });
 
   if (!bookmark || !bookmark.link) {
-    throw new Error("The bookmark either doesn't exist or not a link");
+    throw new Error("The bookmark either doesn't exist or is not a link");
   }
   return {
     url: bookmark.link.url,
@@ -519,6 +519,7 @@ async function crawlAndParseUrl(
   oldScreenshotAssetId: string | undefined,
   oldImageAssetId: string | undefined,
   oldFullPageArchiveAssetId: string | undefined,
+  archiveFullPage: boolean,
 ) {
   const {
     htmlContent,
@@ -578,7 +579,7 @@ async function crawlAndParseUrl(
   ]);
 
   return async () => {
-    if (serverConfig.crawler.fullPageArchive) {
+    if (serverConfig.crawler.fullPageArchive || archiveFullPage) {
       const fullPageArchiveAssetId = await archiveWebpage(
         htmlContent,
         browserUrl,
@@ -615,7 +616,7 @@ async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) {
     return;
   }
 
-  const { bookmarkId } = request.data;
+  const { bookmarkId, archiveFullPage } = request.data;
   const {
     url,
     userId,
@@ -654,6 +655,7 @@ async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) {
       oldScreenshotAssetId,
       oldImageAssetId,
       oldFullPageArchiveAssetId,
+      archiveFullPage,
     );
   }
 
-- 
cgit v1.2.3-70-g09d2