fix: Incremental polling interval for ongoing crawls

2 files changed, 34 insertions, 19 deletions

diff --git a/packages/shared-react/hooks/bookmarks.ts b/packages/shared-react/hooks/bookmarks.ts
index 15b678cc..c4f0c8e7 100644
--- a/packages/shared-react/hooks/bookmarks.ts
+++ b/packages/shared-react/hooks/bookmarks.ts
@@ -1,4 +1,4 @@
-import { isBookmarkStillLoading } from "@karakeep/shared/utils/bookmarkUtils";
+import { getBookmarkRefreshInterval } from "@karakeep/shared/utils/bookmarkUtils";
 
 import { api } from "../trpc";
 import { useBookmarkGridContext } from "./bookmark-grid-context";
@@ -13,11 +13,7 @@ export function useAutoRefreshingBookmarkQuery(
       if (!data) {
         return false;
       }
-      // If the link is not crawled or not tagged
-      if (isBookmarkStillLoading(data)) {
-        return 1000;
-      }
-      return false;
+      return getBookmarkRefreshInterval(data);
     },
   });
 }
diff --git a/packages/shared/utils/bookmarkUtils.ts b/packages/shared/utils/bookmarkUtils.ts
index 97ef08fc..9d4659b1 100644
--- a/packages/shared/utils/bookmarkUtils.ts
+++ b/packages/shared/utils/bookmarkUtils.ts
@@ -1,8 +1,6 @@
 import { BookmarkTypes, ZBookmark, ZBookmarkedLink } from "../types/bookmarks";
 import { getAssetUrl } from "./assetUtils";
 
-const MAX_LOADING_MSEC = 30 * 1000;
-
 export function getBookmarkLinkAssetIdOrUrl(bookmark: ZBookmarkedLink) {
   if (bookmark.imageAssetId) {
     return { assetId: bookmark.imageAssetId, localAsset: true as const };
@@ -31,24 +29,16 @@ export function getBookmarkLinkImageUrl(bookmark: ZBookmarkedLink) {
 
 export function isBookmarkStillCrawling(bookmark: ZBookmark) {
   return (
-    bookmark.content.type == BookmarkTypes.LINK &&
-    !bookmark.content.crawledAt &&
-    Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC
+    bookmark.content.type == BookmarkTypes.LINK && !bookmark.content.crawledAt
   );
 }
 
 export function isBookmarkStillTagging(bookmark: ZBookmark) {
-  return (
-    bookmark.taggingStatus == "pending" &&
-    Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC
-  );
+  return bookmark.taggingStatus == "pending";
 }
 
 export function isBookmarkStillSummarizing(bookmark: ZBookmark) {
-  return (
-    bookmark.summarizationStatus == "pending" &&
-    Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC
-  );
+  return bookmark.summarizationStatus == "pending";
 }
 
 export function isBookmarkStillLoading(bookmark: ZBookmark) {
@@ -59,6 +49,35 @@ export function isBookmarkStillLoading(bookmark: ZBookmark) {
   );
 }
 
+export function getBookmarkRefreshInterval(
+  bookmark: ZBookmark,
+): number | false {
+  if (!isBookmarkStillLoading(bookmark)) {
+    return false;
+  }
+
+  // For the first 30 seconds, we'll refresh the bookmark every second
+  if (Date.now().valueOf() - bookmark.createdAt.valueOf() < 30 * 1000) {
+    return 1000;
+  }
+
+  // Then, we'll refresh it every 10 seconds after than for 10mins
+  if (Date.now().valueOf() - bookmark.createdAt.valueOf() < 10 * 60 * 1000) {
+    return 10_000;
+  }
+
+  // Then, we'll refresh it every minute after than for 6hrs
+  if (
+    Date.now().valueOf() - bookmark.createdAt.valueOf() <
+    6 * 60 * 60 * 1000
+  ) {
+    return 60_000;
+  }
+
+  // Then we'll stop refreshing it
+  return false;
+}
+
 export function getSourceUrl(bookmark: ZBookmark) {
   if (bookmark.content.type === BookmarkTypes.LINK) {
     return bookmark.content.url;