fix: Incremental polling interval for ongoing crawls

1 files changed, 32 insertions, 13 deletions

diff --git a/packages/shared/utils/bookmarkUtils.ts b/packages/shared/utils/bookmarkUtils.ts
index 97ef08fc..9d4659b1 100644
--- a/packages/shared/utils/bookmarkUtils.ts
+++ b/packages/shared/utils/bookmarkUtils.ts
@@ -1,8 +1,6 @@
 import { BookmarkTypes, ZBookmark, ZBookmarkedLink } from "../types/bookmarks";
 import { getAssetUrl } from "./assetUtils";
 
-const MAX_LOADING_MSEC = 30 * 1000;
-
 export function getBookmarkLinkAssetIdOrUrl(bookmark: ZBookmarkedLink) {
   if (bookmark.imageAssetId) {
     return { assetId: bookmark.imageAssetId, localAsset: true as const };
@@ -31,24 +29,16 @@ export function getBookmarkLinkImageUrl(bookmark: ZBookmarkedLink) {
 
 export function isBookmarkStillCrawling(bookmark: ZBookmark) {
   return (
-    bookmark.content.type == BookmarkTypes.LINK &&
-    !bookmark.content.crawledAt &&
-    Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC
+    bookmark.content.type == BookmarkTypes.LINK && !bookmark.content.crawledAt
   );
 }
 
 export function isBookmarkStillTagging(bookmark: ZBookmark) {
-  return (
-    bookmark.taggingStatus == "pending" &&
-    Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC
-  );
+  return bookmark.taggingStatus == "pending";
 }
 
 export function isBookmarkStillSummarizing(bookmark: ZBookmark) {
-  return (
-    bookmark.summarizationStatus == "pending" &&
-    Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC
-  );
+  return bookmark.summarizationStatus == "pending";
 }
 
 export function isBookmarkStillLoading(bookmark: ZBookmark) {
@@ -59,6 +49,35 @@ export function isBookmarkStillLoading(bookmark: ZBookmark) {
   );
 }
 
+export function getBookmarkRefreshInterval(
+  bookmark: ZBookmark,
+): number | false {
+  if (!isBookmarkStillLoading(bookmark)) {
+    return false;
+  }
+
+  // For the first 30 seconds, we'll refresh the bookmark every second
+  if (Date.now().valueOf() - bookmark.createdAt.valueOf() < 30 * 1000) {
+    return 1000;
+  }
+
+  // Then, we'll refresh it every 10 seconds after than for 10mins
+  if (Date.now().valueOf() - bookmark.createdAt.valueOf() < 10 * 60 * 1000) {
+    return 10_000;
+  }
+
+  // Then, we'll refresh it every minute after than for 6hrs
+  if (
+    Date.now().valueOf() - bookmark.createdAt.valueOf() <
+    6 * 60 * 60 * 1000
+  ) {
+    return 60_000;
+  }
+
+  // Then we'll stop refreshing it
+  return false;
+}
+
 export function getSourceUrl(bookmark: ZBookmark) {
   if (bookmark.content.type === BookmarkTypes.LINK) {
     return bookmark.content.url;