Implement metadata fetching logic in the crawler

4 files changed, 23 insertions, 13 deletions

diff --git a/shared/index.ts b/shared/index.ts
index 633b9287..8b93520f 100644
--- a/shared/index.ts
+++ b/shared/index.ts
@@ -1 +1 @@
-export * as Queues from './queues.ts';
+export * as Queues from "./queues.ts";
diff --git a/shared/logger.ts b/shared/logger.ts
index 442304d7..8cd2f808 100644
--- a/shared/logger.ts
+++ b/shared/logger.ts
@@ -1,15 +1,15 @@
 import winston from "winston";
 
 const logger = winston.createLogger({
-    level: process.env.LOG_LEVEL || "debug",
-    format: winston.format.combine(
-        winston.format.timestamp(),
-        winston.format.colorize(),
-        winston.format.printf(
-            (info) => `${info.timestamp} ${info.level}: ${info.message}`,
-        ),
+  level: process.env.LOG_LEVEL || "debug",
+  format: winston.format.combine(
+    winston.format.timestamp(),
+    winston.format.colorize(),
+    winston.format.printf(
+      (info) => `${info.timestamp} ${info.level}: ${info.message}`,
     ),
-    transports: [new winston.transports.Console()],
+  ),
+  transports: [new winston.transports.Console()],
 });
 
 export default logger;
diff --git a/shared/package.json b/shared/package.json
index 9f5ee37b..b75b3ac3 100644
--- a/shared/package.json
+++ b/shared/package.json
@@ -1,8 +1,10 @@
 {
+  "$schema": "https://json.schemastore.org/package.json",
   "name": "@remember/shared",
   "version": "0.1.0",
   "private": true,
   "dependencies": {
+    "winston": "^3.11.0"
   },
   "main": "index.ts"
 }
diff --git a/shared/queues.ts b/shared/queues.ts
index 4303eaa2..ac5acc57 100644
--- a/shared/queues.ts
+++ b/shared/queues.ts
@@ -1,10 +1,18 @@
 import { Queue } from "bullmq";
+import { z } from "zod";
 
 export const queueConnectionDetails = {
-    host: process.env.REDIS_HOST || "localhost",
-    port: parseInt(process.env.REDIS_PORT || "6379"),
+  host: process.env.REDIS_HOST || "localhost",
+  port: parseInt(process.env.REDIS_PORT || "6379"),
 };
 
-export const LinkCrawlerQueue = new Queue("link_crawler_queue", { connection: queueConnectionDetails });
-
+export const zCrawlLinkRequestSchema = z.object({
+  linkId: z.string(),
+  url: z.string().url(),
+});
+export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>;
 
+export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>(
+  "link_crawler_queue",
+  { connection: queueConnectionDetails },
+);