Init package and start bullmq workers

12 files changed, 91 insertions, 8 deletions

diff --git a/bun.lockb b/bun.lockb
index 4a85f011..c5a9f19a 100755
--- a/bun.lockb
+++ b/bun.lockb
diff --git a/crawler/crawler.ts b/crawler/crawler.ts
new file mode 100644
index 00000000..58127331
--- /dev/null
+++ b/crawler/crawler.ts
@@ -0,0 +1,6 @@
+import logger from "@remember/shared/logger";
+import { Job } from "bullmq";
+
+export default async function runCrawler(job: Job) {
+    logger.info(`[Crawler] Got a new job: ${job.name}`);
+}
diff --git a/crawler/main.ts b/crawler/main.ts
new file mode 100644
index 00000000..7d1c0f11
--- /dev/null
+++ b/crawler/main.ts
@@ -0,0 +1,17 @@
+import { Worker } from "bullmq";
+
+import {
+    LinkCrawlerQueue,
+    queueConnectionDetails,
+} from "@remember/shared/queues";
+import logger from "@remember/shared/logger";
+import runCrawler from "./crawler";
+
+logger.info("Starting crawler worker ...");
+
+const crawlerWorker = new Worker(LinkCrawlerQueue.name, runCrawler, {
+    connection: queueConnectionDetails,
+    autorun: false,
+});
+
+await Promise.all([crawlerWorker]);
diff --git a/crawler/package.json b/crawler/package.json
new file mode 100644
index 00000000..67e38cff
--- /dev/null
+++ b/crawler/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "@remember/crawler",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "@remember/shared": "workspace:*"
+  }
+}
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..55bb38ff
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,11 @@
+version: "3.8"
+name: remember
+services:
+  redis:
+    image: redis:7.2-alpine
+    ports:
+      - 6379:6379
+    volumes:
+      - redis:/data
+volumes:
+  redis:
diff --git a/package.json b/package.json
index ef8827d5..e4b183f0 100644
--- a/package.json
+++ b/package.json
@@ -3,15 +3,25 @@
   "version": "0.1.0",
   "private": true,
   "workspaces": [
-    "web"
+    "web",
+    "crawler",
+    "shared"
   ],
   "dependencies": {
     "@next/eslint-plugin-next": "^14.1.0",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
+    "bullmq": "^5.1.9",
     "class-variance-authority": "^0.7.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-react": "^7.33.2",
-    "eslint-plugin-react-hooks": "^4.6.0"
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "winston": "^3.11.0"
+  },
+  "devDependencies": {
+    "typescript": "^5",
+    "@types/node": "^20",
+    "eslint": "^8.56.0",
+    "eslint-config-next": "14.1.0"
   }
 }
diff --git a/shared/index.ts b/shared/index.ts
new file mode 100644
index 00000000..633b9287
--- /dev/null
+++ b/shared/index.ts
@@ -0,0 +1 @@
+export * as Queues from './queues.ts';
diff --git a/shared/logger.ts b/shared/logger.ts
new file mode 100644
index 00000000..442304d7
--- /dev/null
+++ b/shared/logger.ts
@@ -0,0 +1,15 @@
+import winston from "winston";
+
+const logger = winston.createLogger({
+    level: process.env.LOG_LEVEL || "debug",
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.colorize(),
+        winston.format.printf(
+            (info) => `${info.timestamp} ${info.level}: ${info.message}`,
+        ),
+    ),
+    transports: [new winston.transports.Console()],
+});
+
+export default logger;
diff --git a/shared/package.json b/shared/package.json
new file mode 100644
index 00000000..9f5ee37b
--- /dev/null
+++ b/shared/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "@remember/shared",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+  },
+  "main": "index.ts"
+}
diff --git a/shared/queues.ts b/shared/queues.ts
new file mode 100644
index 00000000..4303eaa2
--- /dev/null
+++ b/shared/queues.ts
@@ -0,0 +1,10 @@
+import { Queue } from "bullmq";
+
+export const queueConnectionDetails = {
+    host: process.env.REDIS_HOST || "localhost",
+    port: parseInt(process.env.REDIS_PORT || "6379"),
+};
+
+export const LinkCrawlerQueue = new Queue("link_crawler_queue", { connection: queueConnectionDetails });
+
+
diff --git a/web/package.json b/web/package.json
index e5c99db7..6dcbff47 100644
--- a/web/package.json
+++ b/web/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "remember",
+  "name": "@remember/web",
   "version": "0.1.0",
   "private": true,
   "scripts": {
@@ -26,14 +26,10 @@
     "zod": "^3.22.4"
   },
   "devDependencies": {
-    "typescript": "^5",
-    "@types/node": "^20",
     "@types/react": "^18",
     "@types/react-dom": "^18",
     "autoprefixer": "^10.0.1",
     "postcss": "^8",
-    "tailwindcss": "^3.3.0",
-    "eslint": "^8.56.0",
-    "eslint-config-next": "14.1.0"
+    "tailwindcss": "^3.3.0"
   }
 }
diff --git a/web/tsconfig.json b/web/tsconfig.json
index e7ff90fd..a25dbc14 100644
--- a/web/tsconfig.json
+++ b/web/tsconfig.json
@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json.schemastore.org/tsconfig",
   "compilerOptions": {
     "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,