From 8711bca9db49546d265fc119b5dcf3eb2db22095 Mon Sep 17 00:00:00 2001 From: MohamedBassem Date: Tue, 20 Feb 2024 21:23:28 +0000 Subject: docker: Add a docker-based dev environment --- .github/workflows/docker.yml | 4 +- .github/workflows/main.yml | 1 + Dockerfile | 103 ------------------------------------------ README.md | 13 ++++++ docker-compose.yml | 42 ----------------- docker/Dockerfile | 99 ++++++++++++++++++++++++++++++++++++++++ docker/Dockerfile.dev | 11 +++++ docker/docker-compose.dev.yml | 58 ++++++++++++++++++++++++ docker/docker-compose.yml | 43 ++++++++++++++++++ docker/start-chrome.sh | 7 +++ docker/yarn.lock | 4 ++ 11 files changed, 238 insertions(+), 147 deletions(-) delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.dev create mode 100644 docker/docker-compose.dev.yml create mode 100644 docker/docker-compose.yml create mode 100644 docker/start-chrome.sh create mode 100644 docker/yarn.lock diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 907bb4bc..3735e8d6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,8 +15,8 @@ jobs: - name: Build image run: | - docker build . --target workers --tag hoarder-workers - docker build . --target web --tag hoarder-web + docker build . --target workers --tag hoarder-workers -f docker/Dockerfile + docker build . --target web --tag hoarder-web -f docker/Dockerfile - name: Log in to registry # This is where you will update the PAT to GITHUB_TOKEN diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 44c238f4..28bb13fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,7 @@ on: push: branches: - main + pull_request: jobs: build-lint: name: Build and Lint diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index fc1950b4..00000000 --- a/Dockerfile +++ /dev/null @@ -1,103 +0,0 @@ -################# The Web App ############## - -# Rebuild the source code only when needed -FROM node:21-alpine AS web_builder - -# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. -RUN apk add --no-cache libc6-compat - -WORKDIR /app -COPY packages/db packages/db -COPY packages/web packages/web -COPY packages/shared packages/shared -COPY package.json yarn.lock .yarnrc.yml . - -ENV NEXT_TELEMETRY_DISABLED 1 - -RUN corepack enable && \ - cd /app/packages/db && \ - yarn dlx prisma generate && \ - cd /app/packages/web/ && \ - yarn install --immutable && \ - yarn next experimental-compile - -FROM node:21-alpine AS web -WORKDIR /app - -ENV NODE_ENV production -ENV NEXT_TELEMETRY_DISABLED 1 - -COPY --from=web_builder --chown=node:node /app/packages/web/.next/standalone ./ -COPY --from=web_builder /app/packages/web/public ./packages/web/public - -# Set the correct permission for prerender cache -RUN mkdir -p ./package/web/.next -RUN chown node:node ./packages/web/.next - -# Automatically leverage output traces to reduce image size -# https://nextjs.org/docs/advanced-features/output-file-tracing -COPY --from=web_builder --chown=node:node /app/packages/web/.next/static ./packages/web/.next/static - -WORKDIR /app/packages/web -USER root -EXPOSE 3000 - -ENV PORT 3000 -# set hostname to localhost -ENV HOSTNAME "0.0.0.0" - -# server.js is created by next build from the standalone output -# https://nextjs.org/docs/pages/api-reference/next-config-js/output -CMD ["node", "server.js"] - -################# The workers builder ############## - -FROM node:21-alpine AS workers_builder -WORKDIR /app - -# Install chromium needed for puppeteer -RUN apk add --no-cache make g++ py3-pip linux-headers - -COPY packages/db/package.json packages/db/package.json -COPY packages/shared/package.json packages/shared/package.json -COPY packages/workers/package.json packages/workers/package.json -COPY package.json yarn.lock .yarnrc.yml . - -RUN corepack enable && \ - cd /app/packages/workers && \ - yarn workspaces focus --production && \ - cd /app/packages/db && \ - yarn dlx prisma generate - - -################# The workers ############## - -FROM node:21-alpine AS workers -WORKDIR /app - -# Install chromium needed for puppeteer -RUN apk add --no-cache chromium runuser -ENV PUPPETEER_SKIP_DOWNLOAD true -ENV CHROME_PATH "/usr/bin/chromium-browser" -ENV BROWSER_EXECUTABLE_PATH "/app/start-chrome.sh" -ENV BROWSER_USER_DATA_DIR="/tmp/chrome" - -RUN echo '#!/bin/sh' >> /app/start-chrome.sh && \ - echo 'set -x;' >> /app/start-chrome.sh && \ - echo 'id -u chrome &>/dev/null || adduser -S chrome' >> /app/start-chrome.sh && \ - echo 'mkdir -p /tmp/chrome && chown chrome /tmp/chrome' >> /app/start-chrome.sh && \ - echo 'runuser -u chrome -- $CHROME_PATH $@' >> /app/start-chrome.sh && \ - chmod +x /app/start-chrome.sh - -COPY packages/db packages/db -COPY packages/shared packages/shared -COPY packages/workers packages/workers -COPY package.json yarn.lock .yarnrc.yml . -COPY --from=workers_builder /app/node_modules /app/node_modules - -RUN corepack enable && cd packages/db && yarn dlx prisma generate - -WORKDIR /app/packages/workers -USER root - -CMD ["yarn", "run", "start:prod"] diff --git a/README.md b/README.md index 4a33cb9e..a32fa90f 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,15 @@ I'm a systems engineer in my day job (and have been for the past 7 years). I did ## Development +### Docker + +You can turnup the whole development environment with: +`docker compose -f docker/docker-compose.dev.yml up` + +### Manual + +Or if you have nodejs installed locally, you can do: + - `yarn install` in the root of the repo. - `yarn prisma:migrate:dev` then `yarn prisma:generate` to generate prisma's data and run the database migrations. - `yarn web` to start the web app. @@ -75,3 +84,7 @@ I'm a systems engineer in my day job (and have been for the past 7 years). I did - `packages/shared`: Shared utilities and code between the workers and the web app. - `packages/web`: Where the nextjs based web app lives. - `packages/workers`: Where the background job workers (crawler and openai as of now) run. + +### Submitting PRs + +- Before submitting PRs, you'll want to run `yarn format` and include its changes in the commit. Also make sure `yarn lint` is successful. diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 69c5fead..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: "3.8" -services: - web: - image: ghcr.io/mohamedbassem/hoarder-web:latest - volumes: - - data:/data - ports: - - 3000:3000 - environment: - REDIS_HOST: redis - DATABASE_URL: "file:/data/db.db" - depends_on: - migration: - condition: service_completed_successfully - redis: - image: redis:7.2-alpine - volumes: - - redis:/data - workers: - image: ghcr.io/mohamedbassem/hoarder-workers:latest - volumes: - - data:/data - environment: - REDIS_HOST: redis - DATABASE_URL: "file:/data/db.db" - depends_on: - migration: - condition: service_completed_successfully - migration: - image: ghcr.io/mohamedbassem/hoarder-workers:latest - environment: - DATABASE_URL: "file:/data/db.db" - volumes: - - data:/data - command: - - /bin/sh - - -c - - "cd /app/packages/db && yarn dlx prisma migrate deploy" - -volumes: - redis: - data: diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..0e587790 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,99 @@ +################# The Web App ############## + +# Rebuild the source code only when needed +FROM node:21-alpine AS web_builder + +# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. +RUN apk add --no-cache libc6-compat + +WORKDIR /app +COPY packages/db packages/db +COPY packages/web packages/web +COPY packages/shared packages/shared +COPY package.json yarn.lock .yarnrc.yml . + +ENV NEXT_TELEMETRY_DISABLED 1 + +RUN corepack enable && \ + cd /app/packages/db && \ + yarn dlx prisma generate && \ + cd /app/packages/web/ && \ + yarn install --immutable && \ + yarn next experimental-compile + +FROM node:21-alpine AS web +WORKDIR /app + +ENV NODE_ENV production +ENV NEXT_TELEMETRY_DISABLED 1 + +COPY --from=web_builder --chown=node:node /app/packages/web/.next/standalone ./ +COPY --from=web_builder /app/packages/web/public ./packages/web/public + +# Set the correct permission for prerender cache +RUN mkdir -p ./package/web/.next +RUN chown node:node ./packages/web/.next + +# Automatically leverage output traces to reduce image size +# https://nextjs.org/docs/advanced-features/output-file-tracing +COPY --from=web_builder --chown=node:node /app/packages/web/.next/static ./packages/web/.next/static + +WORKDIR /app/packages/web +USER root +EXPOSE 3000 + +ENV PORT 3000 +# set hostname to localhost +ENV HOSTNAME "0.0.0.0" + +# server.js is created by next build from the standalone output +# https://nextjs.org/docs/pages/api-reference/next-config-js/output +CMD ["node", "server.js"] + +################# The workers builder ############## + +FROM node:21-alpine AS workers_builder +WORKDIR /app + +# Install chromium needed for puppeteer +RUN apk add --no-cache make g++ py3-pip linux-headers + +COPY packages/db/package.json packages/db/package.json +COPY packages/shared/package.json packages/shared/package.json +COPY packages/workers/package.json packages/workers/package.json +COPY package.json yarn.lock .yarnrc.yml . + +RUN corepack enable && \ + cd /app/packages/workers && \ + yarn workspaces focus --production && \ + cd /app/packages/db && \ + yarn dlx prisma generate + + +################# The workers ############## + +FROM node:21-alpine AS workers +WORKDIR /app + +# Install chromium needed for puppeteer +RUN apk add --no-cache chromium runuser +ENV PUPPETEER_SKIP_DOWNLOAD true +ENV CHROME_PATH "/usr/bin/chromium-browser" +ENV BROWSER_EXECUTABLE_PATH "/app/start-chrome.sh" +ENV BROWSER_USER_DATA_DIR="/tmp/chrome" + +ADD docker/start-chrome.sh . +RUN chmod +x /app/start-chrome.sh + +COPY packages/db packages/db +COPY packages/shared packages/shared +COPY packages/workers packages/workers +COPY package.json yarn.lock .yarnrc.yml . +COPY --from=workers_builder /app/node_modules /app/node_modules + +RUN corepack enable && cd packages/db && yarn dlx prisma generate + +WORKDIR /app/packages/workers +USER root + +CMD ["yarn", "run", "start:prod"] diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 00000000..9a8de32b --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,11 @@ +FROM node:21-alpine + +RUN apk add --no-cache libc6-compat chromium runuser make g++ py3-pip linux-headers +ENV PUPPETEER_SKIP_DOWNLOAD true +ENV CHROME_PATH "/usr/bin/chromium-browser" +ENV BROWSER_EXECUTABLE_PATH "/bin/start-chrome.sh" +ENV BROWSER_USER_DATA_DIR="/tmp/chrome" + +WORKDIR /app +ADD start-chrome.sh /bin +RUN chmod +x /bin/start-chrome.sh diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 00000000..a6510939 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,58 @@ +version: "3.8" +services: + web: + build: + dockerfile: Dockerfile.dev + volumes: + - data:/data + - ..:/app + ports: + - 3000:3000 + environment: + REDIS_HOST: redis + DATABASE_URL: "file:/data/db.db" + command: + - yarn + - web + working_dir: /app + depends_on: + prep: + condition: service_completed_successfully + redis: + image: redis:7.2-alpine + volumes: + - redis:/data + workers: + build: + dockerfile: Dockerfile.dev + volumes: + - data:/data + - ..:/app + working_dir: /app + environment: + REDIS_HOST: redis + DATABASE_URL: "file:/data/db.db" + # OPENAI_API_KEY: ... + command: + - yarn + - workers + depends_on: + prep: + condition: service_completed_successfully + prep: + build: + dockerfile: Dockerfile.dev + working_dir: /app + environment: + DATABASE_URL: "file:/data/db.db" + volumes: + - data:/data + - ..:/app + command: + - /bin/sh + - -c + - "corepack enable && yarn install --immutable && yarn prisma:generate && yarn prisma:migrate:dev" + +volumes: + redis: + data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..a0cc7366 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,43 @@ +version: "3.8" +services: + web: + image: ghcr.io/mohamedbassem/hoarder-web:latest + volumes: + - data:/data + ports: + - 3000:3000 + environment: + REDIS_HOST: redis + DATABASE_URL: "file:/data/db.db" + depends_on: + migration: + condition: service_completed_successfully + redis: + image: redis:7.2-alpine + volumes: + - redis:/data + workers: + image: ghcr.io/mohamedbassem/hoarder-workers:latest + volumes: + - data:/data + environment: + REDIS_HOST: redis + DATABASE_URL: "file:/data/db.db" + # OPENAI_API_KEY: ... + depends_on: + migration: + condition: service_completed_successfully + migration: + image: ghcr.io/mohamedbassem/hoarder-workers:latest + environment: + DATABASE_URL: "file:/data/db.db" + volumes: + - data:/data + command: + - /bin/sh + - -c + - "cd /app/packages/db && yarn dlx prisma migrate deploy" + +volumes: + redis: + data: diff --git a/docker/start-chrome.sh b/docker/start-chrome.sh new file mode 100644 index 00000000..9f715906 --- /dev/null +++ b/docker/start-chrome.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x; +id -u chrome &>/dev/null || adduser -S chrome; +mkdir -p $BROWSER_USER_DATA_DIR; +chown chrome $BROWSER_USER_DATA_DIR; +runuser -u chrome -- $CHROME_PATH --no-sandbox $@; diff --git a/docker/yarn.lock b/docker/yarn.lock new file mode 100644 index 00000000..fb57ccd1 --- /dev/null +++ b/docker/yarn.lock @@ -0,0 +1,4 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + -- cgit v1.2.3-70-g09d2