improve worker dockerfile

d.worker.dockerfile

@@ -1,42 +1,67 @@
+## d.worker.dockerfile
+##
+## 'temporal-worker' is already a pod name (temporal helm chart creates it for it's internal use)
+## so our docker image is called fp/worker, not fp/temporal-worker
+## not that we need to name the docker image differently, but
+## the hope with this name is to keep the mental concept of the two pods separate by having different names
+##
+## @todo future improvement might be merging the dockerfiles for the various monorepo packages.
+## this is not an easy task, so I'm not doing it right now.
+## "make it work, make it right, make it fast" (in that order)
+## Right now we are making things work with separate dockerfiles for each package.
+## One thing to determine is build speed. If we're developing in Tilt and have to wait 20 minutes for the build to complete
+## every time we change a file in any dependent package, then merging dockerfiles is not desirable.
+## One of the slow parts of the docker build is copying all package directories into the build context.
+## If we have a lot of packages, it takes a long time.
+## I have yet to determine performance benchmarks, so it's unclear if merging dockerfiles is desirable.
+##
+## @todo another performance improvement would almost certainly be to move strapi, next, and similar packages from `packages/*` into `services/*`
+## this way, when we're building the various @futureporn library-type packages, we don't have to filter and COPY the dependency packages one-by-one.
+## instead, we add the entire `packages/*` directory and then move on to the next step.
+
 FROM node:20 as base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 WORKDIR /app
-RUN corepack enable && corepack prepare pnpm@9.5.0 --activate
+RUN corepack enable && corepack prepare pnpm@9.2.0 --activate
 
-FROM base AS install
+FROM base as build
-COPY pnpm-lock.yaml .npmrc package.json .
+WORKDIR /app
-COPY ./packages/types/ ./packages/types/
+RUN mkdir -p /app/packages/temporal-worker && mkdir -p /prod/temporal-worker
-COPY ./packages/storage/ ./packages/storage/
+
-COPY ./packages/scout/ ./packages/scout/
+## Copy manfiests, lockfiles, and configs into docker context
-COPY ./packages/image/ ./packages/image/
+COPY package.json pnpm-lock.yaml .npmrc .
-COPY ./packages/utils/ ./packages/utils/
+COPY ./packages/image/pnpm-lock.yaml ./packages/image/package.json ./packages/image/
-COPY ./packages/temporal-worker/ ./packages/temporal-worker/
+COPY ./packages/scout/pnpm-lock.yaml ./packages/scout/package.json ./packages/scout/
-COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/
+COPY ./packages/storage/pnpm-lock.yaml ./packages/storage/package.json ./packages/storage/
+COPY ./packages/temporal-workflows/pnpm-lock.yaml ./packages/temporal-workflows/package.json ./packages/temporal-workflows/
+COPY ./packages/temporal-worker/pnpm-lock.yaml ./packages/temporal-worker/package.json ./packages/temporal-worker/
+COPY ./packages/types/pnpm-lock.yaml ./packages/types/package.json ./packages/types/
+COPY ./packages/utils/pnpm-lock.yaml ./packages/utils/package.json ./packages/utils/
+
+## Install npm packages
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm fetch
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline
 
+## Copy package code into docker context
+COPY ./packages/image/ ./packages/image/
+COPY ./packages/scout/ ./packages/scout/
+COPY ./packages/storage/ ./packages/storage/
+COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/
+COPY ./packages/temporal-worker/ ./packages/temporal-worker/
+COPY ./packages/types/ ./packages/types/
+COPY ./packages/utils/ ./packages/utils/
 
-FROM install AS build
+## Transpile TS into JS
-RUN pnpm -r build
+## we have to build temporal-workflows first because other packages depend on it's built js files
-RUN pnpm deploy --filter=temporal-worker --prod /prod/temporal-worker
+RUN pnpm --filter=@futureporn/temporal-workflows build
+RUN pnpm --filter=!@futureporn/temporal-workflows -r build
 
-# FROM base as build
+## Deploy (copy all production code into one place)
-# RUN mkdir -p /prod/worker
+RUN pnpm deploy --filter=@futureporn/temporal-worker --prod /prod/temporal-worker
-# COPY pnpm-workspace.yaml package.json pnpm-lock.yaml .npmrc .
-# COPY ./packages/temporal-workflows/pnpm-lock.yaml ./packages/temporal-workflows/
-# COPY ./packages/temporal-worker/pnpm-lock.yaml ./packages/temporal-worker/
-# # RUN pnpm fetch
-# RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline
-# COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/
-# COPY ./packages/temporal-worker/ ./packages/temporal-worker/
-# RUN ls -la .
-# RUN ls -la ./packages
-# RUN ls -la ./packages/temporal-workflows
-# RUN pnpm -r build
-# RUN pnpm deploy --filter=temporal-worker --prod /prod/worker
 
 FROM base as worker
 COPY --from=build /prod/temporal-worker .
-RUN ls -la ./
+RUN ls -la .
 ENTRYPOINT ["pnpm", "start"]
 

flux/apps/base/fp/release.yaml

@@ -30,12 +30,12 @@ spec:
       ingressClassName: traefik
     capture:
       imageName: gitea.futureporn.net/futureporn/capture:latest
+    worker:
+      imageName: gitea.futureporn.net/futureporn/worker:latest
+      replicas: 2
     scout:
       manager:
         imageName: gitea.futureporn.net/futureporn/scout-manager:latest
-      worker:
-        imageName: gitea.futureporn.net/futureporn/scout-worker:latest
-        replicas: 1
       pubsubServerUrl: https://realtime.futureporn.svc.cluster.local/faye
       hostname: next.futureporn.svc.cluster.local
       cdnBucketUrl: https://fp-dev.b-cdn.net
@@ -55,4 +55,5 @@ spec:
     certManager:
       issuer: letsencrypt-staging
     echo:
-      hostname: echo.fp.sbtp.xyz
+      hostname: echo.fp.sbtp.xyz
+    

packages/temporal-worker/package.json

@@ -11,7 +11,7 @@
     "isolate": "npx isolate-package isolate",
     "lint": "eslint .",
     "dev": "nodemon --ext js,ts,json,yaml --watch ./worker.ts --watch ../temporal-workflows --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./worker.ts\"",
-    "start": "node dist/temporal-worker/worker.js",
+    "start": "node dist/temporal-worker/src/worker.js",
     "clean": "rm -rf dist",
     "superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist"
   },