improve worker dockerfile
ci / build (push) Failing after 2m11s Details

This commit is contained in:
CJ_Clippy 2024-07-17 00:55:08 -08:00
parent d20635f2c4
commit 31f790f6d4
3 changed files with 59 additions and 33 deletions

View File

@ -1,42 +1,67 @@
## d.worker.dockerfile
##
## 'temporal-worker' is already a pod name (temporal helm chart creates it for it's internal use)
## so our docker image is called fp/worker, not fp/temporal-worker
## not that we need to name the docker image differently, but
## the hope with this name is to keep the mental concept of the two pods separate by having different names
##
## @todo future improvement might be merging the dockerfiles for the various monorepo packages.
## this is not an easy task, so I'm not doing it right now.
## "make it work, make it right, make it fast" (in that order)
## Right now we are making things work with separate dockerfiles for each package.
## One thing to determine is build speed. If we're developing in Tilt and have to wait 20 minutes for the build to complete
## every time we change a file in any dependent package, then merging dockerfiles is not desirable.
## One of the slow parts of the docker build is copying all package directories into the build context.
## If we have a lot of packages, it takes a long time.
## I have yet to determine performance benchmarks, so it's unclear if merging dockerfiles is desirable.
##
## @todo another performance improvement would almost certainly be to move strapi, next, and similar packages from `packages/*` into `services/*`
## this way, when we're building the various @futureporn library-type packages, we don't have to filter and COPY the dependency packages one-by-one.
## instead, we add the entire `packages/*` directory and then move on to the next step.
FROM node:20 as base FROM node:20 as base
ENV PNPM_HOME="/pnpm" ENV PNPM_HOME="/pnpm"
ENV PATH="$PNPM_HOME:$PATH" ENV PATH="$PNPM_HOME:$PATH"
WORKDIR /app WORKDIR /app
RUN corepack enable && corepack prepare pnpm@9.5.0 --activate RUN corepack enable && corepack prepare pnpm@9.2.0 --activate
FROM base AS install FROM base as build
COPY pnpm-lock.yaml .npmrc package.json . WORKDIR /app
COPY ./packages/types/ ./packages/types/ RUN mkdir -p /app/packages/temporal-worker && mkdir -p /prod/temporal-worker
COPY ./packages/storage/ ./packages/storage/
COPY ./packages/scout/ ./packages/scout/ ## Copy manfiests, lockfiles, and configs into docker context
COPY ./packages/image/ ./packages/image/ COPY package.json pnpm-lock.yaml .npmrc .
COPY ./packages/utils/ ./packages/utils/ COPY ./packages/image/pnpm-lock.yaml ./packages/image/package.json ./packages/image/
COPY ./packages/temporal-worker/ ./packages/temporal-worker/ COPY ./packages/scout/pnpm-lock.yaml ./packages/scout/package.json ./packages/scout/
COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/ COPY ./packages/storage/pnpm-lock.yaml ./packages/storage/package.json ./packages/storage/
COPY ./packages/temporal-workflows/pnpm-lock.yaml ./packages/temporal-workflows/package.json ./packages/temporal-workflows/
COPY ./packages/temporal-worker/pnpm-lock.yaml ./packages/temporal-worker/package.json ./packages/temporal-worker/
COPY ./packages/types/pnpm-lock.yaml ./packages/types/package.json ./packages/types/
COPY ./packages/utils/pnpm-lock.yaml ./packages/utils/package.json ./packages/utils/
## Install npm packages
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm fetch
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline
## Copy package code into docker context
COPY ./packages/image/ ./packages/image/
COPY ./packages/scout/ ./packages/scout/
COPY ./packages/storage/ ./packages/storage/
COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/
COPY ./packages/temporal-worker/ ./packages/temporal-worker/
COPY ./packages/types/ ./packages/types/
COPY ./packages/utils/ ./packages/utils/
FROM install AS build ## Transpile TS into JS
RUN pnpm -r build ## we have to build temporal-workflows first because other packages depend on it's built js files
RUN pnpm deploy --filter=temporal-worker --prod /prod/temporal-worker RUN pnpm --filter=@futureporn/temporal-workflows build
RUN pnpm --filter=!@futureporn/temporal-workflows -r build
# FROM base as build ## Deploy (copy all production code into one place)
# RUN mkdir -p /prod/worker RUN pnpm deploy --filter=@futureporn/temporal-worker --prod /prod/temporal-worker
# COPY pnpm-workspace.yaml package.json pnpm-lock.yaml .npmrc .
# COPY ./packages/temporal-workflows/pnpm-lock.yaml ./packages/temporal-workflows/
# COPY ./packages/temporal-worker/pnpm-lock.yaml ./packages/temporal-worker/
# # RUN pnpm fetch
# RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline
# COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/
# COPY ./packages/temporal-worker/ ./packages/temporal-worker/
# RUN ls -la .
# RUN ls -la ./packages
# RUN ls -la ./packages/temporal-workflows
# RUN pnpm -r build
# RUN pnpm deploy --filter=temporal-worker --prod /prod/worker
FROM base as worker FROM base as worker
COPY --from=build /prod/temporal-worker . COPY --from=build /prod/temporal-worker .
RUN ls -la ./ RUN ls -la .
ENTRYPOINT ["pnpm", "start"] ENTRYPOINT ["pnpm", "start"]

View File

@ -30,12 +30,12 @@ spec:
ingressClassName: traefik ingressClassName: traefik
capture: capture:
imageName: gitea.futureporn.net/futureporn/capture:latest imageName: gitea.futureporn.net/futureporn/capture:latest
worker:
imageName: gitea.futureporn.net/futureporn/worker:latest
replicas: 2
scout: scout:
manager: manager:
imageName: gitea.futureporn.net/futureporn/scout-manager:latest imageName: gitea.futureporn.net/futureporn/scout-manager:latest
worker:
imageName: gitea.futureporn.net/futureporn/scout-worker:latest
replicas: 1
pubsubServerUrl: https://realtime.futureporn.svc.cluster.local/faye pubsubServerUrl: https://realtime.futureporn.svc.cluster.local/faye
hostname: next.futureporn.svc.cluster.local hostname: next.futureporn.svc.cluster.local
cdnBucketUrl: https://fp-dev.b-cdn.net cdnBucketUrl: https://fp-dev.b-cdn.net
@ -56,3 +56,4 @@ spec:
issuer: letsencrypt-staging issuer: letsencrypt-staging
echo: echo:
hostname: echo.fp.sbtp.xyz hostname: echo.fp.sbtp.xyz

View File

@ -11,7 +11,7 @@
"isolate": "npx isolate-package isolate", "isolate": "npx isolate-package isolate",
"lint": "eslint .", "lint": "eslint .",
"dev": "nodemon --ext js,ts,json,yaml --watch ./worker.ts --watch ../temporal-workflows --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./worker.ts\"", "dev": "nodemon --ext js,ts,json,yaml --watch ./worker.ts --watch ../temporal-workflows --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./worker.ts\"",
"start": "node dist/temporal-worker/worker.js", "start": "node dist/temporal-worker/src/worker.js",
"clean": "rm -rf dist", "clean": "rm -rf dist",
"superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist" "superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist"
}, },