Temporal integration progress
ci / build (push) Waiting to run
Details
ci / build (push) Waiting to run
Details
This commit is contained in:
parent
194377dbd9
commit
7a7db2717e
8
.npmrc
8
.npmrc
|
|
@ -1,2 +1,8 @@
|
||||||
# we are giving every package their own lockfile to keep the docker build process fast
|
# we are giving every package their own lockfile to keep the docker build process fast
|
||||||
shared-workspace-lockfile=false
|
shared-workspace-lockfile=false
|
||||||
|
|
||||||
|
engine-strict=true
|
||||||
|
package-manager-strict=true
|
||||||
|
use-node-version=20.13.1
|
||||||
|
node-version=20.13.1
|
||||||
|
recursive-install=true
|
||||||
|
|
@ -14,6 +14,11 @@ Kubernetes for Production, deployed using FluxCD
|
||||||
|
|
||||||
direnv for loading .envrc
|
direnv for loading .envrc
|
||||||
|
|
||||||
|
Temporal for work queue
|
||||||
|
|
||||||
|
Postgres for data storage
|
||||||
|
|
||||||
|
S3 for media storage
|
||||||
|
|
||||||
Domain Driven Development
|
Domain Driven Development
|
||||||
Test Driven Development
|
Test Driven Development
|
||||||
|
|
|
||||||
11
Makefile
11
Makefile
|
|
@ -12,7 +12,13 @@ cert-manager:
|
||||||
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.yaml
|
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.yaml
|
||||||
|
|
||||||
flux:
|
flux:
|
||||||
flux bootstrap git --url="ssh://git@gitea.futureporn.net:2222/futureporn/fp" --branch=main --path="clusters/production" --private-key-file=/home/chris/.ssh/fp-flux
|
flux bootstrap git --url="ssh://git@gitea.futureporn.net:2222/futureporn/fp" --branch=main --path="clusters/production" --private-key-file=/home/cj/.ssh/fp-flux
|
||||||
|
|
||||||
|
cluster:
|
||||||
|
./scripts/kind-with-local-registry.sh
|
||||||
|
./scripts/k8s-namespaces.sh
|
||||||
|
./scripts/k8s-secrets.sh
|
||||||
|
./scripts/k8s-chisel-operator.sh
|
||||||
|
|
||||||
argo:
|
argo:
|
||||||
helmsman --apply -f ./helmsman.argocd.yaml
|
helmsman --apply -f ./helmsman.argocd.yaml
|
||||||
|
|
@ -49,6 +55,9 @@ minikube:
|
||||||
kind:
|
kind:
|
||||||
bash -x ./scripts/kind-with-local-registry.sh
|
bash -x ./scripts/kind-with-local-registry.sh
|
||||||
|
|
||||||
|
chisel:
|
||||||
|
./scripts/k8s-chisel-operator.sh
|
||||||
|
|
||||||
deps:
|
deps:
|
||||||
sudo pamac install make entr nvm minikube kubectl docker helm
|
sudo pamac install make entr nvm minikube kubectl docker helm
|
||||||
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
|
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ spec:
|
||||||
- name: web
|
- name: web
|
||||||
port: 3000
|
port: 3000
|
||||||
targetPort: 3000
|
targetPort: 3000
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
|
|
|
||||||
|
|
@ -1,53 +0,0 @@
|
||||||
apiVersion: v1
|
|
||||||
kind: Service
|
|
||||||
metadata:
|
|
||||||
name: pgadmin
|
|
||||||
namespace: futureporn
|
|
||||||
spec:
|
|
||||||
selector:
|
|
||||||
app.kubernetes.io/name: pgadmin
|
|
||||||
ports:
|
|
||||||
- name: web
|
|
||||||
protocol: TCP
|
|
||||||
port: 5050
|
|
||||||
targetPort: 5050
|
|
||||||
status:
|
|
||||||
loadBalancer: {}
|
|
||||||
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: pgadmin
|
|
||||||
namespace: futureporn
|
|
||||||
labels:
|
|
||||||
app.kubernetes.io/name: pgadmin
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: pgadmin
|
|
||||||
image: dpage/pgadmin4
|
|
||||||
ports:
|
|
||||||
- containerPort: 5050
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpu: 500m
|
|
||||||
memory: 1Gi
|
|
||||||
env:
|
|
||||||
- name: PGADMIN_LISTEN_PORT
|
|
||||||
value: '5050'
|
|
||||||
- name: POSTGRES_PASSWORD
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: postgres
|
|
||||||
key: password
|
|
||||||
- name: PGADMIN_DEFAULT_PASSWORD
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: pgadmin
|
|
||||||
key: defaultPassword
|
|
||||||
- name: PGADMIN_DEFAULT_EMAIL
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: pgadmin
|
|
||||||
key: defaultEmail
|
|
||||||
restartPolicy: OnFailure
|
|
||||||
|
|
@ -1,61 +1,117 @@
|
||||||
apiVersion: v1
|
apiVersion: apps/v1
|
||||||
kind: Service
|
kind: ReplicaSet
|
||||||
metadata:
|
metadata:
|
||||||
name: scout
|
name: scout-worker
|
||||||
namespace: futureporn
|
namespace: futureporn
|
||||||
|
labels:
|
||||||
|
app: scout-worker
|
||||||
spec:
|
spec:
|
||||||
|
replicas: {{ .Values.scout.worker.replicas }}
|
||||||
selector:
|
selector:
|
||||||
app.kubernetes.io/name: scout
|
matchLabels:
|
||||||
ports:
|
app: scout-worker
|
||||||
- name: http
|
template:
|
||||||
port: 3000
|
metadata:
|
||||||
targetPort: http
|
labels:
|
||||||
protocol: TCP
|
app: scout-worker
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: scout-worker
|
||||||
|
image: fp/scout-worker:latest
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- containerPort: 8080
|
||||||
|
env:
|
||||||
|
- name: TEMPORAL_SERVICE_ADDRESS
|
||||||
|
value: "temporal-frontend.futureporn.svc.cluster.local:7233"
|
||||||
|
- name: TEMPORAL_NAMESPACE
|
||||||
|
value: "futureporn"
|
||||||
|
- name: TEMPORAL_TASK_QUEUE
|
||||||
|
value: "scout"
|
||||||
|
- name: S3_BUCKET_NAME
|
||||||
|
value: "{{ .Values.scout.s3BucketName }}"
|
||||||
|
- name: CDN_BUCKET_URL
|
||||||
|
value: "{{ .Values.scout.cdnBucketUrl }}"
|
||||||
|
- name: STRAPI_URL
|
||||||
|
value: https://strapi.piko.sbtp.xyz
|
||||||
|
- name: SCOUT_NITTER_ACCESS_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: scout
|
||||||
|
key: nitterAccessKey
|
||||||
|
- name: SCOUT_NITTER_URL
|
||||||
|
value: https://nitter.sbtp.xyz
|
||||||
|
- name: SCOUT_RECENTS_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: scout
|
||||||
|
key: recentsToken
|
||||||
|
- name: SCOUT_STRAPI_API_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: scout
|
||||||
|
key: strapiApiKey
|
||||||
|
# - name: SCOUT_IMAP_SERVER
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapServer
|
||||||
|
# - name: SCOUT_IMAP_PORT
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapPort
|
||||||
|
# - name: SCOUT_IMAP_USERNAME
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapUsername
|
||||||
|
# - name: SCOUT_IMAP_PASSWORD
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapPassword
|
||||||
|
# - name: SCOUT_IMAP_ACCESS_TOKEN
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapAccessToken
|
||||||
|
# Add any other necessary environment variables
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "500m"
|
||||||
|
memory: "512Mi"
|
||||||
|
requests:
|
||||||
|
cpu: "250m"
|
||||||
|
memory: "256Mi"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
---
|
---
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: scout
|
name: scout-manager
|
||||||
namespace: futureporn
|
namespace: futureporn
|
||||||
labels:
|
labels:
|
||||||
app: scout
|
app: scout-manager
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: scout
|
app: scout-manager
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app: scout
|
app: scout-manager
|
||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: scout
|
- name: scout-manager
|
||||||
image: "{{ .Values.scout.containerName }}"
|
image: "{{ .Values.scout.manager.containerName }}"
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
containerPort: 3000
|
containerPort: 3000
|
||||||
env:
|
env:
|
||||||
- name: POSTGRES_REALTIME_CONNECTION_STRING
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: realtime
|
|
||||||
key: postgresRealtimeConnectionString
|
|
||||||
- name: STRAPI_URL
|
|
||||||
value: https://strapi.piko.sbtp.xyz
|
|
||||||
- name: SCOUT_NITTER_ACCESS_KEY
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: scout
|
|
||||||
key: nitterAccessKey
|
|
||||||
- NAME: SCOUT_NITTER_URL
|
|
||||||
value: https://nitter.sbtp.xyz
|
|
||||||
- name: SCOUT_RECENTS_TOKEN
|
|
||||||
valueFrom:
|
|
||||||
secretKeyRef:
|
|
||||||
name: scout
|
|
||||||
key: recentsToken
|
|
||||||
- name: SCOUT_IMAP_SERVER
|
- name: SCOUT_IMAP_SERVER
|
||||||
valueFrom:
|
valueFrom:
|
||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
|
|
@ -81,37 +137,56 @@ spec:
|
||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
name: scout
|
name: scout
|
||||||
key: imapAccessToken
|
key: imapAccessToken
|
||||||
- name: SCOUT_STRAPI_API_KEY
|
# env:
|
||||||
valueFrom:
|
# - name: POSTGRES_REALTIME_CONNECTION_STRING
|
||||||
secretKeyRef:
|
# valueFrom:
|
||||||
name: scout
|
# secretKeyRef:
|
||||||
key: strapiApiKey
|
# name: realtime
|
||||||
|
# key: postgresRealtimeConnectionString
|
||||||
|
# - name: CDN_BUCKET_URL
|
||||||
|
# value: "{{ .Values.scout.cdnBucketUrl }}"
|
||||||
|
# - name: STRAPI_URL
|
||||||
|
# value: https://strapi.piko.sbtp.xyz
|
||||||
|
# - name: SCOUT_NITTER_ACCESS_KEY
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: nitterAccessKey
|
||||||
|
# - name: SCOUT_NITTER_URL
|
||||||
|
# value: https://nitter.sbtp.xyz
|
||||||
|
# - name: SCOUT_RECENTS_TOKEN
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: recentsToken
|
||||||
|
# - name: SCOUT_IMAP_SERVER
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapServer
|
||||||
|
# - name: SCOUT_IMAP_PORT
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapPort
|
||||||
|
# - name: SCOUT_IMAP_USERNAME
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapUsername
|
||||||
|
# - name: SCOUT_IMAP_PASSWORD
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapPassword
|
||||||
|
# - name: SCOUT_IMAP_ACCESS_TOKEN
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: imapAccessToken
|
||||||
|
# - name: SCOUT_STRAPI_API_KEY
|
||||||
|
# valueFrom:
|
||||||
|
# secretKeyRef:
|
||||||
|
# name: scout
|
||||||
|
# key: strapiApiKey
|
||||||
|
|
||||||
|
|
||||||
{{ if eq .Values.managedBy "Helm" }}
|
|
||||||
---
|
|
||||||
apiVersion: networking.k8s.io/v1
|
|
||||||
kind: Ingress
|
|
||||||
metadata:
|
|
||||||
name: scout
|
|
||||||
namespace: futureporn
|
|
||||||
annotations:
|
|
||||||
kubernetes.io/ingress.class: nginx
|
|
||||||
cert-manager.io/cluster-issuer: letsencrypt-staging
|
|
||||||
spec:
|
|
||||||
tls:
|
|
||||||
- secretName: scout-tls
|
|
||||||
hosts:
|
|
||||||
- scout.sbtp.xyz
|
|
||||||
rules:
|
|
||||||
- host: scout.sbtp.xyz
|
|
||||||
http:
|
|
||||||
paths:
|
|
||||||
- path: /
|
|
||||||
pathType: Prefix
|
|
||||||
backend:
|
|
||||||
service:
|
|
||||||
name: scout
|
|
||||||
port:
|
|
||||||
number: 3000
|
|
||||||
{{ end }}
|
|
||||||
|
|
@ -10,10 +10,16 @@ next:
|
||||||
capture:
|
capture:
|
||||||
containerName: fp/capture
|
containerName: fp/capture
|
||||||
scout:
|
scout:
|
||||||
containerName: fp/scout
|
manager:
|
||||||
|
containerName: fp/scout-manager
|
||||||
|
worker:
|
||||||
|
containerName: fp/scout-worker
|
||||||
|
replicas: 1
|
||||||
pubsubServerUrl: https://realtime.futureporn.svc.cluster.local/faye
|
pubsubServerUrl: https://realtime.futureporn.svc.cluster.local/faye
|
||||||
certIssuer: letsencrypt-staging
|
certIssuer: letsencrypt-staging
|
||||||
hostname: next.futureporn.svc.cluster.local
|
hostname: next.futureporn.svc.cluster.local
|
||||||
|
cdnBucketUrl: https://fp-dev.b-cdn.net
|
||||||
|
s3BucketName: fp-dev
|
||||||
strapi:
|
strapi:
|
||||||
containerName: fp/strapi
|
containerName: fp/strapi
|
||||||
port: 1339
|
port: 1339
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,14 @@ storageClassName: vultr-block-storage-hdd
|
||||||
link2cid:
|
link2cid:
|
||||||
containerName: gitea.futureporn.net/futureporn/link2cid:latest
|
containerName: gitea.futureporn.net/futureporn/link2cid:latest
|
||||||
scout:
|
scout:
|
||||||
containerName: gitea.futureporn.net/futureporn/scout:latest
|
manager:
|
||||||
|
containerName: gitea.futureporn.net/futureporn/scout-manager:latest
|
||||||
|
worker:
|
||||||
|
containerName: gitea.futureporn.net/futureporn/scout-worker:latest
|
||||||
|
replicas: 2
|
||||||
pubsubServerUrl: https://realtime.futureporn.net/faye
|
pubsubServerUrl: https://realtime.futureporn.net/faye
|
||||||
|
cdnBucketUrl: https://futureporn-b2.b-cdn.net
|
||||||
|
s3BucketName: futureporn
|
||||||
next:
|
next:
|
||||||
containerName: gitea.futureporn.net/futureporn/next:latest
|
containerName: gitea.futureporn.net/futureporn/next:latest
|
||||||
certIssuer: letsencrypt-staging
|
certIssuer: letsencrypt-staging
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,7 @@
|
||||||
FROM node:20-alpine AS base
|
FROM node:20 AS base
|
||||||
ENV PNPM_HOME="/pnpm"
|
ENV PNPM_HOME="/pnpm"
|
||||||
ENV PATH="$PNPM_HOME:$PATH"
|
ENV PATH="$PNPM_HOME:$PATH"
|
||||||
RUN corepack enable
|
RUN corepack enable
|
||||||
RUN apk update
|
|
||||||
|
|
||||||
|
|
||||||
FROM base AS build
|
FROM base AS build
|
||||||
ENV NODE_ENV=production
|
ENV NODE_ENV=production
|
||||||
|
|
@ -12,13 +10,17 @@ WORKDIR /usr/src/app
|
||||||
RUN mkdir -p /prod/scout
|
RUN mkdir -p /prod/scout
|
||||||
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
|
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
|
||||||
RUN pnpm deploy --filter=scout --prod /prod/scout
|
RUN pnpm deploy --filter=scout --prod /prod/scout
|
||||||
# COPY pnpm-lock.yaml ./
|
|
||||||
# RUN pnpm fetch
|
|
||||||
# COPY ./packages/scout /app
|
|
||||||
|
|
||||||
|
|
||||||
FROM base AS scout
|
|
||||||
|
FROM base AS manager
|
||||||
COPY --from=build /prod/scout /app
|
COPY --from=build /prod/scout /app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
ENTRYPOINT ["pnpm"]
|
ENTRYPOINT ["pnpm"]
|
||||||
CMD ["run", "start"]
|
CMD ["run", "start:manager"]
|
||||||
|
|
||||||
|
FROM base AS worker
|
||||||
|
COPY --from=build /prod/scout /app
|
||||||
|
WORKDIR /app
|
||||||
|
ENTRYPOINT ["pnpm"]
|
||||||
|
CMD ["run", "start:worker"]
|
||||||
|
|
|
||||||
|
|
@ -1,24 +1,10 @@
|
||||||
FROM node:18-alpine AS base
|
FROM node:18 AS base
|
||||||
|
WORKDIR /usr/src/app/
|
||||||
|
RUN corepack enable
|
||||||
ENV PNPM_HOME="/pnpm"
|
ENV PNPM_HOME="/pnpm"
|
||||||
ENV PATH="$PNPM_HOME:$PATH"
|
ENV PATH="$PNPM_HOME:$PATH"
|
||||||
RUN corepack enable
|
|
||||||
# Installing libvips-dev for sharp Compatibility
|
|
||||||
RUN apk update && apk add --no-cache build-base gcc autoconf automake zlib-dev libpng-dev nasm bash vips-dev git
|
|
||||||
ARG NODE_ENV=development
|
|
||||||
ENV NODE_ENV=${NODE_ENV}
|
|
||||||
|
|
||||||
FROM base AS build
|
|
||||||
WORKDIR /usr/src/app/
|
|
||||||
COPY ./packages/strapi/package.json ./packages/strapi/pnpm-lock.yaml .
|
COPY ./packages/strapi/package.json ./packages/strapi/pnpm-lock.yaml .
|
||||||
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
|
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
|
||||||
COPY ./packages/strapi/ .
|
COPY ./packages/strapi/ .
|
||||||
|
|
||||||
FROM build AS dev
|
|
||||||
WORKDIR /app
|
|
||||||
ENV PATH /app/node_modules/.bin:$PATH
|
|
||||||
COPY --from=build /usr/src/app/ .
|
|
||||||
RUN chown -R node:node /app
|
|
||||||
USER node
|
|
||||||
RUN ["pnpm", "run", "build"]
|
RUN ["pnpm", "run", "build"]
|
||||||
EXPOSE 1339
|
CMD ["pnpm", "run", "dev"]
|
||||||
CMD ["pnpm", "run", "dev"]
|
|
||||||
|
|
|
||||||
|
|
@ -9,5 +9,6 @@
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
"author": "@CJ_Clippy",
|
"author": "@CJ_Clippy",
|
||||||
"license": "Unlicense"
|
"license": "Unlicense",
|
||||||
|
"packageManager": "pnpm@9.2.0+sha512.98a80fd11c2e7096747762304106432b3ddc67dcf54b5a8c01c93f68a2cd5e05e6821849522a06fb76284d41a2660d5e334f2ee3bbf29183bf2e739b1dafa771"
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,3 @@
|
||||||
engine-strict=true
|
engine-strict=true
|
||||||
use-node-version=20.13.1
|
use-node-version=20.13.1
|
||||||
|
node-version=20.13.1
|
||||||
|
|
@ -8,9 +8,6 @@
|
||||||
"dev": "pnpm nodemon ./index.js",
|
"dev": "pnpm nodemon ./index.js",
|
||||||
"start": "node index.js"
|
"start": "node index.js"
|
||||||
},
|
},
|
||||||
"engines": {
|
|
||||||
"node": ">=20.0.0"
|
|
||||||
},
|
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"IPFS",
|
"IPFS",
|
||||||
"CID",
|
"CID",
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"name": "@futureporn/next",
|
"name": "next",
|
||||||
"version": "2.0.0",
|
"version": "2.0.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|
@ -14,58 +14,58 @@
|
||||||
"@fortawesome/fontawesome-svg-core": "^6.5.2",
|
"@fortawesome/fontawesome-svg-core": "^6.5.2",
|
||||||
"@fortawesome/free-brands-svg-icons": "^6.5.2",
|
"@fortawesome/free-brands-svg-icons": "^6.5.2",
|
||||||
"@fortawesome/free-solid-svg-icons": "^6.5.2",
|
"@fortawesome/free-solid-svg-icons": "^6.5.2",
|
||||||
"@fortawesome/react-fontawesome": "^0.2.0",
|
"@fortawesome/react-fontawesome": "^0.2.2",
|
||||||
"@hookform/error-message": "^2.0.1",
|
"@hookform/error-message": "^2.0.1",
|
||||||
"@hookform/resolvers": "^3.3.4",
|
"@hookform/resolvers": "^3.6.0",
|
||||||
"@mux/blurhash": "^0.1.2",
|
"@mux/blurhash": "^0.1.2",
|
||||||
"@mux/mux-player": "^2.4.1",
|
"@mux/mux-player": "^2.7.0",
|
||||||
"@mux/mux-player-react": "^2.4.1",
|
"@mux/mux-player-react": "^2.7.0",
|
||||||
"@paralleldrive/cuid2": "^2.2.2",
|
"@paralleldrive/cuid2": "^2.2.2",
|
||||||
"@react-hookz/web": "^24.0.4",
|
"@react-hookz/web": "^24.0.4",
|
||||||
"@tanstack/react-query": "^5.32.1",
|
"@tanstack/react-query": "^5.40.1",
|
||||||
"@tanstack/react-table": "^8.15.3",
|
"@tanstack/react-table": "^8.17.3",
|
||||||
"@types/lodash": "^4.17.0",
|
"@types/lodash": "^4.17.4",
|
||||||
"@types/qs": "^6.9.14",
|
"@types/qs": "^6.9.15",
|
||||||
"@types/react": "^18.2.75",
|
"@types/react": "^18.3.3",
|
||||||
"@types/react-dom": "^18.2.24",
|
"@types/react-dom": "^18.3.0",
|
||||||
"@uppy/aws-s3": "^3.6.2",
|
"@uppy/aws-s3": "^3.6.2",
|
||||||
"@uppy/aws-s3-multipart": "^3.11.0",
|
"@uppy/aws-s3-multipart": "^3.12.0",
|
||||||
"@uppy/core": "^3.10.0",
|
"@uppy/core": "^3.12.0",
|
||||||
"@uppy/dashboard": "^3.8.0",
|
"@uppy/dashboard": "^3.8.3",
|
||||||
"@uppy/drag-drop": "^3.1.0",
|
"@uppy/drag-drop": "^3.1.0",
|
||||||
"@uppy/file-input": "^3.1.0",
|
"@uppy/file-input": "^3.1.2",
|
||||||
"@uppy/progress-bar": "^3.1.1",
|
"@uppy/progress-bar": "^3.1.1",
|
||||||
"@uppy/react": "^3.3.0",
|
"@uppy/react": "^3.3.1",
|
||||||
"@uppy/remote-sources": "^1.2.0",
|
"@uppy/remote-sources": "^1.2.0",
|
||||||
"bulma": "^1.0.0",
|
"bulma": "^1.0.1",
|
||||||
"date-fns": "^2.30.0",
|
"date-fns": "^2.30.0",
|
||||||
"date-fns-tz": "^2.0.1",
|
"date-fns-tz": "^2.0.1",
|
||||||
"dayjs": "^1.11.10",
|
"dayjs": "^1.11.11",
|
||||||
"feed": "^4.2.2",
|
"feed": "^4.2.2",
|
||||||
"gray-matter": "^4.0.3",
|
"gray-matter": "^4.0.3",
|
||||||
"hls.js": "^1.5.7",
|
"hls.js": "^1.5.11",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
"lunarphase-js": "^2.0.3",
|
"lunarphase-js": "^2.0.3",
|
||||||
"multiformats": "^13.1.0",
|
"multiformats": "^13.1.1",
|
||||||
"next": "14.0.4",
|
"next": "14.0.4",
|
||||||
"next-goatcounter": "^1.0.5",
|
"next-goatcounter": "^1.0.5",
|
||||||
"nextjs-toploader": "^1.6.11",
|
"nextjs-toploader": "^1.6.12",
|
||||||
"plyr": "^3.7.8",
|
"plyr": "^3.7.8",
|
||||||
"prism-react-renderer": "^2.3.1",
|
"prism-react-renderer": "^2.3.1",
|
||||||
"qs": "^6.12.0",
|
"qs": "^6.12.1",
|
||||||
"react": "^18.2.0",
|
"react": "^18.3.1",
|
||||||
"react-data-table-component": "^7.5.4",
|
"react-data-table-component": "^7.6.2",
|
||||||
"react-dom": "^18.2.0",
|
"react-dom": "^18.3.1",
|
||||||
"react-hook-form": "^7.51.2",
|
"react-hook-form": "^7.51.5",
|
||||||
"react-loading-skeleton": "^3.4.0",
|
"react-loading-skeleton": "^3.4.0",
|
||||||
"react-toastify": "^9.1.3",
|
"react-toastify": "^9.1.3",
|
||||||
"sharp": "^0.33.3",
|
"sharp": "^0.33.4",
|
||||||
"slugify": "^1.6.6",
|
"slugify": "^1.6.6",
|
||||||
"styled-components": "5.3.3",
|
"styled-components": "5.3.3",
|
||||||
"yup": "^1.4.0"
|
"yup": "^1.4.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/node": "^20.12.6",
|
"@types/node": "^20.14.2",
|
||||||
"eslint": "^8.57.0",
|
"eslint": "^8.57.0",
|
||||||
"eslint-config-next": "14.0.4",
|
"eslint-config-next": "14.0.4",
|
||||||
"tsc": "^2.0.4",
|
"tsc": "^2.0.4",
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1 +0,0 @@
|
||||||
Subproject commit 02e159182d462d17866f5dee720c315781c2bdec
|
|
||||||
|
|
@ -0,0 +1,146 @@
|
||||||
|
lib/
|
||||||
|
|
||||||
|
# Created by https://www.toptal.com/developers/gitignore/api/node
|
||||||
|
# Edit at https://www.toptal.com/developers/gitignore?templates=node
|
||||||
|
|
||||||
|
### Node ###
|
||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
.pnpm-debug.log*
|
||||||
|
|
||||||
|
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||||
|
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||||
|
|
||||||
|
# Runtime data
|
||||||
|
pids
|
||||||
|
*.pid
|
||||||
|
*.seed
|
||||||
|
*.pid.lock
|
||||||
|
|
||||||
|
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||||
|
lib-cov
|
||||||
|
|
||||||
|
# Coverage directory used by tools like istanbul
|
||||||
|
coverage
|
||||||
|
*.lcov
|
||||||
|
|
||||||
|
# nyc test coverage
|
||||||
|
.nyc_output
|
||||||
|
|
||||||
|
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||||
|
.grunt
|
||||||
|
|
||||||
|
# Bower dependency directory (https://bower.io/)
|
||||||
|
bower_components
|
||||||
|
|
||||||
|
# node-waf configuration
|
||||||
|
.lock-wscript
|
||||||
|
|
||||||
|
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||||
|
build/Release
|
||||||
|
|
||||||
|
# Dependency directories
|
||||||
|
node_modules/
|
||||||
|
jspm_packages/
|
||||||
|
|
||||||
|
# Snowpack dependency directory (https://snowpack.dev/)
|
||||||
|
web_modules/
|
||||||
|
|
||||||
|
# TypeScript cache
|
||||||
|
*.tsbuildinfo
|
||||||
|
|
||||||
|
# Optional npm cache directory
|
||||||
|
.npm
|
||||||
|
|
||||||
|
# Optional eslint cache
|
||||||
|
.eslintcache
|
||||||
|
|
||||||
|
# Optional stylelint cache
|
||||||
|
.stylelintcache
|
||||||
|
|
||||||
|
# Microbundle cache
|
||||||
|
.rpt2_cache/
|
||||||
|
.rts2_cache_cjs/
|
||||||
|
.rts2_cache_es/
|
||||||
|
.rts2_cache_umd/
|
||||||
|
|
||||||
|
# Optional REPL history
|
||||||
|
.node_repl_history
|
||||||
|
|
||||||
|
# Output of 'npm pack'
|
||||||
|
*.tgz
|
||||||
|
|
||||||
|
# Yarn Integrity file
|
||||||
|
.yarn-integrity
|
||||||
|
|
||||||
|
# dotenv environment variable files
|
||||||
|
.env
|
||||||
|
.env.development.local
|
||||||
|
.env.test.local
|
||||||
|
.env.production.local
|
||||||
|
.env.local
|
||||||
|
|
||||||
|
# parcel-bundler cache (https://parceljs.org/)
|
||||||
|
.cache
|
||||||
|
.parcel-cache
|
||||||
|
|
||||||
|
# Next.js build output
|
||||||
|
.next
|
||||||
|
out
|
||||||
|
|
||||||
|
# Nuxt.js build / generate output
|
||||||
|
.nuxt
|
||||||
|
dist
|
||||||
|
|
||||||
|
# Gatsby files
|
||||||
|
.cache/
|
||||||
|
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||||
|
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||||
|
# public
|
||||||
|
|
||||||
|
# vuepress build output
|
||||||
|
.vuepress/dist
|
||||||
|
|
||||||
|
# vuepress v2.x temp and cache directory
|
||||||
|
.temp
|
||||||
|
|
||||||
|
# Docusaurus cache and generated files
|
||||||
|
.docusaurus
|
||||||
|
|
||||||
|
# Serverless directories
|
||||||
|
.serverless/
|
||||||
|
|
||||||
|
# FuseBox cache
|
||||||
|
.fusebox/
|
||||||
|
|
||||||
|
# DynamoDB Local files
|
||||||
|
.dynamodb/
|
||||||
|
|
||||||
|
# TernJS port file
|
||||||
|
.tern-port
|
||||||
|
|
||||||
|
# Stores VSCode versions used for testing VSCode extensions
|
||||||
|
.vscode-test
|
||||||
|
|
||||||
|
# yarn v2
|
||||||
|
.yarn/cache
|
||||||
|
.yarn/unplugged
|
||||||
|
.yarn/build-state.yml
|
||||||
|
.yarn/install-state.gz
|
||||||
|
.pnp.*
|
||||||
|
|
||||||
|
### Node Patch ###
|
||||||
|
# Serverless Webpack directories
|
||||||
|
.webpack/
|
||||||
|
|
||||||
|
# Optional stylelint cache
|
||||||
|
|
||||||
|
# SvelteKit build / generate output
|
||||||
|
.svelte-kit
|
||||||
|
|
||||||
|
# End of https://www.toptal.com/developers/gitignore/api/node
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
engine-strict=true
|
||||||
|
package-manager-strict=true
|
||||||
|
use-node-version=20.13.1
|
||||||
|
node-version=20.13.1
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
lts/iron
|
|
||||||
|
|
@ -1,18 +1,18 @@
|
||||||
# scout
|
# scout
|
||||||
|
|
||||||
Vtuber data acquisition.
|
Vtuber data acquisition. Anything having to do with external WWW data acquisition goes in this module.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
* [x] Ingests going live notification e-mails
|
* [x] Ingests going live notification e-mails
|
||||||
* [ ] Sends `startRecording` signals to @futureporn/capture
|
* [ ] Sends `startRecording` signals to @futureporn/capture
|
||||||
* [ ] Fetches vtuber data from platform
|
* [x] Fetches vtuber data from platform
|
||||||
* [ ] image
|
* [x] image
|
||||||
* [ ] themeColor
|
* [x] themeColor
|
||||||
* [x] displayName
|
* [x] displayName
|
||||||
* [ ] Platform Support
|
* [x] Platform Support
|
||||||
* [ ] fansly
|
* [x] fansly
|
||||||
* [ ] chaturbate
|
* [x] chaturbate
|
||||||
|
|
||||||
|
|
||||||
## Design requirements
|
## Design requirements
|
||||||
|
|
@ -31,7 +31,6 @@ Vtuber data acquisition.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## Puppeteer
|
## Puppeteer
|
||||||
|
|
||||||
For when we get to the point where we need it, here are the packages we used with success during past testing.
|
For when we get to the point where we need it, here are the packages we used with success during past testing.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
interface ImportMeta {
|
||||||
|
dirname: string;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
@ -3,11 +3,13 @@
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"version": "3.3.0",
|
"version": "3.3.0",
|
||||||
"description": "vtuber data acquisition",
|
"description": "vtuber data acquisition",
|
||||||
"main": "src/index.email.js",
|
"main": "src/index.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "mocha",
|
"test": "mocha",
|
||||||
"start": "node ./src/index.email.js",
|
"build:worker": "tsc --build ./tsconfig.json",
|
||||||
"start:browser": "node ./src/index.browser.js"
|
"start": "echo please use either start:manager or start:worker",
|
||||||
|
"start:manager": "node --loader ts-node/esm ./src/index.ts",
|
||||||
|
"start:worker": "node --loader ts-node/esm ./src/temporal/worker.ts"
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
"author": "@CJ_Clippy",
|
"author": "@CJ_Clippy",
|
||||||
|
|
@ -16,8 +18,12 @@
|
||||||
"@aws-sdk/client-s3": "^3.583.0",
|
"@aws-sdk/client-s3": "^3.583.0",
|
||||||
"@aws-sdk/lib-storage": "^3.588.0",
|
"@aws-sdk/lib-storage": "^3.588.0",
|
||||||
"@aws-sdk/s3-request-presigner": "^3.588.0",
|
"@aws-sdk/s3-request-presigner": "^3.588.0",
|
||||||
"@book000/twitterts": "^0.62.50",
|
|
||||||
"@paralleldrive/cuid2": "^2.2.2",
|
"@paralleldrive/cuid2": "^2.2.2",
|
||||||
|
"@temporalio/client": "^1.9.0",
|
||||||
|
"@temporalio/worker": "^1.9.0",
|
||||||
|
"@temporalio/workflow": "^1.9.0",
|
||||||
|
"@tsconfig/node20": "^20.1.4",
|
||||||
|
"@types/imapflow": "^1.0.18",
|
||||||
"cheerio": "1.0.0-rc.12",
|
"cheerio": "1.0.0-rc.12",
|
||||||
"concurrently": "^8.2.2",
|
"concurrently": "^8.2.2",
|
||||||
"date-fns": "^3.6.0",
|
"date-fns": "^3.6.0",
|
||||||
|
|
@ -28,20 +34,21 @@
|
||||||
"imapflow": "^1.0.160",
|
"imapflow": "^1.0.160",
|
||||||
"limiter": "2.0.1",
|
"limiter": "2.0.1",
|
||||||
"mailparser": "^3.7.1",
|
"mailparser": "^3.7.1",
|
||||||
"node-vibrant": "3.2.1-alpha.1",
|
"node-fetch": "^3.3.0",
|
||||||
|
"nodemon": "^3.1.3",
|
||||||
"p-retry": "^5.1.2",
|
"p-retry": "^5.1.2",
|
||||||
"pg-pubsub": "workspace:*",
|
"next": "workspace:*",
|
||||||
"qs": "^6.12.1",
|
"qs": "^6.12.1",
|
||||||
"sharp": "^0.33.4",
|
"sharp": "^0.33.4",
|
||||||
"slugify": "^1.6.6",
|
"slugify": "^1.6.6",
|
||||||
|
"ts-node": "^10.9.2",
|
||||||
|
"tsx": "^4.7.2",
|
||||||
|
"typescript": "^5.4.5",
|
||||||
"xpath": "^0.0.34"
|
"xpath": "^0.0.34"
|
||||||
},
|
},
|
||||||
"packageManager": "pnpm@9.1.3",
|
"packageManager": "pnpm@9.2.0",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"chai": "^5.1.0",
|
"chai": "^5.1.0",
|
||||||
"mocha": "^10.4.0"
|
"mocha": "^10.4.0"
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": "^20"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,45 +0,0 @@
|
||||||
'use strict'
|
|
||||||
|
|
||||||
/**
|
|
||||||
* watches an e-mail inbox for going live notifications
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { checkEmail } from './parsers.js'
|
|
||||||
import { signalRealtime, createStreamInDb } from './signals.js'
|
|
||||||
import { Email } from './imap.js'
|
|
||||||
import fastq from "fastq";
|
|
||||||
|
|
||||||
|
|
||||||
const q = fastq.promise(handleMessage, 1)
|
|
||||||
|
|
||||||
async function handleMessage({email, msg}) {
|
|
||||||
try {
|
|
||||||
console.log(' ✏️ loading message')
|
|
||||||
const body = await email.loadMessage(msg.uid)
|
|
||||||
|
|
||||||
console.log(' ✏️ checking e-mail')
|
|
||||||
const { isMatch, url, platform, channel, displayName, date, userId, avatar } = (await checkEmail(body))
|
|
||||||
|
|
||||||
if (isMatch) {
|
|
||||||
console.log(' ✏️✏️ signalling realtime')
|
|
||||||
await signalRealtime({ url, platform, channel, displayName, date, userId, avatar })
|
|
||||||
|
|
||||||
console.log(' ✏️✏️ creating stream entry in db')
|
|
||||||
await createStreamInDb({ source: 'email', platform, channel, date, url, userId, avatar })
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(' ✏️ archiving e-mail')
|
|
||||||
await email.archiveMessage(msg.uid)
|
|
||||||
} catch (e) {
|
|
||||||
// console.error('error encoutered')
|
|
||||||
console.error(` An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`)
|
|
||||||
console.error(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
const email = new Email()
|
|
||||||
email.on('message', (msg) => q.push({email, msg}))
|
|
||||||
await email.connect()
|
|
||||||
})()
|
|
||||||
|
|
@ -0,0 +1,63 @@
|
||||||
|
'use strict'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* watches an e-mail inbox for going live notifications
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { checkEmail } from './parsers.js'
|
||||||
|
// import { createStreamInDb } from './signals.js'
|
||||||
|
import { Email } from './imap.js'
|
||||||
|
import { Client, Connection } from '@temporalio/client'
|
||||||
|
import { NotificationData, processEmail } from './temporal/workflows.js'
|
||||||
|
import { FetchMessageObject } from 'imapflow'
|
||||||
|
import { createId } from '@paralleldrive/cuid2'
|
||||||
|
|
||||||
|
const connection = await Connection.connect({ address: 'temporal-frontend.futureporn.svc.cluster.local:7233' });
|
||||||
|
const client = new Client({ connection, namespace: 'futureporn' });
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageObject }) {
|
||||||
|
try {
|
||||||
|
console.log(' ✏️ loading message')
|
||||||
|
const body = await email.loadMessage(msg.uid)
|
||||||
|
|
||||||
|
// console.log(' ✏️ checking e-mail')
|
||||||
|
const { isMatch, url, platform, channel, displayName, date, userId, avatar }: NotificationData = (await checkEmail(body) )
|
||||||
|
|
||||||
|
|
||||||
|
if (isMatch) {
|
||||||
|
const wfId = `process-email-${createId()}`
|
||||||
|
// console.log(` ✏️ [DRY] starting Temporal workflow ${wfId} @todo actually start temporal workflow`)
|
||||||
|
// await signalRealtime({ url, platform, channel, displayName, date, userId, avatar })
|
||||||
|
// @todo invoke a Temporal workflow here
|
||||||
|
const handle = await client.workflow.start(processEmail, {
|
||||||
|
workflowId: wfId,
|
||||||
|
taskQueue: 'scout',
|
||||||
|
args: [{ url, platform, channel, displayName, date, userId, avatar }]
|
||||||
|
});
|
||||||
|
// const handle = client.getHandle(workflowId);
|
||||||
|
const result = await handle.result();
|
||||||
|
console.log(`result of the workflow is as follows`)
|
||||||
|
console.log(result)
|
||||||
|
throw new Error('!todo we are stopping after just one (for now) @todo')
|
||||||
|
|
||||||
|
// console.log(' ✏️✏️ creating stream entry in db')
|
||||||
|
// await createStreamInDb({ source: 'email', platform, channel, date, url, userId, avatar })
|
||||||
|
}
|
||||||
|
|
||||||
|
// console.log(' ✏️ archiving e-mail')
|
||||||
|
// await email.archiveMessage(msg.uid)
|
||||||
|
} catch (e) {
|
||||||
|
// console.error('error encoutered')
|
||||||
|
console.error(` An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`)
|
||||||
|
console.error(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const email = new Email()
|
||||||
|
email.once('message', (msg: FetchMessageObject) => handleMessage({ email, msg }))
|
||||||
|
await email.connect()
|
||||||
|
})()
|
||||||
|
|
@ -9,7 +9,7 @@ import { createId } from '@paralleldrive/cuid2'
|
||||||
import { basename } from 'node:path'
|
import { basename } from 'node:path'
|
||||||
import fs from 'node:fs'
|
import fs from 'node:fs'
|
||||||
|
|
||||||
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET was undefined in env');
|
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
|
||||||
if (!process.env.SCOUT_NITTER_URL) throw new Error('SCOUT_NITTER_URL was undefined in env');
|
if (!process.env.SCOUT_NITTER_URL) throw new Error('SCOUT_NITTER_URL was undefined in env');
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,12 @@
|
||||||
|
|
||||||
import 'dotenv/config'
|
import 'dotenv/config'
|
||||||
// import { PgPubSub } from '@imqueue/pg-pubsub'; // @see https://github.com/imqueue/pg-pubsub/issues/20
|
|
||||||
import { PgPubSub } from 'pg-pubsub'
|
|
||||||
import qs from 'qs'
|
import qs from 'qs'
|
||||||
import { subMinutes, addMinutes } from 'date-fns'
|
import { subMinutes, addMinutes } from 'date-fns'
|
||||||
import { fpSlugify, download } from './utils.js'
|
import { fpSlugify, download } from './utils.js'
|
||||||
import { getProminentColor } from './image.js'
|
import { getProminentColor } from './image.js'
|
||||||
import { RateLimiter } from "limiter"
|
import { RateLimiter } from "limiter"
|
||||||
import { getImage } from './vtuber.js'
|
import { getImage } from './vtuber.js'
|
||||||
|
import fansly from './fansly.js'
|
||||||
|
|
||||||
// alternative js libraries for postgres notify/wait
|
// alternative js libraries for postgres notify/wait
|
||||||
// * https://github.com/imqueue/pg-pubsub
|
// * https://github.com/imqueue/pg-pubsub
|
||||||
|
|
@ -17,28 +16,9 @@ import { getImage } from './vtuber.js'
|
||||||
if (!process.env.SCOUT_STRAPI_API_KEY) throw new Error('SCOUT_STRAPI_API_KEY is missing from env');
|
if (!process.env.SCOUT_STRAPI_API_KEY) throw new Error('SCOUT_STRAPI_API_KEY is missing from env');
|
||||||
if (!process.env.STRAPI_URL) throw new Error('STRAPI_URL is missing from env');
|
if (!process.env.STRAPI_URL) throw new Error('STRAPI_URL is missing from env');
|
||||||
if (!process.env.SCOUT_RECENTS_TOKEN) throw new Error('SCOUT_RECENTS_TOKEN is undefined in env');
|
if (!process.env.SCOUT_RECENTS_TOKEN) throw new Error('SCOUT_RECENTS_TOKEN is undefined in env');
|
||||||
if (!process.env.POSTGRES_REALTIME_CONNECTION_STRING) throw new Error('POSTGRES_REALTIME_CONNECTION_STRING is undefined in env');
|
|
||||||
if (!process.env.CDN_BUCKET_URL) throw new Error('CDN_BUCKET_URL is undefined in env');
|
if (!process.env.CDN_BUCKET_URL) throw new Error('CDN_BUCKET_URL is undefined in env');
|
||||||
|
|
||||||
console.log(`process.env.POSTGRES_REALTIME_CONNECTION_STRING=${process.env.POSTGRES_REALTIME_CONNECTION_STRING}`)
|
|
||||||
|
|
||||||
const pubSub = new PgPubSub({
|
|
||||||
connectionString: process.env.POSTGRES_REALTIME_CONNECTION_STRING,
|
|
||||||
// @see https://github.com/imqueue/pg-pubsub?tab=readme-ov-file#single-listener-inter-process-locking
|
|
||||||
singleListener: true
|
|
||||||
});
|
|
||||||
(async () => {
|
|
||||||
await pubSub.connect();
|
|
||||||
})();
|
|
||||||
|
|
||||||
export async function signalRealtime ({ url, platform, channel, displayName, date}) {
|
|
||||||
// faye.publish('/signals', {
|
|
||||||
// signal: 'startV2',
|
|
||||||
// url: url
|
|
||||||
// })
|
|
||||||
console.log(`📰📰📰 publishing streamStarted event. url=${url}, platform=${platform}, channel=${channel}, displayName=${displayName}, date=${date}`)
|
|
||||||
pubSub.notify('streamStarted', { url, platform, channel, displayName, date });
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -56,7 +36,7 @@ export async function signalRealtime ({ url, platform, channel, displayName, dat
|
||||||
*/
|
*/
|
||||||
export async function createStreamInDb ({ source, platform, channel, date, url, userId }) {
|
export async function createStreamInDb ({ source, platform, channel, date, url, userId }) {
|
||||||
|
|
||||||
const limiter = new RateLimiter({ tokensPerInterval: 0.3, interval: "second" });
|
// const limiter = new RateLimiter({ tokensPerInterval: 0.3, interval: "second" });
|
||||||
let vtuberId, streamId
|
let vtuberId, streamId
|
||||||
|
|
||||||
console.log('>> # Step 1')
|
console.log('>> # Step 1')
|
||||||
|
|
@ -127,8 +107,8 @@ export async function createStreamInDb ({ source, platform, channel, date, url,
|
||||||
fansly: fansly.url.fromUsername(channel)
|
fansly: fansly.url.fromUsername(channel)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const platformImageUrl = await getImage(limiter, dummyVtuber)
|
const platformImageUrl = await getImage(dummyVtuber)
|
||||||
const imageFile = await download({ limiter, url: platformImageUrl })
|
const imageFile = await download({ url: platformImageUrl })
|
||||||
|
|
||||||
// get themeColor from image
|
// get themeColor from image
|
||||||
const themeColor = await getProminentColor(imageFile)
|
const themeColor = await getProminentColor(imageFile)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
export async function greet(name) {
|
||||||
|
return `Hello, ${name}!`;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
import { Client, Connection } from '@temporalio/client';
|
||||||
|
|
||||||
|
import { example } from './temporal.workflow.js';
|
||||||
|
import { createId } from '@paralleldrive/cuid2';
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
// const cert = await fs.readFile('./path-to/your.pem');
|
||||||
|
// const key = await fs.readFile('./path-to/your.key');
|
||||||
|
|
||||||
|
let connectionOptions = {
|
||||||
|
address: 'temporal-frontend.futureporn.svc.cluster.local',
|
||||||
|
};
|
||||||
|
|
||||||
|
const connection = await Connection.connect(connectionOptions);
|
||||||
|
|
||||||
|
const client = new Client({
|
||||||
|
connection,
|
||||||
|
namespace: 'futureporn',
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('>>> WE ARE RUNNING THE WORKFLOW!!!!')
|
||||||
|
|
||||||
|
const handle = await client.workflow.start(example, {
|
||||||
|
taskQueue: 'hello-world',
|
||||||
|
// type inference works! args: [name: string]
|
||||||
|
args: ['Temporal'],
|
||||||
|
// in practice, use a meaningful business ID, like customerId or transactionId
|
||||||
|
workflowId: 'workflow-' + createId(),
|
||||||
|
});
|
||||||
|
console.log(`Started workflow ${handle.workflowId}`);
|
||||||
|
|
||||||
|
// optional: wait for client result
|
||||||
|
console.log(await handle.result()); // Hello, Temporal!
|
||||||
|
|
||||||
|
await client.connection.close();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
import { NativeConnection, Worker } from '@temporalio/worker'
|
||||||
|
import * as activities from './temporal.activities.js'
|
||||||
|
import path from 'node:path'
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
// Step 1: Establish a connection with Temporal server.
|
||||||
|
//
|
||||||
|
// Worker code uses `@temporalio/worker.NativeConnection`.
|
||||||
|
// (But in your application code it's `@temporalio/client.Connection`.)
|
||||||
|
const connection = await NativeConnection.connect({
|
||||||
|
address: 'temporal-frontend.futureporn.svc.cluster.local',
|
||||||
|
// TLS and gRPC metadata configuration goes here.
|
||||||
|
});
|
||||||
|
// Step 2: Register Workflows and Activities with the Worker.
|
||||||
|
const worker = await Worker.create({
|
||||||
|
connection,
|
||||||
|
namespace: 'futureporn',
|
||||||
|
taskQueue: 'hello-world',
|
||||||
|
// Workflows are registered using a path as they run in a separate JS context.
|
||||||
|
workflowsPath: path.join(import.meta.dirname, './temporal.workflow.js'),
|
||||||
|
activities,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Step 3: Start accepting tasks on the `hello-world` queue
|
||||||
|
//
|
||||||
|
// The worker runs until it encounters an unexpected error or the process receives a shutdown signal registered on
|
||||||
|
// the SDK Runtime object.
|
||||||
|
//
|
||||||
|
// By default, worker logs are written via the Runtime logger to STDERR at INFO level.
|
||||||
|
//
|
||||||
|
// See https://typescript.temporal.io/api/classes/worker.Runtime#install to customize these defaults.
|
||||||
|
await worker.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
})
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
import { proxyActivities } from '@temporalio/workflow';
|
||||||
|
|
||||||
|
const { greet } = proxyActivities({
|
||||||
|
startToCloseTimeout: '1 minute',
|
||||||
|
});
|
||||||
|
|
||||||
|
/** A workflow that simply calls an activity */
|
||||||
|
export async function example(name) {
|
||||||
|
return await greet(name);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,202 @@
|
||||||
|
// export function blah(name) {
|
||||||
|
// return `Today is a blah kind of a day. amirite, ${name}?`;
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
import fetch from "node-fetch"
|
||||||
|
import { NotificationData, processEmail } from "./workflows.js"
|
||||||
|
import qs from 'qs'
|
||||||
|
import { IVtuberResponse } from 'next'
|
||||||
|
import { getImage } from '../vtuber.js'
|
||||||
|
import { fpSlugify, download } from '../utils.js'
|
||||||
|
import fansly from '../fansly.js'
|
||||||
|
import { getProminentColor } from '../image.js'
|
||||||
|
import { uploadFile } from '../s3.js'
|
||||||
|
|
||||||
|
export type ChargeResult = {
|
||||||
|
status: string;
|
||||||
|
errorMessage?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
if (!process.env.SCOUT_STRAPI_API_KEY) throw new Error('SCOUT_STRAPI_API_KEY is missing from env');
|
||||||
|
if (!process.env.STRAPI_URL) throw new Error('STRAPI_URL is missing from env');
|
||||||
|
if (!process.env.CDN_BUCKET_URL) throw new Error('CDN_BUCKET_URL is missing from env');
|
||||||
|
if (!process.env.SCOUT_NITTER_URL) throw new Error('SCOUT_NITTER_URL is missing from env');
|
||||||
|
if (!process.env.SCOUT_NITTER_ACCESS_KEY) throw new Error('SCOUT_NITTER_ACCESS_KEY is missing from env');
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* find or create vtuber in Strapi
|
||||||
|
*/
|
||||||
|
export async function upsertVtuber({ platform, userId, url, channel }: NotificationData): Promise<number> {
|
||||||
|
|
||||||
|
let vtuberId
|
||||||
|
console.log('>> # Step 1, upsertVtuber')
|
||||||
|
// # Step 1.
|
||||||
|
// First we find or create the vtuber
|
||||||
|
// The vtuber may already be in the db, so we look for that record. All we need is the Vtuber ID.
|
||||||
|
// If the vtuber is not in the db, we create the vtuber record.
|
||||||
|
|
||||||
|
// GET /api/:pluralApiId?filters[field][operator]=value
|
||||||
|
const findVtubersFilters = (() => {
|
||||||
|
if (platform === 'chaturbate') {
|
||||||
|
return { chaturbate: { $eq: url } }
|
||||||
|
} else if (platform === 'fansly') {
|
||||||
|
if (!userId) throw new Error('Fansly userId was undefined, but it is required.')
|
||||||
|
return { fanslyId: { $eq: userId } }
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
console.log('>>>>> the following is findVtubersFilters.')
|
||||||
|
console.log(findVtubersFilters)
|
||||||
|
|
||||||
|
const findVtubersQueryString = qs.stringify({
|
||||||
|
filters: findVtubersFilters
|
||||||
|
}, { encode: false })
|
||||||
|
console.log(`>>>>> platform=${platform}, url=${url}, userId=${userId}`)
|
||||||
|
|
||||||
|
console.log('>> findVtuber')
|
||||||
|
const findVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers?${findVtubersQueryString}`, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const findVtuberJson = await findVtuberRes.json() as IVtuberResponse
|
||||||
|
console.log('>> here is the vtuber json')
|
||||||
|
console.log(findVtuberJson)
|
||||||
|
if (findVtuberJson?.data && findVtuberJson.data.length > 0) {
|
||||||
|
console.log('>>a vtuber was FOUND')
|
||||||
|
if (findVtuberJson.data.length > 1) throw new Error('There was more than one vtuber match. There must only be one.')
|
||||||
|
vtuberId = findVtuberJson.data[0].id
|
||||||
|
console.log('here is the findVtuberJson (as follows)')
|
||||||
|
console.log(findVtuberJson)
|
||||||
|
console.log(`the matching vtuber has ID=${vtuberId} (${findVtuberJson.data[0].attributes.displayName})`)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!vtuberId) {
|
||||||
|
console.log('>> vtuberId was not found so we create')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* We are creating a vtuber record.
|
||||||
|
* We need a few things.
|
||||||
|
* * image URL
|
||||||
|
* * themeColor
|
||||||
|
*
|
||||||
|
* To get an image, we have to do a few things.
|
||||||
|
* * [x] download image from platform
|
||||||
|
* * [x] get themeColor from image
|
||||||
|
* * [x] upload image to b2
|
||||||
|
* * [x] get B2 cdn link to image
|
||||||
|
*
|
||||||
|
* To get themeColor, we need the image locally where we can then run
|
||||||
|
*/
|
||||||
|
|
||||||
|
// download image from platform
|
||||||
|
// vtuber.getImage expects a vtuber object, which we don't have yet, so we create a dummy one
|
||||||
|
const dummyVtuber = {
|
||||||
|
attributes: {
|
||||||
|
slug: fpSlugify(channel),
|
||||||
|
fansly: fansly.url.fromUsername(channel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const platformImageUrl = await getImage(dummyVtuber)
|
||||||
|
const imageFile = await download({ url: platformImageUrl })
|
||||||
|
|
||||||
|
// get themeColor from image
|
||||||
|
const themeColor = await getProminentColor(imageFile)
|
||||||
|
|
||||||
|
// upload image to b2
|
||||||
|
const b2FileData = await uploadFile(imageFile)
|
||||||
|
|
||||||
|
// get b2 cdn link to image
|
||||||
|
const imageCdnLink = `https://${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
|
||||||
|
|
||||||
|
|
||||||
|
const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
|
||||||
|
'content-type': 'application/json'
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
data: {
|
||||||
|
displayName: channel,
|
||||||
|
fansly: (platform === 'fansly') ? url : null,
|
||||||
|
fanslyId: (platform === 'fansly') ? userId : null,
|
||||||
|
chaturbate: (platform === 'chaturbate') ? url : null,
|
||||||
|
slug: fpSlugify(channel),
|
||||||
|
description1: ' ',
|
||||||
|
image: imageCdnLink,
|
||||||
|
themeColor: themeColor || '#dde1ec'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
const createVtuberJson = await createVtuberRes.json() as IVtuberResponse
|
||||||
|
console.log('>> createVtuberJson as follows')
|
||||||
|
console.log(JSON.stringify(createVtuberJson, null, 2))
|
||||||
|
if (createVtuberJson.data) {
|
||||||
|
vtuberId = createVtuberJson.data.id
|
||||||
|
console.log(`>>> vtuber created with id=${vtuberId}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 777
|
||||||
|
}
|
||||||
|
export async function upsertPlatformNotification(): Promise<number> {
|
||||||
|
return 777
|
||||||
|
}
|
||||||
|
export async function upsertStream(): Promise<number> {
|
||||||
|
return 777
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function chargeUser(
|
||||||
|
userId: string,
|
||||||
|
itemId: string,
|
||||||
|
quantity: number,
|
||||||
|
): Promise<ChargeResult> {
|
||||||
|
// TODO send request to the payments service that looks up the user's saved
|
||||||
|
// payment info and the cost of the item and attempts to charge their payment
|
||||||
|
// method.
|
||||||
|
console.log(`Charging user ${userId} for ${quantity} of item ${itemId}`);
|
||||||
|
try {
|
||||||
|
const response = await fetch("http://httpbin.org/get?status=success");
|
||||||
|
const body: any = await response.json();
|
||||||
|
return { status: body.args.status };
|
||||||
|
} catch (e: any) {
|
||||||
|
return { status: "failure", errorMessage: e.message };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function checkAndDecrementInventory(
|
||||||
|
itemId: string,
|
||||||
|
quantity: number,
|
||||||
|
): Promise<boolean> {
|
||||||
|
// TODO a database request that—in a single operation or transaction—checks
|
||||||
|
// whether there are `quantity` items remaining, and if so, decreases the
|
||||||
|
// total. Something like:
|
||||||
|
// const result = await db.collection('items').updateOne(
|
||||||
|
// { _id: itemId, numAvailable: { $gte: quantity } },
|
||||||
|
// { $inc: { numAvailable: -quantity } }
|
||||||
|
// )
|
||||||
|
// return result.modifiedCount === 1
|
||||||
|
console.log(`Reserving ${quantity} of item ${itemId}`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function incrementInventory(
|
||||||
|
itemId: string,
|
||||||
|
quantity: number,
|
||||||
|
): Promise<boolean> {
|
||||||
|
// TODO increment inventory:
|
||||||
|
// const result = await db.collection('items').updateOne(
|
||||||
|
// { _id: itemId },
|
||||||
|
// { $inc: { numAvailable: quantity } }
|
||||||
|
// )
|
||||||
|
// return result.modifiedCount === 1
|
||||||
|
console.log(`Incrementing ${itemId} inventory by ${quantity}`);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
import path from "path"
|
||||||
|
import { NativeConnection, Worker } from "@temporalio/worker"
|
||||||
|
import * as activities from "./activities.js"
|
||||||
|
|
||||||
|
|
||||||
|
if (!process.env.TEMPORAL_SERVICE_ADDRESS) throw new Error(`TEMPORAL_SERVICE_ADDRESS is missing in env`);
|
||||||
|
if (!process.env.TEMPORAL_NAMESPACE) throw new Error(`TEMPORAL_NAMESPACE is missing in env`);
|
||||||
|
if (!process.env.TEMPORAL_TASK_QUEUE) throw new Error(`TEMPORAL_TASK_QUEUE is missing in env`);
|
||||||
|
|
||||||
|
|
||||||
|
console.log(`
|
||||||
|
process.env.TEMPORAL_SERVICE_ADDRESS=${process.env.TEMPORAL_SERVICE_ADDRESS}
|
||||||
|
process.env.TEMPORAL_NAMESPACE=${process.env.TEMPORAL_NAMESPACE}
|
||||||
|
process.env.TEMPORAL_TASK_QUEUE=${process.env.TEMPORAL_TASK_QUEUE}
|
||||||
|
import.meta.dirname=${import.meta.dirname}
|
||||||
|
`)
|
||||||
|
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
console.log(' scout-worker startup!')
|
||||||
|
// Step 1: Establish a connection with Temporal server.
|
||||||
|
//
|
||||||
|
// Worker code uses `@temporalio/worker.NativeConnection`.
|
||||||
|
// (But in your application code it's `@temporalio/client.Connection`.)
|
||||||
|
const connection = await NativeConnection.connect({
|
||||||
|
address: process.env.TEMPORAL_SERVICE_ADDRESS,
|
||||||
|
// TLS and gRPC metadata configuration goes here.
|
||||||
|
});
|
||||||
|
// Step 2: Register Workflows and Activities with the Worker.
|
||||||
|
const worker = await Worker.create({
|
||||||
|
connection,
|
||||||
|
namespace: process.env.TEMPORAL_NAMESPACE,
|
||||||
|
taskQueue: ''+process.env.TEMPORAL_TASK_QUEUE,
|
||||||
|
// Workflows are registered using a path as they run in a separate JS context.
|
||||||
|
workflowsPath: path.join(import.meta.dirname, './workflows.ts'),
|
||||||
|
activities,
|
||||||
|
maxTaskQueueActivitiesPerSecond: 1 // since we are acessing 3rd party API, we use this as a courtesy throttle
|
||||||
|
});
|
||||||
|
console.log('scout-worker is running.')
|
||||||
|
|
||||||
|
// Step 3: Start accepting tasks on the `hello-world` queue
|
||||||
|
//
|
||||||
|
// The worker runs until it encounters an unexpected error or the process receives a shutdown signal registered on
|
||||||
|
// the SDK Runtime object.
|
||||||
|
//
|
||||||
|
// By default, worker logs are written via the Runtime logger to STDERR at INFO level.
|
||||||
|
//
|
||||||
|
// See https://typescript.temporal.io/api/classes/worker.Runtime#install to customize these defaults.
|
||||||
|
await worker.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
})
|
||||||
|
|
@ -0,0 +1,73 @@
|
||||||
|
import { proxyActivities, continueAsNew, log, sleep } from "@temporalio/workflow"
|
||||||
|
import type * as activities from "./activities.js"
|
||||||
|
import { FetchMessageObject } from 'imapflow'
|
||||||
|
|
||||||
|
// { isMatch, url, platform, channel, displayName, date, userId, avatar }
|
||||||
|
export type NotificationData = {
|
||||||
|
isMatch?: boolean;
|
||||||
|
url: string;
|
||||||
|
platform: string;
|
||||||
|
channel: string;
|
||||||
|
displayName: string;
|
||||||
|
date: string;
|
||||||
|
userId: string | null;
|
||||||
|
avatar: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const {
|
||||||
|
chargeUser,
|
||||||
|
checkAndDecrementInventory,
|
||||||
|
incrementInventory,
|
||||||
|
upsertPlatformNotification,
|
||||||
|
upsertStream,
|
||||||
|
upsertVtuber,
|
||||||
|
} = proxyActivities<typeof activities>({
|
||||||
|
startToCloseTimeout: "1 minute",
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function processEmail({
|
||||||
|
url, platform, channel, displayName, date, userId, avatar
|
||||||
|
}: NotificationData): Promise<{ vtuberId: number, pNotifId: number, streamId: number }> {
|
||||||
|
console.log(`processEmail begin. platform=${platform}, date=${date}, url=${url}`)
|
||||||
|
// * In Strapi, we are finding or updating or creating the following content-types.
|
||||||
|
// * * vtuber
|
||||||
|
// * * platform-notification
|
||||||
|
// * * stream
|
||||||
|
// Step 1
|
||||||
|
|
||||||
|
const vtuberId = await upsertVtuber({ url, platform, channel, displayName, date, userId, avatar })
|
||||||
|
console.log('we have finished upsertVtuber and the vtuberId is '+vtuberId)
|
||||||
|
throw new Error('Error: Error: error: erorreorororr; @todo');
|
||||||
|
const pNotifId = await upsertPlatformNotification()
|
||||||
|
const streamId = await upsertStream()
|
||||||
|
return { vtuberId, pNotifId, streamId }
|
||||||
|
}
|
||||||
|
|
||||||
|
// export async function order(
|
||||||
|
// userId: string,
|
||||||
|
// itemId: string,
|
||||||
|
// quantity: number,
|
||||||
|
// ): Promise<string> {
|
||||||
|
// const haveEnoughInventory: boolean = await checkAndDecrementInventory(
|
||||||
|
// itemId,
|
||||||
|
// quantity,
|
||||||
|
// );
|
||||||
|
// if (haveEnoughInventory) {
|
||||||
|
// const result: activities.ChargeResult = await chargeUser(
|
||||||
|
// userId,
|
||||||
|
// itemId,
|
||||||
|
// quantity,
|
||||||
|
// );
|
||||||
|
// if (result.status === "success") {
|
||||||
|
// return `Order successful!`;
|
||||||
|
// } else {
|
||||||
|
// await incrementInventory(itemId, quantity);
|
||||||
|
// return `Unable to complete payment. Error: ${result.errorMessage}`;
|
||||||
|
// }
|
||||||
|
// } else {
|
||||||
|
// return `Sorry, we don't have enough items in stock to fulfill your order.`;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
// {
|
||||||
|
// "compilerOptions": {
|
||||||
|
// "outDir": "./built",
|
||||||
|
// "allowJs": true,
|
||||||
|
// "target": "ES2017",
|
||||||
|
// "module": "NodeNext"
|
||||||
|
// },
|
||||||
|
// "include": ["./src/**/*"]
|
||||||
|
// }
|
||||||
|
|
||||||
|
{
|
||||||
|
"extends": "@tsconfig/node20/tsconfig.json",
|
||||||
|
"version": "4.4.2",
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "es2017",
|
||||||
|
"module": "esnext",
|
||||||
|
"moduleResolution": "node",
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true,
|
||||||
|
"rootDir": "./src",
|
||||||
|
"outDir": "./lib"
|
||||||
|
},
|
||||||
|
"include": ["**/*.ts"],
|
||||||
|
"ts-node": {
|
||||||
|
"experimentalSpecifierResolution": "node",
|
||||||
|
"transpileOnly": true,
|
||||||
|
"esm": true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
@ -1,2 +1,5 @@
|
||||||
shamefully-hoist=true
|
shamefully-hoist=true
|
||||||
use-node-version=18.20.3
|
engine-strict=true
|
||||||
|
package-manager-strict=true
|
||||||
|
use-node-version=18.20.3
|
||||||
|
node-version=18.20.3
|
||||||
|
|
|
||||||
|
|
@ -20,12 +20,12 @@
|
||||||
"@mux/mux-node": "^7.3.5",
|
"@mux/mux-node": "^7.3.5",
|
||||||
"@paralleldrive/cuid2": "^2.2.2",
|
"@paralleldrive/cuid2": "^2.2.2",
|
||||||
"@radix-ui/react-use-callback-ref": "^1.0.1",
|
"@radix-ui/react-use-callback-ref": "^1.0.1",
|
||||||
"@strapi/plugin-i18n": "4.24.3",
|
"@strapi/plugin-i18n": "4.24.5",
|
||||||
"@strapi/plugin-users-permissions": "4.24.3",
|
"@strapi/plugin-users-permissions": "4.24.5",
|
||||||
"@strapi/provider-email-sendgrid": "4.24.3",
|
"@strapi/provider-email-sendgrid": "4.24.5",
|
||||||
"@strapi/provider-upload-cloudinary": "4.24.3",
|
"@strapi/provider-upload-cloudinary": "4.24.5",
|
||||||
"@strapi/strapi": "4.24.3",
|
"@strapi/strapi": "4.24.5",
|
||||||
"@strapi/utils": "4.24.3",
|
"@strapi/utils": "4.24.5",
|
||||||
"@testing-library/dom": "8.19.0",
|
"@testing-library/dom": "8.19.0",
|
||||||
"@testing-library/react": "12.1.4",
|
"@testing-library/react": "12.1.4",
|
||||||
"@testing-library/react-hooks": "8.0.1",
|
"@testing-library/react-hooks": "8.0.1",
|
||||||
|
|
@ -55,14 +55,14 @@
|
||||||
"prop-types": "^15.8.1",
|
"prop-types": "^15.8.1",
|
||||||
"purest": "4.0.2",
|
"purest": "4.0.2",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
"react-dom": "^18.3.1",
|
"react-dom": "^18.0.0",
|
||||||
"react-intl": "6.3.2",
|
"react-intl": "6.3.2",
|
||||||
"react-query": "3.24.3",
|
"react-query": "3.24.3",
|
||||||
"react-redux": "8.0.5",
|
"react-redux": "8.0.5",
|
||||||
"react-router-dom": "5.3.4",
|
"react-router-dom": "5.3.4",
|
||||||
"react-test-renderer": "^18.3.1",
|
"react-test-renderer": "^18.3.1",
|
||||||
"semver": "^7.6.2",
|
"semver": "^7.6.2",
|
||||||
"sharp": "0.33.3",
|
"sharp": "0.32.6",
|
||||||
"strapi-plugin-fuzzy-search": "^2.2.1",
|
"strapi-plugin-fuzzy-search": "^2.2.1",
|
||||||
"styled-components": "5.3.3",
|
"styled-components": "5.3.3",
|
||||||
"typescript": "^4.7",
|
"typescript": "^4.7",
|
||||||
|
|
@ -74,19 +74,11 @@
|
||||||
"concurrently": "^8.2.2"
|
"concurrently": "^8.2.2"
|
||||||
},
|
},
|
||||||
"author": {
|
"author": {
|
||||||
"name": "CJ_Clippy"
|
"name": "@CJ_Clippy"
|
||||||
},
|
},
|
||||||
"packageManager": "pnpm@9.1.3",
|
"packageManager": "pnpm@9.2.0",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"strapi": {
|
"strapi": {
|
||||||
"uuid": false,
|
"uuid": false
|
||||||
"supportedArchitectures": {
|
|
||||||
"libc": [
|
|
||||||
"musl"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"overrides": {
|
|
||||||
"sharp": "0.33.3"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,50 @@
|
||||||
|
{
|
||||||
|
"kind": "collectionType",
|
||||||
|
"collectionName": "platform_notifications",
|
||||||
|
"info": {
|
||||||
|
"singularName": "platform-notification",
|
||||||
|
"pluralName": "platform-notifications",
|
||||||
|
"displayName": "Platform Notification",
|
||||||
|
"description": ""
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"draftAndPublish": false
|
||||||
|
},
|
||||||
|
"pluginOptions": {},
|
||||||
|
"attributes": {
|
||||||
|
"source": {
|
||||||
|
"type": "enumeration",
|
||||||
|
"enum": [
|
||||||
|
"email",
|
||||||
|
"manual"
|
||||||
|
],
|
||||||
|
"default": "manual",
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"platform": {
|
||||||
|
"type": "enumeration",
|
||||||
|
"enum": [
|
||||||
|
"fansly",
|
||||||
|
"chaturbate"
|
||||||
|
],
|
||||||
|
"default": "chaturbate",
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"date": {
|
||||||
|
"type": "datetime",
|
||||||
|
"required": true,
|
||||||
|
"unique": false
|
||||||
|
},
|
||||||
|
"date2": {
|
||||||
|
"type": "string",
|
||||||
|
"required": true,
|
||||||
|
"unique": false,
|
||||||
|
"regex": "\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d\\.\\d+([+-][0-2]\\d:[0-5]\\d|Z)"
|
||||||
|
},
|
||||||
|
"vtuber": {
|
||||||
|
"type": "relation",
|
||||||
|
"relation": "oneToOne",
|
||||||
|
"target": "api::vtuber.vtuber"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* platform-notification controller
|
||||||
|
*/
|
||||||
|
|
||||||
|
const { createCoreController } = require('@strapi/strapi').factories;
|
||||||
|
|
||||||
|
module.exports = createCoreController('api::platform-notification.platform-notification');
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* platform-notification router
|
||||||
|
*/
|
||||||
|
|
||||||
|
const { createCoreRouter } = require('@strapi/strapi').factories;
|
||||||
|
|
||||||
|
module.exports = createCoreRouter('api::platform-notification.platform-notification');
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* platform-notification service
|
||||||
|
*/
|
||||||
|
|
||||||
|
const { createCoreService } = require('@strapi/strapi').factories;
|
||||||
|
|
||||||
|
module.exports = createCoreService('api::platform-notification.platform-notification');
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
node_modules
|
||||||
|
lib
|
||||||
|
.eslintrc.js
|
||||||
|
|
@ -0,0 +1,48 @@
|
||||||
|
const { builtinModules } = require('module');
|
||||||
|
|
||||||
|
const ALLOWED_NODE_BUILTINS = new Set(['assert']);
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
root: true,
|
||||||
|
parser: '@typescript-eslint/parser',
|
||||||
|
parserOptions: {
|
||||||
|
project: './tsconfig.json',
|
||||||
|
tsconfigRootDir: __dirname,
|
||||||
|
},
|
||||||
|
plugins: ['@typescript-eslint', 'deprecation'],
|
||||||
|
extends: [
|
||||||
|
'eslint:recommended',
|
||||||
|
'plugin:@typescript-eslint/eslint-recommended',
|
||||||
|
'plugin:@typescript-eslint/recommended',
|
||||||
|
'prettier',
|
||||||
|
],
|
||||||
|
rules: {
|
||||||
|
// recommended for safety
|
||||||
|
'@typescript-eslint/no-floating-promises': 'error', // forgetting to await Activities and Workflow APIs is bad
|
||||||
|
'deprecation/deprecation': 'warn',
|
||||||
|
|
||||||
|
// code style preference
|
||||||
|
'object-shorthand': ['error', 'always'],
|
||||||
|
|
||||||
|
// relaxed rules, for convenience
|
||||||
|
'@typescript-eslint/no-unused-vars': [
|
||||||
|
'warn',
|
||||||
|
{
|
||||||
|
argsIgnorePattern: '^_',
|
||||||
|
varsIgnorePattern: '^_',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'@typescript-eslint/no-explicit-any': 'off',
|
||||||
|
},
|
||||||
|
overrides: [
|
||||||
|
{
|
||||||
|
files: ['src/workflows.ts', 'src/workflows-*.ts', 'src/workflows/*.ts'],
|
||||||
|
rules: {
|
||||||
|
'no-restricted-imports': [
|
||||||
|
'error',
|
||||||
|
...builtinModules.filter((m) => !ALLOWED_NODE_BUILTINS.has(m)).flatMap((m) => [m, `node:${m}`]),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
lib
|
||||||
|
node_modules
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
20
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
lib
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
printWidth: 120
|
||||||
|
singleQuote: true
|
||||||
|
|
@ -0,0 +1,46 @@
|
||||||
|
# Cron Workflows
|
||||||
|
|
||||||
|
_DEPRECATED: use [Schedules](https://github.com/temporalio/samples-typescript/tree/main/schedules) instead._
|
||||||
|
|
||||||
|
This example demonstrates a working Cron workflow. Note the limitations and caveats listed in the [docs](https://docs.temporal.io/content/what-is-a-temporal-cron-job/).
|
||||||
|
|
||||||
|
Differences from the hello world demo:
|
||||||
|
|
||||||
|
- The Workflow is started with the `cronSchedule: '* * * * *',` option: [`src/client.ts`](./src/client.ts).
|
||||||
|
- The Activity actually prints a log, instead of returning a string.
|
||||||
|
- The Workflow runs forever, so if we want it to stop, we have to cancel it. In our `client.ts` script, we cancel it using the handle (when `Ctrl/Cmd-C` is hit). Usually, we'd use the Workflow ID to cancel—for example:
|
||||||
|
|
||||||
|
```js
|
||||||
|
const handle = client.getHandle('1e793a6c-31e2-41c9-8139-53d114293a9e');
|
||||||
|
await handle.cancel();
|
||||||
|
```
|
||||||
|
|
||||||
|
Note that when we're changing code and restarting Workers, unless we cancel all previous Workflows, they may get picked up by our Worker (since we likely didn't change our Workflow name or task queue), and their output may conflict/mix with new Workflows we're starting. We can check what is still running in Temporal Web ([localhost:8088](http://localhost:8088) in case we need to kill all previous Workflows.
|
||||||
|
|
||||||
|
### Running this sample
|
||||||
|
|
||||||
|
1. `temporal server start-dev` to start [Temporal Server](https://github.com/temporalio/cli/#installation).
|
||||||
|
2. `npm install` to install dependencies.
|
||||||
|
3. `npm run start.watch` to start the Worker.
|
||||||
|
4. In another shell, `npm run workflow` to run the Workflow.
|
||||||
|
|
||||||
|
Example Worker output:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
Hello from my-schedule, Temporal!
|
||||||
|
Workflow time: 1636333860201
|
||||||
|
Activity time: 1636333860241
|
||||||
|
Hello from my-schedule, Temporal!
|
||||||
|
Workflow time: 1636333920319
|
||||||
|
Activity time: 1636333920340
|
||||||
|
```
|
||||||
|
|
||||||
|
The difference between "Workflow time" and "Activity time" reflects the latency between scheduling an Activity and actually starting it.
|
||||||
|
|
||||||
|
Each new Workflow is `continuedAsNew` under the hood:
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
And you can see the details in the event history:
|
||||||
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,41 @@
|
||||||
|
{
|
||||||
|
"name": "cron-workflows",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc --build",
|
||||||
|
"build.watch": "tsc --build --watch",
|
||||||
|
"lint": "eslint .",
|
||||||
|
"start": "ts-node src/worker.ts",
|
||||||
|
"start.watch": "nodemon src/worker.ts",
|
||||||
|
"workflow": "ts-node src/client.ts"
|
||||||
|
},
|
||||||
|
"nodemonConfig": {
|
||||||
|
"execMap": {
|
||||||
|
"ts": "ts-node"
|
||||||
|
},
|
||||||
|
"ext": "ts",
|
||||||
|
"watch": [
|
||||||
|
"src"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@temporalio/activity": "^1.9.0",
|
||||||
|
"@temporalio/client": "^1.9.0",
|
||||||
|
"@temporalio/worker": "^1.9.0",
|
||||||
|
"@temporalio/workflow": "^1.9.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@tsconfig/node18": "^1.0.0",
|
||||||
|
"@types/node": "^16.11.43",
|
||||||
|
"@typescript-eslint/eslint-plugin": "^5.0.0",
|
||||||
|
"@typescript-eslint/parser": "^5.0.0",
|
||||||
|
"eslint": "^7.32.0",
|
||||||
|
"eslint-config-prettier": "^8.3.0",
|
||||||
|
"eslint-plugin-deprecation": "^1.2.1",
|
||||||
|
"nodemon": "^2.0.12",
|
||||||
|
"prettier": "^2.8.8",
|
||||||
|
"ts-node": "^10.2.1",
|
||||||
|
"typescript": "^4.4.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,6 @@
|
||||||
|
import { log, activityInfo } from '@temporalio/activity';
|
||||||
|
|
||||||
|
export async function logTime(name: string, wfTime: string): Promise<void> {
|
||||||
|
const { workflowExecution } = activityInfo();
|
||||||
|
log.info(`Hello from ${workflowExecution.workflowId}, ${name}!`, { workflowTime: wfTime, activityTime: Date.now() });
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,39 @@
|
||||||
|
import { Client } from '@temporalio/client';
|
||||||
|
import { scheduledWorkflow } from './workflows';
|
||||||
|
|
||||||
|
// Save this to later terminate or cancel this schedule
|
||||||
|
const workflowId = 'my-schedule';
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
const client = new Client();
|
||||||
|
|
||||||
|
const handle = await client.workflow.start(scheduledWorkflow, {
|
||||||
|
taskQueue: 'cron-workflows',
|
||||||
|
workflowId: 'my-schedule', // Save this to later terminate or cancel this schedule
|
||||||
|
cronSchedule: '* * * * *', // start every minute
|
||||||
|
args: ['Temporal'],
|
||||||
|
});
|
||||||
|
console.log('Cron started');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await handle.result(); // await completion of Workflow, which doesn't happen since it's a cron Workflow
|
||||||
|
} catch (err: any) {
|
||||||
|
console.error(err.message + ':' + handle.workflowId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// just for this demo - cancel the workflow on Ctrl+C
|
||||||
|
process.on('SIGINT', async () => {
|
||||||
|
const client = new Client();
|
||||||
|
|
||||||
|
const handle = client.workflow.getHandle(workflowId);
|
||||||
|
await handle.cancel();
|
||||||
|
console.log(`\nCanceled Workflow ${handle.workflowId}`);
|
||||||
|
process.exit(0);
|
||||||
|
});
|
||||||
|
// you cannot catch SIGKILL
|
||||||
|
|
||||||
|
run().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
import { Worker } from '@temporalio/worker';
|
||||||
|
import * as activities from './activities';
|
||||||
|
|
||||||
|
async function run() {
|
||||||
|
// nothing different here compared to hello world
|
||||||
|
const worker = await Worker.create({
|
||||||
|
workflowsPath: require.resolve('./workflows'),
|
||||||
|
activities,
|
||||||
|
taskQueue: 'cron-workflows',
|
||||||
|
});
|
||||||
|
await worker.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
import { proxyActivities } from '@temporalio/workflow';
|
||||||
|
import type * as activities from './activities';
|
||||||
|
|
||||||
|
const { logTime } = proxyActivities<typeof activities>({
|
||||||
|
startToCloseTimeout: '1 minute',
|
||||||
|
});
|
||||||
|
|
||||||
|
/** A workflow that simply calls an activity */
|
||||||
|
export async function scheduledWorkflow(name: string): Promise<void> {
|
||||||
|
await logTime(name, '' + Date.now());
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"extends": "@tsconfig/node18/tsconfig.json",
|
||||||
|
"version": "4.4.2",
|
||||||
|
"compilerOptions": {
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true,
|
||||||
|
"rootDir": "./src",
|
||||||
|
"outDir": "./lib"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts"]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,90 @@
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
bindir=$(dirname "$(readlink -fm "$0")")
|
||||||
|
source "${bindir}/../.env"
|
||||||
|
|
||||||
|
if [ -z $POSTGRES_REALTIME_PASSWORD ]; then
|
||||||
|
echo "POSTGRES_REALTIME_PASSWORD was missing in env"
|
||||||
|
exit 5
|
||||||
|
fi
|
||||||
|
|
||||||
|
## Create the temporal databases
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
||||||
|
CREATE DATABASE temporal_visibility \
|
||||||
|
WITH \
|
||||||
|
OWNER = postgres \
|
||||||
|
ENCODING = 'UTF8' \
|
||||||
|
LOCALE_PROVIDER = 'libc' \
|
||||||
|
CONNECTION LIMIT = -1 \
|
||||||
|
IS_TEMPLATE = False;"
|
||||||
|
|
||||||
|
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
||||||
|
CREATE DATABASE temporal \
|
||||||
|
WITH \
|
||||||
|
OWNER = postgres \
|
||||||
|
ENCODING = 'UTF8' \
|
||||||
|
LOCALE_PROVIDER = 'libc' \
|
||||||
|
CONNECTION LIMIT = -1 \
|
||||||
|
IS_TEMPLATE = False;"
|
||||||
|
|
||||||
|
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(TEMPORAL_DB) drop -f
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(TEMPORAL_DB) create
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(TEMPORAL_DB) setup -v 0.0
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(TEMPORAL_DB) update-schema -d ./schema/postgresql/v12/temporal/versioned
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(VISIBILITY_DB) drop -f
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(VISIBILITY_DB) create
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(VISIBILITY_DB) setup-schema -v 0.0
|
||||||
|
# ./temporal-sql-tool -u $(SQL_USER) --pw $(SQL_PASSWORD) -p 5432 --pl postgres12 --db $(VISIBILITY_DB) update-schema -d ./schema/postgresql/v12/visibility/versioned
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Create the futureporn Strapi database
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
||||||
|
CREATE DATABASE futureporn_db \
|
||||||
|
WITH \
|
||||||
|
OWNER = postgres \
|
||||||
|
ENCODING = 'UTF8' \
|
||||||
|
LOCALE_PROVIDER = 'libc' \
|
||||||
|
CONNECTION LIMIT = -1 \
|
||||||
|
IS_TEMPLATE = False;"
|
||||||
|
|
||||||
|
|
||||||
|
## Create the futureporn realtime database (for NOTIFY/AWAIT pubsub)
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
||||||
|
CREATE DATABASE futureporn_realtime \
|
||||||
|
WITH \
|
||||||
|
OWNER = postgres \
|
||||||
|
ENCODING = 'UTF8' \
|
||||||
|
LOCALE_PROVIDER = 'libc' \
|
||||||
|
CONNECTION LIMIT = -1 \
|
||||||
|
IS_TEMPLATE = False;"
|
||||||
|
|
||||||
|
|
||||||
|
## create futureporn user
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
||||||
|
CREATE ROLE futureporn \
|
||||||
|
WITH \
|
||||||
|
LOGIN \
|
||||||
|
NOSUPERUSER \
|
||||||
|
NOCREATEDB \
|
||||||
|
NOCREATEROLE \
|
||||||
|
INHERIT \
|
||||||
|
NOREPLICATION \
|
||||||
|
NOBYPASSRLS \
|
||||||
|
CONNECTION LIMIT -1 \
|
||||||
|
PASSWORD '$POSTGRES_REALTIME_PASSWORD';"
|
||||||
|
|
||||||
|
|
||||||
|
## grant futureporn user all privs
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
||||||
|
GRANT ALL PRIVILEGES ON DATABASE futureporn_realtime TO futureporn;"
|
||||||
|
|
||||||
|
|
||||||
|
## import schema
|
||||||
|
## I have a file, schema.psql that I want to import. How do I do that?
|
||||||
|
# kubectl -n futureporn exec postgres -- psql -U postgres --command "\ ;"
|
||||||
|
# kubectl -n futureporn exec postgres -- psql -U postgres -f - < "${bindir}/postgres-2024-05-09-futureporn_db-schema-only.psql"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
kubectl -n futureporn exec postgres -- psql -U postgres --command "DROP DATABASE temporal WITH (FORCE);"
|
||||||
|
|
@ -1,58 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
bindir=$(dirname "$(readlink -fm "$0")")
|
|
||||||
source "${bindir}/../.env"
|
|
||||||
|
|
||||||
if [ -z $POSTGRES_REALTIME_PASSWORD ]; then
|
|
||||||
echo "POSTGRES_REALTIME_PASSWORD was missing in env"
|
|
||||||
exit 5
|
|
||||||
fi
|
|
||||||
|
|
||||||
## Create the futureporn Strapi database
|
|
||||||
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
|
||||||
CREATE DATABASE futureporn_db \
|
|
||||||
WITH \
|
|
||||||
OWNER = postgres \
|
|
||||||
ENCODING = 'UTF8' \
|
|
||||||
LOCALE_PROVIDER = 'libc' \
|
|
||||||
CONNECTION LIMIT = -1 \
|
|
||||||
IS_TEMPLATE = False;"
|
|
||||||
|
|
||||||
|
|
||||||
## Create the futureporn realtime database (for NOTIFY/AWAIT pubsub)
|
|
||||||
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
|
||||||
CREATE DATABASE futureporn_realtime \
|
|
||||||
WITH \
|
|
||||||
OWNER = postgres \
|
|
||||||
ENCODING = 'UTF8' \
|
|
||||||
LOCALE_PROVIDER = 'libc' \
|
|
||||||
CONNECTION LIMIT = -1 \
|
|
||||||
IS_TEMPLATE = False;"
|
|
||||||
|
|
||||||
|
|
||||||
## create futureporn user
|
|
||||||
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
|
||||||
CREATE ROLE futureporn \
|
|
||||||
WITH \
|
|
||||||
LOGIN \
|
|
||||||
NOSUPERUSER \
|
|
||||||
NOCREATEDB \
|
|
||||||
NOCREATEROLE \
|
|
||||||
INHERIT \
|
|
||||||
NOREPLICATION \
|
|
||||||
NOBYPASSRLS \
|
|
||||||
CONNECTION LIMIT -1 \
|
|
||||||
PASSWORD '$POSTGRES_REALTIME_PASSWORD';"
|
|
||||||
|
|
||||||
|
|
||||||
## grant futureporn user all privs
|
|
||||||
kubectl -n futureporn exec postgres -- psql -U postgres --command "\
|
|
||||||
GRANT ALL PRIVILEGES ON DATABASE futureporn_realtime TO futureporn;"
|
|
||||||
|
|
||||||
|
|
||||||
## import schema
|
|
||||||
## I have a file, schema.psql that I want to import. How do I do that?
|
|
||||||
# kubectl -n futureporn exec postgres -- psql -U postgres --command "\ ;"
|
|
||||||
kubectl -n futureporn exec postgres -- psql -U postgres -f - < "${bindir}/postgres-2024-05-09-futureporn_db-schema-only.psql"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
temporal operator namespace create futureporn
|
||||||
149
t.wip.tiltfile
149
t.wip.tiltfile
|
|
@ -23,17 +23,49 @@ dotenv(fn='.env')
|
||||||
# )
|
# )
|
||||||
|
|
||||||
# args=['--default_config_file=%s' % os.getenv('TILT_NGROK_DEFAULT_CONFIG_FILE')]
|
# args=['--default_config_file=%s' % os.getenv('TILT_NGROK_DEFAULT_CONFIG_FILE')]
|
||||||
|
|
||||||
load('ext://helm_remote', 'helm_remote')
|
load('ext://helm_remote', 'helm_remote')
|
||||||
|
|
||||||
# helm_remote(
|
# helm_remote(
|
||||||
# 'redis',
|
# 'redis',
|
||||||
# repo_name='redis',
|
# repo_name='redis',
|
||||||
# repo_url='https://charts.bitnami.com/bitnami',
|
# repo_url='https://charts.bitnami.com/bitnami',
|
||||||
# namespace='futureporn',
|
# namespace='futureporn',
|
||||||
# version='19.5.0',
|
# version='19.5.2',
|
||||||
# values=['./charts/nitter/redis.values.yaml']
|
# set=[
|
||||||
|
# 'auth.password=%s' % os.getenv('TRIGGER_REDIS_PASSWORD'),
|
||||||
|
# 'architecture=standalone',
|
||||||
|
# ],
|
||||||
# )
|
# )
|
||||||
|
|
||||||
|
helm_remote(
|
||||||
|
'temporal',
|
||||||
|
repo_name='temporal',
|
||||||
|
repo_url='https://charts.lemontech.engineering',
|
||||||
|
namespace='futureporn',
|
||||||
|
version='0.37.0',
|
||||||
|
set=[
|
||||||
|
'admintools.image.tag=1.24.1-tctl-1.18.1-cli-0.12.0',
|
||||||
|
'web.image.tag=2.27.2',
|
||||||
|
'prometheus.enabled=false',
|
||||||
|
'grafana.enabled=false',
|
||||||
|
'elasticsearch.enabled=false',
|
||||||
|
'web.config.auth.enabled=true',
|
||||||
|
'cassandra.enabled=false',
|
||||||
|
'server.config.persistence.default.driver=sql',
|
||||||
|
'server.config.persistence.default.sql.driver=postgres12',
|
||||||
|
'server.config.persistence.default.sql.host=%s' % os.getenv('POSTGRES_HOST'),
|
||||||
|
'server.config.persistence.default.sql.port=5432',
|
||||||
|
'server.config.persistence.default.sql.user=%s' % os.getenv('POSTGRES_USER'),
|
||||||
|
'server.config.persistence.default.sql.password=%s' % os.getenv('POSTGRES_PASSWORD'),
|
||||||
|
'server.config.persistence.visibility.driver=sql',
|
||||||
|
'server.config.persistence.visibility.sql.driver=postgres12',
|
||||||
|
'server.config.persistence.visibility.sql.host=%s' % os.getenv('POSTGRES_HOST'),
|
||||||
|
'server.config.persistence.visibility.sql.port=5432',
|
||||||
|
'server.config.persistence.visibility.sql.user=%s' % os.getenv('POSTGRES_USER'),
|
||||||
|
'server.config.persistence.visibility.sql.password=%s' % os.getenv('POSTGRES_PASSWORD'),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
# helm_remote(
|
# helm_remote(
|
||||||
# 'nitter',
|
# 'nitter',
|
||||||
# repo_name='truecharts',
|
# repo_name='truecharts',
|
||||||
|
|
@ -70,6 +102,31 @@ k8s_yaml(helm(
|
||||||
'./charts/fp',
|
'./charts/fp',
|
||||||
values=['./charts/fp/values-dev.yaml'],
|
values=['./charts/fp/values-dev.yaml'],
|
||||||
))
|
))
|
||||||
|
# k8s_yaml(helm(
|
||||||
|
# './charts/trigger',
|
||||||
|
# set=[
|
||||||
|
# 'trigger.name=trigger',
|
||||||
|
# 'trigger.replicaCount=2',
|
||||||
|
# 'trigger.image.tag=self-host-rc.2',
|
||||||
|
# 'trigger.image.pullPolicy=IfNotPresent',
|
||||||
|
# 'trigger.env.ENCRYPTION_KEY=%s' % os.getenv('TRIGGER_ENCRYPTION_KEY'),
|
||||||
|
# 'trigger.env.MAGIC_LINK_SECRET=%s' % os.getenv('TRIGGER_MAGIC_LINK_SECRET'),
|
||||||
|
# 'trigger.env.DATABASE_URL=%s' % os.getenv('TRIGGER_DATABASE_URL'),
|
||||||
|
# 'trigger.env.LOGIN_ORIGIN=%s' % os.getenv('TRIGGER_LOGIN_ORIGIN'),
|
||||||
|
# 'trigger.env.APP_ORIGIN=%s' % os.getenv('TRIGGER_APP_ORIGIN'),
|
||||||
|
# 'trigger.env.PORT=%s' % os.getenv('TRIGGER_PORT'),
|
||||||
|
# 'trigger.env.REMIX_APP_PORT=%s' % os.getenv('TRIGGER_REMIX_APP_PORT'),
|
||||||
|
# 'trigger.env.REDIS_HOST=redis-master.futureporn.svc.cluster.local',
|
||||||
|
# 'trigger.env.REDIS_PORT=6379',
|
||||||
|
# 'trigger.ingress.nginx.enabled=false',
|
||||||
|
# 'trigger.ingress.enabled=false',
|
||||||
|
# 'postgres.enabled=false'
|
||||||
|
# ]
|
||||||
|
# ))
|
||||||
|
# k8s_resource(
|
||||||
|
# workload='trigger',
|
||||||
|
# port_forwards=['3030'],
|
||||||
|
# )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -99,23 +156,29 @@ docker_build(
|
||||||
# )
|
# )
|
||||||
|
|
||||||
load('ext://uibutton', 'cmd_button')
|
load('ext://uibutton', 'cmd_button')
|
||||||
# cmd_button('postgres:seed',
|
cmd_button('postgres:create',
|
||||||
# argv=['sh', './scripts/postgres-seed.sh'],
|
argv=['sh', './scripts/postgres-create.sh'],
|
||||||
# resource='postgres',
|
resource='postgres',
|
||||||
# icon_name='dataset',
|
icon_name='dataset',
|
||||||
# text='seed db with schema',
|
text='create (empty) databases',
|
||||||
# )
|
)
|
||||||
cmd_button('postgres:restore',
|
cmd_button('postgres:restore',
|
||||||
argv=['sh', './scripts/postgres-restore.sh'],
|
argv=['sh', './scripts/postgres-restore.sh'],
|
||||||
resource='postgres',
|
resource='postgres',
|
||||||
icon_name='cloud_download',
|
icon_name='upload',
|
||||||
text='restore db from backup',
|
text='restore db from backup',
|
||||||
)
|
)
|
||||||
cmd_button('postgres:drop',
|
cmd_button('postgres:drop',
|
||||||
argv=['sh', './scripts/postgres-drop.sh'],
|
argv=['sh', './scripts/postgres-drop.sh'],
|
||||||
resource='postgres',
|
resource='postgres',
|
||||||
icon_name='delete',
|
icon_name='delete',
|
||||||
text='delete the database'
|
text='DROP futureporn_db'
|
||||||
|
)
|
||||||
|
cmd_button('postgres:drop_temporal',
|
||||||
|
argv=['sh', './scripts/postgres-drop-temporal.sh'],
|
||||||
|
resource='postgres',
|
||||||
|
icon_name='delete',
|
||||||
|
text='DROP temporal'
|
||||||
)
|
)
|
||||||
cmd_button('postgres:backup',
|
cmd_button('postgres:backup',
|
||||||
argv=['sh', './scripts/postgres-backup.sh'],
|
argv=['sh', './scripts/postgres-backup.sh'],
|
||||||
|
|
@ -123,6 +186,12 @@ cmd_button('postgres:backup',
|
||||||
icon_name='download',
|
icon_name='download',
|
||||||
text='backup the database'
|
text='backup the database'
|
||||||
)
|
)
|
||||||
|
cmd_button('temporal-web:namespace',
|
||||||
|
argv=['sh', './scripts/temporal-namespaces.sh'],
|
||||||
|
resource='temporal-web',
|
||||||
|
icon_name='badge',
|
||||||
|
text='create futureporn namespace',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
## Uncomment the following for fp/next in dev mode
|
## Uncomment the following for fp/next in dev mode
|
||||||
|
|
@ -143,14 +212,30 @@ docker_build(
|
||||||
|
|
||||||
|
|
||||||
docker_build(
|
docker_build(
|
||||||
'fp/scout',
|
'fp/scout-manager',
|
||||||
'.',
|
'.',
|
||||||
only=['./pnpm-lock.yaml', './package.json', './packages/scout', './packages/pg-pubsub', './packages/common'],
|
only=['./pnpm-lock.yaml', './package.json', './packages/scout', './packages/next'],
|
||||||
dockerfile='d.scout.dockerfile',
|
dockerfile='d.scout.dockerfile',
|
||||||
target='scout',
|
target='manager',
|
||||||
live_update=[
|
live_update=[
|
||||||
sync('./packages/scout', '/app')
|
sync('./packages/scout', '/app'),
|
||||||
]
|
run('cd /app && pnpm i', trigger=['./packages/scout/package.json', './packages/scout/pnpm-lock.yaml']),
|
||||||
|
],
|
||||||
|
entrypoint='pnpm nodemon --ext js,ts,json,yaml --exec node --no-warnings=ExperimentalWarning --loader ts-node/esm ./src/index.ts'
|
||||||
|
# entrypoint='pnpm tsx watch ./src/index.ts'
|
||||||
|
)
|
||||||
|
|
||||||
|
docker_build(
|
||||||
|
'fp/scout-worker',
|
||||||
|
'.',
|
||||||
|
only=['./pnpm-lock.yaml', './package.json', './packages/scout', './packages/next'],
|
||||||
|
dockerfile='d.scout.dockerfile',
|
||||||
|
target='worker',
|
||||||
|
live_update=[
|
||||||
|
sync('./packages/scout', '/app'),
|
||||||
|
run('cd /app && pnpm i', trigger=['./packages/scout/package.json', './packages/scout/pnpm-lock.yaml']),
|
||||||
|
],
|
||||||
|
entrypoint='pnpm nodemon --ext js,ts,json,yaml --exec node --no-warnings=ExperimentalWarning --loader ts-node/esm ./src/temporal/worker.ts'
|
||||||
)
|
)
|
||||||
# k8s_resource(
|
# k8s_resource(
|
||||||
# workload='kubernetes-ingress-controller-manager',
|
# workload='kubernetes-ingress-controller-manager',
|
||||||
|
|
@ -188,7 +273,8 @@ k8s_resource(
|
||||||
port_forwards=['3000'],
|
port_forwards=['3000'],
|
||||||
links=[
|
links=[
|
||||||
link('https://next.piko.sbtp.xyz'),
|
link('https://next.piko.sbtp.xyz'),
|
||||||
]
|
],
|
||||||
|
resource_deps=['strapi', 'postgres']
|
||||||
)
|
)
|
||||||
k8s_resource(
|
k8s_resource(
|
||||||
workload='strapi',
|
workload='strapi',
|
||||||
|
|
@ -196,7 +282,8 @@ k8s_resource(
|
||||||
links=[
|
links=[
|
||||||
link('http://localhost:1339/admin'),
|
link('http://localhost:1339/admin'),
|
||||||
link('https://strapi.piko.sbtp.xyz'),
|
link('https://strapi.piko.sbtp.xyz'),
|
||||||
]
|
],
|
||||||
|
resource_deps=['postgres']
|
||||||
)
|
)
|
||||||
# k8s_resource(
|
# k8s_resource(
|
||||||
# workload='strapi-app',
|
# workload='strapi-app',
|
||||||
|
|
@ -208,12 +295,34 @@ k8s_resource(
|
||||||
port_forwards=['5432']
|
port_forwards=['5432']
|
||||||
)
|
)
|
||||||
|
|
||||||
|
k8s_resource(
|
||||||
|
workload='scout-worker',
|
||||||
|
resource_deps=['postgres', 'strapi', 'temporal-frontend', 'scout-manager']
|
||||||
|
)
|
||||||
|
|
||||||
|
k8s_resource(
|
||||||
|
workload='scout-manager',
|
||||||
|
resource_deps=['postgres', 'strapi', 'temporal-frontend']
|
||||||
|
)
|
||||||
|
|
||||||
# k8s_resource(
|
# k8s_resource(
|
||||||
# workload='pgadmin',
|
# workload='pgadmin',
|
||||||
# port_forwards=['5050']
|
# port_forwards=['5050'],
|
||||||
|
# resource_deps=['postgres']
|
||||||
# )
|
# )
|
||||||
|
|
||||||
# k8s_resource(
|
# k8s_resource(
|
||||||
# workload='nitter',
|
# workload='nitter',
|
||||||
# port_forwards=['6060:10606'],
|
# port_forwards=['6060:10606'],
|
||||||
# )
|
# )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
k8s_resource(workload='temporal-admintools', labels='temporal', resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-frontend', labels='temporal', port_forwards=['7233'], resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-history', labels='temporal', resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-worker', labels='temporal', resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-web', labels='temporal', port_forwards=['8080'], resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-schema-setup', labels='temporal', resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-schema-update', labels='temporal', resource_deps=['postgres'])
|
||||||
|
k8s_resource(workload='temporal-matching', labels='temporal', resource_deps=['postgres'])
|
||||||
Loading…
Reference in New Issue