capture progress
ci / build (push) Has been cancelled
Details
ci / build (push) Has been cancelled
Details
This commit is contained in:
parent
af8f58940a
commit
4bc11c027e
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"name": "tsx",
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
|
||||
"program": "${file}",
|
||||
|
||||
"runtimeExecutable": "tsx",
|
||||
|
||||
"console": "integratedTerminal",
|
||||
"internalConsoleOptions": "neverOpen",
|
||||
|
||||
"skipFiles": [
|
||||
"<node_internals>/**",
|
||||
|
||||
"${workspaceFolder}/node_modules/**",
|
||||
],
|
||||
}
|
||||
17
Makefile
17
Makefile
|
|
@ -47,22 +47,7 @@ clean:
|
|||
dotenvx run -f .env.${ENV} -- node ./packages/infra/vultr-delete-orphaned-resources.js
|
||||
|
||||
deps:
|
||||
echo "Some of the install methods for these dependencies are not cross-platform compatible. Some of the install methods are not tested. Expect this to fail. Please consult the Makefile for URLs to project sources."
|
||||
sudo pamac install make entr nvm kubectl docker helm expect
|
||||
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
|
||||
echo "go to https://github.com/txn2/kubefwd/releases/latest to get kubefwd"
|
||||
echo "go to https://github.com/tilt-dev/ctlptl/releases/latest to get ctlptl"
|
||||
sudo systemctl enable docker
|
||||
sudo systemctl start docker
|
||||
usermod -aG docker cj
|
||||
newgrp docker
|
||||
npm i -g pnpm
|
||||
pnpm install -g @dotenvx/dotenvx
|
||||
curl -OL 'https://github.com/vmware-tanzu/velero/releases/download/v1.13.2/velero-v1.13.2-linux-amd64.tar.gz'
|
||||
OS=$(go env GOOS); ARCH=$(go env GOARCH); curl -fsSL -o cmctl https://github.com/cert-manager/cmctl/releases/latest/download/cmctl_${OS}_${ARCH}
|
||||
chmod +x cmctl
|
||||
sudo mv cmctl /usr/local/bin
|
||||
echo "use `devbox install`"
|
||||
|
||||
# A gitea act runner which runs locally
|
||||
# https://docs.gitea.com/next/usage/actions/overview
|
||||
|
|
|
|||
24
README.md
24
README.md
|
|
@ -1,9 +1,33 @@
|
|||
# futureporn.net
|
||||
|
||||
[](https://www.jetify.com/devbox/docs/contributor-quickstart/)
|
||||
|
||||
Source Code for https://futureporn.net
|
||||
|
||||
See ./ARCHITECTURE.md for overview
|
||||
|
||||
## Getting Started
|
||||
|
||||
I'm working towards a better development experience with devbox and Tilt. This process is in a state of flux and is likely to be broken.
|
||||
|
||||
The main gist is as follows
|
||||
|
||||
1. Install [devbox](https://www.jetify.com/devbox/docs/installing_devbox/)
|
||||
2. Install development environment & packages using devbox.
|
||||
|
||||
devbox install
|
||||
|
||||
3. Start a local KIND Kubernetes cluster
|
||||
|
||||
make cluster
|
||||
|
||||
4. Start Tilt
|
||||
|
||||
make tilt
|
||||
|
||||
Tilt will manage the KIND cluster, downloading necessary docker containers and building the containers listed in the fp helm chart at ./Charts/fp. Making changes to these charts or the application code will update or re-build the images as necessary.
|
||||
|
||||
|
||||
## Metrics Notes
|
||||
|
||||
Keeping track of metrics we want to scrape using Prometheus
|
||||
|
|
|
|||
16
Tiltfile
16
Tiltfile
|
|
@ -181,14 +181,20 @@ cmd_button('postgres:drop',
|
|||
argv=['sh', './scripts/postgres-drop.sh'],
|
||||
resource='postgresql-primary',
|
||||
icon_name='delete',
|
||||
text='DROP futureporn_db'
|
||||
text='DROP all databases'
|
||||
)
|
||||
|
||||
cmd_button('postgres:backup',
|
||||
argv=['sh', './scripts/postgres-backup.sh'],
|
||||
cmd_button('capture-api:create',
|
||||
argv=['http', '--ignore-stdin', 'POST', 'http://localhost:5003/api/record', "url='https://twitch.tv/ironmouse'", "channel='ironmouse'"],
|
||||
resource='capture-api',
|
||||
icon_name='send',
|
||||
text='Start Recording'
|
||||
)
|
||||
cmd_button('postgres:graphile',
|
||||
argv=['sh', './scripts/postgres-test-graphile.sh'],
|
||||
resource='postgresql-primary',
|
||||
icon_name='download',
|
||||
text='backup the database'
|
||||
icon_name='graph',
|
||||
text='create graphile test job',
|
||||
)
|
||||
cmd_button('postgres:graphile',
|
||||
argv=['sh', './scripts/postgres-test-graphile.sh'],
|
||||
|
|
|
|||
|
|
@ -45,6 +45,22 @@ spec:
|
|||
key: databaseUrl
|
||||
- name: PORT
|
||||
value: "{{ .Values.capture.api.port }}"
|
||||
- name: S3_ENDPOINT
|
||||
value: "{{ .Values.s3.endpoint }}"
|
||||
- name: S3_REGION
|
||||
value: "{{ .Values.s3.region }}"
|
||||
- name: S3_BUCKET_NAME
|
||||
value: "{{ .Values.s3.buckets.usc }}"
|
||||
- name: S3_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: capture
|
||||
key: s3AccessKeyId
|
||||
- name: S3_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: capture
|
||||
key: s3SecretAccessKey
|
||||
resources:
|
||||
limits:
|
||||
cpu: 1000m
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
environment: development
|
||||
# storageClassName: csi-hostpath-sc # used by minikube
|
||||
storageClassName: standard # used by Kind
|
||||
s3:
|
||||
endpoint: https://s3.us-west-000.backblazeb2.com
|
||||
region: us-west-000
|
||||
buckets:
|
||||
main: fp-dev
|
||||
usc: fp-usc-dev
|
||||
link2cid:
|
||||
imageName: fp/link2cid
|
||||
next:
|
||||
|
|
|
|||
|
|
@ -44,7 +44,9 @@ RUN ls -la /prod/capture
|
|||
|
||||
## start the app with dumb init to spawn the Node.js runtime process
|
||||
## with signal support
|
||||
## The mode @futureporn/capture uses when starting is determined by FUNCTION environment variable. (worker|api)
|
||||
FROM base AS capture
|
||||
RUN ls -la /usr/local/bin/yt-dlp
|
||||
ENV HOSTNAME=0.0.0.0 NODE_ENV=production
|
||||
COPY --from=build /prod/capture .
|
||||
CMD [ "dumb-init", "node", "dist/index.js" ]
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.12.0/.schema/devbox.schema.json",
|
||||
"packages": [
|
||||
"nodejs@20",
|
||||
"tilt@latest",
|
||||
"ctlptl@latest",
|
||||
"kubectl@latest",
|
||||
"cmctl@latest"
|
||||
],
|
||||
"env": {
|
||||
"DEVBOX_COREPACK_ENABLED": "true",
|
||||
"ENV": "development"
|
||||
},
|
||||
"shell": {
|
||||
"init_hook": [
|
||||
"pnpm install"
|
||||
],
|
||||
"scripts": {
|
||||
"test": [
|
||||
"echo \"Error: no test specified\" && exit 1"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,298 @@
|
|||
{
|
||||
"lockfile_version": "1",
|
||||
"packages": {
|
||||
"cmctl@latest": {
|
||||
"last_modified": "2024-07-07T07:43:47Z",
|
||||
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#cmctl",
|
||||
"source": "devbox-search",
|
||||
"version": "1.14.7",
|
||||
"systems": {
|
||||
"aarch64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/69lc5y36840ccy6d1pzph994psadk4bm-cmctl-1.14.7",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/69lc5y36840ccy6d1pzph994psadk4bm-cmctl-1.14.7"
|
||||
},
|
||||
"aarch64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/d5l61iil1gaax5sipnzg80mb0p1hqk9f-cmctl-1.14.7",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/d5l61iil1gaax5sipnzg80mb0p1hqk9f-cmctl-1.14.7"
|
||||
},
|
||||
"x86_64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/j0r3wavi836mp31l0s7r1c3rjryw2z62-cmctl-1.14.7",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/j0r3wavi836mp31l0s7r1c3rjryw2z62-cmctl-1.14.7"
|
||||
},
|
||||
"x86_64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/lnmy96wjzmjna7z9f0dbqd16nf2x5qbv-cmctl-1.14.7",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/lnmy96wjzmjna7z9f0dbqd16nf2x5qbv-cmctl-1.14.7"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ctlptl@latest": {
|
||||
"last_modified": "2024-07-07T07:43:47Z",
|
||||
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#ctlptl",
|
||||
"source": "devbox-search",
|
||||
"version": "0.8.29",
|
||||
"systems": {
|
||||
"aarch64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/gvnmvb315zngbg5a0idynlwxcc45gmyd-ctlptl-0.8.29",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/gvnmvb315zngbg5a0idynlwxcc45gmyd-ctlptl-0.8.29"
|
||||
},
|
||||
"aarch64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/vgwbxwaf773mmgcbc1j5h6n5a7g587lf-ctlptl-0.8.29",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/vgwbxwaf773mmgcbc1j5h6n5a7g587lf-ctlptl-0.8.29"
|
||||
},
|
||||
"x86_64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/ga4rp2188c6k24162s2n23nfi4846790-ctlptl-0.8.29",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/ga4rp2188c6k24162s2n23nfi4846790-ctlptl-0.8.29"
|
||||
},
|
||||
"x86_64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/wbfjbk575ny949rfyqpm4vai4ap5rbpp-ctlptl-0.8.29",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/wbfjbk575ny949rfyqpm4vai4ap5rbpp-ctlptl-0.8.29"
|
||||
}
|
||||
}
|
||||
},
|
||||
"kubectl@latest": {
|
||||
"last_modified": "2024-07-07T07:43:47Z",
|
||||
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#kubectl",
|
||||
"source": "devbox-search",
|
||||
"version": "1.30.2",
|
||||
"systems": {
|
||||
"aarch64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/i1zidf41bkfzs2l1pq9fi1frymsfgywc-kubectl-1.30.2",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "man",
|
||||
"path": "/nix/store/dzxnn9mk9plcx3w9862jyd0nxys2yywz-kubectl-1.30.2-man",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "convert",
|
||||
"path": "/nix/store/v9ij5fnxxa02jkzpjvkbxw2jc4p9cbld-kubectl-1.30.2-convert"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/i1zidf41bkfzs2l1pq9fi1frymsfgywc-kubectl-1.30.2"
|
||||
},
|
||||
"aarch64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/k7ql4247qs6ny27m3iz5c9xf5gb248a2-kubectl-1.30.2",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "man",
|
||||
"path": "/nix/store/wy64r4nn3isydw4nx257h95qy2x2z4mx-kubectl-1.30.2-man",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "convert",
|
||||
"path": "/nix/store/ic8za302hvb4kf4zrs55ivr4q2n2lznn-kubectl-1.30.2-convert"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/k7ql4247qs6ny27m3iz5c9xf5gb248a2-kubectl-1.30.2"
|
||||
},
|
||||
"x86_64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/v029n959l5b289br0cq591b04yc48516-kubectl-1.30.2",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "man",
|
||||
"path": "/nix/store/0dvcxn7gsi2ycy9blb7pcy506w4xp2vi-kubectl-1.30.2-man",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "convert",
|
||||
"path": "/nix/store/2nfq4ivwa4a7jwc0183f2wpl1jxbn754-kubectl-1.30.2-convert"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/v029n959l5b289br0cq591b04yc48516-kubectl-1.30.2"
|
||||
},
|
||||
"x86_64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/3vkf0406s1i6l89hk5wrakh4bbn0p1p2-kubectl-1.30.2",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "man",
|
||||
"path": "/nix/store/3wbvgkkka1knkxvyr4c8qbpr448smw8i-kubectl-1.30.2-man",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "convert",
|
||||
"path": "/nix/store/h5zxz8db6wligwhw5gnwk4gbc8j1ixik-kubectl-1.30.2-convert"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/3vkf0406s1i6l89hk5wrakh4bbn0p1p2-kubectl-1.30.2"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nodejs@20": {
|
||||
"last_modified": "2024-07-07T07:43:47Z",
|
||||
"plugin_version": "0.0.2",
|
||||
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#nodejs_20",
|
||||
"source": "devbox-search",
|
||||
"version": "20.14.0",
|
||||
"systems": {
|
||||
"aarch64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/sqnbldm1fjw88v23yq4v6531y4m7v2fh-nodejs-20.14.0",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "libv8",
|
||||
"path": "/nix/store/1i0rb2axkrxvsq5pz8s2q07ard2p36a1-nodejs-20.14.0-libv8"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/sqnbldm1fjw88v23yq4v6531y4m7v2fh-nodejs-20.14.0"
|
||||
},
|
||||
"aarch64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/r1nwmlbsn67f5rhayr7jjjdmiflxpk92-nodejs-20.14.0",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "libv8",
|
||||
"path": "/nix/store/5ii3xkbd3iv0xvqqvjg3agsm0dinidgm-nodejs-20.14.0-libv8"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/r1nwmlbsn67f5rhayr7jjjdmiflxpk92-nodejs-20.14.0"
|
||||
},
|
||||
"x86_64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/wzgnws4r1c98vzj5q6gq4drz2jfq7d5q-nodejs-20.14.0",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "libv8",
|
||||
"path": "/nix/store/gc2gnkc8hvkh51ab3a29fvgzy2qsqb2s-nodejs-20.14.0-libv8"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/wzgnws4r1c98vzj5q6gq4drz2jfq7d5q-nodejs-20.14.0"
|
||||
},
|
||||
"x86_64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/ilkfhnqz4xczrliqjva8770x2svbfznd-nodejs-20.14.0",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"name": "libv8",
|
||||
"path": "/nix/store/2qaf68dzimr8as4bgli0xmsn11c0ah2j-nodejs-20.14.0-libv8"
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/ilkfhnqz4xczrliqjva8770x2svbfznd-nodejs-20.14.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"tilt@latest": {
|
||||
"last_modified": "2024-07-15T21:47:20Z",
|
||||
"resolved": "github:NixOS/nixpkgs/b2c1f10bfbb3f617ea8e8669ac13f3f56ceb2ea2#tilt",
|
||||
"source": "devbox-search",
|
||||
"version": "0.33.17",
|
||||
"systems": {
|
||||
"aarch64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/l19qinywsx7y86xp8vgwr3bgnbi0rfcj-tilt-0.33.17",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/l19qinywsx7y86xp8vgwr3bgnbi0rfcj-tilt-0.33.17"
|
||||
},
|
||||
"aarch64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/f6swxifmvnxjxifxyw4k4aiyxh0dgfyq-tilt-0.33.17",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/f6swxifmvnxjxifxyw4k4aiyxh0dgfyq-tilt-0.33.17"
|
||||
},
|
||||
"x86_64-darwin": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/d59rlsabcqxax6bgw6d30zhmflw65ch0-tilt-0.33.17",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/d59rlsabcqxax6bgw6d30zhmflw65ch0-tilt-0.33.17"
|
||||
},
|
||||
"x86_64-linux": {
|
||||
"outputs": [
|
||||
{
|
||||
"name": "out",
|
||||
"path": "/nix/store/qfv96sjcsslynqbilwj823x8nxvgj5cv-tilt-0.33.17",
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"store_path": "/nix/store/qfv96sjcsslynqbilwj823x8nxvgj5cv-tilt-0.33.17"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -15,6 +15,10 @@ Worker container runs a Graphile Worker which listens for specific tasks related
|
|||
|
||||
## Misc dev notes
|
||||
|
||||
### idea for taking snapshots of stream in progress
|
||||
|
||||
https://ofarukcaki.medium.com/producing-real-time-video-with-node-js-and-ffmpeg-a59ac27461a1
|
||||
|
||||
### youtube-dl end of stream output
|
||||
|
||||
The end-of-stream output from yt-dlp when recording CB looks like this
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@
|
|||
"@futureporn/utils": "workspace:^",
|
||||
"@paralleldrive/cuid2": "^2.2.2",
|
||||
"@types/chai": "^4.3.16",
|
||||
"@types/fluent-ffmpeg": "^2.1.24",
|
||||
"@types/mocha": "^10.0.7",
|
||||
"@types/node": "^20.14.11",
|
||||
"diskusage": "^1.2.0",
|
||||
|
|
@ -46,8 +47,12 @@
|
|||
"youtube-dl-wrap": "github:insanity54/youtube-dl-wrap"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@smithy/util-stream": "^3.1.2",
|
||||
"@types/sinon": "^17.0.3",
|
||||
"@types/sinon-chai": "^3.2.12",
|
||||
"aws-sdk": "^2.1663.0",
|
||||
"aws-sdk-client-mock": "^4.0.1",
|
||||
"aws-sdk-mock": "^6.0.4",
|
||||
"chai": "^4.4.1",
|
||||
"cheerio": "1.0.0-rc.12",
|
||||
"mocha": "^10.7.0",
|
||||
|
|
@ -55,6 +60,8 @@
|
|||
"node-abort-controller": "^3.1.1",
|
||||
"node-fetch": "^3.3.2",
|
||||
"nodemon": "^2.0.22",
|
||||
"pretty-bytes": "^6.1.1",
|
||||
"s3": "link:aws-sdk/clients/s3",
|
||||
"sinon": "^15.2.0",
|
||||
"sinon-chai": "^3.7.0",
|
||||
"sinon-test": "^3.1.6",
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,22 +1,67 @@
|
|||
import Record from "./Record.js"
|
||||
import { expect } from "chai"
|
||||
import { ChildProcess, spawn } from "child_process"
|
||||
import { createReadStream, readFileSync, ReadStream } from "fs"
|
||||
import AWSMock from 'aws-sdk-mock'
|
||||
import sinon from 'sinon';
|
||||
import sinonChai from 'sinon-chai';
|
||||
import { PutObjectCommand, PutObjectCommandInput, S3Client, CreateMultipartUploadCommand, UploadPartCommand, UploadPartCommandInput } from "@aws-sdk/client-s3"
|
||||
import { join, dirname } from "path"
|
||||
import { fileURLToPath } from "url"
|
||||
import { S3 } from 'aws-sdk';
|
||||
import { HeadObjectOutput } from 'aws-sdk/clients/s3';
|
||||
import { Readable } from 'stream';
|
||||
import { mockClient } from 'aws-sdk-client-mock';
|
||||
import { sdkStreamMixin } from '@smithy/util-stream'
|
||||
|
||||
|
||||
// "pay no attention to that man behind the curtain"
|
||||
|
||||
// export function getObjectReadStream(s3Client: S3, Bucket: string, Key: string): Readable {
|
||||
// return s3Client.getObject({ Bucket, Key }).createReadStream();
|
||||
// }
|
||||
|
||||
// export async function waitForObjectExists(
|
||||
// s3Client: S3Client,
|
||||
// Bucket: string,
|
||||
// Key: string
|
||||
// ): Promise<HeadObjectOutput> {
|
||||
// return s3Client.waitFor('objectExists', { Bucket, Key }).promise();
|
||||
// }
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
|
||||
|
||||
const testStreamDir = '/tmp/record-test-stream.m3u8'
|
||||
|
||||
describe('Record', function () {
|
||||
let record: Record
|
||||
this.beforeEach(function () {
|
||||
record = new Record({ url: 'https://example.com/my-cool-stream' })
|
||||
})
|
||||
describe('start()', function () {
|
||||
it('should start the recording', async function () {
|
||||
// mocking @aws-sdk/lib-storage Upload() has some nuances.
|
||||
// @see https://github.com/m-radzikowski/aws-sdk-client-mock?tab=readme-ov-file#lib-storage-upload
|
||||
|
||||
|
||||
it('should accept a {ReadStream} as input', async function () {
|
||||
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) // 192627 bytes
|
||||
const s3ClientMock = mockClient(S3Client)
|
||||
s3ClientMock.on(CreateMultipartUploadCommand).resolves({UploadId: '1'});
|
||||
s3ClientMock.on(UploadPartCommand).resolves({ETag: '1'});
|
||||
const s3Client = new S3Client({ region: 'us-west-000' })
|
||||
const record = new Record({ inputStream, s3Client, channel: 'coolguy_69', bucket: 'test' })
|
||||
await record.start()
|
||||
expect(record).to.have.property('id')
|
||||
expect(record).to.have.property('url')
|
||||
})
|
||||
})
|
||||
describe('stop()', function () {
|
||||
it('should stop the recording', async function () {
|
||||
await record.stop()
|
||||
expect(record).to.have.property('cdnUrl')
|
||||
expect(record).to.have.property('counter', 192627)
|
||||
expect(record).to.have.property('bucket', 'test')
|
||||
})
|
||||
|
||||
xit('should restart if a EPIPE is encountered', async function () {
|
||||
// @todo IDK how to implement this.
|
||||
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4'))
|
||||
const s3ClientMock = mockClient(S3Client)
|
||||
s3ClientMock.on(CreateMultipartUploadCommand).resolves({UploadId: '1'})
|
||||
s3ClientMock.on(UploadPartCommand).rejectsOnce({cause: 'simulated network interruption'}).resolves({ ETag: '1' }) // this rejection is probably not specific enough to simulate EPIPE
|
||||
const s3Client = new S3Client({ region: 'us-west-000' })
|
||||
const record = new Record({ inputStream, s3Client, channel: 'coolguy_69', bucket: 'test' })
|
||||
await record.start()
|
||||
expect(record).to.have.property('counter', 192627)
|
||||
})
|
||||
|
||||
|
||||
})
|
||||
|
|
@ -1,102 +1,184 @@
|
|||
import { createId } from '@paralleldrive/cuid2'
|
||||
import { spawn } from 'child_process';
|
||||
import { ua0 } from '@futureporn/scout/ua.js'
|
||||
import { getTmpFile } from '@futureporn/utils'
|
||||
import { PassThrough, pipeline, Readable } from 'stream';
|
||||
import prettyBytes from 'pretty-bytes';
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import 'dotenv/config'
|
||||
|
||||
|
||||
export interface RecordArgs {
|
||||
url: string;
|
||||
filename?: string;
|
||||
channel?: string;
|
||||
channel: string;
|
||||
s3Client: S3Client;
|
||||
bucket: string;
|
||||
date?: string;
|
||||
inputStream: Readable;
|
||||
}
|
||||
|
||||
interface MakeS3ClientOptions {
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
region: string;
|
||||
endpoint: string
|
||||
}
|
||||
|
||||
interface getFFmpegDownloadOptions {
|
||||
url: string;
|
||||
}
|
||||
|
||||
export default class Record {
|
||||
readonly id: string;
|
||||
readonly url: string;
|
||||
private s3Client: S3Client;
|
||||
private uploadStream: PassThrough;
|
||||
private ticker?: NodeJS.Timeout;
|
||||
inputStream: Readable;
|
||||
counter: number;
|
||||
bucket: string;
|
||||
keyName: string;
|
||||
datestamp: string;
|
||||
filename?: string;
|
||||
channel?: string;
|
||||
channel: string;
|
||||
date?: string;
|
||||
|
||||
constructor({ url }: RecordArgs) {
|
||||
if (!url) throw new Error('url passed to Record constructor was undefined.');
|
||||
constructor({ inputStream, channel, s3Client, bucket }: RecordArgs) {
|
||||
if (!inputStream) throw new Error('Record constructor was missing inputStream.');
|
||||
if (!bucket) throw new Error('Record constructor was missing bucket.');
|
||||
if (!channel) throw new Error('Record constructer was missing channel!');
|
||||
if (!s3Client) throw new Error('Record constructer was missing s3Client');
|
||||
this.inputStream = inputStream
|
||||
this.id = createId()
|
||||
this.url = url
|
||||
this.s3Client = s3Client
|
||||
this.bucket = bucket
|
||||
this.channel = channel
|
||||
this.counter = 0
|
||||
this.datestamp = new Date().toISOString()
|
||||
this.keyName = `${this.datestamp}-${channel}-${createId()}.ts`
|
||||
this.uploadStream = new PassThrough()
|
||||
}
|
||||
|
||||
makeProgressTicker() {
|
||||
this.ticker = setInterval(() => {
|
||||
console.log(`[progress] ${this.counter} bytes (aggregate) (${prettyBytes(this.counter)}) have passed through the pipeline.`)
|
||||
}, 1000 * 30)
|
||||
}
|
||||
|
||||
|
||||
static makeS3Client({
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
region,
|
||||
endpoint
|
||||
}: MakeS3ClientOptions): S3Client {
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region,
|
||||
credentials: {
|
||||
accessKeyId,
|
||||
secretAccessKey
|
||||
}
|
||||
})
|
||||
return client
|
||||
}
|
||||
|
||||
static getFFmpegDownload({ url }: getFFmpegDownloadOptions): Readable {
|
||||
|
||||
|
||||
const ffmpegProc = spawn('ffmpeg', [
|
||||
'-headers', `"User-Agent: ${ua0}"`,
|
||||
'-i', url,
|
||||
'-c:v', 'copy',
|
||||
'-c:a', 'copy',
|
||||
'-movflags', 'faststart',
|
||||
'-y',
|
||||
'-f', 'mpegts',
|
||||
'-loglevel', 'quiet',
|
||||
'pipe:1'
|
||||
], {
|
||||
// ignoring stderr is important because if not, ffmpeg will fill that buffer and node will hang
|
||||
stdio: ['pipe', 'pipe', 'ignore']
|
||||
})
|
||||
return ffmpegProc.stdout
|
||||
|
||||
}
|
||||
|
||||
|
||||
async uploadToS3() {
|
||||
|
||||
const target = {
|
||||
Bucket: this.bucket,
|
||||
Key: this.keyName,
|
||||
// We do this to keep TS happy. Body expects a Readable, not a ReadableStream nor a NodeJS.ReadableStream
|
||||
// Body: new Readable().wrap(this.uploadStream)
|
||||
Body: this.uploadStream
|
||||
}
|
||||
|
||||
// greets https://stackoverflow.com/a/70159394/1004931
|
||||
try {
|
||||
const parallelUploads3 = new Upload({
|
||||
client: this.s3Client,
|
||||
partSize: 1024 * 1024 * 5,
|
||||
queueSize: 1,
|
||||
leavePartsOnError: false,
|
||||
params: target,
|
||||
});
|
||||
|
||||
|
||||
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||
console.log(progress)
|
||||
if (progress?.loaded) {
|
||||
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
|
||||
} else {
|
||||
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
|
||||
}
|
||||
});
|
||||
|
||||
await parallelUploads3.done();
|
||||
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
console.error(`while uploading a file to s3, we encountered an error`)
|
||||
throw new Error(e.message);
|
||||
} else {
|
||||
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
async start() {
|
||||
console.log(`@TODO record start with id=${this.id}, url=${this.url}`)
|
||||
|
||||
const playlistUrlPromise = new Promise<string>((resolve) => {
|
||||
const ytdlp = spawn('yt-dlp', [
|
||||
'-g',
|
||||
this.url
|
||||
])
|
||||
ytdlp.stdout.on('data', (data) => {
|
||||
resolve(data)
|
||||
this.makeProgressTicker()
|
||||
|
||||
// streams setup
|
||||
this.uploadStream.on('data', (data) => {
|
||||
this.counter += data.length
|
||||
})
|
||||
})
|
||||
const playlistUrl = await playlistUrlPromise
|
||||
const filename = getTmpFile(`stream.ts`)
|
||||
|
||||
const ffmpeg = spawn('ffmpeg', [
|
||||
'-headers', `"User-Agent: ${ua0}"`,
|
||||
'-i', playlistUrl,
|
||||
'-c:v', 'copy',
|
||||
'-c:a', 'copy',
|
||||
'-movflags', 'faststart',
|
||||
'-y',
|
||||
'-f', 'mpegts',
|
||||
filename
|
||||
])
|
||||
|
||||
const ffmpegProcess = spawn('ffmpeg', [
|
||||
'-headers', `"User-Agent: ${ua0}"`,
|
||||
'-i', playlistUrl,
|
||||
'-c:v', 'copy',
|
||||
'-c:a', 'copy',
|
||||
'-movflags', 'faststart',
|
||||
'-y',
|
||||
'-f', 'mpegts',
|
||||
filename
|
||||
], {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
const ps = spawn('ps', ['ax']);
|
||||
const grep = spawn('grep', ['ssh']);
|
||||
|
||||
ps.stdout.on('data', (data) => {
|
||||
grep.stdin.write(data);
|
||||
});
|
||||
|
||||
ps.stderr.on('data', (data) => {
|
||||
console.error(`ps stderr: ${data}`);
|
||||
});
|
||||
|
||||
ps.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
console.log(`ps process exited with code ${code}`);
|
||||
// stream pipeline setup
|
||||
pipeline(
|
||||
this.inputStream,
|
||||
this.uploadStream,
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.error(`pipeline errored.`)
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log('pipeline succeeded.')
|
||||
}
|
||||
grep.stdin.end();
|
||||
});
|
||||
|
||||
grep.stdout.on('data', (data) => {
|
||||
console.log(data.toString());
|
||||
});
|
||||
|
||||
grep.stderr.on('data', (data) => {
|
||||
console.error(`grep stderr: ${data}`);
|
||||
});
|
||||
|
||||
grep.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
console.log(`grep process exited with code ${code}`);
|
||||
}
|
||||
});
|
||||
)
|
||||
|
||||
await this.uploadToS3()
|
||||
clearInterval(this.ticker)
|
||||
|
||||
return {
|
||||
id: this.id,
|
||||
url: this.url
|
||||
keyName: this.keyName,
|
||||
channel: this.channel
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ import graphileWorkerPlugin, { type ExtendedFastifyInstance } from './fastify-gr
|
|||
|
||||
const version = getPackageVersion('../package.json')
|
||||
interface RecordBodyType {
|
||||
url: string
|
||||
url: string;
|
||||
channel: string;
|
||||
}
|
||||
|
||||
function build(opts: Record<string, any>={}, connectionString: string) {
|
||||
|
|
@ -17,9 +18,10 @@ function build(opts: Record<string, any>={}, connectionString: string) {
|
|||
return { app: '@futureporn/capture', version }
|
||||
})
|
||||
app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) {
|
||||
const { url } = request.body
|
||||
console.log(`POST /api/record with url=${url}`)
|
||||
const job = await app.graphile.addJob('record', { url })
|
||||
const { url, channel } = request.body
|
||||
console.log(`POST /api/record with url=${url}, channel=${channel}`)
|
||||
|
||||
const job = await app.graphile.addJob('record', { url, channel })
|
||||
return job
|
||||
})
|
||||
return app
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
application/vnd.apple.mpegurl mp4
|
||||
text/x-abc abc
|
||||
|
|
@ -0,0 +1 @@
|
|||
asdfaslfk;sdf
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
|
||||
'use strict'
|
||||
|
||||
import { build } from './app.js'
|
||||
import 'dotenv/config'
|
||||
import { run } from 'graphile-worker'
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'url';
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env');
|
||||
if (!process.env.FUNCTION) throw new Error(`FUNCTION env var was missing. FUNCTION env var must be either 'api' or 'worker'.`);
|
||||
const connectionString = process.env.DATABASE_URL!
|
||||
|
||||
|
||||
|
||||
async function api() {
|
||||
if (!process.env.PORT) throw new Error('PORT is missing in env');
|
||||
const PORT = parseInt(process.env.PORT!)
|
||||
|
||||
const fastifyOpts = {
|
||||
logger: {
|
||||
level: 'info',
|
||||
transport: {
|
||||
target: 'pino-pretty'
|
||||
}
|
||||
}
|
||||
}
|
||||
const server = build(fastifyOpts, connectionString)
|
||||
|
||||
server.listen({ port: PORT }, (err) => {
|
||||
if (err) {
|
||||
server.log.error(err)
|
||||
process.exit(1)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
async function worker() {
|
||||
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
|
||||
|
||||
// Run a worker to execute jobs:
|
||||
const runner = await run({
|
||||
connectionString,
|
||||
concurrency,
|
||||
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
|
||||
noHandleSignals: false,
|
||||
pollInterval: 1000,
|
||||
taskDirectory: `${__dirname}/tasks`,
|
||||
});
|
||||
|
||||
// Immediately await (or otherwise handle) the resulting promise, to avoid
|
||||
// "unhandled rejection" errors causing a process crash in the event of
|
||||
// something going wrong. console.log()
|
||||
|
||||
await runner.promise;
|
||||
|
||||
// If the worker exits (whether through fatal error or otherwise), the above
|
||||
// promise will resolve/reject.
|
||||
}
|
||||
|
||||
|
||||
async function main() {
|
||||
if (process.env.FUNCTION === 'api') {
|
||||
api()
|
||||
} else if (process.env.FUNCTION === 'worker') {
|
||||
worker()
|
||||
} else {
|
||||
throw new Error('process.env.FUNCTION must be either api or worker. got '+process.env.FUNCTION)
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
'use strict'
|
||||
|
||||
import { build } from './app.js'
|
||||
import 'dotenv/config'
|
||||
import { run } from 'graphile-worker'
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'url';
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env');
|
||||
if (!process.env.FUNCTION) throw new Error(`FUNCTION env var was missing. FUNCTION env var must be either 'api' or 'worker'.`);
|
||||
const connectionString = process.env.DATABASE_URL!
|
||||
|
||||
|
||||
|
||||
|
||||
async function api() {
|
||||
if (!process.env.PORT) throw new Error('PORT is missing in env');
|
||||
const PORT = parseInt(process.env.PORT!)
|
||||
|
||||
const fastifyOpts = {
|
||||
logger: {
|
||||
level: 'info',
|
||||
transport: {
|
||||
target: 'pino-pretty'
|
||||
}
|
||||
}
|
||||
}
|
||||
const server = build(fastifyOpts, connectionString)
|
||||
|
||||
server.listen({ port: PORT }, (err) => {
|
||||
if (err) {
|
||||
server.log.error(err)
|
||||
process.exit(1)
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
async function worker() {
|
||||
if (!process.env.WORKER_CONCURRENCY) throw new Error('WORKER_CONCURRENCY is missing in env');
|
||||
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
|
||||
|
||||
// Run a worker to execute jobs:
|
||||
const runner = await run({
|
||||
connectionString: process.env.DATABASE_URL!,
|
||||
concurrency,
|
||||
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
|
||||
noHandleSignals: false,
|
||||
pollInterval: 1000,
|
||||
taskDirectory: `${__dirname}/tasks`,
|
||||
});
|
||||
|
||||
// Immediately await (or otherwise handle) the resulting promise, to avoid
|
||||
// "unhandled rejection" errors causing a process crash in the event of
|
||||
// something going wrong. console.log()
|
||||
|
||||
await runner.promise;
|
||||
|
||||
// If the worker exits (whether through fatal error or otherwise), the above
|
||||
// promise will resolve/reject.
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.env.FUNCTION === 'worker') {
|
||||
worker()
|
||||
} else if (process.env.FUNCTION === 'api') {
|
||||
api()
|
||||
} else {
|
||||
console.error(`FUNCTION environment variable must be 'worker' or 'api', but it was set to ${process.env.FUNCTION}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
'use strict'
|
||||
|
||||
import { run } from 'graphile-worker'
|
||||
import { dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'url';
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is undefined in env');
|
||||
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
|
||||
|
||||
|
||||
|
||||
|
||||
async function main() {
|
||||
// Run a worker to execute jobs:
|
||||
const runner = await run({
|
||||
connectionString: process.env.DATABASE_URL!,
|
||||
concurrency,
|
||||
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
|
||||
noHandleSignals: false,
|
||||
pollInterval: 1000,
|
||||
taskDirectory: `${__dirname}/tasks`,
|
||||
});
|
||||
|
||||
// Immediately await (or otherwise handle) the resulting promise, to avoid
|
||||
// "unhandled rejection" errors causing a process crash in the event of
|
||||
// something going wrong. console.log()
|
||||
|
||||
await runner.promise;
|
||||
|
||||
// If the worker exits (whether through fatal error or otherwise), the above
|
||||
// promise will resolve/reject.
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import { createWriteStream } from 'node:fs'
|
||||
import ffmpeg from 'fluent-ffmpeg'
|
||||
|
||||
// test stream from https://ottverse.com/free-hls-m3u8-test-urls/
|
||||
const playlistUrl = 'https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8'
|
||||
const fileOutputStream = createWriteStream('/tmp/test-stream.ts')
|
||||
|
||||
ffmpeg()
|
||||
.input(playlistUrl)
|
||||
.audioCodec('copy')
|
||||
.videoCodec('copy')
|
||||
.addOutputOptions('-movflags faststart')
|
||||
.output(fileOutputStream)
|
||||
.format('mpegts')
|
||||
.on('end', () => {
|
||||
console.log('Finished');
|
||||
})
|
||||
.on('error', (err, stdout, stderr) => {
|
||||
console.error(`there was an error`);
|
||||
console.error(err);
|
||||
console.error(stdout);
|
||||
console.error(stderr);
|
||||
throw new Error(err.message);
|
||||
})
|
||||
.run();
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
import { PassThrough, pipeline, Readable } from "stream";
|
||||
import { type Progress, Upload } from "@aws-sdk/lib-storage";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { createReadStream, createWriteStream } from 'fs';
|
||||
import { createId } from '@paralleldrive/cuid2';
|
||||
import prettyBytes from 'pretty-bytes';
|
||||
import dotenv from 'dotenv'
|
||||
dotenv.config({
|
||||
path: '../../.env.development'
|
||||
})
|
||||
|
||||
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
|
||||
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
|
||||
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
|
||||
|
||||
async function uploadStream(client: S3Client, stream: NodeJS.ReadableStream, keyName: string) {
|
||||
// const pass = new PassThrough()
|
||||
// Create a stream to the S3 bucket. We use this stream to upload the livestream to Backblaze S3 service
|
||||
console.log(`keyName=${keyName}`)
|
||||
const target = {
|
||||
Bucket: process.env.S3_BUCKET_NAME!,
|
||||
Key: keyName,
|
||||
Body: new Readable().wrap(stream)
|
||||
}
|
||||
console.log(target)
|
||||
// greets https://stackoverflow.com/a/70159394/1004931
|
||||
try {
|
||||
const parallelUploads3 = new Upload({
|
||||
client: client,
|
||||
partSize: 1024 * 1024 * 5,
|
||||
// tags: [...], // optional tags
|
||||
queueSize: 1, // optional concurrency configuration
|
||||
leavePartsOnError: false, // optional manually handle dropped parts
|
||||
params: target,
|
||||
});
|
||||
|
||||
|
||||
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||
if (progress?.loaded) {
|
||||
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
|
||||
} else {
|
||||
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
|
||||
}
|
||||
});
|
||||
|
||||
const res = await parallelUploads3.done();
|
||||
return res
|
||||
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
console.error(`while uploading a file to s3, we encountered an error`)
|
||||
throw new Error(e.message);
|
||||
} else {
|
||||
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const client = new S3Client({
|
||||
endpoint: 'https://s3.us-west-000.backblazeb2.com',
|
||||
region: 'us-west-000',
|
||||
credentials: {
|
||||
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
|
||||
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
// let debugCounter = 0
|
||||
// let uploadStream = new PassThrough()
|
||||
// uploadStream.on('data', (data) => {
|
||||
// debugCounter += data.length
|
||||
// console.log(`[data] uploadStream. ${debugCounter} aggregated bytes (${prettyBytes(debugCounter)}).`)
|
||||
// })
|
||||
// uploadStream.on('drain', () => {
|
||||
// console.log('[drain] uploadStream')
|
||||
// })
|
||||
// uploadStream.on('close', () => {
|
||||
// console.log(`[close] uploadStream closed`)
|
||||
// })
|
||||
// uploadStream.on('error', (err) => {
|
||||
// console.error('[error] uploadStream had an error as follows')
|
||||
// console.error(err)
|
||||
// })
|
||||
// uploadStream.on('exit', (code) => {
|
||||
// console.log(`[exit] uploadStream exited with code ${code}`)
|
||||
// })
|
||||
// uploadStream.on('disconnect', () => {
|
||||
// console.log('[disconnect] uploadStream disconnected')
|
||||
// })
|
||||
// uploadStream.on('message', (msg) => {
|
||||
// console.log('[message] uploadStream sent a message as follows')
|
||||
// console.log(msg)
|
||||
// })
|
||||
|
||||
|
||||
|
||||
const fileStream = createReadStream('/home/cj/Downloads/stream-23894234.ts')
|
||||
const datestamp = new Date().toISOString()
|
||||
const keyName = `${datestamp}-stream3-chaturbate-${createId()}.ts`
|
||||
await uploadStream(client, fileStream, keyName)
|
||||
|
||||
// // we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
|
||||
// pipeline(
|
||||
// fileStream,
|
||||
// uploadStream,
|
||||
// (err) => {
|
||||
// if (err) {
|
||||
// console.error(`pipeline errored.`)
|
||||
// console.error(err)
|
||||
// } else {
|
||||
// console.log('pipeline succeeded.')
|
||||
// }
|
||||
// }
|
||||
// )
|
||||
|
||||
|
||||
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
console.error(`Eyy lookat me, I'm a big nastry try/catch block and I did my jorb!`)
|
||||
console.error(e)
|
||||
} else {
|
||||
console.error('err or some sort')
|
||||
console.error(e)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
main()
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
import { PassThrough, pipeline, Readable } from "stream";
|
||||
import { Upload } from "@aws-sdk/lib-storage";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { createReadStream } from 'fs';
|
||||
import { createId } from '@paralleldrive/cuid2';
|
||||
import prettyBytes from 'pretty-bytes';
|
||||
import dotenv from 'dotenv'
|
||||
dotenv.config({
|
||||
path: '../../.env.development'
|
||||
})
|
||||
|
||||
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
|
||||
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
|
||||
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
|
||||
|
||||
function makeProgressTicker(counter: number) {
|
||||
const ticker = setInterval(() => {
|
||||
console.log(`[progress] ${counter} bytes (aggregate) (${prettyBytes(counter)}) have passed through the pipeline.`)
|
||||
}, 1000 * 30)
|
||||
return ticker
|
||||
}
|
||||
|
||||
function makeS3Client() {
|
||||
const client = new S3Client({
|
||||
endpoint: 'https://s3.us-west-000.backblazeb2.com',
|
||||
region: 'us-west-000',
|
||||
credentials: {
|
||||
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
|
||||
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
|
||||
}
|
||||
})
|
||||
return client
|
||||
}
|
||||
|
||||
|
||||
async function uploadToS3({ client, uploadStream, keyName }: { client: S3Client, uploadStream: NodeJS.ReadableStream, keyName: string }) {
|
||||
|
||||
const target = {
|
||||
Bucket: process.env.S3_BUCKET_NAME!,
|
||||
Key: keyName,
|
||||
Body: new Readable().wrap(uploadStream)
|
||||
}
|
||||
|
||||
// greets https://stackoverflow.com/a/70159394/1004931
|
||||
try {
|
||||
const parallelUploads3 = new Upload({
|
||||
client: client,
|
||||
partSize: 1024 * 1024 * 5,
|
||||
queueSize: 1,
|
||||
leavePartsOnError: false,
|
||||
params: target,
|
||||
});
|
||||
|
||||
|
||||
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||
console.log(progress)
|
||||
if (progress?.loaded) {
|
||||
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
|
||||
} else {
|
||||
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
|
||||
}
|
||||
});
|
||||
|
||||
await parallelUploads3.done();
|
||||
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
console.error(`while uploading a file to s3, we encountered an error`)
|
||||
throw new Error(e.message);
|
||||
} else {
|
||||
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
|
||||
let counter = 0
|
||||
const client = makeS3Client()
|
||||
const ticker = makeProgressTicker(counter)
|
||||
const datestamp = new Date().toISOString()
|
||||
const keyName = `${datestamp}-stream3-chaturbate-${createId()}.ts`
|
||||
console.log(`Uploading ${keyName} to S3`)
|
||||
|
||||
/**
|
||||
* setup the streams which process the data
|
||||
*/
|
||||
const ffmpegStream = createReadStream('/home/cj/Downloads/stream-23894234.ts')
|
||||
const uploadStream = new PassThrough()
|
||||
|
||||
// update the progress ticker data
|
||||
uploadStream.on('data', (data) => {
|
||||
counter += data.length
|
||||
})
|
||||
|
||||
/**
|
||||
* we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
|
||||
*/
|
||||
pipeline(
|
||||
ffmpegStream,
|
||||
uploadStream,
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.error(`pipeline errored.`)
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log('pipeline succeeded.')
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
await uploadToS3({client, uploadStream, keyName })
|
||||
clearInterval(ticker)
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
main()
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
*
|
||||
* @todo if we have the ffmpeg stream send an end event, does the ffmpegStream close?
|
||||
* so far, we have observed the end of a CB stream, and the uploadStream is what shows as closed.
|
||||
* It would be better to have the ffmpeg stream do the closing, amirite? or does it even matter?
|
||||
* Here's what the console.log shows when the CB stream ended while we were not using { end: true } on the ffmpeg stream
|
||||
*
|
||||
*
|
||||
* [data] uploadStream. 118018880 aggregated bytes (118 MB).
|
||||
[data] uploadStream. 118067384 aggregated bytes (118 MB).
|
||||
[data] uploadStream. 118101224 aggregated bytes (118 MB).
|
||||
[data] uploadStream. 118119648 aggregated bytes (118 MB).
|
||||
[close] uploadStream closed
|
||||
pipeline succeeded.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
import { getRandomRoom } from '@futureporn/scout/cb.js'
|
||||
import { ua0 } from "@futureporn/scout/ua.js";
|
||||
import { spawn } from "child_process";
|
||||
import { PassThrough, pipeline } from "stream";
|
||||
import { type Progress, Upload } from "@aws-sdk/lib-storage";
|
||||
import { S3Client } from "@aws-sdk/client-s3";
|
||||
import { createWriteStream } from 'fs';
|
||||
import ffmpeg from 'fluent-ffmpeg'
|
||||
import { createId } from '@paralleldrive/cuid2';
|
||||
import prettyBytes from 'pretty-bytes';
|
||||
import dotenv from 'dotenv'
|
||||
dotenv.config({
|
||||
path: '../../.env.development'
|
||||
})
|
||||
|
||||
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
|
||||
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
|
||||
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
|
||||
|
||||
async function main() {
|
||||
const client = new S3Client({
|
||||
endpoint: 'https://s3.us-west-000.backblazeb2.com',
|
||||
region: 'us-west-000',
|
||||
credentials: {
|
||||
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
|
||||
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
const randomRoom = await getRandomRoom()
|
||||
console.log(`randomRoom=${randomRoom.name}`)
|
||||
|
||||
const playlistUrl: string = await new Promise((resolve, reject) => {
|
||||
|
||||
|
||||
// get the m3u8 playlist for the livestream
|
||||
const ytdlp = spawn('yt-dlp', [ '-g', randomRoom.url ])
|
||||
let output = ''
|
||||
ytdlp.on('error', (err) => {
|
||||
console.error(err)
|
||||
})
|
||||
ytdlp.once('exit', (code) => {
|
||||
console.log(`code=${code}`)
|
||||
if (code !== 0) reject(`yt-dlp exited with code ${code}. stderr as follows ${JSON.stringify(ytdlp.stderr, null, 2)}`);
|
||||
resolve(output)
|
||||
})
|
||||
ytdlp.stderr.on('data', (data) => {
|
||||
console.error('stderr data as follows')
|
||||
console.error(data.toString())
|
||||
})
|
||||
ytdlp.stdout.on('data', (data) => {
|
||||
output = data
|
||||
})
|
||||
|
||||
})
|
||||
console.log(`playlistUrl=${playlistUrl}`)
|
||||
if (!playlistUrl) throw new Error(`failed to get playlistUrl from yt-dlp -g ${randomRoom.url}`);
|
||||
|
||||
|
||||
let debugCounter = 0
|
||||
let fileOutputStream = createWriteStream('/home/cj/Downloads/outputfile.ts');
|
||||
// let ffmpegLogStream = createWriteStream('/tmp/ffmpeg-log.txt')
|
||||
let uploadStream = new PassThrough()
|
||||
uploadStream.on('data', (data) => {
|
||||
debugCounter += data.length
|
||||
console.log(`[data] uploadStream. ${debugCounter} aggregated bytes (${prettyBytes(debugCounter)}).`)
|
||||
})
|
||||
uploadStream.on('drain', () => {
|
||||
console.log('[drain] uploadStream')
|
||||
})
|
||||
uploadStream.on('close', () => {
|
||||
console.log(`[close] uploadStream closed`)
|
||||
})
|
||||
uploadStream.on('error', (err) => {
|
||||
console.error('[error] uploadStream had an error as follows')
|
||||
console.error(err)
|
||||
})
|
||||
uploadStream.on('exit', (code) => {
|
||||
console.log(`[exit] uploadStream exited with code ${code}`)
|
||||
})
|
||||
uploadStream.on('disconnect', () => {
|
||||
console.log('[disconnect] uploadStream disconnected')
|
||||
})
|
||||
uploadStream.on('message', (msg) => {
|
||||
console.log('[message] uploadStream sent a message as follows')
|
||||
console.log(msg)
|
||||
})
|
||||
|
||||
|
||||
const datestamp = new Date().toISOString()
|
||||
|
||||
|
||||
|
||||
const ffmpegProc = spawn('ffmpeg', [
|
||||
'-headers', `"User-Agent: ${ua0}"`,
|
||||
'-i', playlistUrl,
|
||||
'-c:v', 'copy',
|
||||
'-c:a', 'copy',
|
||||
'-movflags', 'faststart',
|
||||
'-y',
|
||||
'-f', 'mpegts',
|
||||
'-loglevel', 'quiet',
|
||||
'pipe:1'
|
||||
], {
|
||||
// ignoring stderr is important because if not, ffmpeg will fill that buffer and node will hang
|
||||
stdio: ['pipe', 'pipe', 'ignore']
|
||||
})
|
||||
|
||||
|
||||
// we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
|
||||
pipeline(
|
||||
ffmpegProc.stdout,
|
||||
uploadStream,
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.error(`pipeline errored.`)
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log('pipeline succeeded.')
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Create a stream to the S3 bucket. We use this stream to upload the livestream to Backblaze S3 service
|
||||
const keyName = `${datestamp}-${randomRoom.name}-chaturbate-${createId()}.ts`
|
||||
console.log(`keyName=${keyName}`)
|
||||
const target = {
|
||||
Bucket: process.env.S3_BUCKET_NAME,
|
||||
Key: keyName,
|
||||
Body: uploadStream
|
||||
}
|
||||
// greets https://stackoverflow.com/a/70159394/1004931
|
||||
try {
|
||||
const parallelUploads3 = new Upload({
|
||||
client: client,
|
||||
//tags: [...], // optional tags
|
||||
queueSize: 4, // optional concurrency configuration
|
||||
leavePartsOnError: false, // optional manually handle dropped parts
|
||||
params: target,
|
||||
});
|
||||
|
||||
await parallelUploads3.done();
|
||||
|
||||
} catch (e) {
|
||||
if (e instanceof Error) {
|
||||
console.error(`while uploading a file to s3, we encountered an error`)
|
||||
throw new Error(e.message);
|
||||
} else {
|
||||
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
main()
|
||||
|
||||
|
||||
|
|
@ -1,15 +1,36 @@
|
|||
import { type Helpers } from 'graphile-worker'
|
||||
import Record from '../Record.ts'
|
||||
import 'dotenv/config'
|
||||
|
||||
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
|
||||
if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was undefined in env');
|
||||
if (!process.env.S3_REGION) throw new Error('S3_REGION was undefined in env');
|
||||
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was undefined in env');
|
||||
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was undefined in env');
|
||||
|
||||
|
||||
type Payload = {
|
||||
url: string
|
||||
url: string;
|
||||
channel: string;
|
||||
}
|
||||
|
||||
export default async function (payload: Payload, helpers: Helpers) {
|
||||
const { url } = payload;
|
||||
helpers.logger.info(`'record' task execution begin with url=${url} (@todo implement)`);
|
||||
const record = new Record({ url: 'https://example.com/stream' })
|
||||
record.start()
|
||||
export default async function (payload: Payload, helpers: Helpers): Promise<string> {
|
||||
const { url, channel } = payload;
|
||||
helpers.logger.info(`'record' task execution begin with url=${url}, channel=${channel}`);
|
||||
|
||||
|
||||
const bucket = process.env.S3_BUCKET_NAME!
|
||||
const endpoint = process.env.S3_ENDPOINT!
|
||||
const region = process.env.S3_REGION!
|
||||
const accessKeyId = process.env.S3_ACCESS_KEY_ID!
|
||||
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY!
|
||||
|
||||
const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
|
||||
const inputStream = Record.getFFmpegDownload({ url })
|
||||
const record = new Record({ inputStream, bucket, s3Client, channel })
|
||||
await record.start()
|
||||
|
||||
|
||||
|
||||
return record.id
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe } from 'mocha'
|
||||
import { expect } from 'chai';
|
||||
import { getInitialRoomDossier } from './cb.js'
|
||||
import { getInitialRoomDossier, getRandomRoom } from './cb.js'
|
||||
|
||||
describe('cb', function () {
|
||||
describe('getInitialRoomDossier', function () {
|
||||
|
|
@ -13,4 +13,14 @@ describe('cb', function () {
|
|||
expect(dossier).to.have.property('wschat_host')
|
||||
})
|
||||
})
|
||||
describe('getRandomRoom', function () {
|
||||
it('should return a Room object of an online room', async function () {
|
||||
this.timeout(1000*60*2)
|
||||
const room = await getRandomRoom()
|
||||
expect(room).to.have.property('url')
|
||||
expect(room).to.have.property('name')
|
||||
expect(room.name).to.match(/[a-z_]/)
|
||||
expect(room.url).to.match(/https:\/\//)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,4 +1,42 @@
|
|||
import * as cheerio from 'cheerio'
|
||||
import fetch from 'node-fetch'
|
||||
|
||||
|
||||
export interface ChaturbateModel {
|
||||
gender: string;
|
||||
location: string;
|
||||
current_show: 'public' | 'private';
|
||||
username: string;
|
||||
room_subject: string;
|
||||
tags: string[];
|
||||
is_new: boolean;
|
||||
num_users: number;
|
||||
num_followers: number;
|
||||
country: string;
|
||||
spoken_languages: string;
|
||||
display_name: string;
|
||||
birthday: string;
|
||||
is_hd: boolean;
|
||||
age: number;
|
||||
seconds_online: number;
|
||||
image_url: string;
|
||||
image_url_360x270: string;
|
||||
chat_room_url_revshare: string;
|
||||
iframe_embed_revshare: string;
|
||||
chat_room_url: string;
|
||||
iframe_embed: string;
|
||||
slug: string;
|
||||
}
|
||||
|
||||
export interface ChaturbateOnlineModelsResponse {
|
||||
results: ChaturbateModel[],
|
||||
count: number
|
||||
}
|
||||
|
||||
export interface Room {
|
||||
name: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
@ -32,3 +70,36 @@ export async function getInitialRoomDossier(roomUrl: string) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
export async function getRandomRoom(): Promise<Room> {
|
||||
try {
|
||||
const res = await fetch('https://chaturbate.com/api/public/affiliates/onlinerooms/?wm=DiPkB&client_ip=request_ip');
|
||||
const data = await res.json() as ChaturbateOnlineModelsResponse;
|
||||
|
||||
if (!data || !Array.isArray(data.results) || data.results.length === 0) {
|
||||
throw new Error('No results found');
|
||||
}
|
||||
|
||||
const results = data.results;
|
||||
const randomIndex = Math.floor(Math.random() * results.length);
|
||||
|
||||
if (!results[randomIndex]) {
|
||||
throw new Error('No result found at random index');
|
||||
}
|
||||
|
||||
const username = results[randomIndex].username;
|
||||
return {
|
||||
url: `https://chaturbate.com/${username}`,
|
||||
name: username
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error) {
|
||||
console.error(`Error in getRandomRoom: ${error.message}`);
|
||||
} else {
|
||||
console.error('An unexpected error occurred');
|
||||
}
|
||||
throw error; // Re-throw the error to propagate it further
|
||||
}
|
||||
}
|
||||
|
|
@ -43,40 +43,6 @@ importers:
|
|||
specifier: ^4.16.2
|
||||
version: 4.16.2
|
||||
|
||||
../..: {}
|
||||
|
||||
../bot: {}
|
||||
|
||||
../capture: {}
|
||||
|
||||
../image: {}
|
||||
|
||||
../infra: {}
|
||||
|
||||
../mailbox: {}
|
||||
|
||||
../meal: {}
|
||||
|
||||
../next: {}
|
||||
|
||||
../old: {}
|
||||
|
||||
../scout: {}
|
||||
|
||||
../storage: {}
|
||||
|
||||
../strapi: {}
|
||||
|
||||
../taco: {}
|
||||
|
||||
../types: {}
|
||||
|
||||
../uppy: {}
|
||||
|
||||
../video: {}
|
||||
|
||||
../worker: {}
|
||||
|
||||
packages:
|
||||
|
||||
'@cspotcode/source-map-support@0.8.1':
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { dirname, basename, join, isAbsolute } from 'node:path';
|
|||
import { fileURLToPath } from 'url';
|
||||
|
||||
export const __filename = fileURLToPath(import.meta.url);
|
||||
export const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function getPackageVersion(packageJsonPath: string): string {
|
||||
if (!isAbsolute(packageJsonPath)) {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,9 @@ EOF
|
|||
|
||||
kubectl --namespace futureporn delete secret capture --ignore-not-found
|
||||
kubectl --namespace futureporn create secret generic capture \
|
||||
--from-literal=databaseUrl=${WORKER_DATABASE_URL}
|
||||
--from-literal=databaseUrl=${WORKER_DATABASE_URL} \
|
||||
--from-literal=s3AccessKeyId=${S3_ACCESS_KEY_ID} \
|
||||
--from-literal=s3SecretAccessKey=${S3_SECRET_ACCESS_KEY}
|
||||
|
||||
kubectl --namespace futureporn delete secret mailbox --ignore-not-found
|
||||
kubectl --namespace futureporn create secret generic mailbox \
|
||||
|
|
|
|||
|
|
@ -1,3 +1,12 @@
|
|||
|
||||
if [ -z $POSTGRES_PASSWORD ]; then
|
||||
echo "POSTGRES_PASSWORD was missing in env"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
## drop futureporn_db
|
||||
kubectl -n futureporn exec postgres-primary-0 -- psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
|
||||
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
|
||||
|
||||
|
||||
## drop graphile_worker
|
||||
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE graphile_worker WITH (FORCE);"
|
||||
|
|
|
|||
Loading…
Reference in New Issue