capture progress
ci / build (push) Has been cancelled
Details
ci / build (push) Has been cancelled
Details
This commit is contained in:
parent
af8f58940a
commit
4bc11c027e
|
|
@ -0,0 +1,18 @@
|
||||||
|
{
|
||||||
|
"name": "tsx",
|
||||||
|
"type": "node",
|
||||||
|
"request": "launch",
|
||||||
|
|
||||||
|
"program": "${file}",
|
||||||
|
|
||||||
|
"runtimeExecutable": "tsx",
|
||||||
|
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"internalConsoleOptions": "neverOpen",
|
||||||
|
|
||||||
|
"skipFiles": [
|
||||||
|
"<node_internals>/**",
|
||||||
|
|
||||||
|
"${workspaceFolder}/node_modules/**",
|
||||||
|
],
|
||||||
|
}
|
||||||
17
Makefile
17
Makefile
|
|
@ -47,22 +47,7 @@ clean:
|
||||||
dotenvx run -f .env.${ENV} -- node ./packages/infra/vultr-delete-orphaned-resources.js
|
dotenvx run -f .env.${ENV} -- node ./packages/infra/vultr-delete-orphaned-resources.js
|
||||||
|
|
||||||
deps:
|
deps:
|
||||||
echo "Some of the install methods for these dependencies are not cross-platform compatible. Some of the install methods are not tested. Expect this to fail. Please consult the Makefile for URLs to project sources."
|
echo "use `devbox install`"
|
||||||
sudo pamac install make entr nvm kubectl docker helm expect
|
|
||||||
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
|
|
||||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
|
|
||||||
echo "go to https://github.com/txn2/kubefwd/releases/latest to get kubefwd"
|
|
||||||
echo "go to https://github.com/tilt-dev/ctlptl/releases/latest to get ctlptl"
|
|
||||||
sudo systemctl enable docker
|
|
||||||
sudo systemctl start docker
|
|
||||||
usermod -aG docker cj
|
|
||||||
newgrp docker
|
|
||||||
npm i -g pnpm
|
|
||||||
pnpm install -g @dotenvx/dotenvx
|
|
||||||
curl -OL 'https://github.com/vmware-tanzu/velero/releases/download/v1.13.2/velero-v1.13.2-linux-amd64.tar.gz'
|
|
||||||
OS=$(go env GOOS); ARCH=$(go env GOARCH); curl -fsSL -o cmctl https://github.com/cert-manager/cmctl/releases/latest/download/cmctl_${OS}_${ARCH}
|
|
||||||
chmod +x cmctl
|
|
||||||
sudo mv cmctl /usr/local/bin
|
|
||||||
|
|
||||||
# A gitea act runner which runs locally
|
# A gitea act runner which runs locally
|
||||||
# https://docs.gitea.com/next/usage/actions/overview
|
# https://docs.gitea.com/next/usage/actions/overview
|
||||||
|
|
|
||||||
24
README.md
24
README.md
|
|
@ -1,9 +1,33 @@
|
||||||
# futureporn.net
|
# futureporn.net
|
||||||
|
|
||||||
|
[](https://www.jetify.com/devbox/docs/contributor-quickstart/)
|
||||||
|
|
||||||
Source Code for https://futureporn.net
|
Source Code for https://futureporn.net
|
||||||
|
|
||||||
See ./ARCHITECTURE.md for overview
|
See ./ARCHITECTURE.md for overview
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
I'm working towards a better development experience with devbox and Tilt. This process is in a state of flux and is likely to be broken.
|
||||||
|
|
||||||
|
The main gist is as follows
|
||||||
|
|
||||||
|
1. Install [devbox](https://www.jetify.com/devbox/docs/installing_devbox/)
|
||||||
|
2. Install development environment & packages using devbox.
|
||||||
|
|
||||||
|
devbox install
|
||||||
|
|
||||||
|
3. Start a local KIND Kubernetes cluster
|
||||||
|
|
||||||
|
make cluster
|
||||||
|
|
||||||
|
4. Start Tilt
|
||||||
|
|
||||||
|
make tilt
|
||||||
|
|
||||||
|
Tilt will manage the KIND cluster, downloading necessary docker containers and building the containers listed in the fp helm chart at ./Charts/fp. Making changes to these charts or the application code will update or re-build the images as necessary.
|
||||||
|
|
||||||
|
|
||||||
## Metrics Notes
|
## Metrics Notes
|
||||||
|
|
||||||
Keeping track of metrics we want to scrape using Prometheus
|
Keeping track of metrics we want to scrape using Prometheus
|
||||||
|
|
|
||||||
16
Tiltfile
16
Tiltfile
|
|
@ -181,14 +181,20 @@ cmd_button('postgres:drop',
|
||||||
argv=['sh', './scripts/postgres-drop.sh'],
|
argv=['sh', './scripts/postgres-drop.sh'],
|
||||||
resource='postgresql-primary',
|
resource='postgresql-primary',
|
||||||
icon_name='delete',
|
icon_name='delete',
|
||||||
text='DROP futureporn_db'
|
text='DROP all databases'
|
||||||
)
|
)
|
||||||
|
|
||||||
cmd_button('postgres:backup',
|
cmd_button('capture-api:create',
|
||||||
argv=['sh', './scripts/postgres-backup.sh'],
|
argv=['http', '--ignore-stdin', 'POST', 'http://localhost:5003/api/record', "url='https://twitch.tv/ironmouse'", "channel='ironmouse'"],
|
||||||
|
resource='capture-api',
|
||||||
|
icon_name='send',
|
||||||
|
text='Start Recording'
|
||||||
|
)
|
||||||
|
cmd_button('postgres:graphile',
|
||||||
|
argv=['sh', './scripts/postgres-test-graphile.sh'],
|
||||||
resource='postgresql-primary',
|
resource='postgresql-primary',
|
||||||
icon_name='download',
|
icon_name='graph',
|
||||||
text='backup the database'
|
text='create graphile test job',
|
||||||
)
|
)
|
||||||
cmd_button('postgres:graphile',
|
cmd_button('postgres:graphile',
|
||||||
argv=['sh', './scripts/postgres-test-graphile.sh'],
|
argv=['sh', './scripts/postgres-test-graphile.sh'],
|
||||||
|
|
|
||||||
|
|
@ -45,6 +45,22 @@ spec:
|
||||||
key: databaseUrl
|
key: databaseUrl
|
||||||
- name: PORT
|
- name: PORT
|
||||||
value: "{{ .Values.capture.api.port }}"
|
value: "{{ .Values.capture.api.port }}"
|
||||||
|
- name: S3_ENDPOINT
|
||||||
|
value: "{{ .Values.s3.endpoint }}"
|
||||||
|
- name: S3_REGION
|
||||||
|
value: "{{ .Values.s3.region }}"
|
||||||
|
- name: S3_BUCKET_NAME
|
||||||
|
value: "{{ .Values.s3.buckets.usc }}"
|
||||||
|
- name: S3_ACCESS_KEY_ID
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: capture
|
||||||
|
key: s3AccessKeyId
|
||||||
|
- name: S3_SECRET_ACCESS_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: capture
|
||||||
|
key: s3SecretAccessKey
|
||||||
resources:
|
resources:
|
||||||
limits:
|
limits:
|
||||||
cpu: 1000m
|
cpu: 1000m
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,12 @@
|
||||||
environment: development
|
environment: development
|
||||||
# storageClassName: csi-hostpath-sc # used by minikube
|
# storageClassName: csi-hostpath-sc # used by minikube
|
||||||
storageClassName: standard # used by Kind
|
storageClassName: standard # used by Kind
|
||||||
|
s3:
|
||||||
|
endpoint: https://s3.us-west-000.backblazeb2.com
|
||||||
|
region: us-west-000
|
||||||
|
buckets:
|
||||||
|
main: fp-dev
|
||||||
|
usc: fp-usc-dev
|
||||||
link2cid:
|
link2cid:
|
||||||
imageName: fp/link2cid
|
imageName: fp/link2cid
|
||||||
next:
|
next:
|
||||||
|
|
|
||||||
|
|
@ -44,7 +44,9 @@ RUN ls -la /prod/capture
|
||||||
|
|
||||||
## start the app with dumb init to spawn the Node.js runtime process
|
## start the app with dumb init to spawn the Node.js runtime process
|
||||||
## with signal support
|
## with signal support
|
||||||
|
## The mode @futureporn/capture uses when starting is determined by FUNCTION environment variable. (worker|api)
|
||||||
FROM base AS capture
|
FROM base AS capture
|
||||||
|
RUN ls -la /usr/local/bin/yt-dlp
|
||||||
ENV HOSTNAME=0.0.0.0 NODE_ENV=production
|
ENV HOSTNAME=0.0.0.0 NODE_ENV=production
|
||||||
COPY --from=build /prod/capture .
|
COPY --from=build /prod/capture .
|
||||||
CMD [ "dumb-init", "node", "dist/index.js" ]
|
CMD [ "dumb-init", "node", "dist/index.js" ]
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
{
|
||||||
|
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.12.0/.schema/devbox.schema.json",
|
||||||
|
"packages": [
|
||||||
|
"nodejs@20",
|
||||||
|
"tilt@latest",
|
||||||
|
"ctlptl@latest",
|
||||||
|
"kubectl@latest",
|
||||||
|
"cmctl@latest"
|
||||||
|
],
|
||||||
|
"env": {
|
||||||
|
"DEVBOX_COREPACK_ENABLED": "true",
|
||||||
|
"ENV": "development"
|
||||||
|
},
|
||||||
|
"shell": {
|
||||||
|
"init_hook": [
|
||||||
|
"pnpm install"
|
||||||
|
],
|
||||||
|
"scripts": {
|
||||||
|
"test": [
|
||||||
|
"echo \"Error: no test specified\" && exit 1"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,298 @@
|
||||||
|
{
|
||||||
|
"lockfile_version": "1",
|
||||||
|
"packages": {
|
||||||
|
"cmctl@latest": {
|
||||||
|
"last_modified": "2024-07-07T07:43:47Z",
|
||||||
|
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#cmctl",
|
||||||
|
"source": "devbox-search",
|
||||||
|
"version": "1.14.7",
|
||||||
|
"systems": {
|
||||||
|
"aarch64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/69lc5y36840ccy6d1pzph994psadk4bm-cmctl-1.14.7",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/69lc5y36840ccy6d1pzph994psadk4bm-cmctl-1.14.7"
|
||||||
|
},
|
||||||
|
"aarch64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/d5l61iil1gaax5sipnzg80mb0p1hqk9f-cmctl-1.14.7",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/d5l61iil1gaax5sipnzg80mb0p1hqk9f-cmctl-1.14.7"
|
||||||
|
},
|
||||||
|
"x86_64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/j0r3wavi836mp31l0s7r1c3rjryw2z62-cmctl-1.14.7",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/j0r3wavi836mp31l0s7r1c3rjryw2z62-cmctl-1.14.7"
|
||||||
|
},
|
||||||
|
"x86_64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/lnmy96wjzmjna7z9f0dbqd16nf2x5qbv-cmctl-1.14.7",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/lnmy96wjzmjna7z9f0dbqd16nf2x5qbv-cmctl-1.14.7"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ctlptl@latest": {
|
||||||
|
"last_modified": "2024-07-07T07:43:47Z",
|
||||||
|
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#ctlptl",
|
||||||
|
"source": "devbox-search",
|
||||||
|
"version": "0.8.29",
|
||||||
|
"systems": {
|
||||||
|
"aarch64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/gvnmvb315zngbg5a0idynlwxcc45gmyd-ctlptl-0.8.29",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/gvnmvb315zngbg5a0idynlwxcc45gmyd-ctlptl-0.8.29"
|
||||||
|
},
|
||||||
|
"aarch64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/vgwbxwaf773mmgcbc1j5h6n5a7g587lf-ctlptl-0.8.29",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/vgwbxwaf773mmgcbc1j5h6n5a7g587lf-ctlptl-0.8.29"
|
||||||
|
},
|
||||||
|
"x86_64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/ga4rp2188c6k24162s2n23nfi4846790-ctlptl-0.8.29",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/ga4rp2188c6k24162s2n23nfi4846790-ctlptl-0.8.29"
|
||||||
|
},
|
||||||
|
"x86_64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/wbfjbk575ny949rfyqpm4vai4ap5rbpp-ctlptl-0.8.29",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/wbfjbk575ny949rfyqpm4vai4ap5rbpp-ctlptl-0.8.29"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"kubectl@latest": {
|
||||||
|
"last_modified": "2024-07-07T07:43:47Z",
|
||||||
|
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#kubectl",
|
||||||
|
"source": "devbox-search",
|
||||||
|
"version": "1.30.2",
|
||||||
|
"systems": {
|
||||||
|
"aarch64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/i1zidf41bkfzs2l1pq9fi1frymsfgywc-kubectl-1.30.2",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "man",
|
||||||
|
"path": "/nix/store/dzxnn9mk9plcx3w9862jyd0nxys2yywz-kubectl-1.30.2-man",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "convert",
|
||||||
|
"path": "/nix/store/v9ij5fnxxa02jkzpjvkbxw2jc4p9cbld-kubectl-1.30.2-convert"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/i1zidf41bkfzs2l1pq9fi1frymsfgywc-kubectl-1.30.2"
|
||||||
|
},
|
||||||
|
"aarch64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/k7ql4247qs6ny27m3iz5c9xf5gb248a2-kubectl-1.30.2",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "man",
|
||||||
|
"path": "/nix/store/wy64r4nn3isydw4nx257h95qy2x2z4mx-kubectl-1.30.2-man",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "convert",
|
||||||
|
"path": "/nix/store/ic8za302hvb4kf4zrs55ivr4q2n2lznn-kubectl-1.30.2-convert"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/k7ql4247qs6ny27m3iz5c9xf5gb248a2-kubectl-1.30.2"
|
||||||
|
},
|
||||||
|
"x86_64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/v029n959l5b289br0cq591b04yc48516-kubectl-1.30.2",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "man",
|
||||||
|
"path": "/nix/store/0dvcxn7gsi2ycy9blb7pcy506w4xp2vi-kubectl-1.30.2-man",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "convert",
|
||||||
|
"path": "/nix/store/2nfq4ivwa4a7jwc0183f2wpl1jxbn754-kubectl-1.30.2-convert"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/v029n959l5b289br0cq591b04yc48516-kubectl-1.30.2"
|
||||||
|
},
|
||||||
|
"x86_64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/3vkf0406s1i6l89hk5wrakh4bbn0p1p2-kubectl-1.30.2",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "man",
|
||||||
|
"path": "/nix/store/3wbvgkkka1knkxvyr4c8qbpr448smw8i-kubectl-1.30.2-man",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "convert",
|
||||||
|
"path": "/nix/store/h5zxz8db6wligwhw5gnwk4gbc8j1ixik-kubectl-1.30.2-convert"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/3vkf0406s1i6l89hk5wrakh4bbn0p1p2-kubectl-1.30.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nodejs@20": {
|
||||||
|
"last_modified": "2024-07-07T07:43:47Z",
|
||||||
|
"plugin_version": "0.0.2",
|
||||||
|
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#nodejs_20",
|
||||||
|
"source": "devbox-search",
|
||||||
|
"version": "20.14.0",
|
||||||
|
"systems": {
|
||||||
|
"aarch64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/sqnbldm1fjw88v23yq4v6531y4m7v2fh-nodejs-20.14.0",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "libv8",
|
||||||
|
"path": "/nix/store/1i0rb2axkrxvsq5pz8s2q07ard2p36a1-nodejs-20.14.0-libv8"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/sqnbldm1fjw88v23yq4v6531y4m7v2fh-nodejs-20.14.0"
|
||||||
|
},
|
||||||
|
"aarch64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/r1nwmlbsn67f5rhayr7jjjdmiflxpk92-nodejs-20.14.0",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "libv8",
|
||||||
|
"path": "/nix/store/5ii3xkbd3iv0xvqqvjg3agsm0dinidgm-nodejs-20.14.0-libv8"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/r1nwmlbsn67f5rhayr7jjjdmiflxpk92-nodejs-20.14.0"
|
||||||
|
},
|
||||||
|
"x86_64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/wzgnws4r1c98vzj5q6gq4drz2jfq7d5q-nodejs-20.14.0",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "libv8",
|
||||||
|
"path": "/nix/store/gc2gnkc8hvkh51ab3a29fvgzy2qsqb2s-nodejs-20.14.0-libv8"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/wzgnws4r1c98vzj5q6gq4drz2jfq7d5q-nodejs-20.14.0"
|
||||||
|
},
|
||||||
|
"x86_64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/ilkfhnqz4xczrliqjva8770x2svbfznd-nodejs-20.14.0",
|
||||||
|
"default": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "libv8",
|
||||||
|
"path": "/nix/store/2qaf68dzimr8as4bgli0xmsn11c0ah2j-nodejs-20.14.0-libv8"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/ilkfhnqz4xczrliqjva8770x2svbfznd-nodejs-20.14.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tilt@latest": {
|
||||||
|
"last_modified": "2024-07-15T21:47:20Z",
|
||||||
|
"resolved": "github:NixOS/nixpkgs/b2c1f10bfbb3f617ea8e8669ac13f3f56ceb2ea2#tilt",
|
||||||
|
"source": "devbox-search",
|
||||||
|
"version": "0.33.17",
|
||||||
|
"systems": {
|
||||||
|
"aarch64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/l19qinywsx7y86xp8vgwr3bgnbi0rfcj-tilt-0.33.17",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/l19qinywsx7y86xp8vgwr3bgnbi0rfcj-tilt-0.33.17"
|
||||||
|
},
|
||||||
|
"aarch64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/f6swxifmvnxjxifxyw4k4aiyxh0dgfyq-tilt-0.33.17",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/f6swxifmvnxjxifxyw4k4aiyxh0dgfyq-tilt-0.33.17"
|
||||||
|
},
|
||||||
|
"x86_64-darwin": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/d59rlsabcqxax6bgw6d30zhmflw65ch0-tilt-0.33.17",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/d59rlsabcqxax6bgw6d30zhmflw65ch0-tilt-0.33.17"
|
||||||
|
},
|
||||||
|
"x86_64-linux": {
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "out",
|
||||||
|
"path": "/nix/store/qfv96sjcsslynqbilwj823x8nxvgj5cv-tilt-0.33.17",
|
||||||
|
"default": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"store_path": "/nix/store/qfv96sjcsslynqbilwj823x8nxvgj5cv-tilt-0.33.17"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -15,6 +15,10 @@ Worker container runs a Graphile Worker which listens for specific tasks related
|
||||||
|
|
||||||
## Misc dev notes
|
## Misc dev notes
|
||||||
|
|
||||||
|
### idea for taking snapshots of stream in progress
|
||||||
|
|
||||||
|
https://ofarukcaki.medium.com/producing-real-time-video-with-node-js-and-ffmpeg-a59ac27461a1
|
||||||
|
|
||||||
### youtube-dl end of stream output
|
### youtube-dl end of stream output
|
||||||
|
|
||||||
The end-of-stream output from yt-dlp when recording CB looks like this
|
The end-of-stream output from yt-dlp when recording CB looks like this
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@
|
||||||
"@futureporn/utils": "workspace:^",
|
"@futureporn/utils": "workspace:^",
|
||||||
"@paralleldrive/cuid2": "^2.2.2",
|
"@paralleldrive/cuid2": "^2.2.2",
|
||||||
"@types/chai": "^4.3.16",
|
"@types/chai": "^4.3.16",
|
||||||
|
"@types/fluent-ffmpeg": "^2.1.24",
|
||||||
"@types/mocha": "^10.0.7",
|
"@types/mocha": "^10.0.7",
|
||||||
"@types/node": "^20.14.11",
|
"@types/node": "^20.14.11",
|
||||||
"diskusage": "^1.2.0",
|
"diskusage": "^1.2.0",
|
||||||
|
|
@ -46,8 +47,12 @@
|
||||||
"youtube-dl-wrap": "github:insanity54/youtube-dl-wrap"
|
"youtube-dl-wrap": "github:insanity54/youtube-dl-wrap"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@smithy/util-stream": "^3.1.2",
|
||||||
"@types/sinon": "^17.0.3",
|
"@types/sinon": "^17.0.3",
|
||||||
"@types/sinon-chai": "^3.2.12",
|
"@types/sinon-chai": "^3.2.12",
|
||||||
|
"aws-sdk": "^2.1663.0",
|
||||||
|
"aws-sdk-client-mock": "^4.0.1",
|
||||||
|
"aws-sdk-mock": "^6.0.4",
|
||||||
"chai": "^4.4.1",
|
"chai": "^4.4.1",
|
||||||
"cheerio": "1.0.0-rc.12",
|
"cheerio": "1.0.0-rc.12",
|
||||||
"mocha": "^10.7.0",
|
"mocha": "^10.7.0",
|
||||||
|
|
@ -55,6 +60,8 @@
|
||||||
"node-abort-controller": "^3.1.1",
|
"node-abort-controller": "^3.1.1",
|
||||||
"node-fetch": "^3.3.2",
|
"node-fetch": "^3.3.2",
|
||||||
"nodemon": "^2.0.22",
|
"nodemon": "^2.0.22",
|
||||||
|
"pretty-bytes": "^6.1.1",
|
||||||
|
"s3": "link:aws-sdk/clients/s3",
|
||||||
"sinon": "^15.2.0",
|
"sinon": "^15.2.0",
|
||||||
"sinon-chai": "^3.7.0",
|
"sinon-chai": "^3.7.0",
|
||||||
"sinon-test": "^3.1.6",
|
"sinon-test": "^3.1.6",
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,22 +1,67 @@
|
||||||
import Record from "./Record.js"
|
import Record from "./Record.js"
|
||||||
import { expect } from "chai"
|
import { expect } from "chai"
|
||||||
|
import { ChildProcess, spawn } from "child_process"
|
||||||
|
import { createReadStream, readFileSync, ReadStream } from "fs"
|
||||||
|
import AWSMock from 'aws-sdk-mock'
|
||||||
|
import sinon from 'sinon';
|
||||||
|
import sinonChai from 'sinon-chai';
|
||||||
|
import { PutObjectCommand, PutObjectCommandInput, S3Client, CreateMultipartUploadCommand, UploadPartCommand, UploadPartCommandInput } from "@aws-sdk/client-s3"
|
||||||
|
import { join, dirname } from "path"
|
||||||
|
import { fileURLToPath } from "url"
|
||||||
|
import { S3 } from 'aws-sdk';
|
||||||
|
import { HeadObjectOutput } from 'aws-sdk/clients/s3';
|
||||||
|
import { Readable } from 'stream';
|
||||||
|
import { mockClient } from 'aws-sdk-client-mock';
|
||||||
|
import { sdkStreamMixin } from '@smithy/util-stream'
|
||||||
|
|
||||||
|
|
||||||
|
// "pay no attention to that man behind the curtain"
|
||||||
|
|
||||||
|
// export function getObjectReadStream(s3Client: S3, Bucket: string, Key: string): Readable {
|
||||||
|
// return s3Client.getObject({ Bucket, Key }).createReadStream();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// export async function waitForObjectExists(
|
||||||
|
// s3Client: S3Client,
|
||||||
|
// Bucket: string,
|
||||||
|
// Key: string
|
||||||
|
// ): Promise<HeadObjectOutput> {
|
||||||
|
// return s3Client.waitFor('objectExists', { Bucket, Key }).promise();
|
||||||
|
// }
|
||||||
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
const testStreamDir = '/tmp/record-test-stream.m3u8'
|
||||||
|
|
||||||
describe('Record', function () {
|
describe('Record', function () {
|
||||||
let record: Record
|
// mocking @aws-sdk/lib-storage Upload() has some nuances.
|
||||||
this.beforeEach(function () {
|
// @see https://github.com/m-radzikowski/aws-sdk-client-mock?tab=readme-ov-file#lib-storage-upload
|
||||||
record = new Record({ url: 'https://example.com/my-cool-stream' })
|
|
||||||
|
|
||||||
|
it('should accept a {ReadStream} as input', async function () {
|
||||||
|
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) // 192627 bytes
|
||||||
|
const s3ClientMock = mockClient(S3Client)
|
||||||
|
s3ClientMock.on(CreateMultipartUploadCommand).resolves({UploadId: '1'});
|
||||||
|
s3ClientMock.on(UploadPartCommand).resolves({ETag: '1'});
|
||||||
|
const s3Client = new S3Client({ region: 'us-west-000' })
|
||||||
|
const record = new Record({ inputStream, s3Client, channel: 'coolguy_69', bucket: 'test' })
|
||||||
|
await record.start()
|
||||||
|
expect(record).to.have.property('counter', 192627)
|
||||||
|
expect(record).to.have.property('bucket', 'test')
|
||||||
})
|
})
|
||||||
describe('start()', function () {
|
|
||||||
it('should start the recording', async function () {
|
xit('should restart if a EPIPE is encountered', async function () {
|
||||||
await record.start()
|
// @todo IDK how to implement this.
|
||||||
expect(record).to.have.property('id')
|
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4'))
|
||||||
expect(record).to.have.property('url')
|
const s3ClientMock = mockClient(S3Client)
|
||||||
})
|
s3ClientMock.on(CreateMultipartUploadCommand).resolves({UploadId: '1'})
|
||||||
})
|
s3ClientMock.on(UploadPartCommand).rejectsOnce({cause: 'simulated network interruption'}).resolves({ ETag: '1' }) // this rejection is probably not specific enough to simulate EPIPE
|
||||||
describe('stop()', function () {
|
const s3Client = new S3Client({ region: 'us-west-000' })
|
||||||
it('should stop the recording', async function () {
|
const record = new Record({ inputStream, s3Client, channel: 'coolguy_69', bucket: 'test' })
|
||||||
await record.stop()
|
await record.start()
|
||||||
expect(record).to.have.property('cdnUrl')
|
expect(record).to.have.property('counter', 192627)
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
})
|
})
|
||||||
|
|
@ -1,102 +1,184 @@
|
||||||
import { createId } from '@paralleldrive/cuid2'
|
import { createId } from '@paralleldrive/cuid2'
|
||||||
import { spawn } from 'child_process';
|
import { spawn } from 'child_process';
|
||||||
import { ua0 } from '@futureporn/scout/ua.js'
|
import { ua0 } from '@futureporn/scout/ua.js'
|
||||||
import { getTmpFile } from '@futureporn/utils'
|
import { PassThrough, pipeline, Readable } from 'stream';
|
||||||
|
import prettyBytes from 'pretty-bytes';
|
||||||
|
import { Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import { S3Client } from "@aws-sdk/client-s3";
|
||||||
|
import 'dotenv/config'
|
||||||
|
|
||||||
|
|
||||||
export interface RecordArgs {
|
export interface RecordArgs {
|
||||||
url: string;
|
|
||||||
filename?: string;
|
filename?: string;
|
||||||
channel?: string;
|
channel: string;
|
||||||
|
s3Client: S3Client;
|
||||||
|
bucket: string;
|
||||||
date?: string;
|
date?: string;
|
||||||
|
inputStream: Readable;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface MakeS3ClientOptions {
|
||||||
|
accessKeyId: string;
|
||||||
|
secretAccessKey: string;
|
||||||
|
region: string;
|
||||||
|
endpoint: string
|
||||||
|
}
|
||||||
|
|
||||||
|
interface getFFmpegDownloadOptions {
|
||||||
|
url: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export default class Record {
|
export default class Record {
|
||||||
readonly id: string;
|
readonly id: string;
|
||||||
readonly url: string;
|
private s3Client: S3Client;
|
||||||
|
private uploadStream: PassThrough;
|
||||||
|
private ticker?: NodeJS.Timeout;
|
||||||
|
inputStream: Readable;
|
||||||
|
counter: number;
|
||||||
|
bucket: string;
|
||||||
|
keyName: string;
|
||||||
|
datestamp: string;
|
||||||
filename?: string;
|
filename?: string;
|
||||||
channel?: string;
|
channel: string;
|
||||||
date?: string;
|
date?: string;
|
||||||
|
|
||||||
constructor({ url }: RecordArgs) {
|
constructor({ inputStream, channel, s3Client, bucket }: RecordArgs) {
|
||||||
if (!url) throw new Error('url passed to Record constructor was undefined.');
|
if (!inputStream) throw new Error('Record constructor was missing inputStream.');
|
||||||
|
if (!bucket) throw new Error('Record constructor was missing bucket.');
|
||||||
|
if (!channel) throw new Error('Record constructer was missing channel!');
|
||||||
|
if (!s3Client) throw new Error('Record constructer was missing s3Client');
|
||||||
|
this.inputStream = inputStream
|
||||||
this.id = createId()
|
this.id = createId()
|
||||||
this.url = url
|
this.s3Client = s3Client
|
||||||
|
this.bucket = bucket
|
||||||
|
this.channel = channel
|
||||||
|
this.counter = 0
|
||||||
|
this.datestamp = new Date().toISOString()
|
||||||
|
this.keyName = `${this.datestamp}-${channel}-${createId()}.ts`
|
||||||
|
this.uploadStream = new PassThrough()
|
||||||
}
|
}
|
||||||
|
|
||||||
async start() {
|
makeProgressTicker() {
|
||||||
console.log(`@TODO record start with id=${this.id}, url=${this.url}`)
|
this.ticker = setInterval(() => {
|
||||||
|
console.log(`[progress] ${this.counter} bytes (aggregate) (${prettyBytes(this.counter)}) have passed through the pipeline.`)
|
||||||
|
}, 1000 * 30)
|
||||||
|
}
|
||||||
|
|
||||||
const playlistUrlPromise = new Promise<string>((resolve) => {
|
|
||||||
const ytdlp = spawn('yt-dlp', [
|
static makeS3Client({
|
||||||
'-g',
|
accessKeyId,
|
||||||
this.url
|
secretAccessKey,
|
||||||
])
|
region,
|
||||||
ytdlp.stdout.on('data', (data) => {
|
endpoint
|
||||||
resolve(data)
|
}: MakeS3ClientOptions): S3Client {
|
||||||
})
|
const client = new S3Client({
|
||||||
|
endpoint,
|
||||||
|
region,
|
||||||
|
credentials: {
|
||||||
|
accessKeyId,
|
||||||
|
secretAccessKey
|
||||||
|
}
|
||||||
})
|
})
|
||||||
const playlistUrl = await playlistUrlPromise
|
return client
|
||||||
const filename = getTmpFile(`stream.ts`)
|
}
|
||||||
|
|
||||||
const ffmpeg = spawn('ffmpeg', [
|
static getFFmpegDownload({ url }: getFFmpegDownloadOptions): Readable {
|
||||||
|
|
||||||
|
|
||||||
|
const ffmpegProc = spawn('ffmpeg', [
|
||||||
'-headers', `"User-Agent: ${ua0}"`,
|
'-headers', `"User-Agent: ${ua0}"`,
|
||||||
'-i', playlistUrl,
|
'-i', url,
|
||||||
'-c:v', 'copy',
|
'-c:v', 'copy',
|
||||||
'-c:a', 'copy',
|
'-c:a', 'copy',
|
||||||
'-movflags', 'faststart',
|
'-movflags', 'faststart',
|
||||||
'-y',
|
'-y',
|
||||||
'-f', 'mpegts',
|
'-f', 'mpegts',
|
||||||
filename
|
'-loglevel', 'quiet',
|
||||||
])
|
'pipe:1'
|
||||||
|
|
||||||
const ffmpegProcess = spawn('ffmpeg', [
|
|
||||||
'-headers', `"User-Agent: ${ua0}"`,
|
|
||||||
'-i', playlistUrl,
|
|
||||||
'-c:v', 'copy',
|
|
||||||
'-c:a', 'copy',
|
|
||||||
'-movflags', 'faststart',
|
|
||||||
'-y',
|
|
||||||
'-f', 'mpegts',
|
|
||||||
filename
|
|
||||||
], {
|
], {
|
||||||
stdio: 'inherit'
|
// ignoring stderr is important because if not, ffmpeg will fill that buffer and node will hang
|
||||||
});
|
stdio: ['pipe', 'pipe', 'ignore']
|
||||||
|
})
|
||||||
|
return ffmpegProc.stdout
|
||||||
|
|
||||||
const ps = spawn('ps', ['ax']);
|
}
|
||||||
const grep = spawn('grep', ['ssh']);
|
|
||||||
|
|
||||||
ps.stdout.on('data', (data) => {
|
async uploadToS3() {
|
||||||
grep.stdin.write(data);
|
|
||||||
});
|
const target = {
|
||||||
|
Bucket: this.bucket,
|
||||||
ps.stderr.on('data', (data) => {
|
Key: this.keyName,
|
||||||
console.error(`ps stderr: ${data}`);
|
// We do this to keep TS happy. Body expects a Readable, not a ReadableStream nor a NodeJS.ReadableStream
|
||||||
});
|
// Body: new Readable().wrap(this.uploadStream)
|
||||||
|
Body: this.uploadStream
|
||||||
ps.on('close', (code) => {
|
}
|
||||||
if (code !== 0) {
|
|
||||||
console.log(`ps process exited with code ${code}`);
|
// greets https://stackoverflow.com/a/70159394/1004931
|
||||||
|
try {
|
||||||
|
const parallelUploads3 = new Upload({
|
||||||
|
client: this.s3Client,
|
||||||
|
partSize: 1024 * 1024 * 5,
|
||||||
|
queueSize: 1,
|
||||||
|
leavePartsOnError: false,
|
||||||
|
params: target,
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||||
|
console.log(progress)
|
||||||
|
if (progress?.loaded) {
|
||||||
|
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
|
||||||
|
} else {
|
||||||
|
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await parallelUploads3.done();
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error) {
|
||||||
|
console.error(`while uploading a file to s3, we encountered an error`)
|
||||||
|
throw new Error(e.message);
|
||||||
|
} else {
|
||||||
|
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||||
}
|
}
|
||||||
grep.stdin.end();
|
}
|
||||||
});
|
}
|
||||||
|
|
||||||
grep.stdout.on('data', (data) => {
|
|
||||||
console.log(data.toString());
|
|
||||||
});
|
|
||||||
|
|
||||||
grep.stderr.on('data', (data) => {
|
|
||||||
console.error(`grep stderr: ${data}`);
|
|
||||||
});
|
|
||||||
|
|
||||||
grep.on('close', (code) => {
|
async start() {
|
||||||
if (code !== 0) {
|
|
||||||
console.log(`grep process exited with code ${code}`);
|
this.makeProgressTicker()
|
||||||
|
|
||||||
|
// streams setup
|
||||||
|
this.uploadStream.on('data', (data) => {
|
||||||
|
this.counter += data.length
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
// stream pipeline setup
|
||||||
|
pipeline(
|
||||||
|
this.inputStream,
|
||||||
|
this.uploadStream,
|
||||||
|
(err) => {
|
||||||
|
if (err) {
|
||||||
|
console.error(`pipeline errored.`)
|
||||||
|
console.error(err)
|
||||||
|
} else {
|
||||||
|
console.log('pipeline succeeded.')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
)
|
||||||
|
|
||||||
|
await this.uploadToS3()
|
||||||
|
clearInterval(this.ticker)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id: this.id,
|
id: this.id,
|
||||||
url: this.url
|
keyName: this.keyName,
|
||||||
|
channel: this.channel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,8 @@ import graphileWorkerPlugin, { type ExtendedFastifyInstance } from './fastify-gr
|
||||||
|
|
||||||
const version = getPackageVersion('../package.json')
|
const version = getPackageVersion('../package.json')
|
||||||
interface RecordBodyType {
|
interface RecordBodyType {
|
||||||
url: string
|
url: string;
|
||||||
|
channel: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function build(opts: Record<string, any>={}, connectionString: string) {
|
function build(opts: Record<string, any>={}, connectionString: string) {
|
||||||
|
|
@ -17,9 +18,10 @@ function build(opts: Record<string, any>={}, connectionString: string) {
|
||||||
return { app: '@futureporn/capture', version }
|
return { app: '@futureporn/capture', version }
|
||||||
})
|
})
|
||||||
app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) {
|
app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) {
|
||||||
const { url } = request.body
|
const { url, channel } = request.body
|
||||||
console.log(`POST /api/record with url=${url}`)
|
console.log(`POST /api/record with url=${url}, channel=${channel}`)
|
||||||
const job = await app.graphile.addJob('record', { url })
|
|
||||||
|
const job = await app.graphile.addJob('record', { url, channel })
|
||||||
return job
|
return job
|
||||||
})
|
})
|
||||||
return app
|
return app
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
application/vnd.apple.mpegurl mp4
|
||||||
|
text/x-abc abc
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
asdfaslfk;sdf
|
||||||
|
|
@ -1,77 +0,0 @@
|
||||||
|
|
||||||
'use strict'
|
|
||||||
|
|
||||||
import { build } from './app.js'
|
|
||||||
import 'dotenv/config'
|
|
||||||
import { run } from 'graphile-worker'
|
|
||||||
import { dirname } from 'node:path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env');
|
|
||||||
if (!process.env.FUNCTION) throw new Error(`FUNCTION env var was missing. FUNCTION env var must be either 'api' or 'worker'.`);
|
|
||||||
const connectionString = process.env.DATABASE_URL!
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async function api() {
|
|
||||||
if (!process.env.PORT) throw new Error('PORT is missing in env');
|
|
||||||
const PORT = parseInt(process.env.PORT!)
|
|
||||||
|
|
||||||
const fastifyOpts = {
|
|
||||||
logger: {
|
|
||||||
level: 'info',
|
|
||||||
transport: {
|
|
||||||
target: 'pino-pretty'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const server = build(fastifyOpts, connectionString)
|
|
||||||
|
|
||||||
server.listen({ port: PORT }, (err) => {
|
|
||||||
if (err) {
|
|
||||||
server.log.error(err)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
async function worker() {
|
|
||||||
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
|
|
||||||
|
|
||||||
// Run a worker to execute jobs:
|
|
||||||
const runner = await run({
|
|
||||||
connectionString,
|
|
||||||
concurrency,
|
|
||||||
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
|
|
||||||
noHandleSignals: false,
|
|
||||||
pollInterval: 1000,
|
|
||||||
taskDirectory: `${__dirname}/tasks`,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Immediately await (or otherwise handle) the resulting promise, to avoid
|
|
||||||
// "unhandled rejection" errors causing a process crash in the event of
|
|
||||||
// something going wrong. console.log()
|
|
||||||
|
|
||||||
await runner.promise;
|
|
||||||
|
|
||||||
// If the worker exits (whether through fatal error or otherwise), the above
|
|
||||||
// promise will resolve/reject.
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
if (process.env.FUNCTION === 'api') {
|
|
||||||
api()
|
|
||||||
} else if (process.env.FUNCTION === 'worker') {
|
|
||||||
worker()
|
|
||||||
} else {
|
|
||||||
throw new Error('process.env.FUNCTION must be either api or worker. got '+process.env.FUNCTION)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error(err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
@ -1,80 +0,0 @@
|
||||||
'use strict'
|
|
||||||
|
|
||||||
import { build } from './app.js'
|
|
||||||
import 'dotenv/config'
|
|
||||||
import { run } from 'graphile-worker'
|
|
||||||
import { dirname } from 'node:path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env');
|
|
||||||
if (!process.env.FUNCTION) throw new Error(`FUNCTION env var was missing. FUNCTION env var must be either 'api' or 'worker'.`);
|
|
||||||
const connectionString = process.env.DATABASE_URL!
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async function api() {
|
|
||||||
if (!process.env.PORT) throw new Error('PORT is missing in env');
|
|
||||||
const PORT = parseInt(process.env.PORT!)
|
|
||||||
|
|
||||||
const fastifyOpts = {
|
|
||||||
logger: {
|
|
||||||
level: 'info',
|
|
||||||
transport: {
|
|
||||||
target: 'pino-pretty'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const server = build(fastifyOpts, connectionString)
|
|
||||||
|
|
||||||
server.listen({ port: PORT }, (err) => {
|
|
||||||
if (err) {
|
|
||||||
server.log.error(err)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async function worker() {
|
|
||||||
if (!process.env.WORKER_CONCURRENCY) throw new Error('WORKER_CONCURRENCY is missing in env');
|
|
||||||
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
|
|
||||||
|
|
||||||
// Run a worker to execute jobs:
|
|
||||||
const runner = await run({
|
|
||||||
connectionString: process.env.DATABASE_URL!,
|
|
||||||
concurrency,
|
|
||||||
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
|
|
||||||
noHandleSignals: false,
|
|
||||||
pollInterval: 1000,
|
|
||||||
taskDirectory: `${__dirname}/tasks`,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Immediately await (or otherwise handle) the resulting promise, to avoid
|
|
||||||
// "unhandled rejection" errors causing a process crash in the event of
|
|
||||||
// something going wrong. console.log()
|
|
||||||
|
|
||||||
await runner.promise;
|
|
||||||
|
|
||||||
// If the worker exits (whether through fatal error or otherwise), the above
|
|
||||||
// promise will resolve/reject.
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
if (process.env.FUNCTION === 'worker') {
|
|
||||||
worker()
|
|
||||||
} else if (process.env.FUNCTION === 'api') {
|
|
||||||
api()
|
|
||||||
} else {
|
|
||||||
console.error(`FUNCTION environment variable must be 'worker' or 'api', but it was set to ${process.env.FUNCTION}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error(err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
'use strict'
|
|
||||||
|
|
||||||
import { run } from 'graphile-worker'
|
|
||||||
import { dirname } from 'node:path';
|
|
||||||
import { fileURLToPath } from 'url';
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
|
|
||||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is undefined in env');
|
|
||||||
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async function main() {
|
|
||||||
// Run a worker to execute jobs:
|
|
||||||
const runner = await run({
|
|
||||||
connectionString: process.env.DATABASE_URL!,
|
|
||||||
concurrency,
|
|
||||||
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
|
|
||||||
noHandleSignals: false,
|
|
||||||
pollInterval: 1000,
|
|
||||||
taskDirectory: `${__dirname}/tasks`,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Immediately await (or otherwise handle) the resulting promise, to avoid
|
|
||||||
// "unhandled rejection" errors causing a process crash in the event of
|
|
||||||
// something going wrong. console.log()
|
|
||||||
|
|
||||||
await runner.promise;
|
|
||||||
|
|
||||||
// If the worker exits (whether through fatal error or otherwise), the above
|
|
||||||
// promise will resolve/reject.
|
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error(err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
import { createWriteStream } from 'node:fs'
|
||||||
|
import ffmpeg from 'fluent-ffmpeg'
|
||||||
|
|
||||||
|
// test stream from https://ottverse.com/free-hls-m3u8-test-urls/
|
||||||
|
const playlistUrl = 'https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8'
|
||||||
|
const fileOutputStream = createWriteStream('/tmp/test-stream.ts')
|
||||||
|
|
||||||
|
ffmpeg()
|
||||||
|
.input(playlistUrl)
|
||||||
|
.audioCodec('copy')
|
||||||
|
.videoCodec('copy')
|
||||||
|
.addOutputOptions('-movflags faststart')
|
||||||
|
.output(fileOutputStream)
|
||||||
|
.format('mpegts')
|
||||||
|
.on('end', () => {
|
||||||
|
console.log('Finished');
|
||||||
|
})
|
||||||
|
.on('error', (err, stdout, stderr) => {
|
||||||
|
console.error(`there was an error`);
|
||||||
|
console.error(err);
|
||||||
|
console.error(stdout);
|
||||||
|
console.error(stderr);
|
||||||
|
throw new Error(err.message);
|
||||||
|
})
|
||||||
|
.run();
|
||||||
|
|
@ -0,0 +1,135 @@
|
||||||
|
import { PassThrough, pipeline, Readable } from "stream";
|
||||||
|
import { type Progress, Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import { S3Client } from "@aws-sdk/client-s3";
|
||||||
|
import { createReadStream, createWriteStream } from 'fs';
|
||||||
|
import { createId } from '@paralleldrive/cuid2';
|
||||||
|
import prettyBytes from 'pretty-bytes';
|
||||||
|
import dotenv from 'dotenv'
|
||||||
|
dotenv.config({
|
||||||
|
path: '../../.env.development'
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
|
||||||
|
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
|
||||||
|
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
|
||||||
|
|
||||||
|
async function uploadStream(client: S3Client, stream: NodeJS.ReadableStream, keyName: string) {
|
||||||
|
// const pass = new PassThrough()
|
||||||
|
// Create a stream to the S3 bucket. We use this stream to upload the livestream to Backblaze S3 service
|
||||||
|
console.log(`keyName=${keyName}`)
|
||||||
|
const target = {
|
||||||
|
Bucket: process.env.S3_BUCKET_NAME!,
|
||||||
|
Key: keyName,
|
||||||
|
Body: new Readable().wrap(stream)
|
||||||
|
}
|
||||||
|
console.log(target)
|
||||||
|
// greets https://stackoverflow.com/a/70159394/1004931
|
||||||
|
try {
|
||||||
|
const parallelUploads3 = new Upload({
|
||||||
|
client: client,
|
||||||
|
partSize: 1024 * 1024 * 5,
|
||||||
|
// tags: [...], // optional tags
|
||||||
|
queueSize: 1, // optional concurrency configuration
|
||||||
|
leavePartsOnError: false, // optional manually handle dropped parts
|
||||||
|
params: target,
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||||
|
if (progress?.loaded) {
|
||||||
|
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
|
||||||
|
} else {
|
||||||
|
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await parallelUploads3.done();
|
||||||
|
return res
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error) {
|
||||||
|
console.error(`while uploading a file to s3, we encountered an error`)
|
||||||
|
throw new Error(e.message);
|
||||||
|
} else {
|
||||||
|
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
try {
|
||||||
|
const client = new S3Client({
|
||||||
|
endpoint: 'https://s3.us-west-000.backblazeb2.com',
|
||||||
|
region: 'us-west-000',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
|
||||||
|
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// let debugCounter = 0
|
||||||
|
// let uploadStream = new PassThrough()
|
||||||
|
// uploadStream.on('data', (data) => {
|
||||||
|
// debugCounter += data.length
|
||||||
|
// console.log(`[data] uploadStream. ${debugCounter} aggregated bytes (${prettyBytes(debugCounter)}).`)
|
||||||
|
// })
|
||||||
|
// uploadStream.on('drain', () => {
|
||||||
|
// console.log('[drain] uploadStream')
|
||||||
|
// })
|
||||||
|
// uploadStream.on('close', () => {
|
||||||
|
// console.log(`[close] uploadStream closed`)
|
||||||
|
// })
|
||||||
|
// uploadStream.on('error', (err) => {
|
||||||
|
// console.error('[error] uploadStream had an error as follows')
|
||||||
|
// console.error(err)
|
||||||
|
// })
|
||||||
|
// uploadStream.on('exit', (code) => {
|
||||||
|
// console.log(`[exit] uploadStream exited with code ${code}`)
|
||||||
|
// })
|
||||||
|
// uploadStream.on('disconnect', () => {
|
||||||
|
// console.log('[disconnect] uploadStream disconnected')
|
||||||
|
// })
|
||||||
|
// uploadStream.on('message', (msg) => {
|
||||||
|
// console.log('[message] uploadStream sent a message as follows')
|
||||||
|
// console.log(msg)
|
||||||
|
// })
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
const fileStream = createReadStream('/home/cj/Downloads/stream-23894234.ts')
|
||||||
|
const datestamp = new Date().toISOString()
|
||||||
|
const keyName = `${datestamp}-stream3-chaturbate-${createId()}.ts`
|
||||||
|
await uploadStream(client, fileStream, keyName)
|
||||||
|
|
||||||
|
// // we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
|
||||||
|
// pipeline(
|
||||||
|
// fileStream,
|
||||||
|
// uploadStream,
|
||||||
|
// (err) => {
|
||||||
|
// if (err) {
|
||||||
|
// console.error(`pipeline errored.`)
|
||||||
|
// console.error(err)
|
||||||
|
// } else {
|
||||||
|
// console.log('pipeline succeeded.')
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// )
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error) {
|
||||||
|
console.error(`Eyy lookat me, I'm a big nastry try/catch block and I did my jorb!`)
|
||||||
|
console.error(e)
|
||||||
|
} else {
|
||||||
|
console.error('err or some sort')
|
||||||
|
console.error(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,122 @@
|
||||||
|
import { PassThrough, pipeline, Readable } from "stream";
|
||||||
|
import { Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import { S3Client } from "@aws-sdk/client-s3";
|
||||||
|
import { createReadStream } from 'fs';
|
||||||
|
import { createId } from '@paralleldrive/cuid2';
|
||||||
|
import prettyBytes from 'pretty-bytes';
|
||||||
|
import dotenv from 'dotenv'
|
||||||
|
dotenv.config({
|
||||||
|
path: '../../.env.development'
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
|
||||||
|
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
|
||||||
|
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
|
||||||
|
|
||||||
|
function makeProgressTicker(counter: number) {
|
||||||
|
const ticker = setInterval(() => {
|
||||||
|
console.log(`[progress] ${counter} bytes (aggregate) (${prettyBytes(counter)}) have passed through the pipeline.`)
|
||||||
|
}, 1000 * 30)
|
||||||
|
return ticker
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeS3Client() {
|
||||||
|
const client = new S3Client({
|
||||||
|
endpoint: 'https://s3.us-west-000.backblazeb2.com',
|
||||||
|
region: 'us-west-000',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
|
||||||
|
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async function uploadToS3({ client, uploadStream, keyName }: { client: S3Client, uploadStream: NodeJS.ReadableStream, keyName: string }) {
|
||||||
|
|
||||||
|
const target = {
|
||||||
|
Bucket: process.env.S3_BUCKET_NAME!,
|
||||||
|
Key: keyName,
|
||||||
|
Body: new Readable().wrap(uploadStream)
|
||||||
|
}
|
||||||
|
|
||||||
|
// greets https://stackoverflow.com/a/70159394/1004931
|
||||||
|
try {
|
||||||
|
const parallelUploads3 = new Upload({
|
||||||
|
client: client,
|
||||||
|
partSize: 1024 * 1024 * 5,
|
||||||
|
queueSize: 1,
|
||||||
|
leavePartsOnError: false,
|
||||||
|
params: target,
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
parallelUploads3.on("httpUploadProgress", (progress) => {
|
||||||
|
console.log(progress)
|
||||||
|
if (progress?.loaded) {
|
||||||
|
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
|
||||||
|
} else {
|
||||||
|
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await parallelUploads3.done();
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error) {
|
||||||
|
console.error(`while uploading a file to s3, we encountered an error`)
|
||||||
|
throw new Error(e.message);
|
||||||
|
} else {
|
||||||
|
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
|
||||||
|
let counter = 0
|
||||||
|
const client = makeS3Client()
|
||||||
|
const ticker = makeProgressTicker(counter)
|
||||||
|
const datestamp = new Date().toISOString()
|
||||||
|
const keyName = `${datestamp}-stream3-chaturbate-${createId()}.ts`
|
||||||
|
console.log(`Uploading ${keyName} to S3`)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* setup the streams which process the data
|
||||||
|
*/
|
||||||
|
const ffmpegStream = createReadStream('/home/cj/Downloads/stream-23894234.ts')
|
||||||
|
const uploadStream = new PassThrough()
|
||||||
|
|
||||||
|
// update the progress ticker data
|
||||||
|
uploadStream.on('data', (data) => {
|
||||||
|
counter += data.length
|
||||||
|
})
|
||||||
|
|
||||||
|
/**
|
||||||
|
* we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
|
||||||
|
*/
|
||||||
|
pipeline(
|
||||||
|
ffmpegStream,
|
||||||
|
uploadStream,
|
||||||
|
(err) => {
|
||||||
|
if (err) {
|
||||||
|
console.error(`pipeline errored.`)
|
||||||
|
console.error(err)
|
||||||
|
} else {
|
||||||
|
console.log('pipeline succeeded.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
await uploadToS3({client, uploadStream, keyName })
|
||||||
|
clearInterval(ticker)
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,178 @@
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @todo if we have the ffmpeg stream send an end event, does the ffmpegStream close?
|
||||||
|
* so far, we have observed the end of a CB stream, and the uploadStream is what shows as closed.
|
||||||
|
* It would be better to have the ffmpeg stream do the closing, amirite? or does it even matter?
|
||||||
|
* Here's what the console.log shows when the CB stream ended while we were not using { end: true } on the ffmpeg stream
|
||||||
|
*
|
||||||
|
*
|
||||||
|
* [data] uploadStream. 118018880 aggregated bytes (118 MB).
|
||||||
|
[data] uploadStream. 118067384 aggregated bytes (118 MB).
|
||||||
|
[data] uploadStream. 118101224 aggregated bytes (118 MB).
|
||||||
|
[data] uploadStream. 118119648 aggregated bytes (118 MB).
|
||||||
|
[close] uploadStream closed
|
||||||
|
pipeline succeeded.
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
import { getRandomRoom } from '@futureporn/scout/cb.js'
|
||||||
|
import { ua0 } from "@futureporn/scout/ua.js";
|
||||||
|
import { spawn } from "child_process";
|
||||||
|
import { PassThrough, pipeline } from "stream";
|
||||||
|
import { type Progress, Upload } from "@aws-sdk/lib-storage";
|
||||||
|
import { S3Client } from "@aws-sdk/client-s3";
|
||||||
|
import { createWriteStream } from 'fs';
|
||||||
|
import ffmpeg from 'fluent-ffmpeg'
|
||||||
|
import { createId } from '@paralleldrive/cuid2';
|
||||||
|
import prettyBytes from 'pretty-bytes';
|
||||||
|
import dotenv from 'dotenv'
|
||||||
|
dotenv.config({
|
||||||
|
path: '../../.env.development'
|
||||||
|
})
|
||||||
|
|
||||||
|
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
|
||||||
|
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
|
||||||
|
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const client = new S3Client({
|
||||||
|
endpoint: 'https://s3.us-west-000.backblazeb2.com',
|
||||||
|
region: 'us-west-000',
|
||||||
|
credentials: {
|
||||||
|
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
|
||||||
|
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
const randomRoom = await getRandomRoom()
|
||||||
|
console.log(`randomRoom=${randomRoom.name}`)
|
||||||
|
|
||||||
|
const playlistUrl: string = await new Promise((resolve, reject) => {
|
||||||
|
|
||||||
|
|
||||||
|
// get the m3u8 playlist for the livestream
|
||||||
|
const ytdlp = spawn('yt-dlp', [ '-g', randomRoom.url ])
|
||||||
|
let output = ''
|
||||||
|
ytdlp.on('error', (err) => {
|
||||||
|
console.error(err)
|
||||||
|
})
|
||||||
|
ytdlp.once('exit', (code) => {
|
||||||
|
console.log(`code=${code}`)
|
||||||
|
if (code !== 0) reject(`yt-dlp exited with code ${code}. stderr as follows ${JSON.stringify(ytdlp.stderr, null, 2)}`);
|
||||||
|
resolve(output)
|
||||||
|
})
|
||||||
|
ytdlp.stderr.on('data', (data) => {
|
||||||
|
console.error('stderr data as follows')
|
||||||
|
console.error(data.toString())
|
||||||
|
})
|
||||||
|
ytdlp.stdout.on('data', (data) => {
|
||||||
|
output = data
|
||||||
|
})
|
||||||
|
|
||||||
|
})
|
||||||
|
console.log(`playlistUrl=${playlistUrl}`)
|
||||||
|
if (!playlistUrl) throw new Error(`failed to get playlistUrl from yt-dlp -g ${randomRoom.url}`);
|
||||||
|
|
||||||
|
|
||||||
|
let debugCounter = 0
|
||||||
|
let fileOutputStream = createWriteStream('/home/cj/Downloads/outputfile.ts');
|
||||||
|
// let ffmpegLogStream = createWriteStream('/tmp/ffmpeg-log.txt')
|
||||||
|
let uploadStream = new PassThrough()
|
||||||
|
uploadStream.on('data', (data) => {
|
||||||
|
debugCounter += data.length
|
||||||
|
console.log(`[data] uploadStream. ${debugCounter} aggregated bytes (${prettyBytes(debugCounter)}).`)
|
||||||
|
})
|
||||||
|
uploadStream.on('drain', () => {
|
||||||
|
console.log('[drain] uploadStream')
|
||||||
|
})
|
||||||
|
uploadStream.on('close', () => {
|
||||||
|
console.log(`[close] uploadStream closed`)
|
||||||
|
})
|
||||||
|
uploadStream.on('error', (err) => {
|
||||||
|
console.error('[error] uploadStream had an error as follows')
|
||||||
|
console.error(err)
|
||||||
|
})
|
||||||
|
uploadStream.on('exit', (code) => {
|
||||||
|
console.log(`[exit] uploadStream exited with code ${code}`)
|
||||||
|
})
|
||||||
|
uploadStream.on('disconnect', () => {
|
||||||
|
console.log('[disconnect] uploadStream disconnected')
|
||||||
|
})
|
||||||
|
uploadStream.on('message', (msg) => {
|
||||||
|
console.log('[message] uploadStream sent a message as follows')
|
||||||
|
console.log(msg)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
const datestamp = new Date().toISOString()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
const ffmpegProc = spawn('ffmpeg', [
|
||||||
|
'-headers', `"User-Agent: ${ua0}"`,
|
||||||
|
'-i', playlistUrl,
|
||||||
|
'-c:v', 'copy',
|
||||||
|
'-c:a', 'copy',
|
||||||
|
'-movflags', 'faststart',
|
||||||
|
'-y',
|
||||||
|
'-f', 'mpegts',
|
||||||
|
'-loglevel', 'quiet',
|
||||||
|
'pipe:1'
|
||||||
|
], {
|
||||||
|
// ignoring stderr is important because if not, ffmpeg will fill that buffer and node will hang
|
||||||
|
stdio: ['pipe', 'pipe', 'ignore']
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
// we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
|
||||||
|
pipeline(
|
||||||
|
ffmpegProc.stdout,
|
||||||
|
uploadStream,
|
||||||
|
(err) => {
|
||||||
|
if (err) {
|
||||||
|
console.error(`pipeline errored.`)
|
||||||
|
console.error(err)
|
||||||
|
} else {
|
||||||
|
console.log('pipeline succeeded.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// Create a stream to the S3 bucket. We use this stream to upload the livestream to Backblaze S3 service
|
||||||
|
const keyName = `${datestamp}-${randomRoom.name}-chaturbate-${createId()}.ts`
|
||||||
|
console.log(`keyName=${keyName}`)
|
||||||
|
const target = {
|
||||||
|
Bucket: process.env.S3_BUCKET_NAME,
|
||||||
|
Key: keyName,
|
||||||
|
Body: uploadStream
|
||||||
|
}
|
||||||
|
// greets https://stackoverflow.com/a/70159394/1004931
|
||||||
|
try {
|
||||||
|
const parallelUploads3 = new Upload({
|
||||||
|
client: client,
|
||||||
|
//tags: [...], // optional tags
|
||||||
|
queueSize: 4, // optional concurrency configuration
|
||||||
|
leavePartsOnError: false, // optional manually handle dropped parts
|
||||||
|
params: target,
|
||||||
|
});
|
||||||
|
|
||||||
|
await parallelUploads3.done();
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error) {
|
||||||
|
console.error(`while uploading a file to s3, we encountered an error`)
|
||||||
|
throw new Error(e.message);
|
||||||
|
} else {
|
||||||
|
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,17 +1,38 @@
|
||||||
import { type Helpers } from 'graphile-worker'
|
import { type Helpers } from 'graphile-worker'
|
||||||
import Record from '../Record.ts'
|
import Record from '../Record.ts'
|
||||||
|
import 'dotenv/config'
|
||||||
|
|
||||||
|
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
|
||||||
|
if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was undefined in env');
|
||||||
|
if (!process.env.S3_REGION) throw new Error('S3_REGION was undefined in env');
|
||||||
|
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was undefined in env');
|
||||||
|
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was undefined in env');
|
||||||
|
|
||||||
|
|
||||||
type Payload = {
|
type Payload = {
|
||||||
url: string
|
url: string;
|
||||||
|
channel: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export default async function (payload: Payload, helpers: Helpers) {
|
export default async function (payload: Payload, helpers: Helpers): Promise<string> {
|
||||||
const { url } = payload;
|
const { url, channel } = payload;
|
||||||
helpers.logger.info(`'record' task execution begin with url=${url} (@todo implement)`);
|
helpers.logger.info(`'record' task execution begin with url=${url}, channel=${channel}`);
|
||||||
const record = new Record({ url: 'https://example.com/stream' })
|
|
||||||
record.start()
|
|
||||||
|
|
||||||
return record.id
|
|
||||||
|
const bucket = process.env.S3_BUCKET_NAME!
|
||||||
|
const endpoint = process.env.S3_ENDPOINT!
|
||||||
|
const region = process.env.S3_REGION!
|
||||||
|
const accessKeyId = process.env.S3_ACCESS_KEY_ID!
|
||||||
|
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY!
|
||||||
|
|
||||||
|
const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
|
||||||
|
const inputStream = Record.getFFmpegDownload({ url })
|
||||||
|
const record = new Record({ inputStream, bucket, s3Client, channel })
|
||||||
|
await record.start()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
return record.id
|
||||||
|
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
@ -1,16 +1,26 @@
|
||||||
import { describe } from 'mocha'
|
import { describe } from 'mocha'
|
||||||
import { expect } from 'chai';
|
import { expect } from 'chai';
|
||||||
import { getInitialRoomDossier } from './cb.js'
|
import { getInitialRoomDossier, getRandomRoom } from './cb.js'
|
||||||
|
|
||||||
describe('cb', function () {
|
describe('cb', function () {
|
||||||
describe('getInitialRoomDossier', function () {
|
describe('getInitialRoomDossier', function () {
|
||||||
/**
|
/**
|
||||||
* this is an integration test that fails in CI due to CB blocking IP ranges
|
* this is an integration test that fails in CI due to CB blocking IP ranges
|
||||||
* @todo use a proxy or something
|
* @todo use a proxy or something
|
||||||
*/
|
*/
|
||||||
xit('should return json', async function () {
|
xit('should return json', async function () {
|
||||||
const dossier = await getInitialRoomDossier('https://chaturbate.com/projektmelody')
|
const dossier = await getInitialRoomDossier('https://chaturbate.com/projektmelody')
|
||||||
expect(dossier).to.have.property('wschat_host')
|
expect(dossier).to.have.property('wschat_host')
|
||||||
})
|
|
||||||
})
|
})
|
||||||
|
})
|
||||||
|
describe('getRandomRoom', function () {
|
||||||
|
it('should return a Room object of an online room', async function () {
|
||||||
|
this.timeout(1000*60*2)
|
||||||
|
const room = await getRandomRoom()
|
||||||
|
expect(room).to.have.property('url')
|
||||||
|
expect(room).to.have.property('name')
|
||||||
|
expect(room.name).to.match(/[a-z_]/)
|
||||||
|
expect(room.url).to.match(/https:\/\//)
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
@ -1,4 +1,42 @@
|
||||||
import * as cheerio from 'cheerio'
|
import * as cheerio from 'cheerio'
|
||||||
|
import fetch from 'node-fetch'
|
||||||
|
|
||||||
|
|
||||||
|
export interface ChaturbateModel {
|
||||||
|
gender: string;
|
||||||
|
location: string;
|
||||||
|
current_show: 'public' | 'private';
|
||||||
|
username: string;
|
||||||
|
room_subject: string;
|
||||||
|
tags: string[];
|
||||||
|
is_new: boolean;
|
||||||
|
num_users: number;
|
||||||
|
num_followers: number;
|
||||||
|
country: string;
|
||||||
|
spoken_languages: string;
|
||||||
|
display_name: string;
|
||||||
|
birthday: string;
|
||||||
|
is_hd: boolean;
|
||||||
|
age: number;
|
||||||
|
seconds_online: number;
|
||||||
|
image_url: string;
|
||||||
|
image_url_360x270: string;
|
||||||
|
chat_room_url_revshare: string;
|
||||||
|
iframe_embed_revshare: string;
|
||||||
|
chat_room_url: string;
|
||||||
|
iframe_embed: string;
|
||||||
|
slug: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ChaturbateOnlineModelsResponse {
|
||||||
|
results: ChaturbateModel[],
|
||||||
|
count: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Room {
|
||||||
|
name: string;
|
||||||
|
url: string;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
|
|
@ -6,29 +44,62 @@ import * as cheerio from 'cheerio'
|
||||||
* @returns {Object} initialRoomDossier
|
* @returns {Object} initialRoomDossier
|
||||||
*/
|
*/
|
||||||
export async function getInitialRoomDossier(roomUrl: string) {
|
export async function getInitialRoomDossier(roomUrl: string) {
|
||||||
try {
|
try {
|
||||||
const res = await fetch(roomUrl, {
|
const res = await fetch(roomUrl, {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
|
||||||
}
|
|
||||||
});
|
|
||||||
const body = await res.text()
|
|
||||||
const $ = cheerio.load(body);
|
|
||||||
let rawScript = $('script:contains(window.initialRoomDossier)').html();
|
|
||||||
if (!rawScript) {
|
|
||||||
throw new Error('window.initialRoomDossier is null. This could mean the channel is in password mode');
|
|
||||||
}
|
}
|
||||||
let rawDossier = rawScript.slice(rawScript.indexOf('"'), rawScript.lastIndexOf('"') + 1);
|
});
|
||||||
let dossier = JSON.parse(JSON.parse(rawDossier));
|
const body = await res.text()
|
||||||
|
const $ = cheerio.load(body);
|
||||||
|
let rawScript = $('script:contains(window.initialRoomDossier)').html();
|
||||||
|
if (!rawScript) {
|
||||||
|
throw new Error('window.initialRoomDossier is null. This could mean the channel is in password mode');
|
||||||
|
}
|
||||||
|
let rawDossier = rawScript.slice(rawScript.indexOf('"'), rawScript.lastIndexOf('"') + 1);
|
||||||
|
let dossier = JSON.parse(JSON.parse(rawDossier));
|
||||||
|
|
||||||
return dossier;
|
return dossier;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof Error) {
|
if (error instanceof Error) {
|
||||||
// Handle the error gracefully
|
// Handle the error gracefully
|
||||||
console.error(`Error fetching initial room dossier: ${error.message}`);
|
console.error(`Error fetching initial room dossier: ${error.message}`);
|
||||||
return null; // Or any other appropriate action you want to take
|
return null; // Or any other appropriate action you want to take
|
||||||
} else {
|
} else {
|
||||||
console.error('caught an exotic error, uh-oh')
|
console.error('caught an exotic error, uh-oh')
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export async function getRandomRoom(): Promise<Room> {
|
||||||
|
try {
|
||||||
|
const res = await fetch('https://chaturbate.com/api/public/affiliates/onlinerooms/?wm=DiPkB&client_ip=request_ip');
|
||||||
|
const data = await res.json() as ChaturbateOnlineModelsResponse;
|
||||||
|
|
||||||
|
if (!data || !Array.isArray(data.results) || data.results.length === 0) {
|
||||||
|
throw new Error('No results found');
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = data.results;
|
||||||
|
const randomIndex = Math.floor(Math.random() * results.length);
|
||||||
|
|
||||||
|
if (!results[randomIndex]) {
|
||||||
|
throw new Error('No result found at random index');
|
||||||
|
}
|
||||||
|
|
||||||
|
const username = results[randomIndex].username;
|
||||||
|
return {
|
||||||
|
url: `https://chaturbate.com/${username}`,
|
||||||
|
name: username
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof Error) {
|
||||||
|
console.error(`Error in getRandomRoom: ${error.message}`);
|
||||||
|
} else {
|
||||||
|
console.error('An unexpected error occurred');
|
||||||
|
}
|
||||||
|
throw error; // Re-throw the error to propagate it further
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -43,40 +43,6 @@ importers:
|
||||||
specifier: ^4.16.2
|
specifier: ^4.16.2
|
||||||
version: 4.16.2
|
version: 4.16.2
|
||||||
|
|
||||||
../..: {}
|
|
||||||
|
|
||||||
../bot: {}
|
|
||||||
|
|
||||||
../capture: {}
|
|
||||||
|
|
||||||
../image: {}
|
|
||||||
|
|
||||||
../infra: {}
|
|
||||||
|
|
||||||
../mailbox: {}
|
|
||||||
|
|
||||||
../meal: {}
|
|
||||||
|
|
||||||
../next: {}
|
|
||||||
|
|
||||||
../old: {}
|
|
||||||
|
|
||||||
../scout: {}
|
|
||||||
|
|
||||||
../storage: {}
|
|
||||||
|
|
||||||
../strapi: {}
|
|
||||||
|
|
||||||
../taco: {}
|
|
||||||
|
|
||||||
../types: {}
|
|
||||||
|
|
||||||
../uppy: {}
|
|
||||||
|
|
||||||
../video: {}
|
|
||||||
|
|
||||||
../worker: {}
|
|
||||||
|
|
||||||
packages:
|
packages:
|
||||||
|
|
||||||
'@cspotcode/source-map-support@0.8.1':
|
'@cspotcode/source-map-support@0.8.1':
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ import { dirname, basename, join, isAbsolute } from 'node:path';
|
||||||
import { fileURLToPath } from 'url';
|
import { fileURLToPath } from 'url';
|
||||||
|
|
||||||
export const __filename = fileURLToPath(import.meta.url);
|
export const __filename = fileURLToPath(import.meta.url);
|
||||||
export const __dirname = dirname(fileURLToPath(import.meta.url));
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
export function getPackageVersion(packageJsonPath: string): string {
|
export function getPackageVersion(packageJsonPath: string): string {
|
||||||
if (!isAbsolute(packageJsonPath)) {
|
if (!isAbsolute(packageJsonPath)) {
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,9 @@ EOF
|
||||||
|
|
||||||
kubectl --namespace futureporn delete secret capture --ignore-not-found
|
kubectl --namespace futureporn delete secret capture --ignore-not-found
|
||||||
kubectl --namespace futureporn create secret generic capture \
|
kubectl --namespace futureporn create secret generic capture \
|
||||||
--from-literal=databaseUrl=${WORKER_DATABASE_URL}
|
--from-literal=databaseUrl=${WORKER_DATABASE_URL} \
|
||||||
|
--from-literal=s3AccessKeyId=${S3_ACCESS_KEY_ID} \
|
||||||
|
--from-literal=s3SecretAccessKey=${S3_SECRET_ACCESS_KEY}
|
||||||
|
|
||||||
kubectl --namespace futureporn delete secret mailbox --ignore-not-found
|
kubectl --namespace futureporn delete secret mailbox --ignore-not-found
|
||||||
kubectl --namespace futureporn create secret generic mailbox \
|
kubectl --namespace futureporn create secret generic mailbox \
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,12 @@
|
||||||
|
|
||||||
|
if [ -z $POSTGRES_PASSWORD ]; then
|
||||||
|
echo "POSTGRES_PASSWORD was missing in env"
|
||||||
|
exit 5
|
||||||
|
fi
|
||||||
|
|
||||||
## drop futureporn_db
|
## drop futureporn_db
|
||||||
kubectl -n futureporn exec postgres-primary-0 -- psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
|
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
|
||||||
|
|
||||||
|
|
||||||
|
## drop graphile_worker
|
||||||
|
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE graphile_worker WITH (FORCE);"
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue