capture progress
ci / build (push) Has been cancelled Details

This commit is contained in:
CJ_Clippy 2024-07-25 05:53:52 -08:00
parent af8f58940a
commit 4bc11c027e
32 changed files with 2147 additions and 386 deletions

18
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,18 @@
{
"name": "tsx",
"type": "node",
"request": "launch",
"program": "${file}",
"runtimeExecutable": "tsx",
"console": "integratedTerminal",
"internalConsoleOptions": "neverOpen",
"skipFiles": [
"<node_internals>/**",
"${workspaceFolder}/node_modules/**",
],
}

View File

@ -47,22 +47,7 @@ clean:
dotenvx run -f .env.${ENV} -- node ./packages/infra/vultr-delete-orphaned-resources.js dotenvx run -f .env.${ENV} -- node ./packages/infra/vultr-delete-orphaned-resources.js
deps: deps:
echo "Some of the install methods for these dependencies are not cross-platform compatible. Some of the install methods are not tested. Expect this to fail. Please consult the Makefile for URLs to project sources." echo "use `devbox install`"
sudo pamac install make entr nvm kubectl docker helm expect
curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
echo "go to https://github.com/txn2/kubefwd/releases/latest to get kubefwd"
echo "go to https://github.com/tilt-dev/ctlptl/releases/latest to get ctlptl"
sudo systemctl enable docker
sudo systemctl start docker
usermod -aG docker cj
newgrp docker
npm i -g pnpm
pnpm install -g @dotenvx/dotenvx
curl -OL 'https://github.com/vmware-tanzu/velero/releases/download/v1.13.2/velero-v1.13.2-linux-amd64.tar.gz'
OS=$(go env GOOS); ARCH=$(go env GOARCH); curl -fsSL -o cmctl https://github.com/cert-manager/cmctl/releases/latest/download/cmctl_${OS}_${ARCH}
chmod +x cmctl
sudo mv cmctl /usr/local/bin
# A gitea act runner which runs locally # A gitea act runner which runs locally
# https://docs.gitea.com/next/usage/actions/overview # https://docs.gitea.com/next/usage/actions/overview

View File

@ -1,9 +1,33 @@
# futureporn.net # futureporn.net
[![Built with Devbox](https://www.jetify.com/img/devbox/shield_galaxy.svg)](https://www.jetify.com/devbox/docs/contributor-quickstart/)
Source Code for https://futureporn.net Source Code for https://futureporn.net
See ./ARCHITECTURE.md for overview See ./ARCHITECTURE.md for overview
## Getting Started
I'm working towards a better development experience with devbox and Tilt. This process is in a state of flux and is likely to be broken.
The main gist is as follows
1. Install [devbox](https://www.jetify.com/devbox/docs/installing_devbox/)
2. Install development environment & packages using devbox.
devbox install
3. Start a local KIND Kubernetes cluster
make cluster
4. Start Tilt
make tilt
Tilt will manage the KIND cluster, downloading necessary docker containers and building the containers listed in the fp helm chart at ./Charts/fp. Making changes to these charts or the application code will update or re-build the images as necessary.
## Metrics Notes ## Metrics Notes
Keeping track of metrics we want to scrape using Prometheus Keeping track of metrics we want to scrape using Prometheus

View File

@ -181,14 +181,20 @@ cmd_button('postgres:drop',
argv=['sh', './scripts/postgres-drop.sh'], argv=['sh', './scripts/postgres-drop.sh'],
resource='postgresql-primary', resource='postgresql-primary',
icon_name='delete', icon_name='delete',
text='DROP futureporn_db' text='DROP all databases'
) )
cmd_button('postgres:backup', cmd_button('capture-api:create',
argv=['sh', './scripts/postgres-backup.sh'], argv=['http', '--ignore-stdin', 'POST', 'http://localhost:5003/api/record', "url='https://twitch.tv/ironmouse'", "channel='ironmouse'"],
resource='capture-api',
icon_name='send',
text='Start Recording'
)
cmd_button('postgres:graphile',
argv=['sh', './scripts/postgres-test-graphile.sh'],
resource='postgresql-primary', resource='postgresql-primary',
icon_name='download', icon_name='graph',
text='backup the database' text='create graphile test job',
) )
cmd_button('postgres:graphile', cmd_button('postgres:graphile',
argv=['sh', './scripts/postgres-test-graphile.sh'], argv=['sh', './scripts/postgres-test-graphile.sh'],

View File

@ -45,6 +45,22 @@ spec:
key: databaseUrl key: databaseUrl
- name: PORT - name: PORT
value: "{{ .Values.capture.api.port }}" value: "{{ .Values.capture.api.port }}"
- name: S3_ENDPOINT
value: "{{ .Values.s3.endpoint }}"
- name: S3_REGION
value: "{{ .Values.s3.region }}"
- name: S3_BUCKET_NAME
value: "{{ .Values.s3.buckets.usc }}"
- name: S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: capture
key: s3AccessKeyId
- name: S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: capture
key: s3SecretAccessKey
resources: resources:
limits: limits:
cpu: 1000m cpu: 1000m

View File

@ -1,6 +1,12 @@
environment: development environment: development
# storageClassName: csi-hostpath-sc # used by minikube # storageClassName: csi-hostpath-sc # used by minikube
storageClassName: standard # used by Kind storageClassName: standard # used by Kind
s3:
endpoint: https://s3.us-west-000.backblazeb2.com
region: us-west-000
buckets:
main: fp-dev
usc: fp-usc-dev
link2cid: link2cid:
imageName: fp/link2cid imageName: fp/link2cid
next: next:

View File

@ -44,7 +44,9 @@ RUN ls -la /prod/capture
## start the app with dumb init to spawn the Node.js runtime process ## start the app with dumb init to spawn the Node.js runtime process
## with signal support ## with signal support
## The mode @futureporn/capture uses when starting is determined by FUNCTION environment variable. (worker|api)
FROM base AS capture FROM base AS capture
RUN ls -la /usr/local/bin/yt-dlp
ENV HOSTNAME=0.0.0.0 NODE_ENV=production ENV HOSTNAME=0.0.0.0 NODE_ENV=production
COPY --from=build /prod/capture . COPY --from=build /prod/capture .
CMD [ "dumb-init", "node", "dist/index.js" ] CMD [ "dumb-init", "node", "dist/index.js" ]

24
devbox.json Normal file
View File

@ -0,0 +1,24 @@
{
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.12.0/.schema/devbox.schema.json",
"packages": [
"nodejs@20",
"tilt@latest",
"ctlptl@latest",
"kubectl@latest",
"cmctl@latest"
],
"env": {
"DEVBOX_COREPACK_ENABLED": "true",
"ENV": "development"
},
"shell": {
"init_hook": [
"pnpm install"
],
"scripts": {
"test": [
"echo \"Error: no test specified\" && exit 1"
]
}
}
}

298
devbox.lock Normal file
View File

@ -0,0 +1,298 @@
{
"lockfile_version": "1",
"packages": {
"cmctl@latest": {
"last_modified": "2024-07-07T07:43:47Z",
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#cmctl",
"source": "devbox-search",
"version": "1.14.7",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/69lc5y36840ccy6d1pzph994psadk4bm-cmctl-1.14.7",
"default": true
}
],
"store_path": "/nix/store/69lc5y36840ccy6d1pzph994psadk4bm-cmctl-1.14.7"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/d5l61iil1gaax5sipnzg80mb0p1hqk9f-cmctl-1.14.7",
"default": true
}
],
"store_path": "/nix/store/d5l61iil1gaax5sipnzg80mb0p1hqk9f-cmctl-1.14.7"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/j0r3wavi836mp31l0s7r1c3rjryw2z62-cmctl-1.14.7",
"default": true
}
],
"store_path": "/nix/store/j0r3wavi836mp31l0s7r1c3rjryw2z62-cmctl-1.14.7"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/lnmy96wjzmjna7z9f0dbqd16nf2x5qbv-cmctl-1.14.7",
"default": true
}
],
"store_path": "/nix/store/lnmy96wjzmjna7z9f0dbqd16nf2x5qbv-cmctl-1.14.7"
}
}
},
"ctlptl@latest": {
"last_modified": "2024-07-07T07:43:47Z",
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#ctlptl",
"source": "devbox-search",
"version": "0.8.29",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/gvnmvb315zngbg5a0idynlwxcc45gmyd-ctlptl-0.8.29",
"default": true
}
],
"store_path": "/nix/store/gvnmvb315zngbg5a0idynlwxcc45gmyd-ctlptl-0.8.29"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/vgwbxwaf773mmgcbc1j5h6n5a7g587lf-ctlptl-0.8.29",
"default": true
}
],
"store_path": "/nix/store/vgwbxwaf773mmgcbc1j5h6n5a7g587lf-ctlptl-0.8.29"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/ga4rp2188c6k24162s2n23nfi4846790-ctlptl-0.8.29",
"default": true
}
],
"store_path": "/nix/store/ga4rp2188c6k24162s2n23nfi4846790-ctlptl-0.8.29"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/wbfjbk575ny949rfyqpm4vai4ap5rbpp-ctlptl-0.8.29",
"default": true
}
],
"store_path": "/nix/store/wbfjbk575ny949rfyqpm4vai4ap5rbpp-ctlptl-0.8.29"
}
}
},
"kubectl@latest": {
"last_modified": "2024-07-07T07:43:47Z",
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#kubectl",
"source": "devbox-search",
"version": "1.30.2",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/i1zidf41bkfzs2l1pq9fi1frymsfgywc-kubectl-1.30.2",
"default": true
},
{
"name": "man",
"path": "/nix/store/dzxnn9mk9plcx3w9862jyd0nxys2yywz-kubectl-1.30.2-man",
"default": true
},
{
"name": "convert",
"path": "/nix/store/v9ij5fnxxa02jkzpjvkbxw2jc4p9cbld-kubectl-1.30.2-convert"
}
],
"store_path": "/nix/store/i1zidf41bkfzs2l1pq9fi1frymsfgywc-kubectl-1.30.2"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/k7ql4247qs6ny27m3iz5c9xf5gb248a2-kubectl-1.30.2",
"default": true
},
{
"name": "man",
"path": "/nix/store/wy64r4nn3isydw4nx257h95qy2x2z4mx-kubectl-1.30.2-man",
"default": true
},
{
"name": "convert",
"path": "/nix/store/ic8za302hvb4kf4zrs55ivr4q2n2lznn-kubectl-1.30.2-convert"
}
],
"store_path": "/nix/store/k7ql4247qs6ny27m3iz5c9xf5gb248a2-kubectl-1.30.2"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/v029n959l5b289br0cq591b04yc48516-kubectl-1.30.2",
"default": true
},
{
"name": "man",
"path": "/nix/store/0dvcxn7gsi2ycy9blb7pcy506w4xp2vi-kubectl-1.30.2-man",
"default": true
},
{
"name": "convert",
"path": "/nix/store/2nfq4ivwa4a7jwc0183f2wpl1jxbn754-kubectl-1.30.2-convert"
}
],
"store_path": "/nix/store/v029n959l5b289br0cq591b04yc48516-kubectl-1.30.2"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/3vkf0406s1i6l89hk5wrakh4bbn0p1p2-kubectl-1.30.2",
"default": true
},
{
"name": "man",
"path": "/nix/store/3wbvgkkka1knkxvyr4c8qbpr448smw8i-kubectl-1.30.2-man",
"default": true
},
{
"name": "convert",
"path": "/nix/store/h5zxz8db6wligwhw5gnwk4gbc8j1ixik-kubectl-1.30.2-convert"
}
],
"store_path": "/nix/store/3vkf0406s1i6l89hk5wrakh4bbn0p1p2-kubectl-1.30.2"
}
}
},
"nodejs@20": {
"last_modified": "2024-07-07T07:43:47Z",
"plugin_version": "0.0.2",
"resolved": "github:NixOS/nixpkgs/b60793b86201040d9dee019a05089a9150d08b5b#nodejs_20",
"source": "devbox-search",
"version": "20.14.0",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/sqnbldm1fjw88v23yq4v6531y4m7v2fh-nodejs-20.14.0",
"default": true
},
{
"name": "libv8",
"path": "/nix/store/1i0rb2axkrxvsq5pz8s2q07ard2p36a1-nodejs-20.14.0-libv8"
}
],
"store_path": "/nix/store/sqnbldm1fjw88v23yq4v6531y4m7v2fh-nodejs-20.14.0"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/r1nwmlbsn67f5rhayr7jjjdmiflxpk92-nodejs-20.14.0",
"default": true
},
{
"name": "libv8",
"path": "/nix/store/5ii3xkbd3iv0xvqqvjg3agsm0dinidgm-nodejs-20.14.0-libv8"
}
],
"store_path": "/nix/store/r1nwmlbsn67f5rhayr7jjjdmiflxpk92-nodejs-20.14.0"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/wzgnws4r1c98vzj5q6gq4drz2jfq7d5q-nodejs-20.14.0",
"default": true
},
{
"name": "libv8",
"path": "/nix/store/gc2gnkc8hvkh51ab3a29fvgzy2qsqb2s-nodejs-20.14.0-libv8"
}
],
"store_path": "/nix/store/wzgnws4r1c98vzj5q6gq4drz2jfq7d5q-nodejs-20.14.0"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/ilkfhnqz4xczrliqjva8770x2svbfznd-nodejs-20.14.0",
"default": true
},
{
"name": "libv8",
"path": "/nix/store/2qaf68dzimr8as4bgli0xmsn11c0ah2j-nodejs-20.14.0-libv8"
}
],
"store_path": "/nix/store/ilkfhnqz4xczrliqjva8770x2svbfznd-nodejs-20.14.0"
}
}
},
"tilt@latest": {
"last_modified": "2024-07-15T21:47:20Z",
"resolved": "github:NixOS/nixpkgs/b2c1f10bfbb3f617ea8e8669ac13f3f56ceb2ea2#tilt",
"source": "devbox-search",
"version": "0.33.17",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/l19qinywsx7y86xp8vgwr3bgnbi0rfcj-tilt-0.33.17",
"default": true
}
],
"store_path": "/nix/store/l19qinywsx7y86xp8vgwr3bgnbi0rfcj-tilt-0.33.17"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/f6swxifmvnxjxifxyw4k4aiyxh0dgfyq-tilt-0.33.17",
"default": true
}
],
"store_path": "/nix/store/f6swxifmvnxjxifxyw4k4aiyxh0dgfyq-tilt-0.33.17"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/d59rlsabcqxax6bgw6d30zhmflw65ch0-tilt-0.33.17",
"default": true
}
],
"store_path": "/nix/store/d59rlsabcqxax6bgw6d30zhmflw65ch0-tilt-0.33.17"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/qfv96sjcsslynqbilwj823x8nxvgj5cv-tilt-0.33.17",
"default": true
}
],
"store_path": "/nix/store/qfv96sjcsslynqbilwj823x8nxvgj5cv-tilt-0.33.17"
}
}
}
}
}

View File

@ -15,6 +15,10 @@ Worker container runs a Graphile Worker which listens for specific tasks related
## Misc dev notes ## Misc dev notes
### idea for taking snapshots of stream in progress
https://ofarukcaki.medium.com/producing-real-time-video-with-node-js-and-ffmpeg-a59ac27461a1
### youtube-dl end of stream output ### youtube-dl end of stream output
The end-of-stream output from yt-dlp when recording CB looks like this The end-of-stream output from yt-dlp when recording CB looks like this

View File

@ -21,6 +21,7 @@
"@futureporn/utils": "workspace:^", "@futureporn/utils": "workspace:^",
"@paralleldrive/cuid2": "^2.2.2", "@paralleldrive/cuid2": "^2.2.2",
"@types/chai": "^4.3.16", "@types/chai": "^4.3.16",
"@types/fluent-ffmpeg": "^2.1.24",
"@types/mocha": "^10.0.7", "@types/mocha": "^10.0.7",
"@types/node": "^20.14.11", "@types/node": "^20.14.11",
"diskusage": "^1.2.0", "diskusage": "^1.2.0",
@ -46,8 +47,12 @@
"youtube-dl-wrap": "github:insanity54/youtube-dl-wrap" "youtube-dl-wrap": "github:insanity54/youtube-dl-wrap"
}, },
"devDependencies": { "devDependencies": {
"@smithy/util-stream": "^3.1.2",
"@types/sinon": "^17.0.3", "@types/sinon": "^17.0.3",
"@types/sinon-chai": "^3.2.12", "@types/sinon-chai": "^3.2.12",
"aws-sdk": "^2.1663.0",
"aws-sdk-client-mock": "^4.0.1",
"aws-sdk-mock": "^6.0.4",
"chai": "^4.4.1", "chai": "^4.4.1",
"cheerio": "1.0.0-rc.12", "cheerio": "1.0.0-rc.12",
"mocha": "^10.7.0", "mocha": "^10.7.0",
@ -55,6 +60,8 @@
"node-abort-controller": "^3.1.1", "node-abort-controller": "^3.1.1",
"node-fetch": "^3.3.2", "node-fetch": "^3.3.2",
"nodemon": "^2.0.22", "nodemon": "^2.0.22",
"pretty-bytes": "^6.1.1",
"s3": "link:aws-sdk/clients/s3",
"sinon": "^15.2.0", "sinon": "^15.2.0",
"sinon-chai": "^3.7.0", "sinon-chai": "^3.7.0",
"sinon-test": "^3.1.6", "sinon-test": "^3.1.6",

File diff suppressed because it is too large Load Diff

View File

@ -1,22 +1,67 @@
import Record from "./Record.js" import Record from "./Record.js"
import { expect } from "chai" import { expect } from "chai"
import { ChildProcess, spawn } from "child_process"
import { createReadStream, readFileSync, ReadStream } from "fs"
import AWSMock from 'aws-sdk-mock'
import sinon from 'sinon';
import sinonChai from 'sinon-chai';
import { PutObjectCommand, PutObjectCommandInput, S3Client, CreateMultipartUploadCommand, UploadPartCommand, UploadPartCommandInput } from "@aws-sdk/client-s3"
import { join, dirname } from "path"
import { fileURLToPath } from "url"
import { S3 } from 'aws-sdk';
import { HeadObjectOutput } from 'aws-sdk/clients/s3';
import { Readable } from 'stream';
import { mockClient } from 'aws-sdk-client-mock';
import { sdkStreamMixin } from '@smithy/util-stream'
// "pay no attention to that man behind the curtain"
// export function getObjectReadStream(s3Client: S3, Bucket: string, Key: string): Readable {
// return s3Client.getObject({ Bucket, Key }).createReadStream();
// }
// export async function waitForObjectExists(
// s3Client: S3Client,
// Bucket: string,
// Key: string
// ): Promise<HeadObjectOutput> {
// return s3Client.waitFor('objectExists', { Bucket, Key }).promise();
// }
const __dirname = dirname(fileURLToPath(import.meta.url));
const testStreamDir = '/tmp/record-test-stream.m3u8'
describe('Record', function () { describe('Record', function () {
let record: Record // mocking @aws-sdk/lib-storage Upload() has some nuances.
this.beforeEach(function () { // @see https://github.com/m-radzikowski/aws-sdk-client-mock?tab=readme-ov-file#lib-storage-upload
record = new Record({ url: 'https://example.com/my-cool-stream' })
})
describe('start()', function () { it('should accept a {ReadStream} as input', async function () {
it('should start the recording', async function () { const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) // 192627 bytes
const s3ClientMock = mockClient(S3Client)
s3ClientMock.on(CreateMultipartUploadCommand).resolves({UploadId: '1'});
s3ClientMock.on(UploadPartCommand).resolves({ETag: '1'});
const s3Client = new S3Client({ region: 'us-west-000' })
const record = new Record({ inputStream, s3Client, channel: 'coolguy_69', bucket: 'test' })
await record.start() await record.start()
expect(record).to.have.property('id') expect(record).to.have.property('counter', 192627)
expect(record).to.have.property('url') expect(record).to.have.property('bucket', 'test')
})
})
describe('stop()', function () {
it('should stop the recording', async function () {
await record.stop()
expect(record).to.have.property('cdnUrl')
}) })
xit('should restart if a EPIPE is encountered', async function () {
// @todo IDK how to implement this.
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4'))
const s3ClientMock = mockClient(S3Client)
s3ClientMock.on(CreateMultipartUploadCommand).resolves({UploadId: '1'})
s3ClientMock.on(UploadPartCommand).rejectsOnce({cause: 'simulated network interruption'}).resolves({ ETag: '1' }) // this rejection is probably not specific enough to simulate EPIPE
const s3Client = new S3Client({ region: 'us-west-000' })
const record = new Record({ inputStream, s3Client, channel: 'coolguy_69', bucket: 'test' })
await record.start()
expect(record).to.have.property('counter', 192627)
}) })
}) })

View File

@ -1,102 +1,184 @@
import { createId } from '@paralleldrive/cuid2' import { createId } from '@paralleldrive/cuid2'
import { spawn } from 'child_process'; import { spawn } from 'child_process';
import { ua0 } from '@futureporn/scout/ua.js' import { ua0 } from '@futureporn/scout/ua.js'
import { getTmpFile } from '@futureporn/utils' import { PassThrough, pipeline, Readable } from 'stream';
import prettyBytes from 'pretty-bytes';
import { Upload } from "@aws-sdk/lib-storage";
import { S3Client } from "@aws-sdk/client-s3";
import 'dotenv/config'
export interface RecordArgs { export interface RecordArgs {
url: string;
filename?: string; filename?: string;
channel?: string; channel: string;
s3Client: S3Client;
bucket: string;
date?: string; date?: string;
inputStream: Readable;
}
interface MakeS3ClientOptions {
accessKeyId: string;
secretAccessKey: string;
region: string;
endpoint: string
}
interface getFFmpegDownloadOptions {
url: string;
} }
export default class Record { export default class Record {
readonly id: string; readonly id: string;
readonly url: string; private s3Client: S3Client;
private uploadStream: PassThrough;
private ticker?: NodeJS.Timeout;
inputStream: Readable;
counter: number;
bucket: string;
keyName: string;
datestamp: string;
filename?: string; filename?: string;
channel?: string; channel: string;
date?: string; date?: string;
constructor({ url }: RecordArgs) { constructor({ inputStream, channel, s3Client, bucket }: RecordArgs) {
if (!url) throw new Error('url passed to Record constructor was undefined.'); if (!inputStream) throw new Error('Record constructor was missing inputStream.');
if (!bucket) throw new Error('Record constructor was missing bucket.');
if (!channel) throw new Error('Record constructer was missing channel!');
if (!s3Client) throw new Error('Record constructer was missing s3Client');
this.inputStream = inputStream
this.id = createId() this.id = createId()
this.url = url this.s3Client = s3Client
this.bucket = bucket
this.channel = channel
this.counter = 0
this.datestamp = new Date().toISOString()
this.keyName = `${this.datestamp}-${channel}-${createId()}.ts`
this.uploadStream = new PassThrough()
} }
makeProgressTicker() {
this.ticker = setInterval(() => {
console.log(`[progress] ${this.counter} bytes (aggregate) (${prettyBytes(this.counter)}) have passed through the pipeline.`)
}, 1000 * 30)
}
static makeS3Client({
accessKeyId,
secretAccessKey,
region,
endpoint
}: MakeS3ClientOptions): S3Client {
const client = new S3Client({
endpoint,
region,
credentials: {
accessKeyId,
secretAccessKey
}
})
return client
}
static getFFmpegDownload({ url }: getFFmpegDownloadOptions): Readable {
const ffmpegProc = spawn('ffmpeg', [
'-headers', `"User-Agent: ${ua0}"`,
'-i', url,
'-c:v', 'copy',
'-c:a', 'copy',
'-movflags', 'faststart',
'-y',
'-f', 'mpegts',
'-loglevel', 'quiet',
'pipe:1'
], {
// ignoring stderr is important because if not, ffmpeg will fill that buffer and node will hang
stdio: ['pipe', 'pipe', 'ignore']
})
return ffmpegProc.stdout
}
async uploadToS3() {
const target = {
Bucket: this.bucket,
Key: this.keyName,
// We do this to keep TS happy. Body expects a Readable, not a ReadableStream nor a NodeJS.ReadableStream
// Body: new Readable().wrap(this.uploadStream)
Body: this.uploadStream
}
// greets https://stackoverflow.com/a/70159394/1004931
try {
const parallelUploads3 = new Upload({
client: this.s3Client,
partSize: 1024 * 1024 * 5,
queueSize: 1,
leavePartsOnError: false,
params: target,
});
parallelUploads3.on("httpUploadProgress", (progress) => {
console.log(progress)
if (progress?.loaded) {
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
} else {
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
}
});
await parallelUploads3.done();
} catch (e) {
if (e instanceof Error) {
console.error(`while uploading a file to s3, we encountered an error`)
throw new Error(e.message);
} else {
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
}
}
}
async start() { async start() {
console.log(`@TODO record start with id=${this.id}, url=${this.url}`)
const playlistUrlPromise = new Promise<string>((resolve) => { this.makeProgressTicker()
const ytdlp = spawn('yt-dlp', [
'-g', // streams setup
this.url this.uploadStream.on('data', (data) => {
]) this.counter += data.length
ytdlp.stdout.on('data', (data) => {
resolve(data)
}) })
})
const playlistUrl = await playlistUrlPromise
const filename = getTmpFile(`stream.ts`)
const ffmpeg = spawn('ffmpeg', [
'-headers', `"User-Agent: ${ua0}"`,
'-i', playlistUrl,
'-c:v', 'copy',
'-c:a', 'copy',
'-movflags', 'faststart',
'-y',
'-f', 'mpegts',
filename
])
const ffmpegProcess = spawn('ffmpeg', [ // stream pipeline setup
'-headers', `"User-Agent: ${ua0}"`, pipeline(
'-i', playlistUrl, this.inputStream,
'-c:v', 'copy', this.uploadStream,
'-c:a', 'copy', (err) => {
'-movflags', 'faststart', if (err) {
'-y', console.error(`pipeline errored.`)
'-f', 'mpegts', console.error(err)
filename } else {
], { console.log('pipeline succeeded.')
stdio: 'inherit'
});
const ps = spawn('ps', ['ax']);
const grep = spawn('grep', ['ssh']);
ps.stdout.on('data', (data) => {
grep.stdin.write(data);
});
ps.stderr.on('data', (data) => {
console.error(`ps stderr: ${data}`);
});
ps.on('close', (code) => {
if (code !== 0) {
console.log(`ps process exited with code ${code}`);
} }
grep.stdin.end();
});
grep.stdout.on('data', (data) => {
console.log(data.toString());
});
grep.stderr.on('data', (data) => {
console.error(`grep stderr: ${data}`);
});
grep.on('close', (code) => {
if (code !== 0) {
console.log(`grep process exited with code ${code}`);
} }
}); )
await this.uploadToS3()
clearInterval(this.ticker)
return { return {
id: this.id, id: this.id,
url: this.url keyName: this.keyName,
channel: this.channel
} }
} }

View File

@ -6,7 +6,8 @@ import graphileWorkerPlugin, { type ExtendedFastifyInstance } from './fastify-gr
const version = getPackageVersion('../package.json') const version = getPackageVersion('../package.json')
interface RecordBodyType { interface RecordBodyType {
url: string url: string;
channel: string;
} }
function build(opts: Record<string, any>={}, connectionString: string) { function build(opts: Record<string, any>={}, connectionString: string) {
@ -17,9 +18,10 @@ function build(opts: Record<string, any>={}, connectionString: string) {
return { app: '@futureporn/capture', version } return { app: '@futureporn/capture', version }
}) })
app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) { app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) {
const { url } = request.body const { url, channel } = request.body
console.log(`POST /api/record with url=${url}`) console.log(`POST /api/record with url=${url}, channel=${channel}`)
const job = await app.graphile.addJob('record', { url })
const job = await app.graphile.addJob('record', { url, channel })
return job return job
}) })
return app return app

View File

@ -0,0 +1,2 @@
application/vnd.apple.mpegurl mp4
text/x-abc abc

View File

@ -0,0 +1 @@
asdfaslfk;sdf

View File

@ -1,77 +0,0 @@
'use strict'
import { build } from './app.js'
import 'dotenv/config'
import { run } from 'graphile-worker'
import { dirname } from 'node:path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env');
if (!process.env.FUNCTION) throw new Error(`FUNCTION env var was missing. FUNCTION env var must be either 'api' or 'worker'.`);
const connectionString = process.env.DATABASE_URL!
async function api() {
if (!process.env.PORT) throw new Error('PORT is missing in env');
const PORT = parseInt(process.env.PORT!)
const fastifyOpts = {
logger: {
level: 'info',
transport: {
target: 'pino-pretty'
}
}
}
const server = build(fastifyOpts, connectionString)
server.listen({ port: PORT }, (err) => {
if (err) {
server.log.error(err)
process.exit(1)
}
})
}
async function worker() {
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
// Run a worker to execute jobs:
const runner = await run({
connectionString,
concurrency,
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
noHandleSignals: false,
pollInterval: 1000,
taskDirectory: `${__dirname}/tasks`,
});
// Immediately await (or otherwise handle) the resulting promise, to avoid
// "unhandled rejection" errors causing a process crash in the event of
// something going wrong. console.log()
await runner.promise;
// If the worker exits (whether through fatal error or otherwise), the above
// promise will resolve/reject.
}
async function main() {
if (process.env.FUNCTION === 'api') {
api()
} else if (process.env.FUNCTION === 'worker') {
worker()
} else {
throw new Error('process.env.FUNCTION must be either api or worker. got '+process.env.FUNCTION)
}
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

View File

@ -1,80 +0,0 @@
'use strict'
import { build } from './app.js'
import 'dotenv/config'
import { run } from 'graphile-worker'
import { dirname } from 'node:path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env');
if (!process.env.FUNCTION) throw new Error(`FUNCTION env var was missing. FUNCTION env var must be either 'api' or 'worker'.`);
const connectionString = process.env.DATABASE_URL!
async function api() {
if (!process.env.PORT) throw new Error('PORT is missing in env');
const PORT = parseInt(process.env.PORT!)
const fastifyOpts = {
logger: {
level: 'info',
transport: {
target: 'pino-pretty'
}
}
}
const server = build(fastifyOpts, connectionString)
server.listen({ port: PORT }, (err) => {
if (err) {
server.log.error(err)
process.exit(1)
}
})
}
async function worker() {
if (!process.env.WORKER_CONCURRENCY) throw new Error('WORKER_CONCURRENCY is missing in env');
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
// Run a worker to execute jobs:
const runner = await run({
connectionString: process.env.DATABASE_URL!,
concurrency,
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
noHandleSignals: false,
pollInterval: 1000,
taskDirectory: `${__dirname}/tasks`,
});
// Immediately await (or otherwise handle) the resulting promise, to avoid
// "unhandled rejection" errors causing a process crash in the event of
// something going wrong. console.log()
await runner.promise;
// If the worker exits (whether through fatal error or otherwise), the above
// promise will resolve/reject.
}
async function main() {
if (process.env.FUNCTION === 'worker') {
worker()
} else if (process.env.FUNCTION === 'api') {
api()
} else {
console.error(`FUNCTION environment variable must be 'worker' or 'api', but it was set to ${process.env.FUNCTION}`)
}
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

View File

@ -1,41 +0,0 @@
'use strict'
import { run } from 'graphile-worker'
import { dirname } from 'node:path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is undefined in env');
const concurrency = (process.env?.WORKER_CONCURRENCY) ? parseInt(process.env.WORKER_CONCURRENCY) : 1
async function main() {
// Run a worker to execute jobs:
const runner = await run({
connectionString: process.env.DATABASE_URL!,
concurrency,
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
noHandleSignals: false,
pollInterval: 1000,
taskDirectory: `${__dirname}/tasks`,
});
// Immediately await (or otherwise handle) the resulting promise, to avoid
// "unhandled rejection" errors causing a process crash in the event of
// something going wrong. console.log()
await runner.promise;
// If the worker exits (whether through fatal error or otherwise), the above
// promise will resolve/reject.
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

View File

@ -0,0 +1,25 @@
import { createWriteStream } from 'node:fs'
import ffmpeg from 'fluent-ffmpeg'
// test stream from https://ottverse.com/free-hls-m3u8-test-urls/
const playlistUrl = 'https://demo.unified-streaming.com/k8s/features/stable/video/tears-of-steel/tears-of-steel.ism/.m3u8'
const fileOutputStream = createWriteStream('/tmp/test-stream.ts')
ffmpeg()
.input(playlistUrl)
.audioCodec('copy')
.videoCodec('copy')
.addOutputOptions('-movflags faststart')
.output(fileOutputStream)
.format('mpegts')
.on('end', () => {
console.log('Finished');
})
.on('error', (err, stdout, stderr) => {
console.error(`there was an error`);
console.error(err);
console.error(stdout);
console.error(stderr);
throw new Error(err.message);
})
.run();

View File

@ -0,0 +1,135 @@
import { PassThrough, pipeline, Readable } from "stream";
import { type Progress, Upload } from "@aws-sdk/lib-storage";
import { S3Client } from "@aws-sdk/client-s3";
import { createReadStream, createWriteStream } from 'fs';
import { createId } from '@paralleldrive/cuid2';
import prettyBytes from 'pretty-bytes';
import dotenv from 'dotenv'
dotenv.config({
path: '../../.env.development'
})
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
async function uploadStream(client: S3Client, stream: NodeJS.ReadableStream, keyName: string) {
// const pass = new PassThrough()
// Create a stream to the S3 bucket. We use this stream to upload the livestream to Backblaze S3 service
console.log(`keyName=${keyName}`)
const target = {
Bucket: process.env.S3_BUCKET_NAME!,
Key: keyName,
Body: new Readable().wrap(stream)
}
console.log(target)
// greets https://stackoverflow.com/a/70159394/1004931
try {
const parallelUploads3 = new Upload({
client: client,
partSize: 1024 * 1024 * 5,
// tags: [...], // optional tags
queueSize: 1, // optional concurrency configuration
leavePartsOnError: false, // optional manually handle dropped parts
params: target,
});
parallelUploads3.on("httpUploadProgress", (progress) => {
if (progress?.loaded) {
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
} else {
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
}
});
const res = await parallelUploads3.done();
return res
} catch (e) {
if (e instanceof Error) {
console.error(`while uploading a file to s3, we encountered an error`)
throw new Error(e.message);
} else {
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
}
}
}
async function main() {
try {
const client = new S3Client({
endpoint: 'https://s3.us-west-000.backblazeb2.com',
region: 'us-west-000',
credentials: {
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
}
});
// let debugCounter = 0
// let uploadStream = new PassThrough()
// uploadStream.on('data', (data) => {
// debugCounter += data.length
// console.log(`[data] uploadStream. ${debugCounter} aggregated bytes (${prettyBytes(debugCounter)}).`)
// })
// uploadStream.on('drain', () => {
// console.log('[drain] uploadStream')
// })
// uploadStream.on('close', () => {
// console.log(`[close] uploadStream closed`)
// })
// uploadStream.on('error', (err) => {
// console.error('[error] uploadStream had an error as follows')
// console.error(err)
// })
// uploadStream.on('exit', (code) => {
// console.log(`[exit] uploadStream exited with code ${code}`)
// })
// uploadStream.on('disconnect', () => {
// console.log('[disconnect] uploadStream disconnected')
// })
// uploadStream.on('message', (msg) => {
// console.log('[message] uploadStream sent a message as follows')
// console.log(msg)
// })
const fileStream = createReadStream('/home/cj/Downloads/stream-23894234.ts')
const datestamp = new Date().toISOString()
const keyName = `${datestamp}-stream3-chaturbate-${createId()}.ts`
await uploadStream(client, fileStream, keyName)
// // we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
// pipeline(
// fileStream,
// uploadStream,
// (err) => {
// if (err) {
// console.error(`pipeline errored.`)
// console.error(err)
// } else {
// console.log('pipeline succeeded.')
// }
// }
// )
} catch (e) {
if (e instanceof Error) {
console.error(`Eyy lookat me, I'm a big nastry try/catch block and I did my jorb!`)
console.error(e)
} else {
console.error('err or some sort')
console.error(e)
}
}
}
main()

View File

@ -0,0 +1,122 @@
import { PassThrough, pipeline, Readable } from "stream";
import { Upload } from "@aws-sdk/lib-storage";
import { S3Client } from "@aws-sdk/client-s3";
import { createReadStream } from 'fs';
import { createId } from '@paralleldrive/cuid2';
import prettyBytes from 'pretty-bytes';
import dotenv from 'dotenv'
dotenv.config({
path: '../../.env.development'
})
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
function makeProgressTicker(counter: number) {
const ticker = setInterval(() => {
console.log(`[progress] ${counter} bytes (aggregate) (${prettyBytes(counter)}) have passed through the pipeline.`)
}, 1000 * 30)
return ticker
}
function makeS3Client() {
const client = new S3Client({
endpoint: 'https://s3.us-west-000.backblazeb2.com',
region: 'us-west-000',
credentials: {
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
}
})
return client
}
async function uploadToS3({ client, uploadStream, keyName }: { client: S3Client, uploadStream: NodeJS.ReadableStream, keyName: string }) {
const target = {
Bucket: process.env.S3_BUCKET_NAME!,
Key: keyName,
Body: new Readable().wrap(uploadStream)
}
// greets https://stackoverflow.com/a/70159394/1004931
try {
const parallelUploads3 = new Upload({
client: client,
partSize: 1024 * 1024 * 5,
queueSize: 1,
leavePartsOnError: false,
params: target,
});
parallelUploads3.on("httpUploadProgress", (progress) => {
console.log(progress)
if (progress?.loaded) {
console.log(`loaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
} else {
console.log(`httpUploadProgress ${JSON.stringify(progress, null, 2)}`)
}
});
await parallelUploads3.done();
} catch (e) {
if (e instanceof Error) {
console.error(`while uploading a file to s3, we encountered an error`)
throw new Error(e.message);
} else {
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
}
}
}
async function main() {
let counter = 0
const client = makeS3Client()
const ticker = makeProgressTicker(counter)
const datestamp = new Date().toISOString()
const keyName = `${datestamp}-stream3-chaturbate-${createId()}.ts`
console.log(`Uploading ${keyName} to S3`)
/**
* setup the streams which process the data
*/
const ffmpegStream = createReadStream('/home/cj/Downloads/stream-23894234.ts')
const uploadStream = new PassThrough()
// update the progress ticker data
uploadStream.on('data', (data) => {
counter += data.length
})
/**
* we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
*/
pipeline(
ffmpegStream,
uploadStream,
(err) => {
if (err) {
console.error(`pipeline errored.`)
console.error(err)
} else {
console.log('pipeline succeeded.')
}
}
)
await uploadToS3({client, uploadStream, keyName })
clearInterval(ticker)
}
main()

178
packages/capture/src/poc.ts Normal file
View File

@ -0,0 +1,178 @@
/**
*
* @todo if we have the ffmpeg stream send an end event, does the ffmpegStream close?
* so far, we have observed the end of a CB stream, and the uploadStream is what shows as closed.
* It would be better to have the ffmpeg stream do the closing, amirite? or does it even matter?
* Here's what the console.log shows when the CB stream ended while we were not using { end: true } on the ffmpeg stream
*
*
* [data] uploadStream. 118018880 aggregated bytes (118 MB).
[data] uploadStream. 118067384 aggregated bytes (118 MB).
[data] uploadStream. 118101224 aggregated bytes (118 MB).
[data] uploadStream. 118119648 aggregated bytes (118 MB).
[close] uploadStream closed
pipeline succeeded.
*/
import { getRandomRoom } from '@futureporn/scout/cb.js'
import { ua0 } from "@futureporn/scout/ua.js";
import { spawn } from "child_process";
import { PassThrough, pipeline } from "stream";
import { type Progress, Upload } from "@aws-sdk/lib-storage";
import { S3Client } from "@aws-sdk/client-s3";
import { createWriteStream } from 'fs';
import ffmpeg from 'fluent-ffmpeg'
import { createId } from '@paralleldrive/cuid2';
import prettyBytes from 'pretty-bytes';
import dotenv from 'dotenv'
dotenv.config({
path: '../../.env.development'
})
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME missing in env');
if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID missing in env');
if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY missing in env');
async function main() {
const client = new S3Client({
endpoint: 'https://s3.us-west-000.backblazeb2.com',
region: 'us-west-000',
credentials: {
accessKeyId: process.env.S3_BUCKET_KEY_ID!,
secretAccessKey: process.env.S3_BUCKET_APPLICATION_KEY!
}
});
const randomRoom = await getRandomRoom()
console.log(`randomRoom=${randomRoom.name}`)
const playlistUrl: string = await new Promise((resolve, reject) => {
// get the m3u8 playlist for the livestream
const ytdlp = spawn('yt-dlp', [ '-g', randomRoom.url ])
let output = ''
ytdlp.on('error', (err) => {
console.error(err)
})
ytdlp.once('exit', (code) => {
console.log(`code=${code}`)
if (code !== 0) reject(`yt-dlp exited with code ${code}. stderr as follows ${JSON.stringify(ytdlp.stderr, null, 2)}`);
resolve(output)
})
ytdlp.stderr.on('data', (data) => {
console.error('stderr data as follows')
console.error(data.toString())
})
ytdlp.stdout.on('data', (data) => {
output = data
})
})
console.log(`playlistUrl=${playlistUrl}`)
if (!playlistUrl) throw new Error(`failed to get playlistUrl from yt-dlp -g ${randomRoom.url}`);
let debugCounter = 0
let fileOutputStream = createWriteStream('/home/cj/Downloads/outputfile.ts');
// let ffmpegLogStream = createWriteStream('/tmp/ffmpeg-log.txt')
let uploadStream = new PassThrough()
uploadStream.on('data', (data) => {
debugCounter += data.length
console.log(`[data] uploadStream. ${debugCounter} aggregated bytes (${prettyBytes(debugCounter)}).`)
})
uploadStream.on('drain', () => {
console.log('[drain] uploadStream')
})
uploadStream.on('close', () => {
console.log(`[close] uploadStream closed`)
})
uploadStream.on('error', (err) => {
console.error('[error] uploadStream had an error as follows')
console.error(err)
})
uploadStream.on('exit', (code) => {
console.log(`[exit] uploadStream exited with code ${code}`)
})
uploadStream.on('disconnect', () => {
console.log('[disconnect] uploadStream disconnected')
})
uploadStream.on('message', (msg) => {
console.log('[message] uploadStream sent a message as follows')
console.log(msg)
})
const datestamp = new Date().toISOString()
const ffmpegProc = spawn('ffmpeg', [
'-headers', `"User-Agent: ${ua0}"`,
'-i', playlistUrl,
'-c:v', 'copy',
'-c:a', 'copy',
'-movflags', 'faststart',
'-y',
'-f', 'mpegts',
'-loglevel', 'quiet',
'pipe:1'
], {
// ignoring stderr is important because if not, ffmpeg will fill that buffer and node will hang
stdio: ['pipe', 'pipe', 'ignore']
})
// we set up a pipeline which has an readable stream (ffmpeg), a transform stream (debug), and a writable stream (s3 Upload)
pipeline(
ffmpegProc.stdout,
uploadStream,
(err) => {
if (err) {
console.error(`pipeline errored.`)
console.error(err)
} else {
console.log('pipeline succeeded.')
}
}
)
// Create a stream to the S3 bucket. We use this stream to upload the livestream to Backblaze S3 service
const keyName = `${datestamp}-${randomRoom.name}-chaturbate-${createId()}.ts`
console.log(`keyName=${keyName}`)
const target = {
Bucket: process.env.S3_BUCKET_NAME,
Key: keyName,
Body: uploadStream
}
// greets https://stackoverflow.com/a/70159394/1004931
try {
const parallelUploads3 = new Upload({
client: client,
//tags: [...], // optional tags
queueSize: 4, // optional concurrency configuration
leavePartsOnError: false, // optional manually handle dropped parts
params: target,
});
await parallelUploads3.done();
} catch (e) {
if (e instanceof Error) {
console.error(`while uploading a file to s3, we encountered an error`)
throw new Error(e.message);
} else {
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
}
}
}
main()

View File

@ -1,15 +1,36 @@
import { type Helpers } from 'graphile-worker' import { type Helpers } from 'graphile-worker'
import Record from '../Record.ts' import Record from '../Record.ts'
import 'dotenv/config'
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was undefined in env');
if (!process.env.S3_REGION) throw new Error('S3_REGION was undefined in env');
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was undefined in env');
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was undefined in env');
type Payload = { type Payload = {
url: string url: string;
channel: string;
} }
export default async function (payload: Payload, helpers: Helpers) { export default async function (payload: Payload, helpers: Helpers): Promise<string> {
const { url } = payload; const { url, channel } = payload;
helpers.logger.info(`'record' task execution begin with url=${url} (@todo implement)`); helpers.logger.info(`'record' task execution begin with url=${url}, channel=${channel}`);
const record = new Record({ url: 'https://example.com/stream' })
record.start()
const bucket = process.env.S3_BUCKET_NAME!
const endpoint = process.env.S3_ENDPOINT!
const region = process.env.S3_REGION!
const accessKeyId = process.env.S3_ACCESS_KEY_ID!
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY!
const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
const inputStream = Record.getFFmpegDownload({ url })
const record = new Record({ inputStream, bucket, s3Client, channel })
await record.start()
return record.id return record.id

View File

@ -1,6 +1,6 @@
import { describe } from 'mocha' import { describe } from 'mocha'
import { expect } from 'chai'; import { expect } from 'chai';
import { getInitialRoomDossier } from './cb.js' import { getInitialRoomDossier, getRandomRoom } from './cb.js'
describe('cb', function () { describe('cb', function () {
describe('getInitialRoomDossier', function () { describe('getInitialRoomDossier', function () {
@ -13,4 +13,14 @@ describe('cb', function () {
expect(dossier).to.have.property('wschat_host') expect(dossier).to.have.property('wschat_host')
}) })
}) })
describe('getRandomRoom', function () {
it('should return a Room object of an online room', async function () {
this.timeout(1000*60*2)
const room = await getRandomRoom()
expect(room).to.have.property('url')
expect(room).to.have.property('name')
expect(room.name).to.match(/[a-z_]/)
expect(room.url).to.match(/https:\/\//)
})
})
}) })

View File

@ -1,4 +1,42 @@
import * as cheerio from 'cheerio' import * as cheerio from 'cheerio'
import fetch from 'node-fetch'
export interface ChaturbateModel {
gender: string;
location: string;
current_show: 'public' | 'private';
username: string;
room_subject: string;
tags: string[];
is_new: boolean;
num_users: number;
num_followers: number;
country: string;
spoken_languages: string;
display_name: string;
birthday: string;
is_hd: boolean;
age: number;
seconds_online: number;
image_url: string;
image_url_360x270: string;
chat_room_url_revshare: string;
iframe_embed_revshare: string;
chat_room_url: string;
iframe_embed: string;
slug: string;
}
export interface ChaturbateOnlineModelsResponse {
results: ChaturbateModel[],
count: number
}
export interface Room {
name: string;
url: string;
}
/** /**
* *
@ -31,4 +69,37 @@ export async function getInitialRoomDossier(roomUrl: string) {
console.error('caught an exotic error, uh-oh') console.error('caught an exotic error, uh-oh')
} }
} }
}
export async function getRandomRoom(): Promise<Room> {
try {
const res = await fetch('https://chaturbate.com/api/public/affiliates/onlinerooms/?wm=DiPkB&client_ip=request_ip');
const data = await res.json() as ChaturbateOnlineModelsResponse;
if (!data || !Array.isArray(data.results) || data.results.length === 0) {
throw new Error('No results found');
} }
const results = data.results;
const randomIndex = Math.floor(Math.random() * results.length);
if (!results[randomIndex]) {
throw new Error('No result found at random index');
}
const username = results[randomIndex].username;
return {
url: `https://chaturbate.com/${username}`,
name: username
}
} catch (error) {
if (error instanceof Error) {
console.error(`Error in getRandomRoom: ${error.message}`);
} else {
console.error('An unexpected error occurred');
}
throw error; // Re-throw the error to propagate it further
}
}

View File

@ -43,40 +43,6 @@ importers:
specifier: ^4.16.2 specifier: ^4.16.2
version: 4.16.2 version: 4.16.2
../..: {}
../bot: {}
../capture: {}
../image: {}
../infra: {}
../mailbox: {}
../meal: {}
../next: {}
../old: {}
../scout: {}
../storage: {}
../strapi: {}
../taco: {}
../types: {}
../uppy: {}
../video: {}
../worker: {}
packages: packages:
'@cspotcode/source-map-support@0.8.1': '@cspotcode/source-map-support@0.8.1':

View File

@ -10,7 +10,7 @@ import { dirname, basename, join, isAbsolute } from 'node:path';
import { fileURLToPath } from 'url'; import { fileURLToPath } from 'url';
export const __filename = fileURLToPath(import.meta.url); export const __filename = fileURLToPath(import.meta.url);
export const __dirname = dirname(fileURLToPath(import.meta.url)); const __dirname = dirname(fileURLToPath(import.meta.url));
export function getPackageVersion(packageJsonPath: string): string { export function getPackageVersion(packageJsonPath: string): string {
if (!isAbsolute(packageJsonPath)) { if (!isAbsolute(packageJsonPath)) {

View File

@ -39,7 +39,9 @@ EOF
kubectl --namespace futureporn delete secret capture --ignore-not-found kubectl --namespace futureporn delete secret capture --ignore-not-found
kubectl --namespace futureporn create secret generic capture \ kubectl --namespace futureporn create secret generic capture \
--from-literal=databaseUrl=${WORKER_DATABASE_URL} --from-literal=databaseUrl=${WORKER_DATABASE_URL} \
--from-literal=s3AccessKeyId=${S3_ACCESS_KEY_ID} \
--from-literal=s3SecretAccessKey=${S3_SECRET_ACCESS_KEY}
kubectl --namespace futureporn delete secret mailbox --ignore-not-found kubectl --namespace futureporn delete secret mailbox --ignore-not-found
kubectl --namespace futureporn create secret generic mailbox \ kubectl --namespace futureporn create secret generic mailbox \

View File

@ -1,3 +1,12 @@
if [ -z $POSTGRES_PASSWORD ]; then
echo "POSTGRES_PASSWORD was missing in env"
exit 5
fi
## drop futureporn_db ## drop futureporn_db
kubectl -n futureporn exec postgres-primary-0 -- psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);" kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
## drop graphile_worker
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE graphile_worker WITH (FORCE);"