futureporn
/
fp
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

chatops progress

This commit is contained in:
CJ_Clippy 2024-08-16 18:42:44 -08:00
parent 7793e38878
commit 9cd9b6a53d
65 changed files with 1562 additions and 1759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
MANTRAS.md
View File

@ -33,7 +33,7 @@ Get through the [OODA loop](https://en.wikipedia.org/wiki/OODA_loop) as many tim
### The computer doesn't care
> "There are 2 hard problems in computer science: cache invalidation, naming things, and off-by-1 errors."
> Leon Bambrick
> -- Leon Bambrick
In other words, pick something for a name and roll with the punches.
@ -44,3 +44,8 @@ In other words, pick something for a name and roll with the punches.
3. Simplify or optimize
4. Accelerate Cycle Time
5. Automate
### Never Settle
> "But it's also about looking at things anew and what they could be instead of what they are"
> -- Rodney Mullen

30
Tiltfile
View File

@ -215,6 +215,12 @@ cmd_button('postgres:drop',
icon_name='delete',
text='DROP all databases'
)
cmd_button('postgres:refresh',
argv=['sh', './scripts/postgres-refresh.sh'],
resource='migrations',
icon_name='refresh',
text='Refresh schema cache'
)
cmd_button('capture-api:create',
argv=['http', '--ignore-stdin', 'POST', 'http://localhost:5003/api/record', "url='https://twitch.tv/ironmouse'", "channel='ironmouse'"],
@ -223,9 +229,9 @@ cmd_button('capture-api:create',
text='Start Recording'
)
cmd_button('postgrest:migrate',
argv=['./scripts/postgrest-migrations.sh'],
resource='postgrest',
cmd_button('postgres:migrate',
argv=['./scripts/postgres-migrations.sh'],
resource='postgresql-primary',
icon_name='directions_run',
text='Run migrations',
)
@ -243,6 +249,16 @@ cmd_button('factory:test',
text='test',
)
## we ignore unused image warnings because we do actually use this image.
## instead of being invoked by helm, we start a container using this image manually via Tilt UI
# update_settings(suppress_unused_image_warnings=["fp/migrations"])
docker_build(
'fp/migrations',
'.',
dockerfile='dockerfiles/migrations.dockerfile',
target='migrations',
pull=False,
)
## Uncomment the following for fp/next in dev mode
## this is useful for changing the UI and seeing results
@ -350,7 +366,7 @@ docker_build(
'./services/capture',
],
live_update=[
sync('./services/capture/dist', '/app/dist'),
sync('./services/capture', '/app/services/capture'),
],
pull=False,
)
@ -513,7 +529,11 @@ k8s_resource(
port_forwards=['5050:80'],
labels=['database'],
)
k8s_resource(
workload='migrations',
labels=['database'],
resource_deps=['postgresql-primary'],
)
k8s_resource(
workload='cert-manager',

21
charts/fp/templates/migrations.yaml Normal file
View File

@ -0,0 +1,21 @@
---
apiVersion: v1
kind: Pod
metadata:
name: migrations
namespace: futureporn
labels:
app.kubernetes.io/name: migrations
spec:
containers:
- name: migrations
image: "{{ .Values.migrations.imageName }}"
resources: {}
env:
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: postgresql
key: password
restartPolicy: Never

2
charts/fp/values.yaml
View File

@ -91,3 +91,5 @@ game2048:
hostname: game-2048.fp.sbtp.xyz
whoami:
hostname: whoami.fp.sbtp.xyz
migrations:
imageName: fp/migrations

43
packages/types/index.d.ts vendored
View File

@ -4,22 +4,45 @@ export as namespace Futureporn;
declare namespace Futureporn {
interface RecordingRecord {
id: number;
recordingState: RecordingState;
fileSize: number;
discordMessageId: string;
isAborted: boolean;
type PlatformNotificationType = 'email' | 'manual' | 'twitter'
type ArchiveStatus = 'good' | 'issue' | 'missing'
type RecordingState = 'pending' | 'recording' | 'stalled' | 'aborted' | 'failed' | 'finished'
type Status = Partial<RecordingState>
interface Stream {
id: string;
url: string;
platform_notification_type: PlatformNotificationType;
date: Date;
created_at: Date;
updated_at: Date;
vtuber: string;
tweet: string;
archive_status: ArchiveStatus;
is_chaturbate_stream: Boolean;
is_fansly_stream: Boolean;
is_recording_aborted: Boolean;
status: Status;
}
interface RawRecordingRecord {
interface RecordingRecord {
id: number;
recording_state: RecordingState;
file_size: number;
discord_message_id: string;
is_aborted: boolean;
is_recording_aborted: boolean;
updated_at: Date;
created_at: Date;
}
interface Segment {
id: number;
s3_key: string;
s3_id: string;
bytes: number;
stream?: Stream[];
}
type RecordingState = 'pending' | 'recording' | 'aborted' | 'ended'
interface IMuxAsset {
@ -70,7 +93,7 @@ declare namespace Futureporn {
attributes: {
date: string;
date2: string;
archiveStatus: 'good' | 'issue' | 'missing';
archiveStatus: ArchiveStatus;
vods: IVodsResponse;
cuid: string;
vtuber: IVtuberResponse;

7
packages/worker/README.md
View File

@ -1,7 +0,0 @@
# @futureporn/worker
The system component which runs background tasks such as thumbnail generation, video encoding, file transfers, etc.
We use [Graphile Worker](https://worker.graphile.org)

26
packages/worker/package.json
View File

@ -1,26 +0,0 @@
{
"name": "@futureporn/worker",
"type": "module",
"version": "1.3.0",
"private": true,
"scripts": {
"bundle": "node ./src/create-workflow-bundle.js",
"build": "tsc --build",
"lint": "eslint .",
"dev": "nodemon --ext js,ts,json,yaml --watch ./src/index.ts --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"",
"start": "node dist/index.js",
"clean": "rm -rf dist",
"superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist"
},
"dependencies": {
"date-fns": "^3.6.0",
"dotenv": "^16.4.5",
"graphile-worker": "^0.16.6",
"qs": "^6.12.3"
},
"packageManager": "pnpm@9.5.0",
"devDependencies": {
"nodemon": "^2.0.15",
"typescript": "^5.5.3"
}
}

1033
packages/worker/pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

35
packages/worker/src/index.ts
View File

@ -1,35 +0,0 @@
import { run } from 'graphile-worker'
import { dirname } from 'node:path';
import { fileURLToPath } from 'url';
const __dirname = dirname(fileURLToPath(import.meta.url));
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is undefined in env');
const connectionString = process.env.DATABASE_URL
console.log(`process.env.DATABASE_URL=${process.env.DATABASE_URL}`)
async function main() {
// Run a worker to execute jobs:
const runner = await run({
connectionString,
concurrency: 5,
// Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
noHandleSignals: false,
pollInterval: 1000,
taskDirectory: `${__dirname}/tasks`,
});
// Immediately await (or otherwise handle) the resulting promise, to avoid
// "unhandled rejection" errors causing a process crash in the event of
// something going wrong.
await runner.promise;
// If the worker exits (whether through fatal error or otherwise), the above
// promise will resolve/reject.
}
main().catch((err) => {
console.error(err);
process.exit(1);
});

4
packages/worker/src/tasks/hello.ts
View File

@ -1,4 +0,0 @@
export default async function (payload: any, helpers: any) {
const { name } = payload;
helpers.logger.info(`Hello, ${name}`);
};

15
packages/worker/src/tasks/identify_image_color.ts
View File

@ -1,15 +0,0 @@
import { download } from "@futureporn/utils";
import { getProminentColor } from "@futureporn/image";
export default async function (payload: any, helpers: any) {
const { url } = payload;
// helpers.logger.info(`Downloading ${url}`)
// const imageFile = await download({ url, filePath: '/tmp/my-image.png' })
// helpers.logger.info(`downloaded to ${imageFile}`)
// if (!imageFile) throw new Error('imageFile was null')
// const color = await getProminentColor(imageFile)
// helpers.logger.info(`prominent color is ${color}`)
// return color
return '#0xffcc00'
}

29
packages/worker/tsconfig.json
View File

@ -1,29 +0,0 @@
{
"compilerOptions": {
// Base Options recommended for all projects
"esModuleInterop": true,
"skipLibCheck": true,
"target": "es2022",
"allowJs": true,
"resolveJsonModule": true,
"moduleDetection": "force",
"isolatedModules": true,
// Enable strict type checking so you can catch bugs early
"strict": true,
"noUncheckedIndexedAccess": true,
"noImplicitOverride": true,
// Transpile our TypeScript code to JavaScript
"module": "NodeNext",
"outDir": "dist",
"lib": [
"es2022"
]
},
// Include the necessary files for your project
"include": [
"**/*.ts"
],
"exclude": [
"node_modules"
]
}

9
scripts/postgres-create.sh
View File

@ -61,14 +61,7 @@ kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PAS
## Create the futureporn Postgrest database
kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
CREATE DATABASE futureporn \
WITH \
OWNER = postgres \
ENCODING = 'UTF8' \
LOCALE_PROVIDER = 'libc' \
CONNECTION LIMIT = -1 \
IS_TEMPLATE = False;"
## !!! Don't create the database here! Allow @services/migrations to create the database.
# @futureporn/migrations takes care of these tasks now

9
scripts/postgres-drop.sh
View File

@ -5,5 +5,12 @@ if [ -z $POSTGRES_PASSWORD ]; then
fi
## drop futureporn_db
kubectl -n futureporn exec postgresql-primary -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);"
## drop futureporn
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn WITH (FORCE);"
## delete postgrest roles
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP ROLE authenticator;"
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP ROLE automation;"
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP ROLE web_anon;"

13
scripts/postgres-migrations.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
if [ -z $POSTGRES_PASSWORD ]; then
echo "POSTGRES_PASSWORD was missing in env. In development environment, runing this command via the UI button in Tilt is recommended as it sets the env var for you."
exit 5
fi
# kubectl -n futureporn run postgrest-migrations -i --rm=true --image=gitea.futureporn.net/futureporn/migrations:latest --env=DATABASE_PASSWORD=${POSTGRES_PASSWORD}
kubectl -n futureporn run postgres-migrations -i --rm=true --image=fp/migrations:latest --env=DATABASE_PASSWORD=${POSTGRES_PASSWORD}

9
scripts/postgres-refresh.sh Normal file
View File

@ -0,0 +1,9 @@
if [ -z $POSTGRES_PASSWORD ]; then
echo "POSTGRES_PASSWORD was missing in env"
exit 5
fi
# reload the schema
# @see https://postgrest.org/en/latest/references/schema_cache.html#schema-reloading
kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "NOTIFY pgrst, 'reload schema'"

12
scripts/postgrest-migrations.sh
View File

@ -1,12 +0,0 @@
#!/bin/bash
if [ -z $POSTGRES_PASSWORD ]; then
echo "POSTGRES_PASSWORD was missing in env. In development environment, runing this command via the UI button in Tilt is recommended as it sets the env var for you."
exit 5
fi
kubectl -n futureporn run postgrest-migrations -i --rm=true --image=gitea.futureporn.net/futureporn/migrations:latest --env=DATABASE_PASSWORD=${POSTGRES_PASSWORD}

4
services/bot/src/crontab → services/bot/crontab
View File

@ -13,6 +13,6 @@
# * * * * * task ?opts {payload}
## every 5 minutes, we see which /records are stale and we mark them as such.
## every n minutes, we see which /records are stale and we mark them as such.
## this prevents stalled Record updates by marking stalled recordings as stopped
*/5 * * * * expire_records
* * * * * expire_stream_recordings ?max=1 { idle_minutes:2 }

16
services/bot/package.json
View File

@ -7,8 +7,10 @@
"scripts": {
"test": "echo \"Warn: no test specified\" && exit 0",
"start": "node ./dist/index.js",
"dev.nodemon": "nodemon --legacy-watch --ext js,ts --watch ./src --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"",
"dev": "tsx --watch ./src/index.ts",
"dev": "pnpm run dev.nodemon # yes this is crazy to have nodemon execute tsx, but it's the only way I have found to get live reloading in TS/ESM/docker with Graphile Worker's way of loading tasks",
"dev.tsx": "tsx ./src/index.ts",
"dev.nodemon": "nodemon --ext ts --exec \"pnpm run dev.tsx\"",
"dev.node": "node --no-warnings=ExperimentalWarning --loader ts-node/esm src/index.ts",
"build": "tsc --build",
"clean": "rm -rf dist",
"superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist",
@ -20,18 +22,22 @@
"license": "Unlicense",
"dependencies": {
"@discordeno/bot": "19.0.0-next.746f0a9",
"@types/node": "^22.2.0",
"@types/qs": "^6.9.15",
"date-fns": "^3.6.0",
"dd-cache-proxy": "^2.1.1",
"dotenv": "^16.4.5",
"graphile-config": "0.0.1-beta.9",
"graphile-worker": "^0.16.6",
"pretty-bytes": "^6.1.1"
"node-fetch": "^3.3.2",
"pretty-bytes": "^6.1.1",
"qs": "^6.13.0"
},
"devDependencies": {
"@futureporn/types": "workspace:^",
"nodemon": "^3.1.4",
"ts-node": "^10.9.2",
"tsx": "^4.16.2",
"typescript": "^5.5.3"
"tsx": "^4.17.0",
"typescript": "^5.5.4"
}
}

495
services/bot/pnpm-lock.yaml
View File

@ -11,6 +11,12 @@ importers:
'@discordeno/bot':
specifier: 19.0.0-next.746f0a9
version: 19.0.0-next.746f0a9
'@types/node':
specifier: ^22.2.0
version: 22.2.0
'@types/qs':
specifier: ^6.9.15
version: 6.9.15
date-fns:
specifier: ^3.6.0
version: 3.6.0
@ -26,9 +32,15 @@ importers:
graphile-worker:
specifier: ^0.16.6
version: 0.16.6(typescript@5.5.4)
node-fetch:
specifier: ^3.3.2
version: 3.3.2
pretty-bytes:
specifier: ^6.1.1
version: 6.1.1
qs:
specifier: ^6.13.0
version: 6.13.0
devDependencies:
'@futureporn/types':
specifier: workspace:^
@ -38,12 +50,12 @@ importers:
version: 3.1.4
ts-node:
specifier: ^10.9.2
version: 10.9.2(@types/node@22.1.0)(typescript@5.5.4)
version: 10.9.2(@types/node@22.2.0)(typescript@5.5.4)
tsx:
specifier: ^4.16.2
version: 4.16.2
specifier: ^4.17.0
version: 4.17.0
typescript:
specifier: ^5.5.3
specifier: ^5.5.4
version: 5.5.4
packages:
@ -79,141 +91,147 @@ packages:
'@discordeno/utils@19.0.0-next.746f0a9':
resolution: {integrity: sha512-UY5GataakuY0yc4SN5qJLexUbTc5y293G3gNAWSaOjaZivEytcdxD4xgeqjNj9c4eN57B3Lfzus6tFZHXwXNOA==}
'@esbuild/aix-ppc64@0.21.5':
resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==}
engines: {node: '>=12'}
'@esbuild/aix-ppc64@0.23.0':
resolution: {integrity: sha512-3sG8Zwa5fMcA9bgqB8AfWPQ+HFke6uD3h1s3RIwUNK8EG7a4buxvuFTs3j1IMs2NXAk9F30C/FF4vxRgQCcmoQ==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [aix]
'@esbuild/android-arm64@0.21.5':
resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==}
engines: {node: '>=12'}
'@esbuild/android-arm64@0.23.0':
resolution: {integrity: sha512-EuHFUYkAVfU4qBdyivULuu03FhJO4IJN9PGuABGrFy4vUuzk91P2d+npxHcFdpUnfYKy0PuV+n6bKIpHOB3prQ==}
engines: {node: '>=18'}
cpu: [arm64]
os: [android]
'@esbuild/android-arm@0.21.5':
resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==}
engines: {node: '>=12'}
'@esbuild/android-arm@0.23.0':
resolution: {integrity: sha512-+KuOHTKKyIKgEEqKbGTK8W7mPp+hKinbMBeEnNzjJGyFcWsfrXjSTNluJHCY1RqhxFurdD8uNXQDei7qDlR6+g==}
engines: {node: '>=18'}
cpu: [arm]
os: [android]
'@esbuild/android-x64@0.21.5':
resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==}
engines: {node: '>=12'}
'@esbuild/android-x64@0.23.0':
resolution: {integrity: sha512-WRrmKidLoKDl56LsbBMhzTTBxrsVwTKdNbKDalbEZr0tcsBgCLbEtoNthOW6PX942YiYq8HzEnb4yWQMLQuipQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [android]
'@esbuild/darwin-arm64@0.21.5':
resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==}
engines: {node: '>=12'}
'@esbuild/darwin-arm64@0.23.0':
resolution: {integrity: sha512-YLntie/IdS31H54Ogdn+v50NuoWF5BDkEUFpiOChVa9UnKpftgwzZRrI4J132ETIi+D8n6xh9IviFV3eXdxfow==}
engines: {node: '>=18'}
cpu: [arm64]
os: [darwin]
'@esbuild/darwin-x64@0.21.5':
resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==}
engines: {node: '>=12'}
'@esbuild/darwin-x64@0.23.0':
resolution: {integrity: sha512-IMQ6eme4AfznElesHUPDZ+teuGwoRmVuuixu7sv92ZkdQcPbsNHzutd+rAfaBKo8YK3IrBEi9SLLKWJdEvJniQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [darwin]
'@esbuild/freebsd-arm64@0.21.5':
resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==}
engines: {node: '>=12'}
'@esbuild/freebsd-arm64@0.23.0':
resolution: {integrity: sha512-0muYWCng5vqaxobq6LB3YNtevDFSAZGlgtLoAc81PjUfiFz36n4KMpwhtAd4he8ToSI3TGyuhyx5xmiWNYZFyw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [freebsd]
'@esbuild/freebsd-x64@0.21.5':
resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==}
engines: {node: '>=12'}
'@esbuild/freebsd-x64@0.23.0':
resolution: {integrity: sha512-XKDVu8IsD0/q3foBzsXGt/KjD/yTKBCIwOHE1XwiXmrRwrX6Hbnd5Eqn/WvDekddK21tfszBSrE/WMaZh+1buQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [freebsd]
'@esbuild/linux-arm64@0.21.5':
resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==}
engines: {node: '>=12'}
'@esbuild/linux-arm64@0.23.0':
resolution: {integrity: sha512-j1t5iG8jE7BhonbsEg5d9qOYcVZv/Rv6tghaXM/Ug9xahM0nX/H2gfu6X6z11QRTMT6+aywOMA8TDkhPo8aCGw==}
engines: {node: '>=18'}
cpu: [arm64]
os: [linux]
'@esbuild/linux-arm@0.21.5':
resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==}
engines: {node: '>=12'}
'@esbuild/linux-arm@0.23.0':
resolution: {integrity: sha512-SEELSTEtOFu5LPykzA395Mc+54RMg1EUgXP+iw2SJ72+ooMwVsgfuwXo5Fn0wXNgWZsTVHwY2cg4Vi/bOD88qw==}
engines: {node: '>=18'}
cpu: [arm]
os: [linux]
'@esbuild/linux-ia32@0.21.5':
resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==}
engines: {node: '>=12'}
'@esbuild/linux-ia32@0.23.0':
resolution: {integrity: sha512-P7O5Tkh2NbgIm2R6x1zGJJsnacDzTFcRWZyTTMgFdVit6E98LTxO+v8LCCLWRvPrjdzXHx9FEOA8oAZPyApWUA==}
engines: {node: '>=18'}
cpu: [ia32]
os: [linux]
'@esbuild/linux-loong64@0.21.5':
resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==}
engines: {node: '>=12'}
'@esbuild/linux-loong64@0.23.0':
resolution: {integrity: sha512-InQwepswq6urikQiIC/kkx412fqUZudBO4SYKu0N+tGhXRWUqAx+Q+341tFV6QdBifpjYgUndV1hhMq3WeJi7A==}
engines: {node: '>=18'}
cpu: [loong64]
os: [linux]
'@esbuild/linux-mips64el@0.21.5':
resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==}
engines: {node: '>=12'}
'@esbuild/linux-mips64el@0.23.0':
resolution: {integrity: sha512-J9rflLtqdYrxHv2FqXE2i1ELgNjT+JFURt/uDMoPQLcjWQA5wDKgQA4t/dTqGa88ZVECKaD0TctwsUfHbVoi4w==}
engines: {node: '>=18'}
cpu: [mips64el]
os: [linux]
'@esbuild/linux-ppc64@0.21.5':
resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==}
engines: {node: '>=12'}
'@esbuild/linux-ppc64@0.23.0':
resolution: {integrity: sha512-cShCXtEOVc5GxU0fM+dsFD10qZ5UpcQ8AM22bYj0u/yaAykWnqXJDpd77ublcX6vdDsWLuweeuSNZk4yUxZwtw==}
engines: {node: '>=18'}
cpu: [ppc64]
os: [linux]
'@esbuild/linux-riscv64@0.21.5':
resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==}
engines: {node: '>=12'}
'@esbuild/linux-riscv64@0.23.0':
resolution: {integrity: sha512-HEtaN7Y5UB4tZPeQmgz/UhzoEyYftbMXrBCUjINGjh3uil+rB/QzzpMshz3cNUxqXN7Vr93zzVtpIDL99t9aRw==}
engines: {node: '>=18'}
cpu: [riscv64]
os: [linux]
'@esbuild/linux-s390x@0.21.5':
resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==}
engines: {node: '>=12'}
'@esbuild/linux-s390x@0.23.0':
resolution: {integrity: sha512-WDi3+NVAuyjg/Wxi+o5KPqRbZY0QhI9TjrEEm+8dmpY9Xir8+HE/HNx2JoLckhKbFopW0RdO2D72w8trZOV+Wg==}
engines: {node: '>=18'}
cpu: [s390x]
os: [linux]
'@esbuild/linux-x64@0.21.5':
resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==}
engines: {node: '>=12'}
'@esbuild/linux-x64@0.23.0':
resolution: {integrity: sha512-a3pMQhUEJkITgAw6e0bWA+F+vFtCciMjW/LPtoj99MhVt+Mfb6bbL9hu2wmTZgNd994qTAEw+U/r6k3qHWWaOQ==}
engines: {node: '>=18'}
cpu: [x64]
os: [linux]
'@esbuild/netbsd-x64@0.21.5':
resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==}
engines: {node: '>=12'}
'@esbuild/netbsd-x64@0.23.0':
resolution: {integrity: sha512-cRK+YDem7lFTs2Q5nEv/HHc4LnrfBCbH5+JHu6wm2eP+d8OZNoSMYgPZJq78vqQ9g+9+nMuIsAO7skzphRXHyw==}
engines: {node: '>=18'}
cpu: [x64]
os: [netbsd]
'@esbuild/openbsd-x64@0.21.5':
resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==}
engines: {node: '>=12'}
'@esbuild/openbsd-arm64@0.23.0':
resolution: {integrity: sha512-suXjq53gERueVWu0OKxzWqk7NxiUWSUlrxoZK7usiF50C6ipColGR5qie2496iKGYNLhDZkPxBI3erbnYkU0rQ==}
engines: {node: '>=18'}
cpu: [arm64]
os: [openbsd]
'@esbuild/openbsd-x64@0.23.0':
resolution: {integrity: sha512-6p3nHpby0DM/v15IFKMjAaayFhqnXV52aEmv1whZHX56pdkK+MEaLoQWj+H42ssFarP1PcomVhbsR4pkz09qBg==}
engines: {node: '>=18'}
cpu: [x64]
os: [openbsd]
'@esbuild/sunos-x64@0.21.5':
resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==}
engines: {node: '>=12'}
'@esbuild/sunos-x64@0.23.0':
resolution: {integrity: sha512-BFelBGfrBwk6LVrmFzCq1u1dZbG4zy/Kp93w2+y83Q5UGYF1d8sCzeLI9NXjKyujjBBniQa8R8PzLFAUrSM9OA==}
engines: {node: '>=18'}
cpu: [x64]
os: [sunos]
'@esbuild/win32-arm64@0.21.5':
resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==}
engines: {node: '>=12'}
'@esbuild/win32-arm64@0.23.0':
resolution: {integrity: sha512-lY6AC8p4Cnb7xYHuIxQ6iYPe6MfO2CC43XXKo9nBXDb35krYt7KGhQnOkRGar5psxYkircpCqfbNDB4uJbS2jQ==}
engines: {node: '>=18'}
cpu: [arm64]
os: [win32]
'@esbuild/win32-ia32@0.21.5':
resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==}
engines: {node: '>=12'}
'@esbuild/win32-ia32@0.23.0':
resolution: {integrity: sha512-7L1bHlOTcO4ByvI7OXVI5pNN6HSu6pUQq9yodga8izeuB1KcT2UkHaH6118QJwopExPn0rMHIseCTx1CRo/uNA==}
engines: {node: '>=18'}
cpu: [ia32]
os: [win32]
'@esbuild/win32-x64@0.21.5':
resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==}
engines: {node: '>=12'}
'@esbuild/win32-x64@0.23.0':
resolution: {integrity: sha512-Arm+WgUFLUATuoxCJcahGuk6Yj9Pzxd6l11Zb/2aAuv5kWWvvfhLFo2fni4uSK5vzlUdCGZ/BdV5tH8klj8p8g==}
engines: {node: '>=18'}
cpu: [x64]
os: [win32]
@ -251,18 +269,18 @@ packages:
'@types/ms@0.7.34':
resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
'@types/node@20.14.13':
resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==}
'@types/node@20.14.15':
resolution: {integrity: sha512-Fz1xDMCF/B00/tYSVMlmK7hVeLh7jE5f3B7X1/hmV0MJBwE27KlS7EvD/Yp+z1lm8mVhwV5w+n8jOZG8AfTlKw==}
'@types/node@22.0.0':
resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==}
'@types/node@22.1.0':
resolution: {integrity: sha512-AOmuRF0R2/5j1knA3c6G3HOk523Ga+l+ZXltX8SF1+5oqcXijjfTd8fY3XRZqSihEu9XhtQnKYLmkFaoxgsJHw==}
'@types/node@22.2.0':
resolution: {integrity: sha512-bm6EG6/pCpkxDf/0gDNDdtDILMOHgaQBVOJGdwsqClnxA3xL6jtMv76rLBc006RVMWbmaf0xbmom4Z/5o2nRkQ==}
'@types/pg@8.11.6':
resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==}
'@types/qs@6.9.15':
resolution: {integrity: sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==}
'@types/semver@7.5.8':
resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==}
@ -311,6 +329,10 @@ packages:
resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==}
engines: {node: '>=8'}
call-bind@1.0.7:
resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}
engines: {node: '>= 0.4'}
callsites@3.1.0:
resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==}
engines: {node: '>=6'}
@ -359,6 +381,10 @@ packages:
create-require@1.1.1:
resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
data-uri-to-buffer@4.0.1:
resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==}
engines: {node: '>= 12'}
date-fns@3.6.0:
resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==}
@ -376,6 +402,10 @@ packages:
supports-color:
optional: true
define-data-property@1.1.4:
resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==}
engines: {node: '>= 0.4'}
diff@4.0.2:
resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
engines: {node: '>=0.3.1'}
@ -390,9 +420,17 @@ packages:
error-ex@1.3.2:
resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==}
esbuild@0.21.5:
resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==}
engines: {node: '>=12'}
es-define-property@1.0.0:
resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==}
engines: {node: '>= 0.4'}
es-errors@1.3.0:
resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
engines: {node: '>= 0.4'}
esbuild@0.23.0:
resolution: {integrity: sha512-1lvV17H2bMYda/WaFb2jLPeHU3zml2k4/yagNMG8Q/YtfMjCwEUZa2eXXMgZTVSL5q1n4H7sQ0X6CdJDqqeCFA==}
engines: {node: '>=18'}
hasBin: true
escalade@3.1.2:
@ -403,19 +441,34 @@ packages:
resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}
engines: {node: '>=0.8.0'}
fetch-blob@3.2.0:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
engines: {node: ^12.20 || >= 14.13}
fill-range@7.1.1:
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
engines: {node: '>=8'}
formdata-polyfill@4.0.10:
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
fsevents@2.3.3:
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
os: [darwin]
function-bind@1.1.2:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
get-caller-file@2.0.5:
resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
engines: {node: 6.* || 8.* || >= 10.*}
get-intrinsic@1.2.4:
resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
engines: {node: '>= 0.4'}
get-tsconfig@4.7.6:
resolution: {integrity: sha512-ZAqrLlu18NbDdRaHq+AKXzAmqIUPswPWKUchfytdAjiRFnCe5ojG2bstg6mRiZabkKfCoL/e98pbBELIV/YCeA==}
@ -423,6 +476,9 @@ packages:
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
engines: {node: '>= 6'}
gopd@1.0.1:
resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
graphile-config@0.0.1-beta.9:
resolution: {integrity: sha512-7vNxXZ24OAgXxDKXYi9JtgWPMuNbBL3057Yf32Ux+/rVP4+EePgySCc+NNnn0tORi8qwqVreN8bdWqGIcSwNXg==}
engines: {node: '>=16'}
@ -440,6 +496,21 @@ packages:
resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
engines: {node: '>=8'}
has-property-descriptors@1.0.2:
resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==}
has-proto@1.0.3:
resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==}
engines: {node: '>= 0.4'}
has-symbols@1.0.3:
resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==}
engines: {node: '>= 0.4'}
hasown@2.0.2:
resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
engines: {node: '>= 0.4'}
ignore-by-default@1.0.1:
resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==}
@ -501,6 +572,14 @@ packages:
ms@2.1.2:
resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
node-domexception@1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
node-fetch@3.3.2:
resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
nodemon@3.1.4:
resolution: {integrity: sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==}
engines: {node: '>=10'}
@ -510,6 +589,10 @@ packages:
resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
engines: {node: '>=0.10.0'}
object-inspect@1.13.2:
resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==}
engines: {node: '>= 0.4'}
obuf@1.1.2:
resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==}
@ -616,6 +699,10 @@ packages:
pstree.remy@1.1.8:
resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}
qs@6.13.0:
resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==}
engines: {node: '>=0.6'}
readdirp@3.6.0:
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
@ -636,6 +723,14 @@ packages:
engines: {node: '>=10'}
hasBin: true
set-function-length@1.2.2:
resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}
engines: {node: '>= 0.4'}
side-channel@1.0.6:
resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==}
engines: {node: '>= 0.4'}
simple-update-notifier@2.0.0:
resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==}
engines: {node: '>=10'}
@ -682,11 +777,11 @@ packages:
'@swc/wasm':
optional: true
tslib@2.6.2:
resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==}
tslib@2.6.3:
resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==}
tsx@4.16.2:
resolution: {integrity: sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==}
tsx@4.17.0:
resolution: {integrity: sha512-eN4mnDA5UMKDt4YZixo9tBioibaMBpoxBkD+rIPAjVmYERSG0/dWEY1CEFuV89CgASlKL499q8AhmkMnnjtOJg==}
engines: {node: '>=18.0.0'}
hasBin: true
@ -701,15 +796,16 @@ packages:
undici-types@5.26.5:
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
undici-types@6.11.1:
resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==}
undici-types@6.13.0:
resolution: {integrity: sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg==}
v8-compile-cache-lib@3.0.1:
resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==}
web-streams-polyfill@3.3.3:
resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==}
engines: {node: '>= 8'}
wrap-ansi@7.0.0:
resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
engines: {node: '>=10'}
@ -796,73 +892,76 @@ snapshots:
dependencies:
'@discordeno/types': 19.0.0-next.746f0a9
'@esbuild/aix-ppc64@0.21.5':
'@esbuild/aix-ppc64@0.23.0':
optional: true
'@esbuild/android-arm64@0.21.5':
'@esbuild/android-arm64@0.23.0':
optional: true
'@esbuild/android-arm@0.21.5':
'@esbuild/android-arm@0.23.0':
optional: true
'@esbuild/android-x64@0.21.5':
'@esbuild/android-x64@0.23.0':
optional: true
'@esbuild/darwin-arm64@0.21.5':
'@esbuild/darwin-arm64@0.23.0':
optional: true
'@esbuild/darwin-x64@0.21.5':
'@esbuild/darwin-x64@0.23.0':
optional: true
'@esbuild/freebsd-arm64@0.21.5':
'@esbuild/freebsd-arm64@0.23.0':
optional: true
'@esbuild/freebsd-x64@0.21.5':
'@esbuild/freebsd-x64@0.23.0':
optional: true
'@esbuild/linux-arm64@0.21.5':
'@esbuild/linux-arm64@0.23.0':
optional: true
'@esbuild/linux-arm@0.21.5':
'@esbuild/linux-arm@0.23.0':
optional: true
'@esbuild/linux-ia32@0.21.5':
'@esbuild/linux-ia32@0.23.0':
optional: true
'@esbuild/linux-loong64@0.21.5':
'@esbuild/linux-loong64@0.23.0':
optional: true
'@esbuild/linux-mips64el@0.21.5':
'@esbuild/linux-mips64el@0.23.0':
optional: true
'@esbuild/linux-ppc64@0.21.5':
'@esbuild/linux-ppc64@0.23.0':
optional: true
'@esbuild/linux-riscv64@0.21.5':
'@esbuild/linux-riscv64@0.23.0':
optional: true
'@esbuild/linux-s390x@0.21.5':
'@esbuild/linux-s390x@0.23.0':
optional: true
'@esbuild/linux-x64@0.21.5':
'@esbuild/linux-x64@0.23.0':
optional: true
'@esbuild/netbsd-x64@0.21.5':
'@esbuild/netbsd-x64@0.23.0':
optional: true
'@esbuild/openbsd-x64@0.21.5':
'@esbuild/openbsd-arm64@0.23.0':
optional: true
'@esbuild/sunos-x64@0.21.5':
'@esbuild/openbsd-x64@0.23.0':
optional: true
'@esbuild/win32-arm64@0.21.5':
'@esbuild/sunos-x64@0.23.0':
optional: true
'@esbuild/win32-ia32@0.21.5':
'@esbuild/win32-arm64@0.23.0':
optional: true
'@esbuild/win32-x64@0.21.5':
'@esbuild/win32-ia32@0.23.0':
optional: true
'@esbuild/win32-x64@0.23.0':
optional: true
'@graphile/logger@0.2.0': {}
@ -890,28 +989,26 @@ snapshots:
'@types/interpret@1.1.3':
dependencies:
'@types/node': 22.0.0
'@types/node': 22.2.0
'@types/ms@0.7.34': {}
'@types/node@20.14.13':
'@types/node@20.14.15':
dependencies:
undici-types: 5.26.5
'@types/node@22.0.0':
dependencies:
undici-types: 6.11.1
'@types/node@22.1.0':
'@types/node@22.2.0':
dependencies:
undici-types: 6.13.0
'@types/pg@8.11.6':
dependencies:
'@types/node': 22.0.0
'@types/node': 22.2.0
pg-protocol: 1.6.1
pg-types: 4.0.2
'@types/qs@6.9.15': {}
'@types/semver@7.5.8': {}
acorn-walk@8.3.3:
@ -952,6 +1049,14 @@ snapshots:
dependencies:
fill-range: 7.1.1
call-bind@1.0.7:
dependencies:
es-define-property: 1.0.0
es-errors: 1.3.0
function-bind: 1.1.2
get-intrinsic: 1.2.4
set-function-length: 1.2.2
callsites@3.1.0: {}
chalk@2.4.2:
@ -1008,6 +1113,8 @@ snapshots:
create-require@1.1.1: {}
data-uri-to-buffer@4.0.1: {}
date-fns@3.6.0: {}
dd-cache-proxy@2.1.1(@discordeno/bot@19.0.0-next.746f0a9):
@ -1020,6 +1127,12 @@ snapshots:
optionalDependencies:
supports-color: 5.5.0
define-data-property@1.1.4:
dependencies:
es-define-property: 1.0.0
es-errors: 1.3.0
gopd: 1.0.1
diff@4.0.2: {}
dotenv@16.4.5: {}
@ -1030,45 +1143,71 @@ snapshots:
dependencies:
is-arrayish: 0.2.1
esbuild@0.21.5:
es-define-property@1.0.0:
dependencies:
get-intrinsic: 1.2.4
es-errors@1.3.0: {}
esbuild@0.23.0:
optionalDependencies:
'@esbuild/aix-ppc64': 0.21.5
'@esbuild/android-arm': 0.21.5
'@esbuild/android-arm64': 0.21.5
'@esbuild/android-x64': 0.21.5
'@esbuild/darwin-arm64': 0.21.5
'@esbuild/darwin-x64': 0.21.5
'@esbuild/freebsd-arm64': 0.21.5
'@esbuild/freebsd-x64': 0.21.5
'@esbuild/linux-arm': 0.21.5
'@esbuild/linux-arm64': 0.21.5
'@esbuild/linux-ia32': 0.21.5
'@esbuild/linux-loong64': 0.21.5
'@esbuild/linux-mips64el': 0.21.5
'@esbuild/linux-ppc64': 0.21.5
'@esbuild/linux-riscv64': 0.21.5
'@esbuild/linux-s390x': 0.21.5
'@esbuild/linux-x64': 0.21.5
'@esbuild/netbsd-x64': 0.21.5
'@esbuild/openbsd-x64': 0.21.5
'@esbuild/sunos-x64': 0.21.5
'@esbuild/win32-arm64': 0.21.5
'@esbuild/win32-ia32': 0.21.5
'@esbuild/win32-x64': 0.21.5
'@esbuild/aix-ppc64': 0.23.0
'@esbuild/android-arm': 0.23.0
'@esbuild/android-arm64': 0.23.0
'@esbuild/android-x64': 0.23.0
'@esbuild/darwin-arm64': 0.23.0
'@esbuild/darwin-x64': 0.23.0
'@esbuild/freebsd-arm64': 0.23.0
'@esbuild/freebsd-x64': 0.23.0
'@esbuild/linux-arm': 0.23.0
'@esbuild/linux-arm64': 0.23.0
'@esbuild/linux-ia32': 0.23.0
'@esbuild/linux-loong64': 0.23.0
'@esbuild/linux-mips64el': 0.23.0
'@esbuild/linux-ppc64': 0.23.0
'@esbuild/linux-riscv64': 0.23.0
'@esbuild/linux-s390x': 0.23.0
'@esbuild/linux-x64': 0.23.0
'@esbuild/netbsd-x64': 0.23.0
'@esbuild/openbsd-arm64': 0.23.0
'@esbuild/openbsd-x64': 0.23.0
'@esbuild/sunos-x64': 0.23.0
'@esbuild/win32-arm64': 0.23.0
'@esbuild/win32-ia32': 0.23.0
'@esbuild/win32-x64': 0.23.0
escalade@3.1.2: {}
escape-string-regexp@1.0.5: {}
fetch-blob@3.2.0:
dependencies:
node-domexception: 1.0.0
web-streams-polyfill: 3.3.3
fill-range@7.1.1:
dependencies:
to-regex-range: 5.0.1
formdata-polyfill@4.0.10:
dependencies:
fetch-blob: 3.2.0
fsevents@2.3.3:
optional: true
function-bind@1.1.2: {}
get-caller-file@2.0.5: {}
get-intrinsic@1.2.4:
dependencies:
es-errors: 1.3.0
function-bind: 1.1.2
has-proto: 1.0.3
has-symbols: 1.0.3
hasown: 2.0.2
get-tsconfig@4.7.6:
dependencies:
resolve-pkg-maps: 1.0.0
@ -1077,16 +1216,20 @@ snapshots:
dependencies:
is-glob: 4.0.3
gopd@1.0.1:
dependencies:
get-intrinsic: 1.2.4
graphile-config@0.0.1-beta.9:
dependencies:
'@types/interpret': 1.1.3
'@types/node': 20.14.13
'@types/node': 20.14.15
'@types/semver': 7.5.8
chalk: 4.1.2
debug: 4.3.6(supports-color@5.5.0)
interpret: 3.1.1
semver: 7.6.3
tslib: 2.6.2
tslib: 2.6.3
yargs: 17.7.2
transitivePeerDependencies:
- supports-color
@ -1100,7 +1243,7 @@ snapshots:
graphile-config: 0.0.1-beta.9
json5: 2.2.3
pg: 8.12.0
tslib: 2.6.2
tslib: 2.6.3
yargs: 17.7.2
transitivePeerDependencies:
- pg-native
@ -1111,6 +1254,18 @@ snapshots:
has-flag@4.0.0: {}
has-property-descriptors@1.0.2:
dependencies:
es-define-property: 1.0.0
has-proto@1.0.3: {}
has-symbols@1.0.3: {}
hasown@2.0.2:
dependencies:
function-bind: 1.1.2
ignore-by-default@1.0.1: {}
import-fresh@3.3.0:
@ -1156,6 +1311,14 @@ snapshots:
ms@2.1.2: {}
node-domexception@1.0.0: {}
node-fetch@3.3.2:
dependencies:
data-uri-to-buffer: 4.0.1
fetch-blob: 3.2.0
formdata-polyfill: 4.0.10
nodemon@3.1.4:
dependencies:
chokidar: 3.6.0
@ -1171,6 +1334,8 @@ snapshots:
normalize-path@3.0.0: {}
object-inspect@1.13.2: {}
obuf@1.1.2: {}
parent-module@1.0.1:
@ -1263,6 +1428,10 @@ snapshots:
pstree.remy@1.1.8: {}
qs@6.13.0:
dependencies:
side-channel: 1.0.6
readdirp@3.6.0:
dependencies:
picomatch: 2.3.1
@ -1275,6 +1444,22 @@ snapshots:
semver@7.6.3: {}
set-function-length@1.2.2:
dependencies:
define-data-property: 1.1.4
es-errors: 1.3.0
function-bind: 1.1.2
get-intrinsic: 1.2.4
gopd: 1.0.1
has-property-descriptors: 1.0.2
side-channel@1.0.6:
dependencies:
call-bind: 1.0.7
es-errors: 1.3.0
get-intrinsic: 1.2.4
object-inspect: 1.13.2
simple-update-notifier@2.0.0:
dependencies:
semver: 7.6.3
@ -1305,14 +1490,14 @@ snapshots:
touch@3.1.1: {}
ts-node@10.9.2(@types/node@22.1.0)(typescript@5.5.4):
ts-node@10.9.2(@types/node@22.2.0)(typescript@5.5.4):
dependencies:
'@cspotcode/source-map-support': 0.8.1
'@tsconfig/node10': 1.0.11
'@tsconfig/node12': 1.0.11
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
'@types/node': 22.1.0
'@types/node': 22.2.0
acorn: 8.12.1
acorn-walk: 8.3.3
arg: 4.1.3
@ -1323,11 +1508,11 @@ snapshots:
v8-compile-cache-lib: 3.0.1
yn: 3.1.1
tslib@2.6.2: {}
tslib@2.6.3: {}
tsx@4.16.2:
tsx@4.17.0:
dependencies:
esbuild: 0.21.5
esbuild: 0.23.0
get-tsconfig: 4.7.6
optionalDependencies:
fsevents: 2.3.3
@ -1338,12 +1523,12 @@ snapshots:
undici-types@5.26.5: {}
undici-types@6.11.1: {}
undici-types@6.13.0: {}
v8-compile-cache-lib@3.0.1: {}
web-streams-polyfill@3.3.3: {}
wrap-ansi@7.0.0:
dependencies:
ansi-styles: 4.3.0

14
services/bot/src/bot.ts
View File

@ -1,12 +1,19 @@
import { createBot, Intents, type Bot } from '@discordeno/bot'
import { createBot, createGatewayManager, createRestManager, Intents, type Bot } from '@discordeno/bot'
import { createProxyCache, } from 'dd-cache-proxy';
import { configs } from './config.ts'
// not sure I need this.
// @see https://github.com/discordeno/discordeno/blob/352887c215cc9d93d7f1fa9c8589e66f47ffb3ea/packages/bot/src/bot.ts#L74
// const getSessionInfoHandler = async () => {
// return await bot.rest.getGatewayBot()
// }
export const bot = createProxyCache(
createBot({
token: configs.token,
intents: Intents.Guilds | Intents.GuildMessages
intents: Intents.Guilds | Intents.GuildMessages,
rest: createRestManager({ token: configs.token, applicationId: configs.discordApplicationId }),
gateway: createGatewayManager({ token: configs.token })
}),
{
desiredProps: {
@ -21,9 +28,6 @@ export const bot = createProxyCache(
},
)
// @todo figure out where this code belongs
// gateway.resharding.getSessionInfo = async () => { // insert code here to fetch getSessionInfo from rest process. }
// Setup desired properties
bot.transformers.desiredProperties.interaction.id = true
bot.transformers.desiredProperties.interaction.type = true

17
services/bot/src/commands/record.ts
View File

@ -9,26 +9,25 @@ import { createCommand } from '../commands.ts'
import { configs } from '../config.ts'
async function createRecordInDatabase(url: string, discordMessageId: string) {
const record = {
async function createStreamInDatabase(url: string, discordMessageId: string) {
const streamPayload = {
url,
recording_state: 'pending',
discord_message_id: discordMessageId,
file_size: 0
status: 'pending_recording',
discord_message_id: discordMessageId
}
const res = await fetch(`${configs.postgrestUrl}/records`, {
const res = await fetch(`${configs.postgrestUrl}/streams`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${configs.automationUserJwt}`,
'Prefer': 'return=headers-only'
},
body: JSON.stringify(record)
body: JSON.stringify(streamPayload)
})
if (!res.ok) {
const status = res.status
const statusText = res.statusText
const msg = `fetch failed to create recording record in database. status=${status}, statusText=${statusText}`
const msg = `fetch failed to create stream in database. status=${status}, statusText=${statusText}`
console.error(msg)
throw new Error(msg)
}
@ -83,7 +82,7 @@ createCommand({
}
// @todo create record in db
const record = await createRecordInDatabase(url, message.id.toString())
const record = await createStreamInDatabase(url, message.id.toString())
// console.log(record)

5
services/bot/src/config.ts
View File

@ -4,6 +4,7 @@ if (!process.env.POSTGREST_URL) throw new Error('Missing POSTGREST_URL env var')
if (!process.env.DISCORD_TOKEN) throw new Error('Missing DISCORD_TOKEN env var');
if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
if (!process.env.DISCORD_GUILD_ID) throw new Error("DISCORD_GUILD_ID was missing from env");
if (!process.env.DISCORD_APPLICATION_ID) throw new Error('DISCORD_APPLICATION_ID was missing from env');
if (!process.env.AUTOMATION_USER_JWT) throw new Error('Missing AUTOMATION_USER_JWT env var');
const token = process.env.DISCORD_TOKEN!
const postgrestUrl = process.env.POSTGREST_URL!
@ -11,8 +12,8 @@ const discordChannelId = process.env.DISCORD_CHANNEL_ID!
const discordGuildId = process.env.DISCORD_GUILD_ID!
const automationUserJwt = process.env.AUTOMATION_USER_JWT!
const connectionString = process.env.WORKER_CONNECTION_STRING!
const discordApplicationId = process.env.DISCORD_APPLICATION_ID!
console.log(`hello i am configs and configs.connectionString=${connectionString}`)
export interface Config {
@ -22,6 +23,7 @@ export interface Config {
discordGuildId: string;
discordChannelId: string;
connectionString: string;
discordApplicationId: string;
}
@ -32,4 +34,5 @@ export const configs: Config = {
discordGuildId,
discordChannelId,
connectionString,
discordApplicationId,
}

8
services/bot/src/index.ts
View File

@ -1,5 +1,5 @@
import updateDiscordMessage from './tasks/update_discord_message.js'
import update_discord_message from './tasks/update_discord_message.js'
import { type WorkerUtils, type RunnerOptions, run } from 'graphile-worker'
import { bot } from './bot.ts'
import type { Interaction } from '@discordeno/bot'
@ -26,15 +26,15 @@ async function setupGraphileWorker() {
taskDirectory: join(__dirname, 'tasks')
},
};
console.log('worker preset as follows')
console.log(preset)
// console.log('worker preset as follows')
// console.log(preset)
const runnerOptions: RunnerOptions = {
preset
// concurrency: 3,
// connectionString: configs.connectionString,
// taskDirectory: join(__dirname, 'tasks'),
// taskList: {
// 'update_discord_message': updateDiscordMessage
// 'update_discord_message': update_discord_message
// }
}

72
services/bot/src/tasks/expire_stream_recordings.ts Normal file
View File

@ -0,0 +1,72 @@
import type { Task, Helpers } from "graphile-worker"
import { sub } from 'date-fns'
import type { RecordingRecord, Stream } from "@futureporn/types"
import qs from 'qs'
import fetch from 'node-fetch'
import { configs } from '../config.ts'
interface Payload {
idle_minutes: number;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (!payload.idle_minutes) throw new Error(`idle_minutes was absent in the payload`);
if (typeof payload.idle_minutes !== 'number') throw new Error(`idle_minutes parameter was not a number`);
}
export const expire_stream_recordings: Task = async function (payload: unknown, helpers: Helpers) {
assertPayload(payload)
const { idle_minutes } = payload
helpers.logger.info(`expire_stream_recordings has begun. Expring 'recording' and 'pending' streams that haven't been updated in ${idle_minutes} minutes.`)
const url = 'http://postgrest.futureporn.svc.cluster.local:9000/streams'
let streams: Stream[] = []
try {
// 1. identify and update stalled /streams
// Any streams that was updated earlier than n minute ago AND is in 'pending_recording' or 'recording' state is marked as stalled.
const timestamp = sub(new Date(), { minutes: idle_minutes }).toISOString()
const queryOptions = {
updated_at: `lt.${timestamp}`,
or: '(status.eq.pending_recording,status.eq.recording)'
}
const updatePayload = {
updated_at: new Date().toISOString(),
status: 'stalled'
}
helpers.logger.info(JSON.stringify(updatePayload))
const query = qs.stringify(queryOptions)
const res = await fetch (`${url}?${query}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${configs.automationUserJwt}`,
'Prefer': 'return=headers-only'
},
body: JSON.stringify(updatePayload)
})
if (!res.ok) {
const body = await res.text()
helpers.logger.info(JSON.stringify(res.headers))
helpers.logger.error(`Response code was not 200. status=${res.status}, statusText=${res.statusText}`)
helpers.logger.error(body)
return;
}
const body = await res.text()
helpers.logger.info('body as follows')
helpers.logger.info(body)
} catch (e: any) {
if (e instanceof Error) {
helpers.logger.error(`hi there we encountered an error while fetching /streams`)
helpers.logger.error(e.message)
} else {
helpers.logger.error(e)
}
}
}
export default expire_stream_recordings

85
services/bot/src/tasks/restart_failed_recordings.ts Normal file
View File

@ -0,0 +1,85 @@
import type { Task, Helpers } from "graphile-worker"
import { sub } from 'date-fns'
import type { RecordingRecord } from "@futureporn/types"
import qs from 'qs'
import fetch from 'node-fetch'
import { configs } from '../config.ts'
interface Payload {
idle_minutes: number;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (!payload.idle_minutes) throw new Error('idle_minutes was missing from payload');
if (typeof payload.idle_minutes !== 'number') throw new Error('idle_minutes must be a number');
}
export const restart_failed_recordings: Task = async function (payload: unknown, helpers: Helpers) {
assertPayload(payload)
const { idle_minutes } = payload
helpers.logger.info(`restart_failed_recordings has begun. Expring 'recording' and 'pending' records that haven't been updated in ${idle_minutes} minutes.`)
const url = 'http://postgrest.futureporn.svc.cluster.local:9000/records'
let records: RecordingRecord[] = []
try {
// 1. identify failed /records
// Any record that was updated earlier than n minute ago AND is in 'pending' or 'recording' state is marked as stalled.
const timestamp = sub(new Date(), { minutes: idle_minutes }).toISOString()
const queryOptions = {
updated_at: `lt.${timestamp}`,
or: '(recording_state.eq.pending,recording_state.eq.recording)'
}
const updatePayload = {
updated_at: new Date().toISOString(),
recording_state: 'stalled'
}
helpers.logger.info(JSON.stringify(updatePayload))
const query = qs.stringify(queryOptions)
const res = await fetch (`${url}?${query}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${configs.automationUserJwt}`,
'Prefer': 'return=headers-only'
},
body: JSON.stringify(updatePayload)
})
if (!res.ok) {
const body = await res.text()
helpers.logger.info(JSON.stringify(res.headers))
helpers.logger.error(`Response code was not 200. status=${res.status}, statusText=${res.statusText}`)
helpers.logger.error(body)
return;
}
const body = await res.text()
helpers.logger.info('body as follows')
helpers.logger.info(body)
// const data = await res.json() as RecordingRecord[]
// if (data.length < 1) return;
// records = data
} catch (e: any) {
if (e instanceof Error) {
helpers.logger.error(`hi there we encountered an error while fetching /records`)
helpers.logger.error(e.message)
} else {
helpers.logger.error(e)
}
}
// // 2. identify and update
// for (const record of records) {
// const res = await fetch(`${url}?`)
// }
// // 3. done
}
export default restart_failed_recordings

116
services/bot/src/tasks/update_discord_message.ts
View File

@ -1,5 +1,5 @@
import 'dotenv/config'
import type { RecordingState } from '@futureporn/types'
import type { Status } from '@futureporn/types'
import { type Task, type Helpers } from 'graphile-worker'
import { add } from 'date-fns'
import prettyBytes from 'pretty-bytes'
@ -18,26 +18,26 @@ import { bot } from '../bot.ts'
import { configs } from '../config.ts'
interface Payload {
record_id: number;
stream_id: number;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (!payload.record_id) throw new Error(`record_id was absent in the payload`);
if (!payload.stream_id) throw new Error(`stream_id was absent in the payload`);
}
async function editDiscordMessage({ helpers, recordingState, discordMessageId, url, fileSize, recordId }: { recordId: number, fileSize: number, url: string, helpers: Helpers, recordingState: RecordingState, discordMessageId: string }) {
async function editDiscordMessage({ helpers, streamStatus, discordMessageId, url, fileSize, streamId }: { streamId: number, fileSize: number, url: string, helpers: Helpers, streamStatus: Status, discordMessageId: string }) {
if (!discordMessageId) throw new Error(`discordMessageId was missing!`);
if (typeof discordMessageId !== 'string') throw new Error(`discordMessageId was not a string!`);
// const { captureJobId } = job.data
helpers.logger.info(`editDiscordMessage has begun with discordMessageId=${discordMessageId}, state=${recordingState}`)
helpers.logger.info(`editDiscordMessage has begun with discordMessageId=${discordMessageId}, streamStatus=${streamStatus}`)
// const guild = await bot.cache.guilds.get(BigInt(configs.discordGuildId))
@ -49,51 +49,19 @@ async function editDiscordMessage({ helpers, recordingState, discordMessageId, u
const channelId = BigInt(configs.discordChannelId)
const updatedMessage: EditMessage = {
embeds: getStatusEmbed({ recordingState, fileSize, recordId, url }),
embeds: getStatusEmbed({ streamStatus, fileSize, streamId, url }),
}
bot.helpers.editMessage(channelId, discordMessageId, updatedMessage)
// channel.
// const guild = await client.guilds.fetch(process.env.DISCORD_GUILD_ID!) as Guild
// if (!guild) throw new Error('guild was undefined');
// helpers.logger.info('here is the guild as follows')
// helpers.logger.info(guild.toString())
// helpers.logger.info(`fetching discord channel id=${process.env.DISCORD_CHANNEL_ID} from discord guild`)
// const channel = await client.channels.fetch(process.env.DISCORD_CHANNEL_ID!) as TextChannel
// if (!channel) throw new Error(`discord channel was undefined`);
// const message = await channel.messages.fetch(discordMessageId)
// helpers.logger.info(`discordMessageId=${discordMessageId}`)
// helpers.logger.info(message as any)
// const statusEmbed = getStatusEmbed({ recordId, recordingState, fileSize, url })
// const buttonRow = getButtonRow(recordingState)
// // const embed = new EmbedBuilder().setTitle('Attachments');
// const updatedMessage = {
// embeds: [
// statusEmbed
// ],
// components: [
// buttonRow
// ]
// };
// message.edit(updatedMessage)
}
async function getRecordFromDatabase(recordId: number) {
const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`)
async function getStreamFromDatabase(streamId: number) {
const res = await fetch(`${process.env.POSTGREST_URL}/streams?select=*,segment:segments(*)&id=eq.${streamId}`)
if (!res.ok) {
throw new Error(`failed fetching record ${recordId}. status=${res.status}, statusText=${res.statusText}`)
throw new Error(`failed fetching stream ${streamId}. status=${res.status}, statusText=${res.statusText}`)
}
const body = await res.json() as any
return body[0];
@ -102,66 +70,74 @@ async function getRecordFromDatabase(recordId: number) {
/**
* updateDiscordMessage is the task where we edit a previously sent discord message to display
* update_discord_message is the task where we edit a previously sent discord message to display
* the most up-to-date status information from the database
*
* Sometimes the update is changing the state, one of Pending|Recording|Aborted|Ended.
* Sometimes the update is updating the Filesize of the recording in-progress
* Sometimes the update is adding a thumbnail image to the message
*/
export const updateDiscordMessage: Task = async function (payload, helpers: Helpers) {
export const update_discord_message: Task = async function (payload, helpers: Helpers) {
try {
assertPayload(payload)
const { record_id } = payload
const recordId = record_id
helpers.logger.info(`updateDiscordMessage() with recordId=${recordId}`)
const record = await getRecordFromDatabase(recordId)
const { discord_message_id, recording_state, file_size, url } = record
const recordingState = recording_state
const { stream_id } = payload
const streamId = stream_id
helpers.logger.info(`update_discord_message() with streamId=${streamId}`)
const stream = await getStreamFromDatabase(streamId)
const { discord_message_id, status, file_size, url } = stream
const streamStatus = status
const discordMessageId = discord_message_id
const fileSize = file_size
editDiscordMessage({ helpers, recordingState, discordMessageId, url, fileSize, recordId })
editDiscordMessage({ helpers, streamStatus, discordMessageId, url, fileSize, streamId })
// schedule the next update 10s from now, but only if the recording is still happening
if (recordingState !== 'ended') {
if (streamStatus !== 'ended') {
const runAt = add(new Date(), { seconds: 10 })
const recordId = record.id
await helpers.addJob('updateDiscordMessage', { recordId }, { jobKey: `record_${recordId}_update_discord_message`, maxAttempts: 3, runAt })
const streamId = stream.id
await helpers.addJob('update_discord_message', { streamId }, { jobKey: `stream_${streamId}_update_discord_message`, maxAttempts: 3, runAt })
}
} catch (e) {
helpers.logger.error(`caught an error during updateDiscordMessage. e=${e}`)
helpers.logger.error(`caught an error during update_discord_message. e=${e}`)
}
}
function getStatusEmbed({
recordingState, recordId, fileSize, url
}: { fileSize: number, recordingState: RecordingState, recordId: number, url: string }) {
streamStatus, streamId, fileSize, url
}: { fileSize: number, streamStatus: Status, streamId: number, url: string }) {
const embeds = new EmbedsBuilder()
.setTitle(`Record ${recordId}`)
.setTitle(`Stream ${streamId}`)
.setFields([
{ name: 'Status', value: recordingState.charAt(0).toUpperCase()+recordingState.slice(1), inline: true },
{ name: 'Status', value: streamStatus.charAt(0).toUpperCase()+streamStatus.slice(1), inline: true },
{ name: 'Filesize', value: prettyBytes(fileSize), inline: true },
{ name: 'URL', value: url, inline: false },
])
if (recordingState === 'pending') {
if (streamStatus === 'pending') {
embeds
.setDescription("Waiting for a worker to accept the job.")
.setColor(2326507)
} else if (recordingState === 'recording') {
} else if (streamStatus === 'recording') {
embeds
.setDescription('The stream is being recorded.')
.setColor(392960)
} else if (recordingState === 'aborted') {
} else if (streamStatus === 'aborted') {
embeds
.setDescription("The recording was stopped by the user.")
.setColor(8289651)
} else if (recordingState === 'ended') {
} else if (streamStatus === 'finished') {
embeds
.setDescription("The recording has stopped.")
.setDescription("The recording has ended nominally.")
.setColor(10855845)
} else if (streamStatus === 'failed') {
embeds
.setDescription("The recording has ended abnorminally.")
.setColor(8289651)
} else if (streamStatus === 'stalled') {
embeds
.setDescription("We have not received a progress update in the past two minutes.")
.setColor(8289651)
} else {
embeds
.setDescription('The recording is in an unknown state? (this is a bug.)')
.setDescription(`The recording is in an unknown state? (streamStatus=${streamStatus} this is a bug.)`)
.setColor(10855845)
}
return embeds
@ -169,10 +145,10 @@ function getStatusEmbed({
function getButtonRow(state: RecordingState): ActionRow {
function getButtonRow(streamStatus: Status): ActionRow {
const components: ButtonComponent[] = []
if (state === 'pending' || state === 'recording') {
if (streamStatus === 'pending' || streamStatus === 'recording') {
const stopButton: ButtonComponent = {
type: MessageComponentTypes.Button,
customId: 'stop',
@ -180,7 +156,7 @@ function getButtonRow(state: RecordingState): ActionRow {
style: ButtonStyles.Danger
}
components.push(stopButton)
} else if (state === 'aborted') {
} else if (streamStatus === 'aborted') {
const retryButton: ButtonComponent = {
type: MessageComponentTypes.Button,
customId: 'retry',
@ -191,7 +167,7 @@ function getButtonRow(state: RecordingState): ActionRow {
style: ButtonStyles.Secondary
}
components.push(retryButton)
} else if (state === 'ended') {
} else if (streamStatus === 'finished') {
const downloadButton: ButtonComponent = {
type: MessageComponentTypes.Button,
customId: 'download',
@ -206,7 +182,7 @@ function getButtonRow(state: RecordingState): ActionRow {
const unknownButton: ButtonComponent = {
type: MessageComponentTypes.Button,
customId: 'unknown',
label: 'Unknown State',
label: 'Unknown Status',
emoji: {
name: 'thinking'
},
@ -225,4 +201,4 @@ function getButtonRow(state: RecordingState): ActionRow {
}
export default updateDiscordMessage
export default update_discord_message

11
services/capture/package.json
View File

@ -10,9 +10,10 @@
"build": "tsup",
"test": "mocha",
"integration": "FUTUREPORN_WORKDIR=/home/cj/Downloads mocha ./integration/**/*.test.js",
"dev": "tsx --watch ./src/index.ts",
"dev.nodemon": "pnpm nodemon --ext ts,json,yaml --ignore ./dist --watch ./src --watch ./node_modules/@futureporn --exec \"pnpm run dev.build\"",
"dev.build": "pnpm run build && pnpm run start",
"dev": "pnpm run dev.nodemon # yes this is crazy to have nodemon execute tsx, but it's the only way I have found to get live reloading in TS/ESM/docker with Graphile Worker's way of loading tasks",
"dev.tsx": "tsx ./src/index.ts",
"dev.nodemon": "nodemon --ext ts --exec \"pnpm run dev.tsx\"",
"dev.node": "node --no-warnings=ExperimentalWarning --loader ts-node/esm src/index.ts",
"clean": "rm -rf dist",
"superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist"
},
@ -25,8 +26,10 @@
"@futureporn/utils": "workspace:^",
"@paralleldrive/cuid2": "^2.2.2",
"@types/chai": "^4.3.16",
"@types/chai-as-promised": "^7.1.8",
"@types/fluent-ffmpeg": "^2.1.24",
"@types/mocha": "^10.0.7",
"@types/qs": "^6.9.15",
"date-fns": "^3.6.0",
"diskusage": "^1.2.0",
"dotenv": "^16.4.5",
@ -47,6 +50,7 @@
"pg-boss": "^9.0.3",
"pino-pretty": "^11.2.1",
"postgres": "^3.4.4",
"qs": "^6.13.0",
"rxjs": "^7.8.1",
"sql": "^0.78.0",
"winston": "^3.13.1",
@ -61,6 +65,7 @@
"aws-sdk-client-mock": "^4.0.1",
"aws-sdk-mock": "^6.0.4",
"chai": "^4.4.1",
"chai-as-promised": "^8.0.0",
"cheerio": "1.0.0-rc.12",
"mocha": "^10.7.0",
"multiformats": "^11.0.2",

48
services/capture/pnpm-lock.yaml
View File

@ -32,12 +32,18 @@ importers:
'@types/chai':
specifier: ^4.3.16
version: 4.3.16
'@types/chai-as-promised':
specifier: ^7.1.8
version: 7.1.8
'@types/fluent-ffmpeg':
specifier: ^2.1.24
version: 2.1.24
'@types/mocha':
specifier: ^10.0.7
version: 10.0.7
'@types/qs':
specifier: ^6.9.15
version: 6.9.15
date-fns:
specifier: ^3.6.0
version: 3.6.0
@ -98,6 +104,9 @@ importers:
postgres:
specifier: ^3.4.4
version: 3.4.4
qs:
specifier: ^6.13.0
version: 6.13.0
rxjs:
specifier: ^7.8.1
version: 7.8.1
@ -135,6 +144,9 @@ importers:
chai:
specifier: ^4.4.1
version: 4.5.0
chai-as-promised:
specifier: ^8.0.0
version: 8.0.0(chai@4.5.0)
cheerio:
specifier: 1.0.0-rc.12
version: 1.0.0-rc.12
@ -1040,6 +1052,9 @@ packages:
'@tsconfig/node16@1.0.4':
resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==}
'@types/chai-as-promised@7.1.8':
resolution: {integrity: sha512-ThlRVIJhr69FLlh6IctTXFkmhtP3NpMZ2QGq69StYLyKZFp/HOp1VdKZj7RvfNWYYcJ1xlbLGLLWj1UvP5u/Gw==}
'@types/chai@4.3.16':
resolution: {integrity: sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==}
@ -1067,6 +1082,9 @@ packages:
'@types/pg@8.11.6':
resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==}
'@types/qs@6.9.15':
resolution: {integrity: sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==}
'@types/retry@0.12.1':
resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==}
@ -1270,6 +1288,11 @@ packages:
resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==}
engines: {node: '>=10'}
chai-as-promised@8.0.0:
resolution: {integrity: sha512-sMsGXTrS3FunP/wbqh/KxM8Kj/aLPXQGkNtvE5wPfSToq8wkkvBpTZo1LIiEVmC4BwkKpag+l5h/20lBMk6nUg==}
peerDependencies:
chai: '>= 2.1.2 < 6'
chai@4.5.0:
resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==}
engines: {node: '>=4'}
@ -1285,6 +1308,10 @@ packages:
check-error@1.0.3:
resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==}
check-error@2.1.1:
resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==}
engines: {node: '>= 16'}
cheerio-select@2.1.0:
resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==}
@ -2408,6 +2435,10 @@ packages:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
qs@6.13.0:
resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==}
engines: {node: '>=0.6'}
querystring@0.2.0:
resolution: {integrity: sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==}
engines: {node: '>=0.4.x'}
@ -4124,6 +4155,10 @@ snapshots:
'@tsconfig/node16@1.0.4': {}
'@types/chai-as-promised@7.1.8':
dependencies:
'@types/chai': 4.3.16
'@types/chai@4.3.16': {}
'@types/debug@4.1.12':
@ -4154,6 +4189,8 @@ snapshots:
pg-protocol: 1.6.1
pg-types: 4.0.2
'@types/qs@6.9.15': {}
'@types/retry@0.12.1': {}
'@types/semver@7.5.8': {}
@ -4356,6 +4393,11 @@ snapshots:
camelcase@6.3.0: {}
chai-as-promised@8.0.0(chai@4.5.0):
dependencies:
chai: 4.5.0
check-error: 2.1.1
chai@4.5.0:
dependencies:
assertion-error: 1.1.0
@ -4381,6 +4423,8 @@ snapshots:
dependencies:
get-func-name: 2.0.2
check-error@2.1.1: {}
cheerio-select@2.1.0:
dependencies:
boolbase: 1.0.0
@ -5642,6 +5686,10 @@ snapshots:
punycode@2.3.1: {}
qs@6.13.0:
dependencies:
side-channel: 1.0.6
querystring@0.2.0: {}
querystringify@2.2.0: {}

24
services/capture/src/Record.spec.ts
View File

@ -1,5 +1,5 @@
import Record from "./Record.js"
import { expect } from "chai"
import Record, { UploadStreamClosedError } from "./Record.js"
import * as chai from 'chai'
import { ChildProcess, spawn } from "child_process"
import { createReadStream, readFileSync, ReadStream } from "fs"
import AWSMock from 'aws-sdk-mock'
@ -13,7 +13,9 @@ import { HeadObjectOutput } from 'aws-sdk/clients/s3';
import { Readable } from 'stream';
import { mockClient } from 'aws-sdk-client-mock';
import { sdkStreamMixin } from '@smithy/util-stream'
import chaiAsPromised from 'chai-as-promised'
chai.use(chaiAsPromised)
const expect = chai.expect
// "pay no attention to that man behind the curtain"
@ -52,7 +54,7 @@ describe('Record', function () {
expect(record).to.have.property('bucket', 'test')
})
it('should be abortable', async function () {
xit('should be abortable', async function () {
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) // 192627 bytes
const s3ClientMock = mockClient(S3Client)
const s3Client = new S3Client({ region: 'us-west-000' })
@ -65,6 +67,20 @@ describe('Record', function () {
await record.abort()
})
xit('should throw if the upload stream closes before the download stream closes', async function () {
const s3Mock = mockClient(S3Client)
// const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4'))
const inputStream = createReadStream('/dev/random') // forever random
// const s3Client = new S3Client({ region: 'us-west-000' })
// s3ClientMock.on()
s3Mock.on(PutObjectCommand).resolvesOnce({}).resolvesOnce({}).rejects({})
const s3 = new S3Client({ region: 'us-west-000' })
return expect(s3.send(new PutObjectCommand({ Body: inputStream, Bucket: 'taco', Key: 'my-cool-taco.mp4' }))).to.be.rejectedWith(UploadStreamClosedError)
})
xit('should restart if a EPIPE is encountered', async function () {
// @todo IDK how to implement this.
const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4'))

22
services/capture/src/Record.ts
View File

@ -7,6 +7,13 @@ import 'dotenv/config'
const ua0 = 'Mozilla/5.0 (X11; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0'
export class UploadStreamClosedError extends Error {
constructor(message: string) {
super(message)
Object.setPrototypeOf(this, UploadStreamClosedError.prototype)
}
}
export interface RecordArgs {
filename?: string;
s3Client: S3Client;
@ -131,6 +138,7 @@ export default class Record {
parallelUploads3.on("httpUploadProgress", (progress) => {
if (progress?.loaded) {
// console.log(progress)
if (this.onProgress) this.onProgress(this.counter);
// console.log(`uploaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`);
} else {
@ -144,8 +152,13 @@ export default class Record {
} catch (e) {
if (e instanceof Error) {
if (e.name === 'AbortError') {
console.error(`We got an error, AbortError which is something we know how to handle. we will NOT throw and instead return gracefully.`)
return
} else {
console.error(`We were uploading a file to S3 but then we encountered an error! ${JSON.stringify(e, null, 2)}`)
throw e
}
} else {
throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`)
}
@ -164,7 +177,14 @@ export default class Record {
this.counter += data.length
})
this.uploadStream.on('close', () => {
console.log('[!!!] upload stream has closed')
// if uploadStream closes before inputStream, throw an error.
if (!this.inputStream.closed) {
const msg = 'upload stream closed before download stream, which suggests the S3 upload failed.'
console.error(msg)
throw new UploadStreamClosedError(msg);
} else {
console.log('upload stream has closed. In this instance it is OK since the input stream is also closed.')
}
})
this.uploadStream.on('error', (e) => {
console.error('there was an error on the uploadStream. error as follows')

40
services/capture/src/config.ts Normal file
View File

@ -0,0 +1,40 @@
import 'dotenv/config'
const requiredEnvVars = [
'S3_ACCESS_KEY_ID',
'S3_SECRET_ACCESS_KEY',
'S3_REGION',
'S3_ENDPOINT',
'S3_BUCKET',
'POSTGREST_URL',
'AUTOMATION_USER_JWT',
] as const;
const getEnvVar = (key: typeof requiredEnvVars[number]): string => {
const value = process.env[key];
if (!value) {
throw new Error(`Missing ${key} env var`);
}
return value;
};
export interface Config {
postgrestUrl: string;
automationUserJwt: string;
s3AccessKeyId: string;
s3SecretAccessKey: string;
s3Region: string;
s3Bucket: string;
s3Endpoint: string;
}
export const configs: Config = {
postgrestUrl: getEnvVar('POSTGREST_URL'),
automationUserJwt: getEnvVar('AUTOMATION_USER_JWT'),
s3AccessKeyId: getEnvVar('S3_ACCESS_KEY_ID'),
s3SecretAccessKey: getEnvVar('S3_SECRET_ACCESS_KEY'),
s3Region: getEnvVar('S3_REGION'),
s3Bucket: getEnvVar('S3_BUCKET'),
s3Endpoint: getEnvVar('S3_ENDPOINT'),
}

13
services/capture/src/index.ts
View File

@ -10,9 +10,6 @@ import { fileURLToPath } from 'url';
import { getPackageVersion } from '@futureporn/utils';
import type { GraphileConfig } from "graphile-config";
import type {} from "graphile-worker";
import start_recording from './tasks/start_recording.ts';
import { stop_recording } from './tasks/stop_recording.ts';
import record from './tasks/record.ts'
const __dirname = dirname(fileURLToPath(import.meta.url));
const version = getPackageVersion(join(__dirname, '../package.json'))
@ -58,12 +55,10 @@ async function worker(workerUtils: WorkerUtils) {
const runnerOptions: RunnerOptions = {
preset,
concurrency,
// taskDirectory: join(__dirname, 'tasks'),
taskList: {
'record': record,
'start_recording': start_recording,
'stop_recording': stop_recording
}
taskDirectory: join(__dirname, 'tasks'),
// taskList: {
// 'record': record,
// }
}
const runner = await graphileRun(runnerOptions)

418
services/capture/src/tasks/record.ts
View File

@ -1,9 +1,115 @@
/**
*
* # notes
*
* # creation
*
* ## api.records
*
* id: 2
* url: 'https://chaturbate.com/example'
* discord_message_id: 238492348324
* recording_state: 'pending'
* is_aborted: false
* created_at: 2024-08-15T21:36:27.796Z
* updated_at: 2024-08-15T21:36:27.796Z
*
* ## api.segments
*
* id: 5
* s3_key: example-date-cuid.mp4
* s3_id: 2342309492348324
* bytes: 0
* created_at: 2024-08-15T21:36:27.796Z
* updated_at: 2024-08-15T21:36:27.796Z
*
* ## api.records_segments_links
*
* id: 9
* stream_id: 2
* segment_id: 5
* segment_order: 0
* created_at: 2024-08-15T21:36:27.796Z
* updated_at: 2024-08-15T21:36:27.796Z
*
* # progress
*
* ## api.records
*
* id: 2
* url: 'https://chaturbate.com/example'
* discord_message_id: 238492348324
* recording_state: 'recording'
* is_aborted: false
* created_at: 2024-08-15T21:36:27.796Z
* updated_at: 2024-08-15T21:37:37.168Z
*
* ## api.segments
*
* id: 5
* s3_key: example-2024-08-15-72ff4b5ae7dae73b.mp4
* s3_id: 2342309492348324
* bytes: 8384
* created_at: 2024-08-15T21:36:27.796Z
* updated_at: 2024-08-15T21:37:37.168Z
*
*
* # new segment
*
* ## api.segments
*
* id: 6
* s3_key: example-2024-08-15-cda21be5e54621f2.mp4
* s3_id: a974eb6e194b7987
* byte: 0
* created_at: 2024-08-15T21:38:34.878Z
* updated_at: 2024-08-15T21:38:34.878Z
*
* ## api.records_segments_links
*
* id: 10
* stream_id: 2
* segment_id: 6
* segment_order: 1
* created_at: 2024-08-15T21:38:34.878Z
* updated_at: 2024-08-15T21:38:34.878Z
*
* # progress
*
* ## api.segments
*
* id: 6
* s3_key: example-2024-08-15-cda21be5e54621f2.mp4
* s3_id: a974eb6e194b7987
* byte: 1024
* created_at: 2024-08-15T21:38:34.878Z
* updated_at: 2024-08-15T21:39:11.437Z
*
* # completion
*
* ## api.records
*
* id: 2
* url: 'https://chaturbate.com/example'
* discord_message_id: 238492348324
* recording_state: 'finished'
* is_aborted: false
* created_at: 2024-08-15T21:36:27.796Z
* updated_at: 2024-08-15T21:39:41.692Z
*
*/
import querystring from 'node:querystring'
import { Helpers, type Task } from 'graphile-worker'
import Record from '../Record.ts'
import { getPlaylistUrl } from '@futureporn/scout/ytdlp.ts'
import type { RecordingState } from '@futureporn/types'
import type { RecordingState, RecordingRecord, Segment } from '@futureporn/types'
import { add } from 'date-fns'
import { backOff } from "exponential-backoff";
import { configs } from '../config.ts'
import qs from 'qs'
import { createId } from '@paralleldrive/cuid2'
/**
* url is the URL to be recorded. Ex: chaturbate.com/projektmelody
@ -12,189 +118,229 @@ import { add } from 'date-fns'
*/
interface Payload {
url: string;
record_id: number;
}
interface RecordingRecord {
id: number;
recordingState: RecordingState;
fileSize: number;
discordMessageId: string;
isAborted: boolean;
}
interface RawRecordingRecord {
id: number;
recording_state: RecordingState;
file_size: number;
discord_message_id: string;
is_aborted: boolean;
stream_id: string;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.url !== "string") throw new Error("invalid url");
if (typeof payload.record_id !== "number") throw new Error(`invalid record_id=${payload.record_id}`);
}
function assertEnv() {
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was missing in env');
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was missing in env');
if (!process.env.S3_REGION) throw new Error('S3_REGION was missing in env');
if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was missing in env');
if (!process.env.S3_BUCKET) throw new Error('S3_BUCKET was missing in env');
if (!process.env.POSTGREST_URL) throw new Error('POSTGREST_URL was missing in env');
if (!process.env.AUTOMATION_USER_JWT) throw new Error('AUTOMATION_USER_JWT was missing in env');
if (typeof payload.stream_id !== "string") throw new Error(`invalid stream_id=${payload.stream_id}`);
}
async function getRecording(url: string, recordId: number, helpers: Helpers) {
async function getRecordInstance(url: string, segment_id: number, helpers: Helpers) {
const abortController = new AbortController()
const abortSignal = abortController.signal
const accessKeyId = process.env.S3_ACCESS_KEY_ID!;
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY!;
const region = process.env.S3_REGION!;
const endpoint = process.env.S3_ENDPOINT!;
const bucket = process.env.S3_BUCKET!;
const accessKeyId = configs.s3AccessKeyId;
const secretAccessKey = configs.s3SecretAccessKey;
const region = configs.s3Region;
const endpoint = configs.s3Endpoint;
const bucket = configs.s3Bucket;
const playlistUrl = await getPlaylistUrl(url)
const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
const inputStream = Record.getFFmpegStream({ url: playlistUrl })
const onProgress = (fileSize: number) => {
updateDatabaseRecord({ recordId, recordingState: 'recording', fileSize }).then(checkIfAborted).then((isAborted) => isAborted ? abortController.abort() : null)
updateDatabaseRecord({ segment_id, fileSize, helpers })
.then((reee) => {
helpers.logger.info(JSON.stringify(reee))
return reee
})
.then(checkIfAborted)
.then((isAborted) => {
helpers.logger.info(`isAborted=${isAborted}`)
isAborted ? abortController.abort() : null
})
.catch((e) => {
helpers.logger.error('caught error while updatingDatabaseRecord inside onProgress inside getRecordInstance')
helpers.logger.error(e)
})
}
const record = new Record({ inputStream, onProgress, bucket, s3Client, jobId: ''+recordId, abortSignal })
const record = new Record({ inputStream, onProgress, bucket, s3Client, jobId: ''+segment_id, abortSignal })
return record
}
function checkIfAborted(record: RawRecordingRecord): boolean {
return (record.is_aborted)
function checkIfAborted(segment: Partial<Segment>): boolean {
return (!!segment?.stream?.at(0)?.is_recording_aborted)
}
async function updateDatabaseRecord({
recordId,
recordingState,
fileSize
segment_id,
fileSize,
helpers
}: {
recordId: number,
recordingState: RecordingState,
fileSize: number
}): Promise<RawRecordingRecord> {
// console.log(`updating database record with recordId=${recordId}, recordingState=${recordingState}, fileSize=${fileSize}`)
segment_id: number,
fileSize: number,
helpers: Helpers
}): Promise<Segment> {
const payload: any = {
file_size: fileSize
bytes: fileSize
}
if (recordingState) payload.recording_state = recordingState;
const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`, {
const res = await fetch(`${configs.postgrestUrl}/segments?id=eq.${segment_id}&select=stream:streams(is_recording_aborted)`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
'Accepts': 'application/json',
'Accept': 'application/json',
'Prefer': 'return=representation',
'Authorization': `Bearer ${process.env.AUTOMATION_USER_JWT}`
'Authorization': `Bearer ${configs.automationUserJwt}`
},
body: JSON.stringify(payload)
})
if (!res.ok) {
const body = await res.text()
throw new Error(`failed to updateDatabaseRecord. status=${res.status}, statusText=${res.statusText}, body=${body}`);
const msg = `failed to updateDatabaseRecord. status=${res.status}, statusText=${res.statusText}, body=${body}`
helpers.logger.error(msg)
throw new Error(msg);
}
const body = await res.json() as RawRecordingRecord[];
if (!body[0]) throw new Error(`failed to get a record that matched recordId=${recordId}`)
// helpers.logger.info(`response was OK~`)
const body = await res.json() as Segment[];
if (!body[0]) throw new Error(`failed to get a segment that matched segment_id=${segment_id}`);
const bod = body[0]
// helpers.logger.info('the following was the response from PATCH-ing /segments')
// helpers.logger.info(JSON.stringify(bod))
return bod
}
const getSegments = async function getSegments(stream_id: string): Promise<Segment> {
if (!stream_id) throw new Error('getSegments requires {String} stream_id as first arg');
const res = await fetch(`${configs.postgrestUrl}/segments_stream_links?stream_id=eq.${stream_id}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Prefer': 'return=representation'
},
})
if (!res.ok) {
const body = await res.text()
throw new Error(`failed to getSegments. status=${res.status}, statusText=${res.statusText}, body=${body}`);
}
const body = await res.json() as Segment[];
if (!body[0]) throw new Error(`failed to get segments that matched stream_id=${stream_id}`)
return body[0]
}
export const record: Task = async function (payload, helpers) {
console.log(payload)
assertPayload(payload)
assertEnv()
const { url, record_id } = payload
// let interval
try {
// every 30s, we
// 1. update the db record with the filesize
// 2. poll db to see if our job has been aborted by the user
// interval = setInterval(async () => {
// try {
// helpers.logger.info(`updateDatabaseRecord()`)
// const recordingState: RecordingState = 'recording'
// const fileSize = record.counter
// const updatePayload = { recordingState, recordId, fileSize }
// const updatedRecord = await updateDatabaseRecord(updatePayload)
// if (updatedRecord.isAborted) {
// helpers.logger.info(`record ${recordId} has been aborted by a user so we stop the recording now.`)
// abortController.abort()
// }
// } catch (e) {
// helpers.logger.error(`error while updating database. For sake of the recording in progress we are ignoring the following error. ${e}`)
// }
// }, 3000)
// start recording and await the S3 upload being finished
const recordId = record_id
const record = await getRecording(url, recordId, helpers)
const createSegment = async function createSegment(s3_key: string, helpers: Helpers): Promise<number> {
if (!s3_key) throw new Error('getSegments requires {string} s3_key as first arg');
const segmentPayload = {
s3_key
}
helpers.logger.info(`Creating segment with s3_key=${s3_key}. payload as follows`)
helpers.logger.info(JSON.stringify(segmentPayload))
const res = await fetch(`${configs.postgrestUrl}/segments`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Prefer': 'return=headers-only',
'Authorization': `Bearer ${configs.automationUserJwt}`
},
body: JSON.stringify(segmentPayload)
})
if (!res.ok) {
const body = await res.text()
const msg = `failed to create Segment. status=${res.status}, statusText=${res.statusText}, body=${body}`
helpers.logger.error(msg)
throw new Error(msg);
}
const location = res.headers.get('location')
if (!location) throw new Error(`failed to get location header in response from postgrest`);
const parsedQuery = querystring.parse(location)
const segmentsId = parsedQuery['/segments?id']
if (!segmentsId) throw new Error('segmentsId was undefined which is unexpected');
if (Array.isArray(segmentsId)) throw new Error('segmentsId was an array which is unexpected');
const id = segmentsId.split('.').at(-1)
if (!id) throw new Error('failed to get id ');
return parseInt(id)
}
const createSegmentsStreamLink = async function createSegmentsStreamLink(stream_id: string, segment_id: number, helpers: Helpers): Promise<number> {
if (!stream_id) throw new Error('createSegmentsStreamLink requires {string} stream_id as first arg');
if (!segment_id) throw new Error('createSegmentsStreamLink requires {Number} segment_id as second arg');
const segmentStreamLinkPayload = {
stream_id,
segment_id
}
const res = await fetch(`${configs.postgrestUrl}/segments_stream_links`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
'Prefer': 'return=headers-only',
'Authorization': `Bearer ${configs.automationUserJwt}`,
},
body: JSON.stringify(segmentStreamLinkPayload)
})
if (!res.ok) {
const body = await res.text()
throw new Error(`failed to create SegmentsStreamLink. status=${res.status}, statusText=${res.statusText}, body=${body}`);
}
const location = res.headers.get('location')
if (!location) throw new Error(`failed to get location header in response from postgrest`);
const parsedQuery = querystring.parse(location)
const segmentsId = parsedQuery['/segments_stream_links?id']
if (!segmentsId) throw new Error('segments_stream_links?id was undefined which is unexpected');
if (Array.isArray(segmentsId)) throw new Error('segments_stream_links was an array which is unexpected');
const id = segmentsId.split('.').at(-1)
if (!id) throw new Error('failed to get id ');
return parseInt(id)
}
/**
* # doRecordSegment
*
* Record a segment of a livestream using ffmpeg.
*
* Ideally, we record the entire livestream, but the universe is not so kind. Network interruptions are common, so we handle the situation as best as we can.
*
* This function creates a new segments and segments_streams_links entry in the db via Postgrest REST API.
*
* This function also names the S3 file (s3_key) with a datestamp and a cuid.
*/
const doRecordSegment = async function doRecordSegment(url: string, stream_id: string, helpers: Helpers): Promise<void> {
const s3_key = `${new Date().toISOString()}-${createId()}.ts`
helpers.logger.info(`let's create a segment...`)
const segment_id = await createSegment(s3_key, helpers)
helpers.logger.info(`let's create a segmentsStreamLink...`)
const segmentsStreamLinkId = await createSegmentsStreamLink(stream_id, segment_id, helpers)
helpers.logger.info(`doTheRecording with segmentsStreamLinkId=${segmentsStreamLinkId}, stream_id=${stream_id}, segment_id=${segment_id}, url=${url}`)
const record = await getRecordInstance(url, segment_id, helpers)
await record.start()
}
export const record: Task = async function (payload: unknown, helpers: Helpers) {
assertPayload(payload)
const { url, stream_id } = payload
const recordId = stream_id
try {
/**
* We do an exponential backoff timer when we record. If the Record() instance throws an error, we try again after a delay.
* This will take effect only when Record() throws an error.
* If however Record() returns, as is the case when the stream ends, this backoff timer will not retry.
* This does not handle the corner case where the streamer's internet temporarliy goes down, and their stream drops.
*
* @todo We must implement retrying at a higher level, and retry a few times to handle this type of corner-case.
*/
// await backOff(() => doRecordSegment(url, recordId, helpers))
await doRecordSegment(url, recordId, helpers)
} catch (e) {
helpers.logger.error(`caught an error duing record(). error as follows`)
// await updateDatabaseRecord({ recordId: stream_id, recordingState: 'failed' })
helpers.logger.error(`caught an error during record Task`)
if (e instanceof Error) {
helpers.logger.error(e.message)
} else {
helpers.logger.error(JSON.stringify(e))
}
// throw e // @todo uncomment this for production
}
// const recordId = await createRecordingRecord(payload, helpers)
// const { url } = payload;
// console.log(`@todo simulated start_recording with url=${url}, recordId=${recordId}`)
// await helpers.addJob('record', { url, recordId })
}
/**
* Here we middleman the stream from FFmpeg --> S3,
* counting bits and creating graphile jobs to inform the UI of our progress
*/
// const transformStreamFactory = (recordId: number, helpers: Helpers): PassThrough => {
// let counter = 0
// return new PassThrough ({
// async transform(chunk, controller) {
// controller.enqueue(chunk) // we don't actually transform anything here. we're only gathering statistics.
// counter += chunk.length
// if (counter % (1 * 1024 * 1024) <= 1024) {
// helpers.logger.info(`Updating record ${recordId}`)
// try {
// await updateDatabaseRecord({ fileSize: counter, recordId, recordingState: 'recording' })
// } catch (e) {
// helpers.logger.warn(`We are ignoring the following error which occured while updating db record ${e}`)
// }
// }
// },
// flush() {
// helpers.logger.info(`transformStream has flushed.`)
// }
// })
// }
// export const recordNg: Task = async function (payload, helpers) {
// assertPayload(payload)
// const { url, recordId } = payload
// try {
// const abortController = new AbortController()
// const abortSignal = abortController.signal
// const inputStream =
// const transformStream = transformStreamFactory(recordId, helpers)
// const record = new Record({ inputStream, abortSignal, transformStream })
// await record.done()
// } catch (e) {
// console.error(`error during recording. error as follows`)
// console.error(e)
// } finally {
// helpers.addJob('updateDiscordMessage', { recordId }, { maxAttempts: 3, runAt: add(new Date(), { seconds: 5 }) })
// }
// }
export default record

77
services/capture/src/tasks/recording_start.ts.old
View File

@ -1,77 +0,0 @@
import Record from '../Record.ts'
import { getPlaylistUrl } from '@futureporn/scout/ytdlp.ts'
import 'dotenv/config'
import { type Job } from 'pg-boss'
import { backOff } from 'exponential-backoff'
export interface RecordJob extends Job {
data: {
url: string;
}
}
async function _record (job: RecordJob, retries?: number): Promise<string> {
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was undefined in env');
if (!process.env.S3_REGION) throw new Error('S3_REGION was undefined in env');
if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was undefined in env');
if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was undefined in env');
if (!job) throw new Error('Job sent to job worker execution callback was empty!!!');
const { url } = job.data;
console.log(`'record' job ${job!.id} begin with url=${url}`)
const bucket = process.env.S3_BUCKET_NAME!
const endpoint = process.env.S3_ENDPOINT!
const region = process.env.S3_REGION!
const accessKeyId = process.env.S3_ACCESS_KEY_ID!
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY!
let playlistUrl
try {
playlistUrl = await getPlaylistUrl(url)
console.log(`playlistUrl=${playlistUrl}`)
} catch (e) {
console.error('error during getPlaylistUrl()')
console.error(e)
throw e
}
const jobId = job.id
const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
const inputStream = Record.getFFmpegStream({ url: playlistUrl })
const record = new Record({ inputStream, bucket, s3Client, jobId })
await record.start()
console.log(`record job ${job.id} complete`)
return job.id
}
export default async function main (jobs: RecordJob[]): Promise<any> {
// @todo why are we passed multiple jobs? I'm expecting only one.
const backOffOptions = {
numOfAttempts: 5,
startingDelay: 5000,
retry: (e: any, attemptNumber: number) => {
console.log(`Record Job is retrying. Attempt number ${attemptNumber}. e=${JSON.stringify(e, null, 2)}`)
return true
}
}
for (const j of jobs) {
console.log(`record job ${j.id} GO GO GO`)
try {
await backOff(() => _record(j), backOffOptions)
} catch (e) {
console.warn(`record job ${j.id} encountered the following error.`)
console.error(e)
}
console.log(`record job ${j.id} is finished.`)
}
};

67
services/capture/src/tasks/start_recording.ts
View File

@ -1,67 +0,0 @@
import { Helpers, type Task } from 'graphile-worker'
import { add } from 'date-fns'
/**
* url is the URL to be recorded. Ex: chaturbate.com/projektmelody
* discordMessageId is the ID of the discord messate which displays recording status.
* we use the ID to update the message later, and/or relate button press events to this record task
*/
interface Payload {
url: string;
discordMessageId: string;
isAborted: boolean;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.url !== "string") throw new Error("invalid url");
if (typeof payload.discordMessageId !== "string") throw new Error(`invalid discordMessageId=${payload.discordMessageId}`);
}
function assertEnv() {
if (!process.env.AUTOMATION_USER_JWT) throw new Error('AUTOMATION_USER_JWT was missing in env');
if (!process.env.POSTGREST_URL) throw new Error('POSTGREST_URL was missing in env');
}
async function createRecordingRecord(payload: Payload, helpers: Helpers): Promise<number> {
const { url, discordMessageId } = payload
const record = {
url,
discord_message_id: discordMessageId,
recording_state: 'pending',
file_size: 0
}
const res = await fetch(`${process.env.POSTGREST_URL}/records`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${process.env.AUTOMATION_USER_JWT}`,
'Prefer': 'return=headers-only'
},
body: JSON.stringify(record)
})
if (!res.ok) {
const status = res.status
const statusText = res.statusText
throw new Error(`fetch failed to create recording record in database. status=${status}, statusText=${statusText}`)
}
helpers.logger.info('res.headers.location as follows.')
helpers.logger.info(res.headers.get('location')!)
const id = res.headers.get('location')?.split('.').at(-1)
if (!id) throw new Error('id could not be parsed from location header');
return parseInt(id)
}
export const start_recording: Task = async function (payload, helpers) {
assertPayload(payload)
assertEnv()
const recordId = await createRecordingRecord(payload, helpers)
const { url } = payload;
await helpers.addJob('record', { url, recordId }, { maxAttempts: 3, jobKey: `record_${recordId}` })
const runAt = add(new Date(), { seconds: 10 })
await helpers.addJob('updateDiscordMessage', { recordId }, { jobKey: `record_${recordId}_update_discord_message`, maxAttempts: 3, runAt })
helpers.logger.info(`startRecording() with url=${url}, recordId=${recordId}, (updateDiscordMessage runAt=${runAt})`)
}
export default start_recording

18
services/capture/src/tasks/stop_recording.ts
View File

@ -1,18 +0,0 @@
import { type Task } from 'graphile-worker'
interface Payload {
id: string
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.id !== "string") throw new Error("invalid id");
}
export const stop_recording: Task = async function (payload) {
assertPayload(payload)
const { id } = payload;
console.log(`@todo simulated stop_recording with id=${id}`)
}

2
services/capture/tsconfig.json
View File

@ -25,7 +25,7 @@
// Include the necessary files for your project
"include": [
"src/**/*.ts"
],
, "../bot/src/tasks/restart_failed_recordings.ts" ],
"exclude": [
"node_modules"
]

15
services/migrations/README.md
View File

@ -4,9 +4,24 @@ Here we handle migrations for the postgrest database.
@see https://github.com/thomwright/postgres-migrations
Reminder: only write migrations that affect schema. (don't write migrations that affect data)
## K.I.S.S.
Keep It Stupidly Simple.
We are keeping this module as simple as possible. This means pure JS (no typescript!)
## troubleshooting
If you see the following error, graphile_worker likely hasn't had a chance to create it's functions. Make sure that a graphile_worker is running, so it can automatically create the necessary functions.
```json
{
"code": "42883",
"details": null,
"hint": "No function matches the given name and argument types. You might need to add explicit type casts.",
"message": "function graphile_worker.add_job(text, json, max_attempts => integer) does not exist"
}
```

4
services/migrations/index.js
View File

@ -2,6 +2,8 @@ import {migrate} from 'postgres-migrations'
import path, { dirname } from 'node:path'
import { fileURLToPath } from 'url';
import 'dotenv/config'
const __dirname = dirname(fileURLToPath(import.meta.url));
if (!process.env.DATABASE_PASSWORD) throw new Error('DATABASE_PASSWORD is missing in env');
@ -23,7 +25,7 @@ async function main() {
defaultDatabase: "postgres"
}
await migrate(dbConfig, path.join(__dirname, "./migrations/"))
await migrate(dbConfig, path.join(__dirname, "./migrations/"), { logger: console.log })
}

5
services/migrations/migrations/00006_add-updated-at-to-records.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE IF EXISTS api.records
ADD COLUMN created_at timestamp(6) without time zone;
ALTER TABLE IF EXISTS api.records
ADD COLUMN updated_at timestamp(6) without time zone;

7
services/migrations/migrations/00007_add-default-records-timestamps.sql Normal file
View File

@ -0,0 +1,7 @@
ALTER TABLE IF EXISTS api.records
ADD CONSTRAINT created_at_not_null
CHECK (created_at IS NOT NULL) NOT VALID;
ALTER TABLE IF EXISTS api.records
ADD CONSTRAINT updated_at_not_null
CHECK (updated_at IS NOT NULL) NOT VALID;

26
services/migrations/migrations/00008_add-default-records-timestamp.sql Normal file
View File

@ -0,0 +1,26 @@
-- In the prev. migration I added a CHECK, but I forgot to add the default
ALTER TABLE IF EXISTS api.records
ALTER COLUMN created_at SET DEFAULT now();
ALTER TABLE IF EXISTS api.records
ALTER COLUMN updated_at SET DEFAULT now();
-- create a function which updates the row's updated_at
CREATE FUNCTION public.tg__updated_at() RETURNS trigger
LANGUAGE plpgsql
SET search_path TO 'pg_catalog', 'public', 'pg_temp'
AS $$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$$;
-- create a trigger which runs the above function when a /record is updated
CREATE TRIGGER record_updated_at
AFTER UPDATE ON api.records
FOR EACH ROW
EXECUTE PROCEDURE public.tg__updated_at();

140
services/migrations/migrations/00009_add-streams-vods-vtubers.sql Normal file
View File

@ -0,0 +1,140 @@
-- vtubers table
CREATE TABLE api.vtubers (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
display_name TEXT NOT NULL,
chaturbate TEXT,
twitter TEXT,
patreon TEXT,
twitch TEXT,
tiktok TEXT,
onlyfans TEXT,
youtube TEXT,
linktree TEXT,
carrd TEXT,
fansly TEXT,
pornhub TEXT,
discord TEXT,
reddit TEXT,
throne TEXT,
instagram TEXT,
facebook TEXT,
merch TEXT,
slug TEXT NOT NULL,
description1 TEXT,
description2 TEXT,
image TEXT NOT NULL,
theme_color VARCHAR(7) NOT NULL,
image_blur TEXT DEFAULT 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABmJLR0QA/wD/AP+gvaeTAAAADUlEQVQImWMwtf//HwAEkwJzh0T9qwAAAABJRU5ErkJggg==',
fansly_id TEXT,
chaturbate_id TEXT,
twitter_id TEXT
-- F.Y.I., relations as follows
-- toys (one-to-many)
-- vods (one-to-many)
-- streams (one-to-many)
);
GRANT all ON api.vtubers TO automation;
GRANT SELECT ON api.vtubers TO web_anon;
-- streams table
CREATE TABLE api.streams (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
url TEXT NOT NULL,
platform_notification_type TEXT,
date timestamp(6) without time zone,
created_at timestamp(6) without time zone,
vtuber uuid,
FOREIGN KEY (vtuber) REFERENCES api.vtubers(id),
tweet TEXT,
archive_status TEXT,
is_chaturbate_stream BOOLEAN,
is_fansly_stream BOOLEAN
);
GRANT all ON api.streams TO automation;
GRANT SELECT ON api.streams TO web_anon;
-- toys table
CREATE TABLE api.toys (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-- relation. one toy to many tags
-- relation. one toy to many vtubers
make TEXT NOT NULL,
model TEXT NOT NULL,
image TEXT NOT NULL DEFAULT 'https://futureporn-b2.b-cdn.net/default-thumbnail.webp'
);
GRANT all ON api.toys TO automation;
GRANT SELECT ON api.toys TO web_anon;
-- tags table
CREATE TABLE api.tags (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL UNIQUE,
toy_id uuid,
FOREIGN KEY (toy_id) REFERENCES api.toys
);
GRANT all ON api.tags TO automation;
GRANT SELECT ON api.tags TO web_anon;
-- toys-tags junction table
CREATE TABLE api.toys_tags(
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
toy_id uuid,
tag_id uuid,
CONSTRAINT fk_toys FOREIGN KEY(toy_id) REFERENCES api.toys(id),
CONSTRAINT fk_tags FOREIGN KEY(tag_id) REFERENCES api.tags(id)
);
GRANT all ON api.toys_tags TO automation;
GRANT SELECT ON api.toys_tags TO web_anon;
-- tags-vods junction table
-- toys-vtubers junction table
CREATE TABLE api.toys_vtubers(
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
toy_id uuid,
vtuber_id uuid,
CONSTRAINT fk_toys FOREIGN KEY(toy_id) REFERENCES api.toys(id),
CONSTRAINT fk_vtubers FOREIGN KEY(vtuber_id) REFERENCES api.vtubers(id)
);
GRANT all ON api.toys_vtubers TO automation;
GRANT SELECT ON api.toys_vtubers TO web_anon;
-- vods table
CREATE TABLE api.vods (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
stream_id uuid NOT NULL,
FOREIGN KEY (stream_id) REFERENCES api.streams(id),
video_cid TEXT UNIQUE,
CONSTRAINT check_video_cid CHECK (video_cid ~ 'Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}'),
announce_title TEXT,
announce_url TEXT,
note TEXT,
date timestamp(6) without time zone,
spoilers TEXT,
title TEXT,
uploader uuid,
mux_asset_id TEXT,
mux_playback_id TEXT,
s3_key TEXT,
s3_id TEXT,
thumbnail TEXT
);
GRANT all ON api.vods TO automation;
GRANT SELECT ON api.vods TO web_anon;
-- tags-vods junction table
CREATE TABLE api.tags_vods(
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
tag_id uuid,
vod_id uuid,
CONSTRAINT fk_tags FOREIGN KEY(tag_id) REFERENCES api.tags(id),
CONSTRAINT fk_vods FOREIGN KEY(vod_id) REFERENCES api.vods(id)
);
GRANT all ON api.tags_vods TO automation;
GRANT SELECT ON api.tags_vods TO web_anon;

7
services/migrations/migrations/00010_record-segments.sql Normal file
View File

@ -0,0 +1,7 @@
-- we add the concept of segments to api.records
-- implemented as a multidimensional text array, s3_segments.
-- the first value is the s3 id, the second value is the s3 key
-- [id, key]
ALTER TABLE IF EXISTS api.records
ADD COLUMN s3_segments text[][];

28
services/migrations/migrations/00011_use-composite-primary-keys.sql Normal file
View File

@ -0,0 +1,28 @@
-- we don't need s3_segments multidimential array. we're moving it's functionality to a new table
ALTER TABLE IF EXISTS api.records
DROP COLUMN s3_segments;
-- segments table
CREATE TABLE api.segments (
id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
s3_key TEXT NOT NULL,
s3_id TEXT NOT NULL,
bytes bigint DEFAULT 0
);
GRANT all ON api.segments TO automation;
GRANT SELECT ON api.segments TO web_anon;
-- records-segments join table
CREATE TABLE api.records_segments(
id INT GENERATED ALWAYS AS IDENTITY,
record_id INT NOT NULL,
segment_id INT NOT NULL,
CONSTRAINT fk_record FOREIGN KEY(record_id) REFERENCES api.records(id),
CONSTRAINT fk_segment FOREIGN KEY(segment_id) REFERENCES api.segments(id),
PRIMARY KEY(id, record_id, segment_id)
);
GRANT all ON api.records_segments TO automation;
GRANT SELECT ON api.records_segments TO web_anon;

2
services/migrations/migrations/00012_order-records_segments.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE IF EXISTS api.records_segments
ADD COLUMN segments_order INT NOT NULL DEFAULT 0;

5
services/migrations/migrations/00013_rename-segments-order.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE IF EXISTS api.records_segments
DROP COLUMN segments_order;
ALTER TABLE IF EXISTS api.records_segments
ADD COLUMN segment_order INT NOT NULL DEFAULT 0;

2
services/migrations/migrations/00014_create-segments-stream-links.sql Normal file
View File

@ -0,0 +1,2 @@
DROP TABLE IF EXISTS api.records CASCADE;
DROP TABLE IF EXISTS api.records_segments CASCADE;

16
services/migrations/migrations/00015_create-segments-stream-links-2.sql Normal file
View File

@ -0,0 +1,16 @@
-- I forgot to actually create the new table
CREATE TABLE api.segments_stream_links (
id int PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
discord_message_id text NOT NULL,
capture_job_id text NOT NULL
);
-- roles & permissions
GRANT all ON api.segments_stream_links TO automation;
GRANT SELECT ON api.segments_stream_links TO web_anon;
-- there is no s3_id in the segments run context so we don't need a column for it
ALTER TABLE IF EXISTS api.segments
DROP COLUMN s3_id;

8
services/migrations/migrations/00016_remove-unecessary-columns.sql Normal file
View File

@ -0,0 +1,8 @@
-- oops. bit by unfinished copy-paste
-- there is no s3_id in the segments run context so we don't need a column for it
ALTER TABLE IF EXISTS api.segments_stream_links
DROP COLUMN discord_message_id;
ALTER TABLE IF EXISTS api.segments_stream_links
DROP COLUMN capture_job_id;

5
services/migrations/migrations/00017_add-stream-status-col.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE IF EXISTS api.streams
ADD COLUMN updated_at timestamp(6) without time zone;
ALTER TABLE IF EXISTS api.streams
ADD COLUMN status TEXT NOT NULL;

5
services/migrations/migrations/00018_add-stream-status-default.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE IF EXISTS api.streams
DROP COLUMN IF EXISTS status;
ALTER TABLE api.streams
ADD COLUMN status TEXT NOT NULL DEFAULT 'pending_recording';

1
services/migrations/migrations/00019_drop-discord-interactions.sql Normal file
View File

@ -0,0 +1 @@
DROP TABLE IF EXISTS api.discord_interactions CASCADE;

38
services/migrations/migrations/00020_add-streams-update-trigger.sql Normal file
View File

@ -0,0 +1,38 @@
-- delete outdated
DROP FUNCTION IF EXISTS public.tg__add_job();
-- We create a function which lets Postgrest's automation user create jobs in Graphile Worker.
-- Normally only the database owner, in our case `postgres`, can add jobs due to RLS in graphile_worker tables.
-- Under the advice of graphile_worker author, we can use a SECURITY DEFINER wrapper function.
-- @see https://worker.graphile.org/docs/sql-add-job#graphile_workeradd_job:~:text=graphile_worker.add_job(...),that%20are%20necessary.)
-- @see https://discord.com/channels/489127045289476126/1179293106336694333/1179605043729670306
-- @see https://discord.com/channels/489127045289476126/498852330754801666/1067707497235873822
CREATE FUNCTION public.tg__add_record_job() RETURNS trigger
LANGUAGE plpgsql SECURITY DEFINER
SET search_path TO 'pg_catalog', 'public', 'pg_temp'
AS $$
begin
PERFORM graphile_worker.add_job('record', json_build_object(
'url', NEW.url,
'stream_id', NEW.id
), max_attempts := 12);
return NEW;
end;
$$;
-- when a stream is updated, we add a job in graphile to update_discord_message
CREATE TRIGGER stream_update
AFTER UPDATE ON api.streams
FOR EACH ROW
EXECUTE PROCEDURE public.tg__update_discord_message('update_discord_message');
-- when a stream is created, we add a 'record' job in graphile-worker
CREATE TRIGGER stream_create
AFTER INSERT ON api.streams
FOR EACH ROW
EXECUTE PROCEDURE public.tg__add_record_job('record');

9
services/migrations/migrations/00021_add-foreign-key-to-segments_stream.sql Normal file
View File

@ -0,0 +1,9 @@
DROP TABLE api.segments_stream_links;
CREATE TABLE api.segments_stream_links (
id int GENERATED ALWAYS AS IDENTITY,
stream_id UUID NOT NULL REFERENCES api.streams(id),
segment_id INT NOT NULL REFERENCES api.segments(id),
capture_job_id text NOT NULL,
PRIMARY KEY(id, stream_id, segment_id)
);

2
services/migrations/migrations/00022_add-permissions-for-segments_stream_links.sql Normal file
View File

@ -0,0 +1,2 @@
GRANT all ON api.segments_stream_links TO automation;
GRANT SELECT ON api.segments_stream_links TO web_anon;

3
services/migrations/migrations/00023_drop-capture_job_id-column.sql Normal file
View File

@ -0,0 +1,3 @@
ALTER TABLE IF EXISTS api.segments_stream_links
DROP COLUMN IF EXISTS capture_job_id;

39
services/migrations/migrations/00024_add-updated_at-for-segments.sql Normal file
View File

@ -0,0 +1,39 @@
ALTER TABLE api.segments
ADD COLUMN created_at TIMESTAMP(6) WITHOUT TIME ZONE;
ALTER TABLE api.segments
ADD COLUMN updated_at TIMESTAMP(6) WITHOUT TIME ZONE;
-- in migration 8, we already created tg__updated_at() so we don't need to create that,
-- but we do need to create a function which will the row's created_at
CREATE FUNCTION public.tg__created_at() RETURNS trigger
LANGUAGE plpgsql
SET search_path TO 'pg_catalog', 'public', 'pg_temp'
AS $$
BEGIN
NEW.created_at = now();
RETURN NEW;
END;
$$;
-- create a trigger which runs the tg__updated_at() function when a /segment is updated
CREATE TRIGGER segment_updated_at
AFTER UPDATE ON api.segments
FOR EACH ROW
EXECUTE PROCEDURE public.tg__updated_at();
-- create a trigger which runs the tg__created_at() function when a /segment is created
CREATE TRIGGER segment_created_at
AFTER INSERT ON api.segments
FOR EACH ROW
EXECUTE PROCEDURE public.tg__created_at();
-- create a trigger which runs the tg__created_at() function when a /stream is created
CREATE TRIGGER stream_created_at
AFTER INSERT ON api.streams
FOR EACH ROW
EXECUTE PROCEDURE public.tg__created_at();

2
services/migrations/migrations/00025_add-is_recording_aborted-to-streams.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER TABLE api.streams
ADD COLUMN is_recording_aborted BOOLEAN DEFAULT FALSE;

2
services/migrations/migrations/00026_use-moddatetime.sql Normal file
View File

@ -0,0 +1,2 @@
CREATE EXTENSION moddatetime;

49
services/migrations/migrations/00027_create-triggers-for-moddatetime.sql Normal file
View File

@ -0,0 +1,49 @@
-- now we set up the triggers
-- streams created_at
ALTER TABLE api.streams
ALTER created_at SET DEFAULT now();
DROP TRIGGER stream_created_at ON api.streams;
CREATE TRIGGER stream_created_at
BEFORE INSERT ON api.streams
FOR EACH ROW
EXECUTE PROCEDURE moddatetime (created_at);
-- streams updated_at
ALTER TABLE api.streams
ALTER updated_at SET DEFAULT now();
CREATE TRIGGER stream_updated_at
BEFORE UPDATE ON api.streams
FOR EACH ROW
EXECUTE PROCEDURE moddatetime (updated_at);
-- segments created_at
ALTER TABLE api.segments
ALTER created_at SET DEFAULT now();
DROP TRIGGER segment_created_at ON api.segments;
CREATE TRIGGER segment_created_at
BEFORE INSERT ON api.segments
FOR EACH ROW
EXECUTE PROCEDURE moddatetime(created_at);
-- segments updated_at
ALTER TABLE api.segments
ALTER updated_at SET DEFAULT now();
DROP TRIGGER segment_updated_at ON api.segments;
CREATE TRIGGER segment_updated_at
BEFORE UPDATE ON api.segments
FOR EACH ROW
EXECUTE PROCEDURE moddatetime(updated_at);

9
services/migrations/migrations/00028_remove-moddate-on-insert.sql Normal file
View File

@ -0,0 +1,9 @@
-- A fix for the following error
-- moddatetime: cannot process INSERT events
--
-- We don't need moddatetime for INSERT events because we have column defaults set the time when the row is created.
DROP TRIGGER segment_created_at ON api.segments;
DROP TRIGGER stream_created_at ON api.streams;

5
services/migrations/migrations/00029_add-discord-message-id.sql Normal file
View File

@ -0,0 +1,5 @@
-- streams needs discord_message_id for chatops
ALTER TABLE api.streams
ADD COLUMN discord_message_id TEXT;

1
services/migrations/package.json
View File

@ -8,6 +8,7 @@
"test": "echo \"Error: no test specified\" && exit 0",
"start": "node index.js"
},
"packageManager": "pnpm@9.6.0",
"keywords": [],
"author": "@CJ_Clippy",
"license": "Unlicense",