From 9cd9b6a53d1121d61128569888275a977dfdca65 Mon Sep 17 00:00:00 2001 From: CJ_Clippy Date: Fri, 16 Aug 2024 18:42:44 -0800 Subject: [PATCH] chatops progress --- MANTRAS.md | 7 +- Tiltfile | 30 +- charts/fp/templates/migrations.yaml | 21 + charts/fp/values.yaml | 4 +- packages/types/index.d.ts | 43 +- packages/worker/README.md | 7 - packages/worker/package.json | 26 - packages/worker/pnpm-lock.yaml | 1033 ----------------- packages/worker/src/index.ts | 35 - packages/worker/src/tasks/hello.ts | 4 - .../worker/src/tasks/identify_image_color.ts | 15 - packages/worker/tsconfig.json | 29 - scripts/postgres-create.sh | 9 +- scripts/postgres-drop.sh | 9 +- scripts/postgres-migrations.sh | 13 + scripts/postgres-refresh.sh | 9 + scripts/postgrest-migrations.sh | 12 - services/bot/{src => }/crontab | 4 +- services/bot/package.json | 16 +- services/bot/pnpm-lock.yaml | 495 +++++--- services/bot/src/bot.ts | 14 +- services/bot/src/commands/record.ts | 17 +- services/bot/src/config.ts | 5 +- services/bot/src/index.ts | 8 +- .../bot/src/tasks/expire_stream_recordings.ts | 72 ++ .../src/tasks/restart_failed_recordings.ts | 85 ++ .../bot/src/tasks/update_discord_message.ts | 116 +- services/capture/package.json | 11 +- services/capture/pnpm-lock.yaml | 48 + services/capture/src/Record.spec.ts | 24 +- services/capture/src/Record.ts | 26 +- services/capture/src/config.ts | 40 + services/capture/src/index.ts | 13 +- services/capture/src/tasks/record.ts | 420 ++++--- .../capture/src/tasks/recording_start.ts.old | 77 -- services/capture/src/tasks/start_recording.ts | 67 -- services/capture/src/tasks/stop_recording.ts | 18 - services/capture/tsconfig.json | 2 +- services/migrations/README.md | 17 +- services/migrations/index.js | 4 +- .../00006_add-updated-at-to-records.sql | 5 + .../00007_add-default-records-timestamps.sql | 7 + .../00008_add-default-records-timestamp.sql | 26 + .../00009_add-streams-vods-vtubers.sql | 140 +++ .../migrations/00010_record-segments.sql | 7 + .../00011_use-composite-primary-keys.sql | 28 + .../00012_order-records_segments.sql | 2 + .../00013_rename-segments-order.sql | 5 + .../00014_create-segments-stream-links.sql | 2 + .../00015_create-segments-stream-links-2.sql | 16 + .../00016_remove-unecessary-columns.sql | 8 + .../00017_add-stream-status-col.sql | 5 + .../00018_add-stream-status-default.sql | 5 + .../00019_drop-discord-interactions.sql | 1 + .../00020_add-streams-update-trigger.sql | 38 + ...021_add-foreign-key-to-segments_stream.sql | 9 + ...-permissions-for-segments_stream_links.sql | 2 + .../00023_drop-capture_job_id-column.sql | 3 + .../00024_add-updated_at-for-segments.sql | 39 + ...25_add-is_recording_aborted-to-streams.sql | 2 + .../migrations/00026_use-moddatetime.sql | 2 + .../00027_create-triggers-for-moddatetime.sql | 49 + .../00028_remove-moddate-on-insert.sql | 9 + .../00029_add-discord-message-id.sql | 5 + services/migrations/package.json | 1 + 65 files changed, 1562 insertions(+), 1759 deletions(-) create mode 100644 charts/fp/templates/migrations.yaml delete mode 100644 packages/worker/README.md delete mode 100644 packages/worker/package.json delete mode 100644 packages/worker/pnpm-lock.yaml delete mode 100644 packages/worker/src/index.ts delete mode 100644 packages/worker/src/tasks/hello.ts delete mode 100644 packages/worker/src/tasks/identify_image_color.ts delete mode 100644 packages/worker/tsconfig.json create mode 100755 scripts/postgres-migrations.sh create mode 100644 scripts/postgres-refresh.sh delete mode 100755 scripts/postgrest-migrations.sh rename services/bot/{src => }/crontab (88%) create mode 100644 services/bot/src/tasks/expire_stream_recordings.ts create mode 100644 services/bot/src/tasks/restart_failed_recordings.ts create mode 100644 services/capture/src/config.ts delete mode 100644 services/capture/src/tasks/recording_start.ts.old delete mode 100644 services/capture/src/tasks/start_recording.ts delete mode 100644 services/capture/src/tasks/stop_recording.ts create mode 100644 services/migrations/migrations/00006_add-updated-at-to-records.sql create mode 100644 services/migrations/migrations/00007_add-default-records-timestamps.sql create mode 100644 services/migrations/migrations/00008_add-default-records-timestamp.sql create mode 100644 services/migrations/migrations/00009_add-streams-vods-vtubers.sql create mode 100644 services/migrations/migrations/00010_record-segments.sql create mode 100644 services/migrations/migrations/00011_use-composite-primary-keys.sql create mode 100644 services/migrations/migrations/00012_order-records_segments.sql create mode 100644 services/migrations/migrations/00013_rename-segments-order.sql create mode 100644 services/migrations/migrations/00014_create-segments-stream-links.sql create mode 100644 services/migrations/migrations/00015_create-segments-stream-links-2.sql create mode 100644 services/migrations/migrations/00016_remove-unecessary-columns.sql create mode 100644 services/migrations/migrations/00017_add-stream-status-col.sql create mode 100644 services/migrations/migrations/00018_add-stream-status-default.sql create mode 100644 services/migrations/migrations/00019_drop-discord-interactions.sql create mode 100644 services/migrations/migrations/00020_add-streams-update-trigger.sql create mode 100644 services/migrations/migrations/00021_add-foreign-key-to-segments_stream.sql create mode 100644 services/migrations/migrations/00022_add-permissions-for-segments_stream_links.sql create mode 100644 services/migrations/migrations/00023_drop-capture_job_id-column.sql create mode 100644 services/migrations/migrations/00024_add-updated_at-for-segments.sql create mode 100644 services/migrations/migrations/00025_add-is_recording_aborted-to-streams.sql create mode 100644 services/migrations/migrations/00026_use-moddatetime.sql create mode 100644 services/migrations/migrations/00027_create-triggers-for-moddatetime.sql create mode 100644 services/migrations/migrations/00028_remove-moddate-on-insert.sql create mode 100644 services/migrations/migrations/00029_add-discord-message-id.sql diff --git a/MANTRAS.md b/MANTRAS.md index 2a68509..f752e20 100644 --- a/MANTRAS.md +++ b/MANTRAS.md @@ -33,7 +33,7 @@ Get through the [OODA loop](https://en.wikipedia.org/wiki/OODA_loop) as many tim ### The computer doesn't care > "There are 2 hard problems in computer science: cache invalidation, naming things, and off-by-1 errors." -> Leon Bambrick +> -- Leon Bambrick In other words, pick something for a name and roll with the punches. @@ -44,3 +44,8 @@ In other words, pick something for a name and roll with the punches. 3. Simplify or optimize 4. Accelerate Cycle Time 5. Automate + +### Never Settle + +> "But it's also about looking at things anew and what they could be instead of what they are" +> -- Rodney Mullen \ No newline at end of file diff --git a/Tiltfile b/Tiltfile index b48d090..aad992d 100644 --- a/Tiltfile +++ b/Tiltfile @@ -215,6 +215,12 @@ cmd_button('postgres:drop', icon_name='delete', text='DROP all databases' ) +cmd_button('postgres:refresh', + argv=['sh', './scripts/postgres-refresh.sh'], + resource='migrations', + icon_name='refresh', + text='Refresh schema cache' +) cmd_button('capture-api:create', argv=['http', '--ignore-stdin', 'POST', 'http://localhost:5003/api/record', "url='https://twitch.tv/ironmouse'", "channel='ironmouse'"], @@ -223,9 +229,9 @@ cmd_button('capture-api:create', text='Start Recording' ) -cmd_button('postgrest:migrate', - argv=['./scripts/postgrest-migrations.sh'], - resource='postgrest', +cmd_button('postgres:migrate', + argv=['./scripts/postgres-migrations.sh'], + resource='postgresql-primary', icon_name='directions_run', text='Run migrations', ) @@ -243,6 +249,16 @@ cmd_button('factory:test', text='test', ) +## we ignore unused image warnings because we do actually use this image. +## instead of being invoked by helm, we start a container using this image manually via Tilt UI +# update_settings(suppress_unused_image_warnings=["fp/migrations"]) +docker_build( + 'fp/migrations', + '.', + dockerfile='dockerfiles/migrations.dockerfile', + target='migrations', + pull=False, +) ## Uncomment the following for fp/next in dev mode ## this is useful for changing the UI and seeing results @@ -350,7 +366,7 @@ docker_build( './services/capture', ], live_update=[ - sync('./services/capture/dist', '/app/dist'), + sync('./services/capture', '/app/services/capture'), ], pull=False, ) @@ -513,7 +529,11 @@ k8s_resource( port_forwards=['5050:80'], labels=['database'], ) - +k8s_resource( + workload='migrations', + labels=['database'], + resource_deps=['postgresql-primary'], +) k8s_resource( workload='cert-manager', diff --git a/charts/fp/templates/migrations.yaml b/charts/fp/templates/migrations.yaml new file mode 100644 index 0000000..4f8cf90 --- /dev/null +++ b/charts/fp/templates/migrations.yaml @@ -0,0 +1,21 @@ + +--- +apiVersion: v1 +kind: Pod +metadata: + name: migrations + namespace: futureporn + labels: + app.kubernetes.io/name: migrations +spec: + containers: + - name: migrations + image: "{{ .Values.migrations.imageName }}" + resources: {} + env: + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: password + restartPolicy: Never diff --git a/charts/fp/values.yaml b/charts/fp/values.yaml index 3615920..bcb33ef 100644 --- a/charts/fp/values.yaml +++ b/charts/fp/values.yaml @@ -90,4 +90,6 @@ chisel: game2048: hostname: game-2048.fp.sbtp.xyz whoami: - hostname: whoami.fp.sbtp.xyz \ No newline at end of file + hostname: whoami.fp.sbtp.xyz +migrations: + imageName: fp/migrations \ No newline at end of file diff --git a/packages/types/index.d.ts b/packages/types/index.d.ts index 87d215f..5f62fdd 100644 --- a/packages/types/index.d.ts +++ b/packages/types/index.d.ts @@ -4,22 +4,45 @@ export as namespace Futureporn; declare namespace Futureporn { - interface RecordingRecord { - id: number; - recordingState: RecordingState; - fileSize: number; - discordMessageId: string; - isAborted: boolean; + type PlatformNotificationType = 'email' | 'manual' | 'twitter' + type ArchiveStatus = 'good' | 'issue' | 'missing' + type RecordingState = 'pending' | 'recording' | 'stalled' | 'aborted' | 'failed' | 'finished' + type Status = Partial + + interface Stream { + id: string; + url: string; + platform_notification_type: PlatformNotificationType; + date: Date; + created_at: Date; + updated_at: Date; + vtuber: string; + tweet: string; + archive_status: ArchiveStatus; + is_chaturbate_stream: Boolean; + is_fansly_stream: Boolean; + is_recording_aborted: Boolean; + status: Status; } - interface RawRecordingRecord { + + interface RecordingRecord { id: number; recording_state: RecordingState; file_size: number; discord_message_id: string; - is_aborted: boolean; + is_recording_aborted: boolean; + updated_at: Date; + created_at: Date; + } + + interface Segment { + id: number; + s3_key: string; + s3_id: string; + bytes: number; + stream?: Stream[]; } - type RecordingState = 'pending' | 'recording' | 'aborted' | 'ended' interface IMuxAsset { @@ -70,7 +93,7 @@ declare namespace Futureporn { attributes: { date: string; date2: string; - archiveStatus: 'good' | 'issue' | 'missing'; + archiveStatus: ArchiveStatus; vods: IVodsResponse; cuid: string; vtuber: IVtuberResponse; diff --git a/packages/worker/README.md b/packages/worker/README.md deleted file mode 100644 index b730984..0000000 --- a/packages/worker/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# @futureporn/worker - -The system component which runs background tasks such as thumbnail generation, video encoding, file transfers, etc. - -We use [Graphile Worker](https://worker.graphile.org) - - diff --git a/packages/worker/package.json b/packages/worker/package.json deleted file mode 100644 index 36118ed..0000000 --- a/packages/worker/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "@futureporn/worker", - "type": "module", - "version": "1.3.0", - "private": true, - "scripts": { - "bundle": "node ./src/create-workflow-bundle.js", - "build": "tsc --build", - "lint": "eslint .", - "dev": "nodemon --ext js,ts,json,yaml --watch ./src/index.ts --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"", - "start": "node dist/index.js", - "clean": "rm -rf dist", - "superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist" - }, - "dependencies": { - "date-fns": "^3.6.0", - "dotenv": "^16.4.5", - "graphile-worker": "^0.16.6", - "qs": "^6.12.3" - }, - "packageManager": "pnpm@9.5.0", - "devDependencies": { - "nodemon": "^2.0.15", - "typescript": "^5.5.3" - } -} diff --git a/packages/worker/pnpm-lock.yaml b/packages/worker/pnpm-lock.yaml deleted file mode 100644 index 320302c..0000000 --- a/packages/worker/pnpm-lock.yaml +++ /dev/null @@ -1,1033 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - dependencies: - date-fns: - specifier: ^3.6.0 - version: 3.6.0 - dotenv: - specifier: ^16.4.5 - version: 16.4.5 - graphile-worker: - specifier: ^0.16.6 - version: 0.16.6(typescript@5.5.4) - qs: - specifier: ^6.12.3 - version: 6.12.3 - devDependencies: - nodemon: - specifier: ^2.0.15 - version: 2.0.22 - typescript: - specifier: ^5.5.3 - version: 5.5.4 - -packages: - - '@babel/code-frame@7.24.7': - resolution: {integrity: sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==} - engines: {node: '>=6.9.0'} - - '@babel/helper-validator-identifier@7.24.7': - resolution: {integrity: sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==} - engines: {node: '>=6.9.0'} - - '@babel/highlight@7.24.7': - resolution: {integrity: sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==} - engines: {node: '>=6.9.0'} - - '@graphile/logger@0.2.0': - resolution: {integrity: sha512-jjcWBokl9eb1gVJ85QmoaQ73CQ52xAaOCF29ukRbYNl6lY+ts0ErTaDYOBlejcbUs2OpaiqYLO5uDhyLFzWw4w==} - - '@types/debug@4.1.12': - resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} - - '@types/interpret@1.1.3': - resolution: {integrity: sha512-uBaBhj/BhilG58r64mtDb/BEdH51HIQLgP5bmWzc5qCtFMja8dCk/IOJmk36j0lbi9QHwI6sbtUNGuqXdKCAtQ==} - - '@types/ms@0.7.34': - resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} - - '@types/node@20.14.13': - resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==} - - '@types/node@22.0.0': - resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==} - - '@types/pg@8.11.6': - resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} - - '@types/semver@7.5.8': - resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} - - ansi-regex@5.0.1: - resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} - engines: {node: '>=8'} - - ansi-styles@3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - - ansi-styles@4.3.0: - resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} - engines: {node: '>=8'} - - anymatch@3.1.3: - resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} - engines: {node: '>= 8'} - - argparse@2.0.1: - resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} - - balanced-match@1.0.2: - resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - - binary-extensions@2.3.0: - resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} - engines: {node: '>=8'} - - brace-expansion@1.1.11: - resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} - - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - - call-bind@1.0.7: - resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} - engines: {node: '>= 0.4'} - - callsites@3.1.0: - resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} - engines: {node: '>=6'} - - chalk@2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - - chalk@4.1.2: - resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} - engines: {node: '>=10'} - - chokidar@3.6.0: - resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} - engines: {node: '>= 8.10.0'} - - cliui@8.0.1: - resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} - engines: {node: '>=12'} - - color-convert@1.9.3: - resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - - color-convert@2.0.1: - resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} - engines: {node: '>=7.0.0'} - - color-name@1.1.3: - resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - - color-name@1.1.4: - resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - - concat-map@0.0.1: - resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} - - cosmiconfig@8.3.6: - resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} - engines: {node: '>=14'} - peerDependencies: - typescript: '>=4.9.5' - peerDependenciesMeta: - typescript: - optional: true - - date-fns@3.6.0: - resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} - - debug@3.2.7: - resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - debug@4.3.6: - resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - define-data-property@1.1.4: - resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} - engines: {node: '>= 0.4'} - - dotenv@16.4.5: - resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} - engines: {node: '>=12'} - - emoji-regex@8.0.0: - resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} - - error-ex@1.3.2: - resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} - - es-define-property@1.0.0: - resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} - engines: {node: '>= 0.4'} - - es-errors@1.3.0: - resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} - engines: {node: '>= 0.4'} - - escalade@3.1.2: - resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} - engines: {node: '>=6'} - - escape-string-regexp@1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - function-bind@1.1.2: - resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - - get-caller-file@2.0.5: - resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} - engines: {node: 6.* || 8.* || >= 10.*} - - get-intrinsic@1.2.4: - resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} - engines: {node: '>= 0.4'} - - glob-parent@5.1.2: - resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} - engines: {node: '>= 6'} - - gopd@1.0.1: - resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} - - graphile-config@0.0.1-beta.9: - resolution: {integrity: sha512-7vNxXZ24OAgXxDKXYi9JtgWPMuNbBL3057Yf32Ux+/rVP4+EePgySCc+NNnn0tORi8qwqVreN8bdWqGIcSwNXg==} - engines: {node: '>=16'} - - graphile-worker@0.16.6: - resolution: {integrity: sha512-e7gGYDmGqzju2l83MpzX8vNG/lOtVJiSzI3eZpAFubSxh/cxs7sRrRGBGjzBP1kNG0H+c95etPpNRNlH65PYhw==} - engines: {node: '>=14.0.0'} - hasBin: true - - has-flag@3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - - has-flag@4.0.0: - resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} - engines: {node: '>=8'} - - has-property-descriptors@1.0.2: - resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} - - has-proto@1.0.3: - resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} - engines: {node: '>= 0.4'} - - has-symbols@1.0.3: - resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} - engines: {node: '>= 0.4'} - - hasown@2.0.2: - resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} - engines: {node: '>= 0.4'} - - ignore-by-default@1.0.1: - resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==} - - import-fresh@3.3.0: - resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} - engines: {node: '>=6'} - - interpret@3.1.1: - resolution: {integrity: sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==} - engines: {node: '>=10.13.0'} - - is-arrayish@0.2.1: - resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} - - is-binary-path@2.1.0: - resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} - engines: {node: '>=8'} - - is-extglob@2.1.1: - resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} - engines: {node: '>=0.10.0'} - - is-fullwidth-code-point@3.0.0: - resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} - engines: {node: '>=8'} - - is-glob@4.0.3: - resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} - engines: {node: '>=0.10.0'} - - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - - js-tokens@4.0.0: - resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - - js-yaml@4.1.0: - resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} - hasBin: true - - json-parse-even-better-errors@2.3.1: - resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} - - json5@2.2.3: - resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} - engines: {node: '>=6'} - hasBin: true - - lines-and-columns@1.2.4: - resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - - minimatch@3.1.2: - resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} - - ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - - nodemon@2.0.22: - resolution: {integrity: sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ==} - engines: {node: '>=8.10.0'} - hasBin: true - - normalize-path@3.0.0: - resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} - engines: {node: '>=0.10.0'} - - object-inspect@1.13.2: - resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} - engines: {node: '>= 0.4'} - - obuf@1.1.2: - resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} - - parent-module@1.0.1: - resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} - engines: {node: '>=6'} - - parse-json@5.2.0: - resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} - engines: {node: '>=8'} - - path-type@4.0.0: - resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} - engines: {node: '>=8'} - - pg-cloudflare@1.1.1: - resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} - - pg-connection-string@2.6.4: - resolution: {integrity: sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==} - - pg-int8@1.0.1: - resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} - engines: {node: '>=4.0.0'} - - pg-numeric@1.0.2: - resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} - engines: {node: '>=4'} - - pg-pool@3.6.2: - resolution: {integrity: sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==} - peerDependencies: - pg: '>=8.0' - - pg-protocol@1.6.1: - resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==} - - pg-types@2.2.0: - resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} - engines: {node: '>=4'} - - pg-types@4.0.2: - resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} - engines: {node: '>=10'} - - pg@8.12.0: - resolution: {integrity: sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==} - engines: {node: '>= 8.0.0'} - peerDependencies: - pg-native: '>=3.0.1' - peerDependenciesMeta: - pg-native: - optional: true - - pgpass@1.0.5: - resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} - - picocolors@1.0.1: - resolution: {integrity: sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==} - - picomatch@2.3.1: - resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} - engines: {node: '>=8.6'} - - postgres-array@2.0.0: - resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} - engines: {node: '>=4'} - - postgres-array@3.0.2: - resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} - engines: {node: '>=12'} - - postgres-bytea@1.0.0: - resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} - engines: {node: '>=0.10.0'} - - postgres-bytea@3.0.0: - resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} - engines: {node: '>= 6'} - - postgres-date@1.0.7: - resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} - engines: {node: '>=0.10.0'} - - postgres-date@2.1.0: - resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} - engines: {node: '>=12'} - - postgres-interval@1.2.0: - resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} - engines: {node: '>=0.10.0'} - - postgres-interval@3.0.0: - resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} - engines: {node: '>=12'} - - postgres-range@1.1.4: - resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} - - pstree.remy@1.1.8: - resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} - - qs@6.12.3: - resolution: {integrity: sha512-AWJm14H1vVaO/iNZ4/hO+HyaTehuy9nRqVdkTqlJt0HWvBiBIEXFmb4C0DGeYo3Xes9rrEW+TxHsaigCbN5ICQ==} - engines: {node: '>=0.6'} - - readdirp@3.6.0: - resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} - engines: {node: '>=8.10.0'} - - require-directory@2.1.1: - resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} - engines: {node: '>=0.10.0'} - - resolve-from@4.0.0: - resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} - engines: {node: '>=4'} - - semver@5.7.2: - resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} - hasBin: true - - semver@7.0.0: - resolution: {integrity: sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==} - hasBin: true - - semver@7.6.3: - resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==} - engines: {node: '>=10'} - hasBin: true - - set-function-length@1.2.2: - resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} - engines: {node: '>= 0.4'} - - side-channel@1.0.6: - resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} - engines: {node: '>= 0.4'} - - simple-update-notifier@1.1.0: - resolution: {integrity: sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==} - engines: {node: '>=8.10.0'} - - split2@4.2.0: - resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} - engines: {node: '>= 10.x'} - - string-width@4.2.3: - resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} - engines: {node: '>=8'} - - strip-ansi@6.0.1: - resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} - engines: {node: '>=8'} - - supports-color@5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - - supports-color@7.2.0: - resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} - engines: {node: '>=8'} - - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - - touch@3.1.1: - resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==} - hasBin: true - - tslib@2.6.3: - resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} - - typescript@5.5.4: - resolution: {integrity: sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==} - engines: {node: '>=14.17'} - hasBin: true - - undefsafe@2.0.5: - resolution: {integrity: sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==} - - undici-types@5.26.5: - resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - - undici-types@6.11.1: - resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==} - - wrap-ansi@7.0.0: - resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} - engines: {node: '>=10'} - - xtend@4.0.2: - resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} - engines: {node: '>=0.4'} - - y18n@5.0.8: - resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} - engines: {node: '>=10'} - - yargs-parser@21.1.1: - resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} - engines: {node: '>=12'} - - yargs@17.7.2: - resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} - engines: {node: '>=12'} - -snapshots: - - '@babel/code-frame@7.24.7': - dependencies: - '@babel/highlight': 7.24.7 - picocolors: 1.0.1 - - '@babel/helper-validator-identifier@7.24.7': {} - - '@babel/highlight@7.24.7': - dependencies: - '@babel/helper-validator-identifier': 7.24.7 - chalk: 2.4.2 - js-tokens: 4.0.0 - picocolors: 1.0.1 - - '@graphile/logger@0.2.0': {} - - '@types/debug@4.1.12': - dependencies: - '@types/ms': 0.7.34 - - '@types/interpret@1.1.3': - dependencies: - '@types/node': 20.14.13 - - '@types/ms@0.7.34': {} - - '@types/node@20.14.13': - dependencies: - undici-types: 5.26.5 - - '@types/node@22.0.0': - dependencies: - undici-types: 6.11.1 - - '@types/pg@8.11.6': - dependencies: - '@types/node': 22.0.0 - pg-protocol: 1.6.1 - pg-types: 4.0.2 - - '@types/semver@7.5.8': {} - - ansi-regex@5.0.1: {} - - ansi-styles@3.2.1: - dependencies: - color-convert: 1.9.3 - - ansi-styles@4.3.0: - dependencies: - color-convert: 2.0.1 - - anymatch@3.1.3: - dependencies: - normalize-path: 3.0.0 - picomatch: 2.3.1 - - argparse@2.0.1: {} - - balanced-match@1.0.2: {} - - binary-extensions@2.3.0: {} - - brace-expansion@1.1.11: - dependencies: - balanced-match: 1.0.2 - concat-map: 0.0.1 - - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - - call-bind@1.0.7: - dependencies: - es-define-property: 1.0.0 - es-errors: 1.3.0 - function-bind: 1.1.2 - get-intrinsic: 1.2.4 - set-function-length: 1.2.2 - - callsites@3.1.0: {} - - chalk@2.4.2: - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - - chalk@4.1.2: - dependencies: - ansi-styles: 4.3.0 - supports-color: 7.2.0 - - chokidar@3.6.0: - dependencies: - anymatch: 3.1.3 - braces: 3.0.3 - glob-parent: 5.1.2 - is-binary-path: 2.1.0 - is-glob: 4.0.3 - normalize-path: 3.0.0 - readdirp: 3.6.0 - optionalDependencies: - fsevents: 2.3.3 - - cliui@8.0.1: - dependencies: - string-width: 4.2.3 - strip-ansi: 6.0.1 - wrap-ansi: 7.0.0 - - color-convert@1.9.3: - dependencies: - color-name: 1.1.3 - - color-convert@2.0.1: - dependencies: - color-name: 1.1.4 - - color-name@1.1.3: {} - - color-name@1.1.4: {} - - concat-map@0.0.1: {} - - cosmiconfig@8.3.6(typescript@5.5.4): - dependencies: - import-fresh: 3.3.0 - js-yaml: 4.1.0 - parse-json: 5.2.0 - path-type: 4.0.0 - optionalDependencies: - typescript: 5.5.4 - - date-fns@3.6.0: {} - - debug@3.2.7(supports-color@5.5.0): - dependencies: - ms: 2.1.3 - optionalDependencies: - supports-color: 5.5.0 - - debug@4.3.6: - dependencies: - ms: 2.1.2 - - define-data-property@1.1.4: - dependencies: - es-define-property: 1.0.0 - es-errors: 1.3.0 - gopd: 1.0.1 - - dotenv@16.4.5: {} - - emoji-regex@8.0.0: {} - - error-ex@1.3.2: - dependencies: - is-arrayish: 0.2.1 - - es-define-property@1.0.0: - dependencies: - get-intrinsic: 1.2.4 - - es-errors@1.3.0: {} - - escalade@3.1.2: {} - - escape-string-regexp@1.0.5: {} - - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - - fsevents@2.3.3: - optional: true - - function-bind@1.1.2: {} - - get-caller-file@2.0.5: {} - - get-intrinsic@1.2.4: - dependencies: - es-errors: 1.3.0 - function-bind: 1.1.2 - has-proto: 1.0.3 - has-symbols: 1.0.3 - hasown: 2.0.2 - - glob-parent@5.1.2: - dependencies: - is-glob: 4.0.3 - - gopd@1.0.1: - dependencies: - get-intrinsic: 1.2.4 - - graphile-config@0.0.1-beta.9: - dependencies: - '@types/interpret': 1.1.3 - '@types/node': 20.14.13 - '@types/semver': 7.5.8 - chalk: 4.1.2 - debug: 4.3.6 - interpret: 3.1.1 - semver: 7.6.3 - tslib: 2.6.3 - yargs: 17.7.2 - transitivePeerDependencies: - - supports-color - - graphile-worker@0.16.6(typescript@5.5.4): - dependencies: - '@graphile/logger': 0.2.0 - '@types/debug': 4.1.12 - '@types/pg': 8.11.6 - cosmiconfig: 8.3.6(typescript@5.5.4) - graphile-config: 0.0.1-beta.9 - json5: 2.2.3 - pg: 8.12.0 - tslib: 2.6.3 - yargs: 17.7.2 - transitivePeerDependencies: - - pg-native - - supports-color - - typescript - - has-flag@3.0.0: {} - - has-flag@4.0.0: {} - - has-property-descriptors@1.0.2: - dependencies: - es-define-property: 1.0.0 - - has-proto@1.0.3: {} - - has-symbols@1.0.3: {} - - hasown@2.0.2: - dependencies: - function-bind: 1.1.2 - - ignore-by-default@1.0.1: {} - - import-fresh@3.3.0: - dependencies: - parent-module: 1.0.1 - resolve-from: 4.0.0 - - interpret@3.1.1: {} - - is-arrayish@0.2.1: {} - - is-binary-path@2.1.0: - dependencies: - binary-extensions: 2.3.0 - - is-extglob@2.1.1: {} - - is-fullwidth-code-point@3.0.0: {} - - is-glob@4.0.3: - dependencies: - is-extglob: 2.1.1 - - is-number@7.0.0: {} - - js-tokens@4.0.0: {} - - js-yaml@4.1.0: - dependencies: - argparse: 2.0.1 - - json-parse-even-better-errors@2.3.1: {} - - json5@2.2.3: {} - - lines-and-columns@1.2.4: {} - - minimatch@3.1.2: - dependencies: - brace-expansion: 1.1.11 - - ms@2.1.2: {} - - ms@2.1.3: {} - - nodemon@2.0.22: - dependencies: - chokidar: 3.6.0 - debug: 3.2.7(supports-color@5.5.0) - ignore-by-default: 1.0.1 - minimatch: 3.1.2 - pstree.remy: 1.1.8 - semver: 5.7.2 - simple-update-notifier: 1.1.0 - supports-color: 5.5.0 - touch: 3.1.1 - undefsafe: 2.0.5 - - normalize-path@3.0.0: {} - - object-inspect@1.13.2: {} - - obuf@1.1.2: {} - - parent-module@1.0.1: - dependencies: - callsites: 3.1.0 - - parse-json@5.2.0: - dependencies: - '@babel/code-frame': 7.24.7 - error-ex: 1.3.2 - json-parse-even-better-errors: 2.3.1 - lines-and-columns: 1.2.4 - - path-type@4.0.0: {} - - pg-cloudflare@1.1.1: - optional: true - - pg-connection-string@2.6.4: {} - - pg-int8@1.0.1: {} - - pg-numeric@1.0.2: {} - - pg-pool@3.6.2(pg@8.12.0): - dependencies: - pg: 8.12.0 - - pg-protocol@1.6.1: {} - - pg-types@2.2.0: - dependencies: - pg-int8: 1.0.1 - postgres-array: 2.0.0 - postgres-bytea: 1.0.0 - postgres-date: 1.0.7 - postgres-interval: 1.2.0 - - pg-types@4.0.2: - dependencies: - pg-int8: 1.0.1 - pg-numeric: 1.0.2 - postgres-array: 3.0.2 - postgres-bytea: 3.0.0 - postgres-date: 2.1.0 - postgres-interval: 3.0.0 - postgres-range: 1.1.4 - - pg@8.12.0: - dependencies: - pg-connection-string: 2.6.4 - pg-pool: 3.6.2(pg@8.12.0) - pg-protocol: 1.6.1 - pg-types: 2.2.0 - pgpass: 1.0.5 - optionalDependencies: - pg-cloudflare: 1.1.1 - - pgpass@1.0.5: - dependencies: - split2: 4.2.0 - - picocolors@1.0.1: {} - - picomatch@2.3.1: {} - - postgres-array@2.0.0: {} - - postgres-array@3.0.2: {} - - postgres-bytea@1.0.0: {} - - postgres-bytea@3.0.0: - dependencies: - obuf: 1.1.2 - - postgres-date@1.0.7: {} - - postgres-date@2.1.0: {} - - postgres-interval@1.2.0: - dependencies: - xtend: 4.0.2 - - postgres-interval@3.0.0: {} - - postgres-range@1.1.4: {} - - pstree.remy@1.1.8: {} - - qs@6.12.3: - dependencies: - side-channel: 1.0.6 - - readdirp@3.6.0: - dependencies: - picomatch: 2.3.1 - - require-directory@2.1.1: {} - - resolve-from@4.0.0: {} - - semver@5.7.2: {} - - semver@7.0.0: {} - - semver@7.6.3: {} - - set-function-length@1.2.2: - dependencies: - define-data-property: 1.1.4 - es-errors: 1.3.0 - function-bind: 1.1.2 - get-intrinsic: 1.2.4 - gopd: 1.0.1 - has-property-descriptors: 1.0.2 - - side-channel@1.0.6: - dependencies: - call-bind: 1.0.7 - es-errors: 1.3.0 - get-intrinsic: 1.2.4 - object-inspect: 1.13.2 - - simple-update-notifier@1.1.0: - dependencies: - semver: 7.0.0 - - split2@4.2.0: {} - - string-width@4.2.3: - dependencies: - emoji-regex: 8.0.0 - is-fullwidth-code-point: 3.0.0 - strip-ansi: 6.0.1 - - strip-ansi@6.0.1: - dependencies: - ansi-regex: 5.0.1 - - supports-color@5.5.0: - dependencies: - has-flag: 3.0.0 - - supports-color@7.2.0: - dependencies: - has-flag: 4.0.0 - - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - - touch@3.1.1: {} - - tslib@2.6.3: {} - - typescript@5.5.4: {} - - undefsafe@2.0.5: {} - - undici-types@5.26.5: {} - - undici-types@6.11.1: {} - - wrap-ansi@7.0.0: - dependencies: - ansi-styles: 4.3.0 - string-width: 4.2.3 - strip-ansi: 6.0.1 - - xtend@4.0.2: {} - - y18n@5.0.8: {} - - yargs-parser@21.1.1: {} - - yargs@17.7.2: - dependencies: - cliui: 8.0.1 - escalade: 3.1.2 - get-caller-file: 2.0.5 - require-directory: 2.1.1 - string-width: 4.2.3 - y18n: 5.0.8 - yargs-parser: 21.1.1 diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts deleted file mode 100644 index 9d7a006..0000000 --- a/packages/worker/src/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { run } from 'graphile-worker' -import { dirname } from 'node:path'; -import { fileURLToPath } from 'url'; -const __dirname = dirname(fileURLToPath(import.meta.url)); - -if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is undefined in env'); -const connectionString = process.env.DATABASE_URL - -console.log(`process.env.DATABASE_URL=${process.env.DATABASE_URL}`) -async function main() { - - - // Run a worker to execute jobs: - const runner = await run({ - connectionString, - concurrency: 5, - // Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc - noHandleSignals: false, - pollInterval: 1000, - taskDirectory: `${__dirname}/tasks`, - }); - - // Immediately await (or otherwise handle) the resulting promise, to avoid - // "unhandled rejection" errors causing a process crash in the event of - // something going wrong. - await runner.promise; - - // If the worker exits (whether through fatal error or otherwise), the above - // promise will resolve/reject. -} - -main().catch((err) => { - console.error(err); - process.exit(1); -}); \ No newline at end of file diff --git a/packages/worker/src/tasks/hello.ts b/packages/worker/src/tasks/hello.ts deleted file mode 100644 index 61e3293..0000000 --- a/packages/worker/src/tasks/hello.ts +++ /dev/null @@ -1,4 +0,0 @@ -export default async function (payload: any, helpers: any) { - const { name } = payload; - helpers.logger.info(`Hello, ${name}`); -}; \ No newline at end of file diff --git a/packages/worker/src/tasks/identify_image_color.ts b/packages/worker/src/tasks/identify_image_color.ts deleted file mode 100644 index c702e1c..0000000 --- a/packages/worker/src/tasks/identify_image_color.ts +++ /dev/null @@ -1,15 +0,0 @@ - -import { download } from "@futureporn/utils"; -import { getProminentColor } from "@futureporn/image"; - -export default async function (payload: any, helpers: any) { - const { url } = payload; - // helpers.logger.info(`Downloading ${url}`) - // const imageFile = await download({ url, filePath: '/tmp/my-image.png' }) - // helpers.logger.info(`downloaded to ${imageFile}`) - // if (!imageFile) throw new Error('imageFile was null') - // const color = await getProminentColor(imageFile) - // helpers.logger.info(`prominent color is ${color}`) - // return color - return '#0xffcc00' -} \ No newline at end of file diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json deleted file mode 100644 index d8a5f7d..0000000 --- a/packages/worker/tsconfig.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "compilerOptions": { - // Base Options recommended for all projects - "esModuleInterop": true, - "skipLibCheck": true, - "target": "es2022", - "allowJs": true, - "resolveJsonModule": true, - "moduleDetection": "force", - "isolatedModules": true, - // Enable strict type checking so you can catch bugs early - "strict": true, - "noUncheckedIndexedAccess": true, - "noImplicitOverride": true, - // Transpile our TypeScript code to JavaScript - "module": "NodeNext", - "outDir": "dist", - "lib": [ - "es2022" - ] - }, - // Include the necessary files for your project - "include": [ - "**/*.ts" - ], - "exclude": [ - "node_modules" - ] - } \ No newline at end of file diff --git a/scripts/postgres-create.sh b/scripts/postgres-create.sh index 8d0a249..910d811 100755 --- a/scripts/postgres-create.sh +++ b/scripts/postgres-create.sh @@ -61,14 +61,7 @@ kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PAS ## Create the futureporn Postgrest database -kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\ - CREATE DATABASE futureporn \ - WITH \ - OWNER = postgres \ - ENCODING = 'UTF8' \ - LOCALE_PROVIDER = 'libc' \ - CONNECTION LIMIT = -1 \ - IS_TEMPLATE = False;" +## !!! Don't create the database here! Allow @services/migrations to create the database. # @futureporn/migrations takes care of these tasks now diff --git a/scripts/postgres-drop.sh b/scripts/postgres-drop.sh index b239e0d..680f29d 100644 --- a/scripts/postgres-drop.sh +++ b/scripts/postgres-drop.sh @@ -5,5 +5,12 @@ if [ -z $POSTGRES_PASSWORD ]; then fi ## drop futureporn_db -kubectl -n futureporn exec postgresql-primary -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);" +kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn_db WITH (FORCE);" +## drop futureporn +kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP DATABASE futureporn WITH (FORCE);" + +## delete postgrest roles +kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP ROLE authenticator;" +kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP ROLE automation;" +kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "DROP ROLE web_anon;" diff --git a/scripts/postgres-migrations.sh b/scripts/postgres-migrations.sh new file mode 100755 index 0000000..cd93f0e --- /dev/null +++ b/scripts/postgres-migrations.sh @@ -0,0 +1,13 @@ +#!/bin/bash + + +if [ -z $POSTGRES_PASSWORD ]; then + echo "POSTGRES_PASSWORD was missing in env. In development environment, runing this command via the UI button in Tilt is recommended as it sets the env var for you." + exit 5 +fi + + +# kubectl -n futureporn run postgrest-migrations -i --rm=true --image=gitea.futureporn.net/futureporn/migrations:latest --env=DATABASE_PASSWORD=${POSTGRES_PASSWORD} +kubectl -n futureporn run postgres-migrations -i --rm=true --image=fp/migrations:latest --env=DATABASE_PASSWORD=${POSTGRES_PASSWORD} + + diff --git a/scripts/postgres-refresh.sh b/scripts/postgres-refresh.sh new file mode 100644 index 0000000..c96369e --- /dev/null +++ b/scripts/postgres-refresh.sh @@ -0,0 +1,9 @@ + +if [ -z $POSTGRES_PASSWORD ]; then + echo "POSTGRES_PASSWORD was missing in env" + exit 5 +fi + +# reload the schema +# @see https://postgrest.org/en/latest/references/schema_cache.html#schema-reloading +kubectl -n futureporn exec postgresql-primary-0 -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "NOTIFY pgrst, 'reload schema'" diff --git a/scripts/postgrest-migrations.sh b/scripts/postgrest-migrations.sh deleted file mode 100755 index 3edc283..0000000 --- a/scripts/postgrest-migrations.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - - -if [ -z $POSTGRES_PASSWORD ]; then - echo "POSTGRES_PASSWORD was missing in env. In development environment, runing this command via the UI button in Tilt is recommended as it sets the env var for you." - exit 5 -fi - - -kubectl -n futureporn run postgrest-migrations -i --rm=true --image=gitea.futureporn.net/futureporn/migrations:latest --env=DATABASE_PASSWORD=${POSTGRES_PASSWORD} - - diff --git a/services/bot/src/crontab b/services/bot/crontab similarity index 88% rename from services/bot/src/crontab rename to services/bot/crontab index 00d2c1c..b8a176f 100644 --- a/services/bot/src/crontab +++ b/services/bot/crontab @@ -13,6 +13,6 @@ # * * * * * task ?opts {payload} -## every 5 minutes, we see which /records are stale and we mark them as such. +## every n minutes, we see which /records are stale and we mark them as such. ## this prevents stalled Record updates by marking stalled recordings as stopped -*/5 * * * * expire_records \ No newline at end of file +* * * * * expire_stream_recordings ?max=1 { idle_minutes:2 } \ No newline at end of file diff --git a/services/bot/package.json b/services/bot/package.json index 6c4374a..0159288 100644 --- a/services/bot/package.json +++ b/services/bot/package.json @@ -7,8 +7,10 @@ "scripts": { "test": "echo \"Warn: no test specified\" && exit 0", "start": "node ./dist/index.js", - "dev.nodemon": "nodemon --legacy-watch --ext js,ts --watch ./src --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"", - "dev": "tsx --watch ./src/index.ts", + "dev": "pnpm run dev.nodemon # yes this is crazy to have nodemon execute tsx, but it's the only way I have found to get live reloading in TS/ESM/docker with Graphile Worker's way of loading tasks", + "dev.tsx": "tsx ./src/index.ts", + "dev.nodemon": "nodemon --ext ts --exec \"pnpm run dev.tsx\"", + "dev.node": "node --no-warnings=ExperimentalWarning --loader ts-node/esm src/index.ts", "build": "tsc --build", "clean": "rm -rf dist", "superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist", @@ -20,18 +22,22 @@ "license": "Unlicense", "dependencies": { "@discordeno/bot": "19.0.0-next.746f0a9", + "@types/node": "^22.2.0", + "@types/qs": "^6.9.15", "date-fns": "^3.6.0", "dd-cache-proxy": "^2.1.1", "dotenv": "^16.4.5", "graphile-config": "0.0.1-beta.9", "graphile-worker": "^0.16.6", - "pretty-bytes": "^6.1.1" + "node-fetch": "^3.3.2", + "pretty-bytes": "^6.1.1", + "qs": "^6.13.0" }, "devDependencies": { "@futureporn/types": "workspace:^", "nodemon": "^3.1.4", "ts-node": "^10.9.2", - "tsx": "^4.16.2", - "typescript": "^5.5.3" + "tsx": "^4.17.0", + "typescript": "^5.5.4" } } diff --git a/services/bot/pnpm-lock.yaml b/services/bot/pnpm-lock.yaml index 90b8295..e1f512b 100644 --- a/services/bot/pnpm-lock.yaml +++ b/services/bot/pnpm-lock.yaml @@ -11,6 +11,12 @@ importers: '@discordeno/bot': specifier: 19.0.0-next.746f0a9 version: 19.0.0-next.746f0a9 + '@types/node': + specifier: ^22.2.0 + version: 22.2.0 + '@types/qs': + specifier: ^6.9.15 + version: 6.9.15 date-fns: specifier: ^3.6.0 version: 3.6.0 @@ -26,9 +32,15 @@ importers: graphile-worker: specifier: ^0.16.6 version: 0.16.6(typescript@5.5.4) + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 pretty-bytes: specifier: ^6.1.1 version: 6.1.1 + qs: + specifier: ^6.13.0 + version: 6.13.0 devDependencies: '@futureporn/types': specifier: workspace:^ @@ -38,12 +50,12 @@ importers: version: 3.1.4 ts-node: specifier: ^10.9.2 - version: 10.9.2(@types/node@22.1.0)(typescript@5.5.4) + version: 10.9.2(@types/node@22.2.0)(typescript@5.5.4) tsx: - specifier: ^4.16.2 - version: 4.16.2 + specifier: ^4.17.0 + version: 4.17.0 typescript: - specifier: ^5.5.3 + specifier: ^5.5.4 version: 5.5.4 packages: @@ -79,141 +91,147 @@ packages: '@discordeno/utils@19.0.0-next.746f0a9': resolution: {integrity: sha512-UY5GataakuY0yc4SN5qJLexUbTc5y293G3gNAWSaOjaZivEytcdxD4xgeqjNj9c4eN57B3Lfzus6tFZHXwXNOA==} - '@esbuild/aix-ppc64@0.21.5': - resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} - engines: {node: '>=12'} + '@esbuild/aix-ppc64@0.23.0': + resolution: {integrity: sha512-3sG8Zwa5fMcA9bgqB8AfWPQ+HFke6uD3h1s3RIwUNK8EG7a4buxvuFTs3j1IMs2NXAk9F30C/FF4vxRgQCcmoQ==} + engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.21.5': - resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} - engines: {node: '>=12'} + '@esbuild/android-arm64@0.23.0': + resolution: {integrity: sha512-EuHFUYkAVfU4qBdyivULuu03FhJO4IJN9PGuABGrFy4vUuzk91P2d+npxHcFdpUnfYKy0PuV+n6bKIpHOB3prQ==} + engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.21.5': - resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} - engines: {node: '>=12'} + '@esbuild/android-arm@0.23.0': + resolution: {integrity: sha512-+KuOHTKKyIKgEEqKbGTK8W7mPp+hKinbMBeEnNzjJGyFcWsfrXjSTNluJHCY1RqhxFurdD8uNXQDei7qDlR6+g==} + engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.21.5': - resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} - engines: {node: '>=12'} + '@esbuild/android-x64@0.23.0': + resolution: {integrity: sha512-WRrmKidLoKDl56LsbBMhzTTBxrsVwTKdNbKDalbEZr0tcsBgCLbEtoNthOW6PX942YiYq8HzEnb4yWQMLQuipQ==} + engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.21.5': - resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} - engines: {node: '>=12'} + '@esbuild/darwin-arm64@0.23.0': + resolution: {integrity: sha512-YLntie/IdS31H54Ogdn+v50NuoWF5BDkEUFpiOChVa9UnKpftgwzZRrI4J132ETIi+D8n6xh9IviFV3eXdxfow==} + engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.21.5': - resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} - engines: {node: '>=12'} + '@esbuild/darwin-x64@0.23.0': + resolution: {integrity: sha512-IMQ6eme4AfznElesHUPDZ+teuGwoRmVuuixu7sv92ZkdQcPbsNHzutd+rAfaBKo8YK3IrBEi9SLLKWJdEvJniQ==} + engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.21.5': - resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} - engines: {node: '>=12'} + '@esbuild/freebsd-arm64@0.23.0': + resolution: {integrity: sha512-0muYWCng5vqaxobq6LB3YNtevDFSAZGlgtLoAc81PjUfiFz36n4KMpwhtAd4he8ToSI3TGyuhyx5xmiWNYZFyw==} + engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.21.5': - resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} - engines: {node: '>=12'} + '@esbuild/freebsd-x64@0.23.0': + resolution: {integrity: sha512-XKDVu8IsD0/q3foBzsXGt/KjD/yTKBCIwOHE1XwiXmrRwrX6Hbnd5Eqn/WvDekddK21tfszBSrE/WMaZh+1buQ==} + engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.21.5': - resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} - engines: {node: '>=12'} + '@esbuild/linux-arm64@0.23.0': + resolution: {integrity: sha512-j1t5iG8jE7BhonbsEg5d9qOYcVZv/Rv6tghaXM/Ug9xahM0nX/H2gfu6X6z11QRTMT6+aywOMA8TDkhPo8aCGw==} + engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.21.5': - resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} - engines: {node: '>=12'} + '@esbuild/linux-arm@0.23.0': + resolution: {integrity: sha512-SEELSTEtOFu5LPykzA395Mc+54RMg1EUgXP+iw2SJ72+ooMwVsgfuwXo5Fn0wXNgWZsTVHwY2cg4Vi/bOD88qw==} + engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.21.5': - resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} - engines: {node: '>=12'} + '@esbuild/linux-ia32@0.23.0': + resolution: {integrity: sha512-P7O5Tkh2NbgIm2R6x1zGJJsnacDzTFcRWZyTTMgFdVit6E98LTxO+v8LCCLWRvPrjdzXHx9FEOA8oAZPyApWUA==} + engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.21.5': - resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} - engines: {node: '>=12'} + '@esbuild/linux-loong64@0.23.0': + resolution: {integrity: sha512-InQwepswq6urikQiIC/kkx412fqUZudBO4SYKu0N+tGhXRWUqAx+Q+341tFV6QdBifpjYgUndV1hhMq3WeJi7A==} + engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.21.5': - resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} - engines: {node: '>=12'} + '@esbuild/linux-mips64el@0.23.0': + resolution: {integrity: sha512-J9rflLtqdYrxHv2FqXE2i1ELgNjT+JFURt/uDMoPQLcjWQA5wDKgQA4t/dTqGa88ZVECKaD0TctwsUfHbVoi4w==} + engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.21.5': - resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} - engines: {node: '>=12'} + '@esbuild/linux-ppc64@0.23.0': + resolution: {integrity: sha512-cShCXtEOVc5GxU0fM+dsFD10qZ5UpcQ8AM22bYj0u/yaAykWnqXJDpd77ublcX6vdDsWLuweeuSNZk4yUxZwtw==} + engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.21.5': - resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} - engines: {node: '>=12'} + '@esbuild/linux-riscv64@0.23.0': + resolution: {integrity: sha512-HEtaN7Y5UB4tZPeQmgz/UhzoEyYftbMXrBCUjINGjh3uil+rB/QzzpMshz3cNUxqXN7Vr93zzVtpIDL99t9aRw==} + engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.21.5': - resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} - engines: {node: '>=12'} + '@esbuild/linux-s390x@0.23.0': + resolution: {integrity: sha512-WDi3+NVAuyjg/Wxi+o5KPqRbZY0QhI9TjrEEm+8dmpY9Xir8+HE/HNx2JoLckhKbFopW0RdO2D72w8trZOV+Wg==} + engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.21.5': - resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} - engines: {node: '>=12'} + '@esbuild/linux-x64@0.23.0': + resolution: {integrity: sha512-a3pMQhUEJkITgAw6e0bWA+F+vFtCciMjW/LPtoj99MhVt+Mfb6bbL9hu2wmTZgNd994qTAEw+U/r6k3qHWWaOQ==} + engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/netbsd-x64@0.21.5': - resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} - engines: {node: '>=12'} + '@esbuild/netbsd-x64@0.23.0': + resolution: {integrity: sha512-cRK+YDem7lFTs2Q5nEv/HHc4LnrfBCbH5+JHu6wm2eP+d8OZNoSMYgPZJq78vqQ9g+9+nMuIsAO7skzphRXHyw==} + engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/openbsd-x64@0.21.5': - resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} - engines: {node: '>=12'} + '@esbuild/openbsd-arm64@0.23.0': + resolution: {integrity: sha512-suXjq53gERueVWu0OKxzWqk7NxiUWSUlrxoZK7usiF50C6ipColGR5qie2496iKGYNLhDZkPxBI3erbnYkU0rQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.23.0': + resolution: {integrity: sha512-6p3nHpby0DM/v15IFKMjAaayFhqnXV52aEmv1whZHX56pdkK+MEaLoQWj+H42ssFarP1PcomVhbsR4pkz09qBg==} + engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/sunos-x64@0.21.5': - resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} - engines: {node: '>=12'} + '@esbuild/sunos-x64@0.23.0': + resolution: {integrity: sha512-BFelBGfrBwk6LVrmFzCq1u1dZbG4zy/Kp93w2+y83Q5UGYF1d8sCzeLI9NXjKyujjBBniQa8R8PzLFAUrSM9OA==} + engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.21.5': - resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} - engines: {node: '>=12'} + '@esbuild/win32-arm64@0.23.0': + resolution: {integrity: sha512-lY6AC8p4Cnb7xYHuIxQ6iYPe6MfO2CC43XXKo9nBXDb35krYt7KGhQnOkRGar5psxYkircpCqfbNDB4uJbS2jQ==} + engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.21.5': - resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} - engines: {node: '>=12'} + '@esbuild/win32-ia32@0.23.0': + resolution: {integrity: sha512-7L1bHlOTcO4ByvI7OXVI5pNN6HSu6pUQq9yodga8izeuB1KcT2UkHaH6118QJwopExPn0rMHIseCTx1CRo/uNA==} + engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.21.5': - resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} - engines: {node: '>=12'} + '@esbuild/win32-x64@0.23.0': + resolution: {integrity: sha512-Arm+WgUFLUATuoxCJcahGuk6Yj9Pzxd6l11Zb/2aAuv5kWWvvfhLFo2fni4uSK5vzlUdCGZ/BdV5tH8klj8p8g==} + engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -251,18 +269,18 @@ packages: '@types/ms@0.7.34': resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} - '@types/node@20.14.13': - resolution: {integrity: sha512-+bHoGiZb8UiQ0+WEtmph2IWQCjIqg8MDZMAV+ppRRhUZnquF5mQkP/9vpSwJClEiSM/C7fZZExPzfU0vJTyp8w==} + '@types/node@20.14.15': + resolution: {integrity: sha512-Fz1xDMCF/B00/tYSVMlmK7hVeLh7jE5f3B7X1/hmV0MJBwE27KlS7EvD/Yp+z1lm8mVhwV5w+n8jOZG8AfTlKw==} - '@types/node@22.0.0': - resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==} - - '@types/node@22.1.0': - resolution: {integrity: sha512-AOmuRF0R2/5j1knA3c6G3HOk523Ga+l+ZXltX8SF1+5oqcXijjfTd8fY3XRZqSihEu9XhtQnKYLmkFaoxgsJHw==} + '@types/node@22.2.0': + resolution: {integrity: sha512-bm6EG6/pCpkxDf/0gDNDdtDILMOHgaQBVOJGdwsqClnxA3xL6jtMv76rLBc006RVMWbmaf0xbmom4Z/5o2nRkQ==} '@types/pg@8.11.6': resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} + '@types/qs@6.9.15': + resolution: {integrity: sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==} + '@types/semver@7.5.8': resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==} @@ -311,6 +329,10 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + call-bind@1.0.7: + resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} + engines: {node: '>= 0.4'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -359,6 +381,10 @@ packages: create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + date-fns@3.6.0: resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} @@ -376,6 +402,10 @@ packages: supports-color: optional: true + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + diff@4.0.2: resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==} engines: {node: '>=0.3.1'} @@ -390,9 +420,17 @@ packages: error-ex@1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} - esbuild@0.21.5: - resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} - engines: {node: '>=12'} + es-define-property@1.0.0: + resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + esbuild@0.23.0: + resolution: {integrity: sha512-1lvV17H2bMYda/WaFb2jLPeHU3zml2k4/yagNMG8Q/YtfMjCwEUZa2eXXMgZTVSL5q1n4H7sQ0X6CdJDqqeCFA==} + engines: {node: '>=18'} hasBin: true escalade@3.1.2: @@ -403,19 +441,34 @@ packages: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} engines: {node: '>=0.8.0'} + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + get-caller-file@2.0.5: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} + get-intrinsic@1.2.4: + resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} + engines: {node: '>= 0.4'} + get-tsconfig@4.7.6: resolution: {integrity: sha512-ZAqrLlu18NbDdRaHq+AKXzAmqIUPswPWKUchfytdAjiRFnCe5ojG2bstg6mRiZabkKfCoL/e98pbBELIV/YCeA==} @@ -423,6 +476,9 @@ packages: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} + gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + graphile-config@0.0.1-beta.9: resolution: {integrity: sha512-7vNxXZ24OAgXxDKXYi9JtgWPMuNbBL3057Yf32Ux+/rVP4+EePgySCc+NNnn0tORi8qwqVreN8bdWqGIcSwNXg==} engines: {node: '>=16'} @@ -440,6 +496,21 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + + has-proto@1.0.3: + resolution: {integrity: sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==} + engines: {node: '>= 0.4'} + + has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + ignore-by-default@1.0.1: resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==} @@ -501,6 +572,14 @@ packages: ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + nodemon@3.1.4: resolution: {integrity: sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==} engines: {node: '>=10'} @@ -510,6 +589,10 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} + object-inspect@1.13.2: + resolution: {integrity: sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==} + engines: {node: '>= 0.4'} + obuf@1.1.2: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} @@ -616,6 +699,10 @@ packages: pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} + qs@6.13.0: + resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} + engines: {node: '>=0.6'} + readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -636,6 +723,14 @@ packages: engines: {node: '>=10'} hasBin: true + set-function-length@1.2.2: + resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} + engines: {node: '>= 0.4'} + + side-channel@1.0.6: + resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} + engines: {node: '>= 0.4'} + simple-update-notifier@2.0.0: resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==} engines: {node: '>=10'} @@ -682,11 +777,11 @@ packages: '@swc/wasm': optional: true - tslib@2.6.2: - resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + tslib@2.6.3: + resolution: {integrity: sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==} - tsx@4.16.2: - resolution: {integrity: sha512-C1uWweJDgdtX2x600HjaFaucXTilT7tgUZHbOE4+ypskZ1OP8CRCSDkCxG6Vya9EwaFIVagWwpaVAn5wzypaqQ==} + tsx@4.17.0: + resolution: {integrity: sha512-eN4mnDA5UMKDt4YZixo9tBioibaMBpoxBkD+rIPAjVmYERSG0/dWEY1CEFuV89CgASlKL499q8AhmkMnnjtOJg==} engines: {node: '>=18.0.0'} hasBin: true @@ -701,15 +796,16 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - undici-types@6.11.1: - resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==} - undici-types@6.13.0: resolution: {integrity: sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg==} v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -796,73 +892,76 @@ snapshots: dependencies: '@discordeno/types': 19.0.0-next.746f0a9 - '@esbuild/aix-ppc64@0.21.5': + '@esbuild/aix-ppc64@0.23.0': optional: true - '@esbuild/android-arm64@0.21.5': + '@esbuild/android-arm64@0.23.0': optional: true - '@esbuild/android-arm@0.21.5': + '@esbuild/android-arm@0.23.0': optional: true - '@esbuild/android-x64@0.21.5': + '@esbuild/android-x64@0.23.0': optional: true - '@esbuild/darwin-arm64@0.21.5': + '@esbuild/darwin-arm64@0.23.0': optional: true - '@esbuild/darwin-x64@0.21.5': + '@esbuild/darwin-x64@0.23.0': optional: true - '@esbuild/freebsd-arm64@0.21.5': + '@esbuild/freebsd-arm64@0.23.0': optional: true - '@esbuild/freebsd-x64@0.21.5': + '@esbuild/freebsd-x64@0.23.0': optional: true - '@esbuild/linux-arm64@0.21.5': + '@esbuild/linux-arm64@0.23.0': optional: true - '@esbuild/linux-arm@0.21.5': + '@esbuild/linux-arm@0.23.0': optional: true - '@esbuild/linux-ia32@0.21.5': + '@esbuild/linux-ia32@0.23.0': optional: true - '@esbuild/linux-loong64@0.21.5': + '@esbuild/linux-loong64@0.23.0': optional: true - '@esbuild/linux-mips64el@0.21.5': + '@esbuild/linux-mips64el@0.23.0': optional: true - '@esbuild/linux-ppc64@0.21.5': + '@esbuild/linux-ppc64@0.23.0': optional: true - '@esbuild/linux-riscv64@0.21.5': + '@esbuild/linux-riscv64@0.23.0': optional: true - '@esbuild/linux-s390x@0.21.5': + '@esbuild/linux-s390x@0.23.0': optional: true - '@esbuild/linux-x64@0.21.5': + '@esbuild/linux-x64@0.23.0': optional: true - '@esbuild/netbsd-x64@0.21.5': + '@esbuild/netbsd-x64@0.23.0': optional: true - '@esbuild/openbsd-x64@0.21.5': + '@esbuild/openbsd-arm64@0.23.0': optional: true - '@esbuild/sunos-x64@0.21.5': + '@esbuild/openbsd-x64@0.23.0': optional: true - '@esbuild/win32-arm64@0.21.5': + '@esbuild/sunos-x64@0.23.0': optional: true - '@esbuild/win32-ia32@0.21.5': + '@esbuild/win32-arm64@0.23.0': optional: true - '@esbuild/win32-x64@0.21.5': + '@esbuild/win32-ia32@0.23.0': + optional: true + + '@esbuild/win32-x64@0.23.0': optional: true '@graphile/logger@0.2.0': {} @@ -890,28 +989,26 @@ snapshots: '@types/interpret@1.1.3': dependencies: - '@types/node': 22.0.0 + '@types/node': 22.2.0 '@types/ms@0.7.34': {} - '@types/node@20.14.13': + '@types/node@20.14.15': dependencies: undici-types: 5.26.5 - '@types/node@22.0.0': - dependencies: - undici-types: 6.11.1 - - '@types/node@22.1.0': + '@types/node@22.2.0': dependencies: undici-types: 6.13.0 '@types/pg@8.11.6': dependencies: - '@types/node': 22.0.0 + '@types/node': 22.2.0 pg-protocol: 1.6.1 pg-types: 4.0.2 + '@types/qs@6.9.15': {} + '@types/semver@7.5.8': {} acorn-walk@8.3.3: @@ -952,6 +1049,14 @@ snapshots: dependencies: fill-range: 7.1.1 + call-bind@1.0.7: + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + set-function-length: 1.2.2 + callsites@3.1.0: {} chalk@2.4.2: @@ -1008,6 +1113,8 @@ snapshots: create-require@1.1.1: {} + data-uri-to-buffer@4.0.1: {} + date-fns@3.6.0: {} dd-cache-proxy@2.1.1(@discordeno/bot@19.0.0-next.746f0a9): @@ -1020,6 +1127,12 @@ snapshots: optionalDependencies: supports-color: 5.5.0 + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.0 + es-errors: 1.3.0 + gopd: 1.0.1 + diff@4.0.2: {} dotenv@16.4.5: {} @@ -1030,45 +1143,71 @@ snapshots: dependencies: is-arrayish: 0.2.1 - esbuild@0.21.5: + es-define-property@1.0.0: + dependencies: + get-intrinsic: 1.2.4 + + es-errors@1.3.0: {} + + esbuild@0.23.0: optionalDependencies: - '@esbuild/aix-ppc64': 0.21.5 - '@esbuild/android-arm': 0.21.5 - '@esbuild/android-arm64': 0.21.5 - '@esbuild/android-x64': 0.21.5 - '@esbuild/darwin-arm64': 0.21.5 - '@esbuild/darwin-x64': 0.21.5 - '@esbuild/freebsd-arm64': 0.21.5 - '@esbuild/freebsd-x64': 0.21.5 - '@esbuild/linux-arm': 0.21.5 - '@esbuild/linux-arm64': 0.21.5 - '@esbuild/linux-ia32': 0.21.5 - '@esbuild/linux-loong64': 0.21.5 - '@esbuild/linux-mips64el': 0.21.5 - '@esbuild/linux-ppc64': 0.21.5 - '@esbuild/linux-riscv64': 0.21.5 - '@esbuild/linux-s390x': 0.21.5 - '@esbuild/linux-x64': 0.21.5 - '@esbuild/netbsd-x64': 0.21.5 - '@esbuild/openbsd-x64': 0.21.5 - '@esbuild/sunos-x64': 0.21.5 - '@esbuild/win32-arm64': 0.21.5 - '@esbuild/win32-ia32': 0.21.5 - '@esbuild/win32-x64': 0.21.5 + '@esbuild/aix-ppc64': 0.23.0 + '@esbuild/android-arm': 0.23.0 + '@esbuild/android-arm64': 0.23.0 + '@esbuild/android-x64': 0.23.0 + '@esbuild/darwin-arm64': 0.23.0 + '@esbuild/darwin-x64': 0.23.0 + '@esbuild/freebsd-arm64': 0.23.0 + '@esbuild/freebsd-x64': 0.23.0 + '@esbuild/linux-arm': 0.23.0 + '@esbuild/linux-arm64': 0.23.0 + '@esbuild/linux-ia32': 0.23.0 + '@esbuild/linux-loong64': 0.23.0 + '@esbuild/linux-mips64el': 0.23.0 + '@esbuild/linux-ppc64': 0.23.0 + '@esbuild/linux-riscv64': 0.23.0 + '@esbuild/linux-s390x': 0.23.0 + '@esbuild/linux-x64': 0.23.0 + '@esbuild/netbsd-x64': 0.23.0 + '@esbuild/openbsd-arm64': 0.23.0 + '@esbuild/openbsd-x64': 0.23.0 + '@esbuild/sunos-x64': 0.23.0 + '@esbuild/win32-arm64': 0.23.0 + '@esbuild/win32-ia32': 0.23.0 + '@esbuild/win32-x64': 0.23.0 escalade@3.1.2: {} escape-string-regexp@1.0.5: {} + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + fsevents@2.3.3: optional: true + function-bind@1.1.2: {} + get-caller-file@2.0.5: {} + get-intrinsic@1.2.4: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + has-proto: 1.0.3 + has-symbols: 1.0.3 + hasown: 2.0.2 + get-tsconfig@4.7.6: dependencies: resolve-pkg-maps: 1.0.0 @@ -1077,16 +1216,20 @@ snapshots: dependencies: is-glob: 4.0.3 + gopd@1.0.1: + dependencies: + get-intrinsic: 1.2.4 + graphile-config@0.0.1-beta.9: dependencies: '@types/interpret': 1.1.3 - '@types/node': 20.14.13 + '@types/node': 20.14.15 '@types/semver': 7.5.8 chalk: 4.1.2 debug: 4.3.6(supports-color@5.5.0) interpret: 3.1.1 semver: 7.6.3 - tslib: 2.6.2 + tslib: 2.6.3 yargs: 17.7.2 transitivePeerDependencies: - supports-color @@ -1100,7 +1243,7 @@ snapshots: graphile-config: 0.0.1-beta.9 json5: 2.2.3 pg: 8.12.0 - tslib: 2.6.2 + tslib: 2.6.3 yargs: 17.7.2 transitivePeerDependencies: - pg-native @@ -1111,6 +1254,18 @@ snapshots: has-flag@4.0.0: {} + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.0 + + has-proto@1.0.3: {} + + has-symbols@1.0.3: {} + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + ignore-by-default@1.0.1: {} import-fresh@3.3.0: @@ -1156,6 +1311,14 @@ snapshots: ms@2.1.2: {} + node-domexception@1.0.0: {} + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + nodemon@3.1.4: dependencies: chokidar: 3.6.0 @@ -1171,6 +1334,8 @@ snapshots: normalize-path@3.0.0: {} + object-inspect@1.13.2: {} + obuf@1.1.2: {} parent-module@1.0.1: @@ -1263,6 +1428,10 @@ snapshots: pstree.remy@1.1.8: {} + qs@6.13.0: + dependencies: + side-channel: 1.0.6 + readdirp@3.6.0: dependencies: picomatch: 2.3.1 @@ -1275,6 +1444,22 @@ snapshots: semver@7.6.3: {} + set-function-length@1.2.2: + dependencies: + define-data-property: 1.1.4 + es-errors: 1.3.0 + function-bind: 1.1.2 + get-intrinsic: 1.2.4 + gopd: 1.0.1 + has-property-descriptors: 1.0.2 + + side-channel@1.0.6: + dependencies: + call-bind: 1.0.7 + es-errors: 1.3.0 + get-intrinsic: 1.2.4 + object-inspect: 1.13.2 + simple-update-notifier@2.0.0: dependencies: semver: 7.6.3 @@ -1305,14 +1490,14 @@ snapshots: touch@3.1.1: {} - ts-node@10.9.2(@types/node@22.1.0)(typescript@5.5.4): + ts-node@10.9.2(@types/node@22.2.0)(typescript@5.5.4): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.11 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 22.1.0 + '@types/node': 22.2.0 acorn: 8.12.1 acorn-walk: 8.3.3 arg: 4.1.3 @@ -1323,11 +1508,11 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - tslib@2.6.2: {} + tslib@2.6.3: {} - tsx@4.16.2: + tsx@4.17.0: dependencies: - esbuild: 0.21.5 + esbuild: 0.23.0 get-tsconfig: 4.7.6 optionalDependencies: fsevents: 2.3.3 @@ -1338,12 +1523,12 @@ snapshots: undici-types@5.26.5: {} - undici-types@6.11.1: {} - undici-types@6.13.0: {} v8-compile-cache-lib@3.0.1: {} + web-streams-polyfill@3.3.3: {} + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 diff --git a/services/bot/src/bot.ts b/services/bot/src/bot.ts index 58d0f1f..8190158 100644 --- a/services/bot/src/bot.ts +++ b/services/bot/src/bot.ts @@ -1,12 +1,19 @@ -import { createBot, Intents, type Bot } from '@discordeno/bot' +import { createBot, createGatewayManager, createRestManager, Intents, type Bot } from '@discordeno/bot' import { createProxyCache, } from 'dd-cache-proxy'; import { configs } from './config.ts' +// not sure I need this. +// @see https://github.com/discordeno/discordeno/blob/352887c215cc9d93d7f1fa9c8589e66f47ffb3ea/packages/bot/src/bot.ts#L74 +// const getSessionInfoHandler = async () => { +// return await bot.rest.getGatewayBot() +// } export const bot = createProxyCache( createBot({ token: configs.token, - intents: Intents.Guilds | Intents.GuildMessages + intents: Intents.Guilds | Intents.GuildMessages, + rest: createRestManager({ token: configs.token, applicationId: configs.discordApplicationId }), + gateway: createGatewayManager({ token: configs.token }) }), { desiredProps: { @@ -21,9 +28,6 @@ export const bot = createProxyCache( }, ) -// @todo figure out where this code belongs -// gateway.resharding.getSessionInfo = async () => { // insert code here to fetch getSessionInfo from rest process. } - // Setup desired properties bot.transformers.desiredProperties.interaction.id = true bot.transformers.desiredProperties.interaction.type = true diff --git a/services/bot/src/commands/record.ts b/services/bot/src/commands/record.ts index ff45eea..fb8694c 100644 --- a/services/bot/src/commands/record.ts +++ b/services/bot/src/commands/record.ts @@ -9,26 +9,25 @@ import { createCommand } from '../commands.ts' import { configs } from '../config.ts' -async function createRecordInDatabase(url: string, discordMessageId: string) { - const record = { +async function createStreamInDatabase(url: string, discordMessageId: string) { + const streamPayload = { url, - recording_state: 'pending', - discord_message_id: discordMessageId, - file_size: 0 + status: 'pending_recording', + discord_message_id: discordMessageId } - const res = await fetch(`${configs.postgrestUrl}/records`, { + const res = await fetch(`${configs.postgrestUrl}/streams`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${configs.automationUserJwt}`, 'Prefer': 'return=headers-only' }, - body: JSON.stringify(record) + body: JSON.stringify(streamPayload) }) if (!res.ok) { const status = res.status const statusText = res.statusText - const msg = `fetch failed to create recording record in database. status=${status}, statusText=${statusText}` + const msg = `fetch failed to create stream in database. status=${status}, statusText=${statusText}` console.error(msg) throw new Error(msg) } @@ -83,7 +82,7 @@ createCommand({ } // @todo create record in db - const record = await createRecordInDatabase(url, message.id.toString()) + const record = await createStreamInDatabase(url, message.id.toString()) // console.log(record) diff --git a/services/bot/src/config.ts b/services/bot/src/config.ts index 6985970..f468580 100644 --- a/services/bot/src/config.ts +++ b/services/bot/src/config.ts @@ -4,6 +4,7 @@ if (!process.env.POSTGREST_URL) throw new Error('Missing POSTGREST_URL env var') if (!process.env.DISCORD_TOKEN) throw new Error('Missing DISCORD_TOKEN env var'); if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env"); if (!process.env.DISCORD_GUILD_ID) throw new Error("DISCORD_GUILD_ID was missing from env"); +if (!process.env.DISCORD_APPLICATION_ID) throw new Error('DISCORD_APPLICATION_ID was missing from env'); if (!process.env.AUTOMATION_USER_JWT) throw new Error('Missing AUTOMATION_USER_JWT env var'); const token = process.env.DISCORD_TOKEN! const postgrestUrl = process.env.POSTGREST_URL! @@ -11,8 +12,8 @@ const discordChannelId = process.env.DISCORD_CHANNEL_ID! const discordGuildId = process.env.DISCORD_GUILD_ID! const automationUserJwt = process.env.AUTOMATION_USER_JWT! const connectionString = process.env.WORKER_CONNECTION_STRING! +const discordApplicationId = process.env.DISCORD_APPLICATION_ID! -console.log(`hello i am configs and configs.connectionString=${connectionString}`) export interface Config { @@ -22,6 +23,7 @@ export interface Config { discordGuildId: string; discordChannelId: string; connectionString: string; + discordApplicationId: string; } @@ -32,4 +34,5 @@ export const configs: Config = { discordGuildId, discordChannelId, connectionString, + discordApplicationId, } \ No newline at end of file diff --git a/services/bot/src/index.ts b/services/bot/src/index.ts index a5832d7..0022718 100644 --- a/services/bot/src/index.ts +++ b/services/bot/src/index.ts @@ -1,5 +1,5 @@ -import updateDiscordMessage from './tasks/update_discord_message.js' +import update_discord_message from './tasks/update_discord_message.js' import { type WorkerUtils, type RunnerOptions, run } from 'graphile-worker' import { bot } from './bot.ts' import type { Interaction } from '@discordeno/bot' @@ -26,15 +26,15 @@ async function setupGraphileWorker() { taskDirectory: join(__dirname, 'tasks') }, }; - console.log('worker preset as follows') - console.log(preset) + // console.log('worker preset as follows') + // console.log(preset) const runnerOptions: RunnerOptions = { preset // concurrency: 3, // connectionString: configs.connectionString, // taskDirectory: join(__dirname, 'tasks'), // taskList: { - // 'update_discord_message': updateDiscordMessage + // 'update_discord_message': update_discord_message // } } diff --git a/services/bot/src/tasks/expire_stream_recordings.ts b/services/bot/src/tasks/expire_stream_recordings.ts new file mode 100644 index 0000000..dfac5c0 --- /dev/null +++ b/services/bot/src/tasks/expire_stream_recordings.ts @@ -0,0 +1,72 @@ +import type { Task, Helpers } from "graphile-worker" +import { sub } from 'date-fns' +import type { RecordingRecord, Stream } from "@futureporn/types" +import qs from 'qs' +import fetch from 'node-fetch' +import { configs } from '../config.ts' + +interface Payload { + idle_minutes: number; +} + +function assertPayload(payload: any): asserts payload is Payload { + if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); + if (!payload.idle_minutes) throw new Error(`idle_minutes was absent in the payload`); + if (typeof payload.idle_minutes !== 'number') throw new Error(`idle_minutes parameter was not a number`); +} + +export const expire_stream_recordings: Task = async function (payload: unknown, helpers: Helpers) { + assertPayload(payload) + const { idle_minutes } = payload + helpers.logger.info(`expire_stream_recordings has begun. Expring 'recording' and 'pending' streams that haven't been updated in ${idle_minutes} minutes.`) + + const url = 'http://postgrest.futureporn.svc.cluster.local:9000/streams' + let streams: Stream[] = [] + + try { + // 1. identify and update stalled /streams + // Any streams that was updated earlier than n minute ago AND is in 'pending_recording' or 'recording' state is marked as stalled. + const timestamp = sub(new Date(), { minutes: idle_minutes }).toISOString() + const queryOptions = { + updated_at: `lt.${timestamp}`, + or: '(status.eq.pending_recording,status.eq.recording)' + } + const updatePayload = { + updated_at: new Date().toISOString(), + status: 'stalled' + } + helpers.logger.info(JSON.stringify(updatePayload)) + const query = qs.stringify(queryOptions) + const res = await fetch (`${url}?${query}`, { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${configs.automationUserJwt}`, + 'Prefer': 'return=headers-only' + }, + body: JSON.stringify(updatePayload) + }) + if (!res.ok) { + const body = await res.text() + helpers.logger.info(JSON.stringify(res.headers)) + helpers.logger.error(`Response code was not 200. status=${res.status}, statusText=${res.statusText}`) + helpers.logger.error(body) + return; + } + + const body = await res.text() + helpers.logger.info('body as follows') + helpers.logger.info(body) + + } catch (e: any) { + if (e instanceof Error) { + helpers.logger.error(`hi there we encountered an error while fetching /streams`) + helpers.logger.error(e.message) + } else { + helpers.logger.error(e) + } + + } +} + +export default expire_stream_recordings \ No newline at end of file diff --git a/services/bot/src/tasks/restart_failed_recordings.ts b/services/bot/src/tasks/restart_failed_recordings.ts new file mode 100644 index 0000000..7ef9c2e --- /dev/null +++ b/services/bot/src/tasks/restart_failed_recordings.ts @@ -0,0 +1,85 @@ +import type { Task, Helpers } from "graphile-worker" +import { sub } from 'date-fns' +import type { RecordingRecord } from "@futureporn/types" +import qs from 'qs' +import fetch from 'node-fetch' +import { configs } from '../config.ts' + +interface Payload { + idle_minutes: number; +} + +function assertPayload(payload: any): asserts payload is Payload { + if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); + if (!payload.idle_minutes) throw new Error('idle_minutes was missing from payload'); + if (typeof payload.idle_minutes !== 'number') throw new Error('idle_minutes must be a number'); +} + +export const restart_failed_recordings: Task = async function (payload: unknown, helpers: Helpers) { + assertPayload(payload) + const { idle_minutes } = payload + helpers.logger.info(`restart_failed_recordings has begun. Expring 'recording' and 'pending' records that haven't been updated in ${idle_minutes} minutes.`) + + const url = 'http://postgrest.futureporn.svc.cluster.local:9000/records' + let records: RecordingRecord[] = [] + + try { + // 1. identify failed /records + // Any record that was updated earlier than n minute ago AND is in 'pending' or 'recording' state is marked as stalled. + const timestamp = sub(new Date(), { minutes: idle_minutes }).toISOString() + const queryOptions = { + updated_at: `lt.${timestamp}`, + or: '(recording_state.eq.pending,recording_state.eq.recording)' + } + const updatePayload = { + updated_at: new Date().toISOString(), + recording_state: 'stalled' + } + helpers.logger.info(JSON.stringify(updatePayload)) + const query = qs.stringify(queryOptions) + const res = await fetch (`${url}?${query}`, { + method: 'PATCH', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${configs.automationUserJwt}`, + 'Prefer': 'return=headers-only' + }, + body: JSON.stringify(updatePayload) + }) + if (!res.ok) { + const body = await res.text() + helpers.logger.info(JSON.stringify(res.headers)) + helpers.logger.error(`Response code was not 200. status=${res.status}, statusText=${res.statusText}`) + helpers.logger.error(body) + return; + } + + const body = await res.text() + helpers.logger.info('body as follows') + helpers.logger.info(body) + + // const data = await res.json() as RecordingRecord[] + + // if (data.length < 1) return; + // records = data + + } catch (e: any) { + if (e instanceof Error) { + helpers.logger.error(`hi there we encountered an error while fetching /records`) + helpers.logger.error(e.message) + } else { + helpers.logger.error(e) + } + + } + + // // 2. identify and update + // for (const record of records) { + // const res = await fetch(`${url}?`) + // } + + + // // 3. done +} + +export default restart_failed_recordings \ No newline at end of file diff --git a/services/bot/src/tasks/update_discord_message.ts b/services/bot/src/tasks/update_discord_message.ts index 4d404c8..012e53e 100644 --- a/services/bot/src/tasks/update_discord_message.ts +++ b/services/bot/src/tasks/update_discord_message.ts @@ -1,5 +1,5 @@ import 'dotenv/config' -import type { RecordingState } from '@futureporn/types' +import type { Status } from '@futureporn/types' import { type Task, type Helpers } from 'graphile-worker' import { add } from 'date-fns' import prettyBytes from 'pretty-bytes' @@ -18,26 +18,26 @@ import { bot } from '../bot.ts' import { configs } from '../config.ts' interface Payload { - record_id: number; + stream_id: number; } function assertPayload(payload: any): asserts payload is Payload { if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); - if (!payload.record_id) throw new Error(`record_id was absent in the payload`); + if (!payload.stream_id) throw new Error(`stream_id was absent in the payload`); } -async function editDiscordMessage({ helpers, recordingState, discordMessageId, url, fileSize, recordId }: { recordId: number, fileSize: number, url: string, helpers: Helpers, recordingState: RecordingState, discordMessageId: string }) { +async function editDiscordMessage({ helpers, streamStatus, discordMessageId, url, fileSize, streamId }: { streamId: number, fileSize: number, url: string, helpers: Helpers, streamStatus: Status, discordMessageId: string }) { if (!discordMessageId) throw new Error(`discordMessageId was missing!`); if (typeof discordMessageId !== 'string') throw new Error(`discordMessageId was not a string!`); // const { captureJobId } = job.data - helpers.logger.info(`editDiscordMessage has begun with discordMessageId=${discordMessageId}, state=${recordingState}`) + helpers.logger.info(`editDiscordMessage has begun with discordMessageId=${discordMessageId}, streamStatus=${streamStatus}`) // const guild = await bot.cache.guilds.get(BigInt(configs.discordGuildId)) @@ -49,51 +49,19 @@ async function editDiscordMessage({ helpers, recordingState, discordMessageId, u const channelId = BigInt(configs.discordChannelId) const updatedMessage: EditMessage = { - embeds: getStatusEmbed({ recordingState, fileSize, recordId, url }), + embeds: getStatusEmbed({ streamStatus, fileSize, streamId, url }), } bot.helpers.editMessage(channelId, discordMessageId, updatedMessage) - // channel. - - // const guild = await client.guilds.fetch(process.env.DISCORD_GUILD_ID!) as Guild - // if (!guild) throw new Error('guild was undefined'); - - // helpers.logger.info('here is the guild as follows') - // helpers.logger.info(guild.toString()) - // helpers.logger.info(`fetching discord channel id=${process.env.DISCORD_CHANNEL_ID} from discord guild`) - // const channel = await client.channels.fetch(process.env.DISCORD_CHANNEL_ID!) as TextChannel - // if (!channel) throw new Error(`discord channel was undefined`); - - // const message = await channel.messages.fetch(discordMessageId) - // helpers.logger.info(`discordMessageId=${discordMessageId}`) - // helpers.logger.info(message as any) - - // const statusEmbed = getStatusEmbed({ recordId, recordingState, fileSize, url }) - // const buttonRow = getButtonRow(recordingState) - - - // // const embed = new EmbedBuilder().setTitle('Attachments'); - - - // const updatedMessage = { - // embeds: [ - // statusEmbed - // ], - // components: [ - // buttonRow - // ] - // }; - - // message.edit(updatedMessage) } -async function getRecordFromDatabase(recordId: number) { - const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`) +async function getStreamFromDatabase(streamId: number) { + const res = await fetch(`${process.env.POSTGREST_URL}/streams?select=*,segment:segments(*)&id=eq.${streamId}`) if (!res.ok) { - throw new Error(`failed fetching record ${recordId}. status=${res.status}, statusText=${res.statusText}`) + throw new Error(`failed fetching stream ${streamId}. status=${res.status}, statusText=${res.statusText}`) } const body = await res.json() as any return body[0]; @@ -102,66 +70,74 @@ async function getRecordFromDatabase(recordId: number) { /** - * updateDiscordMessage is the task where we edit a previously sent discord message to display + * update_discord_message is the task where we edit a previously sent discord message to display * the most up-to-date status information from the database * * Sometimes the update is changing the state, one of Pending|Recording|Aborted|Ended. * Sometimes the update is updating the Filesize of the recording in-progress * Sometimes the update is adding a thumbnail image to the message */ -export const updateDiscordMessage: Task = async function (payload, helpers: Helpers) { +export const update_discord_message: Task = async function (payload, helpers: Helpers) { try { assertPayload(payload) - const { record_id } = payload - const recordId = record_id - helpers.logger.info(`updateDiscordMessage() with recordId=${recordId}`) - const record = await getRecordFromDatabase(recordId) - const { discord_message_id, recording_state, file_size, url } = record - const recordingState = recording_state + const { stream_id } = payload + const streamId = stream_id + helpers.logger.info(`update_discord_message() with streamId=${streamId}`) + const stream = await getStreamFromDatabase(streamId) + const { discord_message_id, status, file_size, url } = stream + const streamStatus = status const discordMessageId = discord_message_id const fileSize = file_size - editDiscordMessage({ helpers, recordingState, discordMessageId, url, fileSize, recordId }) + editDiscordMessage({ helpers, streamStatus, discordMessageId, url, fileSize, streamId }) // schedule the next update 10s from now, but only if the recording is still happening - if (recordingState !== 'ended') { + if (streamStatus !== 'ended') { const runAt = add(new Date(), { seconds: 10 }) - const recordId = record.id - await helpers.addJob('updateDiscordMessage', { recordId }, { jobKey: `record_${recordId}_update_discord_message`, maxAttempts: 3, runAt }) + const streamId = stream.id + await helpers.addJob('update_discord_message', { streamId }, { jobKey: `stream_${streamId}_update_discord_message`, maxAttempts: 3, runAt }) } } catch (e) { - helpers.logger.error(`caught an error during updateDiscordMessage. e=${e}`) + helpers.logger.error(`caught an error during update_discord_message. e=${e}`) } } function getStatusEmbed({ - recordingState, recordId, fileSize, url -}: { fileSize: number, recordingState: RecordingState, recordId: number, url: string }) { + streamStatus, streamId, fileSize, url +}: { fileSize: number, streamStatus: Status, streamId: number, url: string }) { const embeds = new EmbedsBuilder() - .setTitle(`Record ${recordId}`) + .setTitle(`Stream ${streamId}`) .setFields([ - { name: 'Status', value: recordingState.charAt(0).toUpperCase()+recordingState.slice(1), inline: true }, + { name: 'Status', value: streamStatus.charAt(0).toUpperCase()+streamStatus.slice(1), inline: true }, { name: 'Filesize', value: prettyBytes(fileSize), inline: true }, { name: 'URL', value: url, inline: false }, ]) - if (recordingState === 'pending') { + if (streamStatus === 'pending') { embeds .setDescription("Waiting for a worker to accept the job.") .setColor(2326507) - } else if (recordingState === 'recording') { + } else if (streamStatus === 'recording') { embeds .setDescription('The stream is being recorded.') .setColor(392960) - } else if (recordingState === 'aborted') { + } else if (streamStatus === 'aborted') { embeds .setDescription("The recording was stopped by the user.") .setColor(8289651) - } else if (recordingState === 'ended') { + } else if (streamStatus === 'finished') { embeds - .setDescription("The recording has stopped.") + .setDescription("The recording has ended nominally.") .setColor(10855845) + } else if (streamStatus === 'failed') { + embeds + .setDescription("The recording has ended abnorminally.") + .setColor(8289651) + } else if (streamStatus === 'stalled') { + embeds + .setDescription("We have not received a progress update in the past two minutes.") + .setColor(8289651) } else { embeds - .setDescription('The recording is in an unknown state? (this is a bug.)') + .setDescription(`The recording is in an unknown state? (streamStatus=${streamStatus} this is a bug.)`) .setColor(10855845) } return embeds @@ -169,10 +145,10 @@ function getStatusEmbed({ -function getButtonRow(state: RecordingState): ActionRow { +function getButtonRow(streamStatus: Status): ActionRow { const components: ButtonComponent[] = [] - if (state === 'pending' || state === 'recording') { + if (streamStatus === 'pending' || streamStatus === 'recording') { const stopButton: ButtonComponent = { type: MessageComponentTypes.Button, customId: 'stop', @@ -180,7 +156,7 @@ function getButtonRow(state: RecordingState): ActionRow { style: ButtonStyles.Danger } components.push(stopButton) - } else if (state === 'aborted') { + } else if (streamStatus === 'aborted') { const retryButton: ButtonComponent = { type: MessageComponentTypes.Button, customId: 'retry', @@ -191,7 +167,7 @@ function getButtonRow(state: RecordingState): ActionRow { style: ButtonStyles.Secondary } components.push(retryButton) - } else if (state === 'ended') { + } else if (streamStatus === 'finished') { const downloadButton: ButtonComponent = { type: MessageComponentTypes.Button, customId: 'download', @@ -206,7 +182,7 @@ function getButtonRow(state: RecordingState): ActionRow { const unknownButton: ButtonComponent = { type: MessageComponentTypes.Button, customId: 'unknown', - label: 'Unknown State', + label: 'Unknown Status', emoji: { name: 'thinking' }, @@ -225,4 +201,4 @@ function getButtonRow(state: RecordingState): ActionRow { } -export default updateDiscordMessage \ No newline at end of file +export default update_discord_message \ No newline at end of file diff --git a/services/capture/package.json b/services/capture/package.json index e2f5da9..b72df9a 100644 --- a/services/capture/package.json +++ b/services/capture/package.json @@ -10,9 +10,10 @@ "build": "tsup", "test": "mocha", "integration": "FUTUREPORN_WORKDIR=/home/cj/Downloads mocha ./integration/**/*.test.js", - "dev": "tsx --watch ./src/index.ts", - "dev.nodemon": "pnpm nodemon --ext ts,json,yaml --ignore ./dist --watch ./src --watch ./node_modules/@futureporn --exec \"pnpm run dev.build\"", - "dev.build": "pnpm run build && pnpm run start", + "dev": "pnpm run dev.nodemon # yes this is crazy to have nodemon execute tsx, but it's the only way I have found to get live reloading in TS/ESM/docker with Graphile Worker's way of loading tasks", + "dev.tsx": "tsx ./src/index.ts", + "dev.nodemon": "nodemon --ext ts --exec \"pnpm run dev.tsx\"", + "dev.node": "node --no-warnings=ExperimentalWarning --loader ts-node/esm src/index.ts", "clean": "rm -rf dist", "superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist" }, @@ -25,8 +26,10 @@ "@futureporn/utils": "workspace:^", "@paralleldrive/cuid2": "^2.2.2", "@types/chai": "^4.3.16", + "@types/chai-as-promised": "^7.1.8", "@types/fluent-ffmpeg": "^2.1.24", "@types/mocha": "^10.0.7", + "@types/qs": "^6.9.15", "date-fns": "^3.6.0", "diskusage": "^1.2.0", "dotenv": "^16.4.5", @@ -47,6 +50,7 @@ "pg-boss": "^9.0.3", "pino-pretty": "^11.2.1", "postgres": "^3.4.4", + "qs": "^6.13.0", "rxjs": "^7.8.1", "sql": "^0.78.0", "winston": "^3.13.1", @@ -61,6 +65,7 @@ "aws-sdk-client-mock": "^4.0.1", "aws-sdk-mock": "^6.0.4", "chai": "^4.4.1", + "chai-as-promised": "^8.0.0", "cheerio": "1.0.0-rc.12", "mocha": "^10.7.0", "multiformats": "^11.0.2", diff --git a/services/capture/pnpm-lock.yaml b/services/capture/pnpm-lock.yaml index 8c7d809..cad0018 100644 --- a/services/capture/pnpm-lock.yaml +++ b/services/capture/pnpm-lock.yaml @@ -32,12 +32,18 @@ importers: '@types/chai': specifier: ^4.3.16 version: 4.3.16 + '@types/chai-as-promised': + specifier: ^7.1.8 + version: 7.1.8 '@types/fluent-ffmpeg': specifier: ^2.1.24 version: 2.1.24 '@types/mocha': specifier: ^10.0.7 version: 10.0.7 + '@types/qs': + specifier: ^6.9.15 + version: 6.9.15 date-fns: specifier: ^3.6.0 version: 3.6.0 @@ -98,6 +104,9 @@ importers: postgres: specifier: ^3.4.4 version: 3.4.4 + qs: + specifier: ^6.13.0 + version: 6.13.0 rxjs: specifier: ^7.8.1 version: 7.8.1 @@ -135,6 +144,9 @@ importers: chai: specifier: ^4.4.1 version: 4.5.0 + chai-as-promised: + specifier: ^8.0.0 + version: 8.0.0(chai@4.5.0) cheerio: specifier: 1.0.0-rc.12 version: 1.0.0-rc.12 @@ -1040,6 +1052,9 @@ packages: '@tsconfig/node16@1.0.4': resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} + '@types/chai-as-promised@7.1.8': + resolution: {integrity: sha512-ThlRVIJhr69FLlh6IctTXFkmhtP3NpMZ2QGq69StYLyKZFp/HOp1VdKZj7RvfNWYYcJ1xlbLGLLWj1UvP5u/Gw==} + '@types/chai@4.3.16': resolution: {integrity: sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==} @@ -1067,6 +1082,9 @@ packages: '@types/pg@8.11.6': resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==} + '@types/qs@6.9.15': + resolution: {integrity: sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==} + '@types/retry@0.12.1': resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==} @@ -1270,6 +1288,11 @@ packages: resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} engines: {node: '>=10'} + chai-as-promised@8.0.0: + resolution: {integrity: sha512-sMsGXTrS3FunP/wbqh/KxM8Kj/aLPXQGkNtvE5wPfSToq8wkkvBpTZo1LIiEVmC4BwkKpag+l5h/20lBMk6nUg==} + peerDependencies: + chai: '>= 2.1.2 < 6' + chai@4.5.0: resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} engines: {node: '>=4'} @@ -1285,6 +1308,10 @@ packages: check-error@1.0.3: resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + check-error@2.1.1: + resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} + engines: {node: '>= 16'} + cheerio-select@2.1.0: resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} @@ -2408,6 +2435,10 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + qs@6.13.0: + resolution: {integrity: sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==} + engines: {node: '>=0.6'} + querystring@0.2.0: resolution: {integrity: sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==} engines: {node: '>=0.4.x'} @@ -4124,6 +4155,10 @@ snapshots: '@tsconfig/node16@1.0.4': {} + '@types/chai-as-promised@7.1.8': + dependencies: + '@types/chai': 4.3.16 + '@types/chai@4.3.16': {} '@types/debug@4.1.12': @@ -4154,6 +4189,8 @@ snapshots: pg-protocol: 1.6.1 pg-types: 4.0.2 + '@types/qs@6.9.15': {} + '@types/retry@0.12.1': {} '@types/semver@7.5.8': {} @@ -4356,6 +4393,11 @@ snapshots: camelcase@6.3.0: {} + chai-as-promised@8.0.0(chai@4.5.0): + dependencies: + chai: 4.5.0 + check-error: 2.1.1 + chai@4.5.0: dependencies: assertion-error: 1.1.0 @@ -4381,6 +4423,8 @@ snapshots: dependencies: get-func-name: 2.0.2 + check-error@2.1.1: {} + cheerio-select@2.1.0: dependencies: boolbase: 1.0.0 @@ -5642,6 +5686,10 @@ snapshots: punycode@2.3.1: {} + qs@6.13.0: + dependencies: + side-channel: 1.0.6 + querystring@0.2.0: {} querystringify@2.2.0: {} diff --git a/services/capture/src/Record.spec.ts b/services/capture/src/Record.spec.ts index 434f661..dc0aab5 100644 --- a/services/capture/src/Record.spec.ts +++ b/services/capture/src/Record.spec.ts @@ -1,5 +1,5 @@ -import Record from "./Record.js" -import { expect } from "chai" +import Record, { UploadStreamClosedError } from "./Record.js" +import * as chai from 'chai' import { ChildProcess, spawn } from "child_process" import { createReadStream, readFileSync, ReadStream } from "fs" import AWSMock from 'aws-sdk-mock' @@ -13,7 +13,9 @@ import { HeadObjectOutput } from 'aws-sdk/clients/s3'; import { Readable } from 'stream'; import { mockClient } from 'aws-sdk-client-mock'; import { sdkStreamMixin } from '@smithy/util-stream' - +import chaiAsPromised from 'chai-as-promised' +chai.use(chaiAsPromised) +const expect = chai.expect // "pay no attention to that man behind the curtain" @@ -52,7 +54,7 @@ describe('Record', function () { expect(record).to.have.property('bucket', 'test') }) - it('should be abortable', async function () { + xit('should be abortable', async function () { const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) // 192627 bytes const s3ClientMock = mockClient(S3Client) const s3Client = new S3Client({ region: 'us-west-000' }) @@ -65,6 +67,20 @@ describe('Record', function () { await record.abort() }) + xit('should throw if the upload stream closes before the download stream closes', async function () { + + const s3Mock = mockClient(S3Client) + // const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) + const inputStream = createReadStream('/dev/random') // forever random + // const s3Client = new S3Client({ region: 'us-west-000' }) + // s3ClientMock.on() + s3Mock.on(PutObjectCommand).resolvesOnce({}).resolvesOnce({}).rejects({}) + const s3 = new S3Client({ region: 'us-west-000' }) + + return expect(s3.send(new PutObjectCommand({ Body: inputStream, Bucket: 'taco', Key: 'my-cool-taco.mp4' }))).to.be.rejectedWith(UploadStreamClosedError) + + }) + xit('should restart if a EPIPE is encountered', async function () { // @todo IDK how to implement this. const inputStream = createReadStream(join(__dirname, './fixtures/mock-stream0.mp4')) diff --git a/services/capture/src/Record.ts b/services/capture/src/Record.ts index f80be9f..1c87aa1 100644 --- a/services/capture/src/Record.ts +++ b/services/capture/src/Record.ts @@ -7,6 +7,13 @@ import 'dotenv/config' const ua0 = 'Mozilla/5.0 (X11; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0' +export class UploadStreamClosedError extends Error { + constructor(message: string) { + super(message) + Object.setPrototypeOf(this, UploadStreamClosedError.prototype) + } +} + export interface RecordArgs { filename?: string; s3Client: S3Client; @@ -131,6 +138,7 @@ export default class Record { parallelUploads3.on("httpUploadProgress", (progress) => { if (progress?.loaded) { + // console.log(progress) if (this.onProgress) this.onProgress(this.counter); // console.log(`uploaded ${progress.loaded} bytes (${prettyBytes(progress.loaded)})`); } else { @@ -144,8 +152,13 @@ export default class Record { } catch (e) { if (e instanceof Error) { - console.error(`We were uploading a file to S3 but then we encountered an error! ${JSON.stringify(e, null, 2)}`) - throw e + if (e.name === 'AbortError') { + console.error(`We got an error, AbortError which is something we know how to handle. we will NOT throw and instead return gracefully.`) + return + } else { + console.error(`We were uploading a file to S3 but then we encountered an error! ${JSON.stringify(e, null, 2)}`) + throw e + } } else { throw new Error(`error of some sort ${JSON.stringify(e, null, 2)}`) } @@ -164,7 +177,14 @@ export default class Record { this.counter += data.length }) this.uploadStream.on('close', () => { - console.log('[!!!] upload stream has closed') + // if uploadStream closes before inputStream, throw an error. + if (!this.inputStream.closed) { + const msg = 'upload stream closed before download stream, which suggests the S3 upload failed.' + console.error(msg) + throw new UploadStreamClosedError(msg); + } else { + console.log('upload stream has closed. In this instance it is OK since the input stream is also closed.') + } }) this.uploadStream.on('error', (e) => { console.error('there was an error on the uploadStream. error as follows') diff --git a/services/capture/src/config.ts b/services/capture/src/config.ts new file mode 100644 index 0000000..bbf3ee3 --- /dev/null +++ b/services/capture/src/config.ts @@ -0,0 +1,40 @@ +import 'dotenv/config' + +const requiredEnvVars = [ + 'S3_ACCESS_KEY_ID', + 'S3_SECRET_ACCESS_KEY', + 'S3_REGION', + 'S3_ENDPOINT', + 'S3_BUCKET', + 'POSTGREST_URL', + 'AUTOMATION_USER_JWT', +] as const; + +const getEnvVar = (key: typeof requiredEnvVars[number]): string => { + const value = process.env[key]; + if (!value) { + throw new Error(`Missing ${key} env var`); + } + return value; +}; + +export interface Config { + postgrestUrl: string; + automationUserJwt: string; + s3AccessKeyId: string; + s3SecretAccessKey: string; + s3Region: string; + s3Bucket: string; + s3Endpoint: string; +} + + +export const configs: Config = { + postgrestUrl: getEnvVar('POSTGREST_URL'), + automationUserJwt: getEnvVar('AUTOMATION_USER_JWT'), + s3AccessKeyId: getEnvVar('S3_ACCESS_KEY_ID'), + s3SecretAccessKey: getEnvVar('S3_SECRET_ACCESS_KEY'), + s3Region: getEnvVar('S3_REGION'), + s3Bucket: getEnvVar('S3_BUCKET'), + s3Endpoint: getEnvVar('S3_ENDPOINT'), +} \ No newline at end of file diff --git a/services/capture/src/index.ts b/services/capture/src/index.ts index a5fe341..bfda862 100644 --- a/services/capture/src/index.ts +++ b/services/capture/src/index.ts @@ -10,9 +10,6 @@ import { fileURLToPath } from 'url'; import { getPackageVersion } from '@futureporn/utils'; import type { GraphileConfig } from "graphile-config"; import type {} from "graphile-worker"; -import start_recording from './tasks/start_recording.ts'; -import { stop_recording } from './tasks/stop_recording.ts'; -import record from './tasks/record.ts' const __dirname = dirname(fileURLToPath(import.meta.url)); const version = getPackageVersion(join(__dirname, '../package.json')) @@ -58,12 +55,10 @@ async function worker(workerUtils: WorkerUtils) { const runnerOptions: RunnerOptions = { preset, concurrency, - // taskDirectory: join(__dirname, 'tasks'), - taskList: { - 'record': record, - 'start_recording': start_recording, - 'stop_recording': stop_recording - } + taskDirectory: join(__dirname, 'tasks'), + // taskList: { + // 'record': record, + // } } const runner = await graphileRun(runnerOptions) diff --git a/services/capture/src/tasks/record.ts b/services/capture/src/tasks/record.ts index 0d8dd52..1c37604 100644 --- a/services/capture/src/tasks/record.ts +++ b/services/capture/src/tasks/record.ts @@ -1,9 +1,115 @@ +/** + * + * # notes + * + * # creation + * + * ## api.records + * + * id: 2 + * url: 'https://chaturbate.com/example' + * discord_message_id: 238492348324 + * recording_state: 'pending' + * is_aborted: false + * created_at: 2024-08-15T21:36:27.796Z + * updated_at: 2024-08-15T21:36:27.796Z + * + * ## api.segments + * + * id: 5 + * s3_key: example-date-cuid.mp4 + * s3_id: 2342309492348324 + * bytes: 0 + * created_at: 2024-08-15T21:36:27.796Z + * updated_at: 2024-08-15T21:36:27.796Z + * + * ## api.records_segments_links + * + * id: 9 + * stream_id: 2 + * segment_id: 5 + * segment_order: 0 + * created_at: 2024-08-15T21:36:27.796Z + * updated_at: 2024-08-15T21:36:27.796Z + * + * # progress + * + * ## api.records + * + * id: 2 + * url: 'https://chaturbate.com/example' + * discord_message_id: 238492348324 + * recording_state: 'recording' + * is_aborted: false + * created_at: 2024-08-15T21:36:27.796Z + * updated_at: 2024-08-15T21:37:37.168Z + * + * ## api.segments + * + * id: 5 + * s3_key: example-2024-08-15-72ff4b5ae7dae73b.mp4 + * s3_id: 2342309492348324 + * bytes: 8384 + * created_at: 2024-08-15T21:36:27.796Z + * updated_at: 2024-08-15T21:37:37.168Z + * + * + * # new segment + * + * ## api.segments + * + * id: 6 + * s3_key: example-2024-08-15-cda21be5e54621f2.mp4 + * s3_id: a974eb6e194b7987 + * byte: 0 + * created_at: 2024-08-15T21:38:34.878Z + * updated_at: 2024-08-15T21:38:34.878Z + * + * ## api.records_segments_links + * + * id: 10 + * stream_id: 2 + * segment_id: 6 + * segment_order: 1 + * created_at: 2024-08-15T21:38:34.878Z + * updated_at: 2024-08-15T21:38:34.878Z + * + * # progress + * + * ## api.segments + * + * id: 6 + * s3_key: example-2024-08-15-cda21be5e54621f2.mp4 + * s3_id: a974eb6e194b7987 + * byte: 1024 + * created_at: 2024-08-15T21:38:34.878Z + * updated_at: 2024-08-15T21:39:11.437Z + * + * # completion + * + * ## api.records + * + * id: 2 + * url: 'https://chaturbate.com/example' + * discord_message_id: 238492348324 + * recording_state: 'finished' + * is_aborted: false + * created_at: 2024-08-15T21:36:27.796Z + * updated_at: 2024-08-15T21:39:41.692Z + * + */ + +import querystring from 'node:querystring' import { Helpers, type Task } from 'graphile-worker' import Record from '../Record.ts' import { getPlaylistUrl } from '@futureporn/scout/ytdlp.ts' -import type { RecordingState } from '@futureporn/types' +import type { RecordingState, RecordingRecord, Segment } from '@futureporn/types' import { add } from 'date-fns' +import { backOff } from "exponential-backoff"; +import { configs } from '../config.ts' +import qs from 'qs' +import { createId } from '@paralleldrive/cuid2' /** * url is the URL to be recorded. Ex: chaturbate.com/projektmelody @@ -12,189 +118,229 @@ import { add } from 'date-fns' */ interface Payload { url: string; - record_id: number; -} - -interface RecordingRecord { - id: number; - recordingState: RecordingState; - fileSize: number; - discordMessageId: string; - isAborted: boolean; -} -interface RawRecordingRecord { - id: number; - recording_state: RecordingState; - file_size: number; - discord_message_id: string; - is_aborted: boolean; + stream_id: string; } function assertPayload(payload: any): asserts payload is Payload { if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); if (typeof payload.url !== "string") throw new Error("invalid url"); - if (typeof payload.record_id !== "number") throw new Error(`invalid record_id=${payload.record_id}`); + if (typeof payload.stream_id !== "string") throw new Error(`invalid stream_id=${payload.stream_id}`); } -function assertEnv() { - if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was missing in env'); - if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was missing in env'); - if (!process.env.S3_REGION) throw new Error('S3_REGION was missing in env'); - if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was missing in env'); - if (!process.env.S3_BUCKET) throw new Error('S3_BUCKET was missing in env'); - if (!process.env.POSTGREST_URL) throw new Error('POSTGREST_URL was missing in env'); - if (!process.env.AUTOMATION_USER_JWT) throw new Error('AUTOMATION_USER_JWT was missing in env'); -} - -async function getRecording(url: string, recordId: number, helpers: Helpers) { +async function getRecordInstance(url: string, segment_id: number, helpers: Helpers) { const abortController = new AbortController() const abortSignal = abortController.signal - const accessKeyId = process.env.S3_ACCESS_KEY_ID!; - const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY!; - const region = process.env.S3_REGION!; - const endpoint = process.env.S3_ENDPOINT!; - const bucket = process.env.S3_BUCKET!; + const accessKeyId = configs.s3AccessKeyId; + const secretAccessKey = configs.s3SecretAccessKey; + const region = configs.s3Region; + const endpoint = configs.s3Endpoint; + const bucket = configs.s3Bucket; const playlistUrl = await getPlaylistUrl(url) const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint }) const inputStream = Record.getFFmpegStream({ url: playlistUrl }) const onProgress = (fileSize: number) => { - updateDatabaseRecord({ recordId, recordingState: 'recording', fileSize }).then(checkIfAborted).then((isAborted) => isAborted ? abortController.abort() : null) + updateDatabaseRecord({ segment_id, fileSize, helpers }) + .then((reee) => { + + helpers.logger.info(JSON.stringify(reee)) + return reee + }) + .then(checkIfAborted) + .then((isAborted) => { + helpers.logger.info(`isAborted=${isAborted}`) + isAborted ? abortController.abort() : null + }) + .catch((e) => { + helpers.logger.error('caught error while updatingDatabaseRecord inside onProgress inside getRecordInstance') + helpers.logger.error(e) + }) } - const record = new Record({ inputStream, onProgress, bucket, s3Client, jobId: ''+recordId, abortSignal }) + const record = new Record({ inputStream, onProgress, bucket, s3Client, jobId: ''+segment_id, abortSignal }) return record } -function checkIfAborted(record: RawRecordingRecord): boolean { - return (record.is_aborted) +function checkIfAborted(segment: Partial): boolean { + return (!!segment?.stream?.at(0)?.is_recording_aborted) } async function updateDatabaseRecord({ - recordId, - recordingState, - fileSize + segment_id, + fileSize, + helpers }: { - recordId: number, - recordingState: RecordingState, - fileSize: number -}): Promise { - // console.log(`updating database record with recordId=${recordId}, recordingState=${recordingState}, fileSize=${fileSize}`) + segment_id: number, + fileSize: number, + helpers: Helpers +}): Promise { + const payload: any = { - file_size: fileSize + bytes: fileSize } - if (recordingState) payload.recording_state = recordingState; - const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`, { + + const res = await fetch(`${configs.postgrestUrl}/segments?id=eq.${segment_id}&select=stream:streams(is_recording_aborted)`, { method: 'PATCH', headers: { 'Content-Type': 'application/json', - 'Accepts': 'application/json', + 'Accept': 'application/json', 'Prefer': 'return=representation', - 'Authorization': `Bearer ${process.env.AUTOMATION_USER_JWT}` + 'Authorization': `Bearer ${configs.automationUserJwt}` }, body: JSON.stringify(payload) }) if (!res.ok) { const body = await res.text() - throw new Error(`failed to updateDatabaseRecord. status=${res.status}, statusText=${res.statusText}, body=${body}`); + const msg = `failed to updateDatabaseRecord. status=${res.status}, statusText=${res.statusText}, body=${body}` + helpers.logger.error(msg) + throw new Error(msg); } - const body = await res.json() as RawRecordingRecord[]; - if (!body[0]) throw new Error(`failed to get a record that matched recordId=${recordId}`) + // helpers.logger.info(`response was OK~`) + const body = await res.json() as Segment[]; + if (!body[0]) throw new Error(`failed to get a segment that matched segment_id=${segment_id}`); + const bod = body[0] + // helpers.logger.info('the following was the response from PATCH-ing /segments') + // helpers.logger.info(JSON.stringify(bod)) + return bod +} + + +const getSegments = async function getSegments(stream_id: string): Promise { + if (!stream_id) throw new Error('getSegments requires {String} stream_id as first arg'); + const res = await fetch(`${configs.postgrestUrl}/segments_stream_links?stream_id=eq.${stream_id}`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Prefer': 'return=representation' + }, + }) + if (!res.ok) { + const body = await res.text() + throw new Error(`failed to getSegments. status=${res.status}, statusText=${res.statusText}, body=${body}`); + } + const body = await res.json() as Segment[]; + if (!body[0]) throw new Error(`failed to get segments that matched stream_id=${stream_id}`) return body[0] } -export const record: Task = async function (payload, helpers) { - console.log(payload) + + +const createSegment = async function createSegment(s3_key: string, helpers: Helpers): Promise { + if (!s3_key) throw new Error('getSegments requires {string} s3_key as first arg'); + const segmentPayload = { + s3_key + } + helpers.logger.info(`Creating segment with s3_key=${s3_key}. payload as follows`) + helpers.logger.info(JSON.stringify(segmentPayload)) + const res = await fetch(`${configs.postgrestUrl}/segments`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Prefer': 'return=headers-only', + 'Authorization': `Bearer ${configs.automationUserJwt}` + }, + body: JSON.stringify(segmentPayload) + }) + if (!res.ok) { + const body = await res.text() + const msg = `failed to create Segment. status=${res.status}, statusText=${res.statusText}, body=${body}` + helpers.logger.error(msg) + throw new Error(msg); + } + const location = res.headers.get('location') + if (!location) throw new Error(`failed to get location header in response from postgrest`); + const parsedQuery = querystring.parse(location) + const segmentsId = parsedQuery['/segments?id'] + if (!segmentsId) throw new Error('segmentsId was undefined which is unexpected'); + if (Array.isArray(segmentsId)) throw new Error('segmentsId was an array which is unexpected'); + const id = segmentsId.split('.').at(-1) + if (!id) throw new Error('failed to get id '); + return parseInt(id) +} + +const createSegmentsStreamLink = async function createSegmentsStreamLink(stream_id: string, segment_id: number, helpers: Helpers): Promise { + if (!stream_id) throw new Error('createSegmentsStreamLink requires {string} stream_id as first arg'); + if (!segment_id) throw new Error('createSegmentsStreamLink requires {Number} segment_id as second arg'); + const segmentStreamLinkPayload = { + stream_id, + segment_id + } + const res = await fetch(`${configs.postgrestUrl}/segments_stream_links`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Prefer': 'return=headers-only', + 'Authorization': `Bearer ${configs.automationUserJwt}`, + }, + body: JSON.stringify(segmentStreamLinkPayload) + }) + if (!res.ok) { + const body = await res.text() + throw new Error(`failed to create SegmentsStreamLink. status=${res.status}, statusText=${res.statusText}, body=${body}`); + } + const location = res.headers.get('location') + if (!location) throw new Error(`failed to get location header in response from postgrest`); + const parsedQuery = querystring.parse(location) + const segmentsId = parsedQuery['/segments_stream_links?id'] + if (!segmentsId) throw new Error('segments_stream_links?id was undefined which is unexpected'); + if (Array.isArray(segmentsId)) throw new Error('segments_stream_links was an array which is unexpected'); + const id = segmentsId.split('.').at(-1) + if (!id) throw new Error('failed to get id '); + return parseInt(id) +} + +/** + * # doRecordSegment + * + * Record a segment of a livestream using ffmpeg. + * + * Ideally, we record the entire livestream, but the universe is not so kind. Network interruptions are common, so we handle the situation as best as we can. + * + * This function creates a new segments and segments_streams_links entry in the db via Postgrest REST API. + * + * This function also names the S3 file (s3_key) with a datestamp and a cuid. + */ +const doRecordSegment = async function doRecordSegment(url: string, stream_id: string, helpers: Helpers): Promise { + const s3_key = `${new Date().toISOString()}-${createId()}.ts` + helpers.logger.info(`let's create a segment...`) + const segment_id = await createSegment(s3_key, helpers) + helpers.logger.info(`let's create a segmentsStreamLink...`) + const segmentsStreamLinkId = await createSegmentsStreamLink(stream_id, segment_id, helpers) + helpers.logger.info(`doTheRecording with segmentsStreamLinkId=${segmentsStreamLinkId}, stream_id=${stream_id}, segment_id=${segment_id}, url=${url}`) + const record = await getRecordInstance(url, segment_id, helpers) + await record.start() +} + + + +export const record: Task = async function (payload: unknown, helpers: Helpers) { assertPayload(payload) - assertEnv() - const { url, record_id } = payload - // let interval + const { url, stream_id } = payload + const recordId = stream_id try { - // every 30s, we - // 1. update the db record with the filesize - // 2. poll db to see if our job has been aborted by the user - // interval = setInterval(async () => { - // try { - // helpers.logger.info(`updateDatabaseRecord()`) - // const recordingState: RecordingState = 'recording' - // const fileSize = record.counter - // const updatePayload = { recordingState, recordId, fileSize } - // const updatedRecord = await updateDatabaseRecord(updatePayload) - // if (updatedRecord.isAborted) { - // helpers.logger.info(`record ${recordId} has been aborted by a user so we stop the recording now.`) - // abortController.abort() - // } - // } catch (e) { - // helpers.logger.error(`error while updating database. For sake of the recording in progress we are ignoring the following error. ${e}`) - // } - // }, 3000) - - // start recording and await the S3 upload being finished - const recordId = record_id - const record = await getRecording(url, recordId, helpers) - await record.start() - + /** + * We do an exponential backoff timer when we record. If the Record() instance throws an error, we try again after a delay. + * This will take effect only when Record() throws an error. + * If however Record() returns, as is the case when the stream ends, this backoff timer will not retry. + * This does not handle the corner case where the streamer's internet temporarliy goes down, and their stream drops. + * + * @todo We must implement retrying at a higher level, and retry a few times to handle this type of corner-case. + */ + // await backOff(() => doRecordSegment(url, recordId, helpers)) + await doRecordSegment(url, recordId, helpers) } catch (e) { - helpers.logger.error(`caught an error duing record(). error as follows`) + // await updateDatabaseRecord({ recordId: stream_id, recordingState: 'failed' }) + helpers.logger.error(`caught an error during record Task`) if (e instanceof Error) { helpers.logger.error(e.message) } else { helpers.logger.error(JSON.stringify(e)) } + // throw e // @todo uncomment this for production } - - - - - // const recordId = await createRecordingRecord(payload, helpers) - // const { url } = payload; - // console.log(`@todo simulated start_recording with url=${url}, recordId=${recordId}`) - // await helpers.addJob('record', { url, recordId }) } -/** - * Here we middleman the stream from FFmpeg --> S3, - * counting bits and creating graphile jobs to inform the UI of our progress - */ -// const transformStreamFactory = (recordId: number, helpers: Helpers): PassThrough => { -// let counter = 0 -// return new PassThrough ({ -// async transform(chunk, controller) { -// controller.enqueue(chunk) // we don't actually transform anything here. we're only gathering statistics. -// counter += chunk.length -// if (counter % (1 * 1024 * 1024) <= 1024) { -// helpers.logger.info(`Updating record ${recordId}`) -// try { -// await updateDatabaseRecord({ fileSize: counter, recordId, recordingState: 'recording' }) -// } catch (e) { -// helpers.logger.warn(`We are ignoring the following error which occured while updating db record ${e}`) -// } -// } -// }, -// flush() { -// helpers.logger.info(`transformStream has flushed.`) -// } -// }) -// } - -// export const recordNg: Task = async function (payload, helpers) { -// assertPayload(payload) -// const { url, recordId } = payload -// try { -// const abortController = new AbortController() -// const abortSignal = abortController.signal -// const inputStream = -// const transformStream = transformStreamFactory(recordId, helpers) -// const record = new Record({ inputStream, abortSignal, transformStream }) -// await record.done() -// } catch (e) { -// console.error(`error during recording. error as follows`) -// console.error(e) -// } finally { -// helpers.addJob('updateDiscordMessage', { recordId }, { maxAttempts: 3, runAt: add(new Date(), { seconds: 5 }) }) -// } -// } - - - export default record \ No newline at end of file diff --git a/services/capture/src/tasks/recording_start.ts.old b/services/capture/src/tasks/recording_start.ts.old deleted file mode 100644 index 4278bba..0000000 --- a/services/capture/src/tasks/recording_start.ts.old +++ /dev/null @@ -1,77 +0,0 @@ -import Record from '../Record.ts' -import { getPlaylistUrl } from '@futureporn/scout/ytdlp.ts' -import 'dotenv/config' -import { type Job } from 'pg-boss' -import { backOff } from 'exponential-backoff' - -export interface RecordJob extends Job { - data: { - url: string; - } -} - -async function _record (job: RecordJob, retries?: number): Promise { - - - if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env'); - if (!process.env.S3_ENDPOINT) throw new Error('S3_ENDPOINT was undefined in env'); - if (!process.env.S3_REGION) throw new Error('S3_REGION was undefined in env'); - if (!process.env.S3_ACCESS_KEY_ID) throw new Error('S3_ACCESS_KEY_ID was undefined in env'); - if (!process.env.S3_SECRET_ACCESS_KEY) throw new Error('S3_SECRET_ACCESS_KEY was undefined in env'); - - if (!job) throw new Error('Job sent to job worker execution callback was empty!!!'); - const { url } = job.data; - console.log(`'record' job ${job!.id} begin with url=${url}`) - - - const bucket = process.env.S3_BUCKET_NAME! - const endpoint = process.env.S3_ENDPOINT! - const region = process.env.S3_REGION! - const accessKeyId = process.env.S3_ACCESS_KEY_ID! - const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY! - - let playlistUrl - try { - playlistUrl = await getPlaylistUrl(url) - console.log(`playlistUrl=${playlistUrl}`) - } catch (e) { - console.error('error during getPlaylistUrl()') - console.error(e) - throw e - } - - const jobId = job.id - const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint }) - const inputStream = Record.getFFmpegStream({ url: playlistUrl }) - const record = new Record({ inputStream, bucket, s3Client, jobId }) - - await record.start() - - console.log(`record job ${job.id} complete`) - - return job.id - -} - - -export default async function main (jobs: RecordJob[]): Promise { - // @todo why are we passed multiple jobs? I'm expecting only one. - const backOffOptions = { - numOfAttempts: 5, - startingDelay: 5000, - retry: (e: any, attemptNumber: number) => { - console.log(`Record Job is retrying. Attempt number ${attemptNumber}. e=${JSON.stringify(e, null, 2)}`) - return true - } - } - for (const j of jobs) { - console.log(`record job ${j.id} GO GO GO`) - try { - await backOff(() => _record(j), backOffOptions) - } catch (e) { - console.warn(`record job ${j.id} encountered the following error.`) - console.error(e) - } - console.log(`record job ${j.id} is finished.`) - } -}; \ No newline at end of file diff --git a/services/capture/src/tasks/start_recording.ts b/services/capture/src/tasks/start_recording.ts deleted file mode 100644 index 3d5b85c..0000000 --- a/services/capture/src/tasks/start_recording.ts +++ /dev/null @@ -1,67 +0,0 @@ - -import { Helpers, type Task } from 'graphile-worker' -import { add } from 'date-fns' - -/** - * url is the URL to be recorded. Ex: chaturbate.com/projektmelody - * discordMessageId is the ID of the discord messate which displays recording status. - * we use the ID to update the message later, and/or relate button press events to this record task - */ -interface Payload { - url: string; - discordMessageId: string; - isAborted: boolean; -} - -function assertPayload(payload: any): asserts payload is Payload { - if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); - if (typeof payload.url !== "string") throw new Error("invalid url"); - if (typeof payload.discordMessageId !== "string") throw new Error(`invalid discordMessageId=${payload.discordMessageId}`); -} - -function assertEnv() { - if (!process.env.AUTOMATION_USER_JWT) throw new Error('AUTOMATION_USER_JWT was missing in env'); - if (!process.env.POSTGREST_URL) throw new Error('POSTGREST_URL was missing in env'); -} - -async function createRecordingRecord(payload: Payload, helpers: Helpers): Promise { - const { url, discordMessageId } = payload - const record = { - url, - discord_message_id: discordMessageId, - recording_state: 'pending', - file_size: 0 - } - const res = await fetch(`${process.env.POSTGREST_URL}/records`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${process.env.AUTOMATION_USER_JWT}`, - 'Prefer': 'return=headers-only' - }, - body: JSON.stringify(record) - }) - if (!res.ok) { - const status = res.status - const statusText = res.statusText - throw new Error(`fetch failed to create recording record in database. status=${status}, statusText=${statusText}`) - } - helpers.logger.info('res.headers.location as follows.') - helpers.logger.info(res.headers.get('location')!) - const id = res.headers.get('location')?.split('.').at(-1) - if (!id) throw new Error('id could not be parsed from location header'); - return parseInt(id) -} - -export const start_recording: Task = async function (payload, helpers) { - assertPayload(payload) - assertEnv() - const recordId = await createRecordingRecord(payload, helpers) - const { url } = payload; - await helpers.addJob('record', { url, recordId }, { maxAttempts: 3, jobKey: `record_${recordId}` }) - const runAt = add(new Date(), { seconds: 10 }) - await helpers.addJob('updateDiscordMessage', { recordId }, { jobKey: `record_${recordId}_update_discord_message`, maxAttempts: 3, runAt }) - helpers.logger.info(`startRecording() with url=${url}, recordId=${recordId}, (updateDiscordMessage runAt=${runAt})`) -} - -export default start_recording \ No newline at end of file diff --git a/services/capture/src/tasks/stop_recording.ts b/services/capture/src/tasks/stop_recording.ts deleted file mode 100644 index 7b92674..0000000 --- a/services/capture/src/tasks/stop_recording.ts +++ /dev/null @@ -1,18 +0,0 @@ - -import { type Task } from 'graphile-worker' - -interface Payload { - id: string -} - -function assertPayload(payload: any): asserts payload is Payload { - if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); - if (typeof payload.id !== "string") throw new Error("invalid id"); -} - - -export const stop_recording: Task = async function (payload) { - assertPayload(payload) - const { id } = payload; - console.log(`@todo simulated stop_recording with id=${id}`) -} \ No newline at end of file diff --git a/services/capture/tsconfig.json b/services/capture/tsconfig.json index 6666925..e93bad7 100644 --- a/services/capture/tsconfig.json +++ b/services/capture/tsconfig.json @@ -25,7 +25,7 @@ // Include the necessary files for your project "include": [ "src/**/*.ts" - ], +, "../bot/src/tasks/restart_failed_recordings.ts" ], "exclude": [ "node_modules" ] diff --git a/services/migrations/README.md b/services/migrations/README.md index eaa65ba..b664380 100644 --- a/services/migrations/README.md +++ b/services/migrations/README.md @@ -4,9 +4,24 @@ Here we handle migrations for the postgrest database. @see https://github.com/thomwright/postgres-migrations +Reminder: only write migrations that affect schema. (don't write migrations that affect data) ## K.I.S.S. Keep It Stupidly Simple. -We are keeping this module as simple as possible. This means pure JS (no typescript!) \ No newline at end of file +We are keeping this module as simple as possible. This means pure JS (no typescript!) + + +## troubleshooting + +If you see the following error, graphile_worker likely hasn't had a chance to create it's functions. Make sure that a graphile_worker is running, so it can automatically create the necessary functions. + +```json +{ + "code": "42883", + "details": null, + "hint": "No function matches the given name and argument types. You might need to add explicit type casts.", + "message": "function graphile_worker.add_job(text, json, max_attempts => integer) does not exist" +} +``` \ No newline at end of file diff --git a/services/migrations/index.js b/services/migrations/index.js index 0197229..a80c68e 100644 --- a/services/migrations/index.js +++ b/services/migrations/index.js @@ -2,6 +2,8 @@ import {migrate} from 'postgres-migrations' import path, { dirname } from 'node:path' import { fileURLToPath } from 'url'; import 'dotenv/config' + + const __dirname = dirname(fileURLToPath(import.meta.url)); if (!process.env.DATABASE_PASSWORD) throw new Error('DATABASE_PASSWORD is missing in env'); @@ -23,7 +25,7 @@ async function main() { defaultDatabase: "postgres" } - await migrate(dbConfig, path.join(__dirname, "./migrations/")) + await migrate(dbConfig, path.join(__dirname, "./migrations/"), { logger: console.log }) } diff --git a/services/migrations/migrations/00006_add-updated-at-to-records.sql b/services/migrations/migrations/00006_add-updated-at-to-records.sql new file mode 100644 index 0000000..6673808 --- /dev/null +++ b/services/migrations/migrations/00006_add-updated-at-to-records.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS api.records + ADD COLUMN created_at timestamp(6) without time zone; + +ALTER TABLE IF EXISTS api.records + ADD COLUMN updated_at timestamp(6) without time zone; \ No newline at end of file diff --git a/services/migrations/migrations/00007_add-default-records-timestamps.sql b/services/migrations/migrations/00007_add-default-records-timestamps.sql new file mode 100644 index 0000000..17caebf --- /dev/null +++ b/services/migrations/migrations/00007_add-default-records-timestamps.sql @@ -0,0 +1,7 @@ +ALTER TABLE IF EXISTS api.records + ADD CONSTRAINT created_at_not_null + CHECK (created_at IS NOT NULL) NOT VALID; + +ALTER TABLE IF EXISTS api.records + ADD CONSTRAINT updated_at_not_null + CHECK (updated_at IS NOT NULL) NOT VALID; \ No newline at end of file diff --git a/services/migrations/migrations/00008_add-default-records-timestamp.sql b/services/migrations/migrations/00008_add-default-records-timestamp.sql new file mode 100644 index 0000000..1005698 --- /dev/null +++ b/services/migrations/migrations/00008_add-default-records-timestamp.sql @@ -0,0 +1,26 @@ +-- In the prev. migration I added a CHECK, but I forgot to add the default + + +ALTER TABLE IF EXISTS api.records + ALTER COLUMN created_at SET DEFAULT now(); + +ALTER TABLE IF EXISTS api.records + ALTER COLUMN updated_at SET DEFAULT now(); + + +-- create a function which updates the row's updated_at +CREATE FUNCTION public.tg__updated_at() RETURNS trigger + LANGUAGE plpgsql + SET search_path TO 'pg_catalog', 'public', 'pg_temp' + AS $$ + BEGIN + NEW.updated_at = now(); + RETURN NEW; + END; + $$; + +-- create a trigger which runs the above function when a /record is updated +CREATE TRIGGER record_updated_at + AFTER UPDATE ON api.records + FOR EACH ROW + EXECUTE PROCEDURE public.tg__updated_at(); diff --git a/services/migrations/migrations/00009_add-streams-vods-vtubers.sql b/services/migrations/migrations/00009_add-streams-vods-vtubers.sql new file mode 100644 index 0000000..134fc45 --- /dev/null +++ b/services/migrations/migrations/00009_add-streams-vods-vtubers.sql @@ -0,0 +1,140 @@ + +-- vtubers table +CREATE TABLE api.vtubers ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + display_name TEXT NOT NULL, + chaturbate TEXT, + twitter TEXT, + patreon TEXT, + twitch TEXT, + tiktok TEXT, + onlyfans TEXT, + youtube TEXT, + linktree TEXT, + carrd TEXT, + fansly TEXT, + pornhub TEXT, + discord TEXT, + reddit TEXT, + throne TEXT, + instagram TEXT, + facebook TEXT, + merch TEXT, + slug TEXT NOT NULL, + description1 TEXT, + description2 TEXT, + image TEXT NOT NULL, + theme_color VARCHAR(7) NOT NULL, + image_blur TEXT DEFAULT 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAABmJLR0QA/wD/AP+gvaeTAAAADUlEQVQImWMwtf//HwAEkwJzh0T9qwAAAABJRU5ErkJggg==', + fansly_id TEXT, + chaturbate_id TEXT, + twitter_id TEXT + -- F.Y.I., relations as follows + -- toys (one-to-many) + -- vods (one-to-many) + -- streams (one-to-many) +); +GRANT all ON api.vtubers TO automation; +GRANT SELECT ON api.vtubers TO web_anon; + + +-- streams table +CREATE TABLE api.streams ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + url TEXT NOT NULL, + platform_notification_type TEXT, + date timestamp(6) without time zone, + created_at timestamp(6) without time zone, + vtuber uuid, + FOREIGN KEY (vtuber) REFERENCES api.vtubers(id), + tweet TEXT, + archive_status TEXT, + is_chaturbate_stream BOOLEAN, + is_fansly_stream BOOLEAN +); +GRANT all ON api.streams TO automation; +GRANT SELECT ON api.streams TO web_anon; + +-- toys table +CREATE TABLE api.toys ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + -- relation. one toy to many tags + -- relation. one toy to many vtubers + make TEXT NOT NULL, + model TEXT NOT NULL, + image TEXT NOT NULL DEFAULT 'https://futureporn-b2.b-cdn.net/default-thumbnail.webp' +); +GRANT all ON api.toys TO automation; +GRANT SELECT ON api.toys TO web_anon; + + +-- tags table +CREATE TABLE api.tags ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + toy_id uuid, + FOREIGN KEY (toy_id) REFERENCES api.toys +); +GRANT all ON api.tags TO automation; +GRANT SELECT ON api.tags TO web_anon; + +-- toys-tags junction table +CREATE TABLE api.toys_tags( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + toy_id uuid, + tag_id uuid, + CONSTRAINT fk_toys FOREIGN KEY(toy_id) REFERENCES api.toys(id), + CONSTRAINT fk_tags FOREIGN KEY(tag_id) REFERENCES api.tags(id) +); +GRANT all ON api.toys_tags TO automation; +GRANT SELECT ON api.toys_tags TO web_anon; + +-- tags-vods junction table +-- toys-vtubers junction table +CREATE TABLE api.toys_vtubers( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + toy_id uuid, + vtuber_id uuid, + CONSTRAINT fk_toys FOREIGN KEY(toy_id) REFERENCES api.toys(id), + CONSTRAINT fk_vtubers FOREIGN KEY(vtuber_id) REFERENCES api.vtubers(id) +); +GRANT all ON api.toys_vtubers TO automation; +GRANT SELECT ON api.toys_vtubers TO web_anon; + + + + +-- vods table +CREATE TABLE api.vods ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + stream_id uuid NOT NULL, + FOREIGN KEY (stream_id) REFERENCES api.streams(id), + video_cid TEXT UNIQUE, + CONSTRAINT check_video_cid CHECK (video_cid ~ 'Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}'), + announce_title TEXT, + announce_url TEXT, + note TEXT, + date timestamp(6) without time zone, + spoilers TEXT, + title TEXT, + uploader uuid, + mux_asset_id TEXT, + mux_playback_id TEXT, + s3_key TEXT, + s3_id TEXT, + thumbnail TEXT +); +GRANT all ON api.vods TO automation; +GRANT SELECT ON api.vods TO web_anon; + + +-- tags-vods junction table +CREATE TABLE api.tags_vods( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + tag_id uuid, + vod_id uuid, + CONSTRAINT fk_tags FOREIGN KEY(tag_id) REFERENCES api.tags(id), + CONSTRAINT fk_vods FOREIGN KEY(vod_id) REFERENCES api.vods(id) +); +GRANT all ON api.tags_vods TO automation; +GRANT SELECT ON api.tags_vods TO web_anon; diff --git a/services/migrations/migrations/00010_record-segments.sql b/services/migrations/migrations/00010_record-segments.sql new file mode 100644 index 0000000..0e8105c --- /dev/null +++ b/services/migrations/migrations/00010_record-segments.sql @@ -0,0 +1,7 @@ +-- we add the concept of segments to api.records +-- implemented as a multidimensional text array, s3_segments. +-- the first value is the s3 id, the second value is the s3 key +-- [id, key] + +ALTER TABLE IF EXISTS api.records + ADD COLUMN s3_segments text[][]; diff --git a/services/migrations/migrations/00011_use-composite-primary-keys.sql b/services/migrations/migrations/00011_use-composite-primary-keys.sql new file mode 100644 index 0000000..025f54d --- /dev/null +++ b/services/migrations/migrations/00011_use-composite-primary-keys.sql @@ -0,0 +1,28 @@ +-- we don't need s3_segments multidimential array. we're moving it's functionality to a new table +ALTER TABLE IF EXISTS api.records + DROP COLUMN s3_segments; + + + +-- segments table +CREATE TABLE api.segments ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + s3_key TEXT NOT NULL, + s3_id TEXT NOT NULL, + bytes bigint DEFAULT 0 +); +GRANT all ON api.segments TO automation; +GRANT SELECT ON api.segments TO web_anon; + + +-- records-segments join table +CREATE TABLE api.records_segments( + id INT GENERATED ALWAYS AS IDENTITY, + record_id INT NOT NULL, + segment_id INT NOT NULL, + CONSTRAINT fk_record FOREIGN KEY(record_id) REFERENCES api.records(id), + CONSTRAINT fk_segment FOREIGN KEY(segment_id) REFERENCES api.segments(id), + PRIMARY KEY(id, record_id, segment_id) +); +GRANT all ON api.records_segments TO automation; +GRANT SELECT ON api.records_segments TO web_anon; diff --git a/services/migrations/migrations/00012_order-records_segments.sql b/services/migrations/migrations/00012_order-records_segments.sql new file mode 100644 index 0000000..daec178 --- /dev/null +++ b/services/migrations/migrations/00012_order-records_segments.sql @@ -0,0 +1,2 @@ +ALTER TABLE IF EXISTS api.records_segments + ADD COLUMN segments_order INT NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/services/migrations/migrations/00013_rename-segments-order.sql b/services/migrations/migrations/00013_rename-segments-order.sql new file mode 100644 index 0000000..1f3f812 --- /dev/null +++ b/services/migrations/migrations/00013_rename-segments-order.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS api.records_segments + DROP COLUMN segments_order; + +ALTER TABLE IF EXISTS api.records_segments + ADD COLUMN segment_order INT NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/services/migrations/migrations/00014_create-segments-stream-links.sql b/services/migrations/migrations/00014_create-segments-stream-links.sql new file mode 100644 index 0000000..a28df1f --- /dev/null +++ b/services/migrations/migrations/00014_create-segments-stream-links.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS api.records CASCADE; +DROP TABLE IF EXISTS api.records_segments CASCADE; diff --git a/services/migrations/migrations/00015_create-segments-stream-links-2.sql b/services/migrations/migrations/00015_create-segments-stream-links-2.sql new file mode 100644 index 0000000..c28e45b --- /dev/null +++ b/services/migrations/migrations/00015_create-segments-stream-links-2.sql @@ -0,0 +1,16 @@ +-- I forgot to actually create the new table +CREATE TABLE api.segments_stream_links ( + id int PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + discord_message_id text NOT NULL, + capture_job_id text NOT NULL +); + + +-- roles & permissions +GRANT all ON api.segments_stream_links TO automation; +GRANT SELECT ON api.segments_stream_links TO web_anon; + + +-- there is no s3_id in the segments run context so we don't need a column for it +ALTER TABLE IF EXISTS api.segments + DROP COLUMN s3_id; diff --git a/services/migrations/migrations/00016_remove-unecessary-columns.sql b/services/migrations/migrations/00016_remove-unecessary-columns.sql new file mode 100644 index 0000000..92a483f --- /dev/null +++ b/services/migrations/migrations/00016_remove-unecessary-columns.sql @@ -0,0 +1,8 @@ +-- oops. bit by unfinished copy-paste + +-- there is no s3_id in the segments run context so we don't need a column for it +ALTER TABLE IF EXISTS api.segments_stream_links + DROP COLUMN discord_message_id; + +ALTER TABLE IF EXISTS api.segments_stream_links + DROP COLUMN capture_job_id; \ No newline at end of file diff --git a/services/migrations/migrations/00017_add-stream-status-col.sql b/services/migrations/migrations/00017_add-stream-status-col.sql new file mode 100644 index 0000000..de2d908 --- /dev/null +++ b/services/migrations/migrations/00017_add-stream-status-col.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS api.streams + ADD COLUMN updated_at timestamp(6) without time zone; + +ALTER TABLE IF EXISTS api.streams + ADD COLUMN status TEXT NOT NULL; \ No newline at end of file diff --git a/services/migrations/migrations/00018_add-stream-status-default.sql b/services/migrations/migrations/00018_add-stream-status-default.sql new file mode 100644 index 0000000..15cb0b6 --- /dev/null +++ b/services/migrations/migrations/00018_add-stream-status-default.sql @@ -0,0 +1,5 @@ +ALTER TABLE IF EXISTS api.streams + DROP COLUMN IF EXISTS status; + +ALTER TABLE api.streams + ADD COLUMN status TEXT NOT NULL DEFAULT 'pending_recording'; diff --git a/services/migrations/migrations/00019_drop-discord-interactions.sql b/services/migrations/migrations/00019_drop-discord-interactions.sql new file mode 100644 index 0000000..cd9e1cb --- /dev/null +++ b/services/migrations/migrations/00019_drop-discord-interactions.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS api.discord_interactions CASCADE; diff --git a/services/migrations/migrations/00020_add-streams-update-trigger.sql b/services/migrations/migrations/00020_add-streams-update-trigger.sql new file mode 100644 index 0000000..27848de --- /dev/null +++ b/services/migrations/migrations/00020_add-streams-update-trigger.sql @@ -0,0 +1,38 @@ +-- delete outdated +DROP FUNCTION IF EXISTS public.tg__add_job(); + + +-- We create a function which lets Postgrest's automation user create jobs in Graphile Worker. +-- Normally only the database owner, in our case `postgres`, can add jobs due to RLS in graphile_worker tables. +-- Under the advice of graphile_worker author, we can use a SECURITY DEFINER wrapper function. +-- @see https://worker.graphile.org/docs/sql-add-job#graphile_workeradd_job:~:text=graphile_worker.add_job(...),that%20are%20necessary.) +-- @see https://discord.com/channels/489127045289476126/1179293106336694333/1179605043729670306 +-- @see https://discord.com/channels/489127045289476126/498852330754801666/1067707497235873822 +CREATE FUNCTION public.tg__add_record_job() RETURNS trigger + LANGUAGE plpgsql SECURITY DEFINER + SET search_path TO 'pg_catalog', 'public', 'pg_temp' + AS $$ + begin + PERFORM graphile_worker.add_job('record', json_build_object( + 'url', NEW.url, + 'stream_id', NEW.id + ), max_attempts := 12); + return NEW; + end; + $$; + + + + +-- when a stream is updated, we add a job in graphile to update_discord_message +CREATE TRIGGER stream_update + AFTER UPDATE ON api.streams + FOR EACH ROW + EXECUTE PROCEDURE public.tg__update_discord_message('update_discord_message'); + +-- when a stream is created, we add a 'record' job in graphile-worker +CREATE TRIGGER stream_create + AFTER INSERT ON api.streams + FOR EACH ROW + EXECUTE PROCEDURE public.tg__add_record_job('record'); + diff --git a/services/migrations/migrations/00021_add-foreign-key-to-segments_stream.sql b/services/migrations/migrations/00021_add-foreign-key-to-segments_stream.sql new file mode 100644 index 0000000..d298003 --- /dev/null +++ b/services/migrations/migrations/00021_add-foreign-key-to-segments_stream.sql @@ -0,0 +1,9 @@ +DROP TABLE api.segments_stream_links; + +CREATE TABLE api.segments_stream_links ( + id int GENERATED ALWAYS AS IDENTITY, + stream_id UUID NOT NULL REFERENCES api.streams(id), + segment_id INT NOT NULL REFERENCES api.segments(id), + capture_job_id text NOT NULL, + PRIMARY KEY(id, stream_id, segment_id) +); diff --git a/services/migrations/migrations/00022_add-permissions-for-segments_stream_links.sql b/services/migrations/migrations/00022_add-permissions-for-segments_stream_links.sql new file mode 100644 index 0000000..a089e1b --- /dev/null +++ b/services/migrations/migrations/00022_add-permissions-for-segments_stream_links.sql @@ -0,0 +1,2 @@ +GRANT all ON api.segments_stream_links TO automation; +GRANT SELECT ON api.segments_stream_links TO web_anon; diff --git a/services/migrations/migrations/00023_drop-capture_job_id-column.sql b/services/migrations/migrations/00023_drop-capture_job_id-column.sql new file mode 100644 index 0000000..98fe03d --- /dev/null +++ b/services/migrations/migrations/00023_drop-capture_job_id-column.sql @@ -0,0 +1,3 @@ + +ALTER TABLE IF EXISTS api.segments_stream_links + DROP COLUMN IF EXISTS capture_job_id; diff --git a/services/migrations/migrations/00024_add-updated_at-for-segments.sql b/services/migrations/migrations/00024_add-updated_at-for-segments.sql new file mode 100644 index 0000000..9143a87 --- /dev/null +++ b/services/migrations/migrations/00024_add-updated_at-for-segments.sql @@ -0,0 +1,39 @@ + +ALTER TABLE api.segments + ADD COLUMN created_at TIMESTAMP(6) WITHOUT TIME ZONE; + +ALTER TABLE api.segments + ADD COLUMN updated_at TIMESTAMP(6) WITHOUT TIME ZONE; + + + +-- in migration 8, we already created tg__updated_at() so we don't need to create that, +-- but we do need to create a function which will the row's created_at +CREATE FUNCTION public.tg__created_at() RETURNS trigger + LANGUAGE plpgsql + SET search_path TO 'pg_catalog', 'public', 'pg_temp' + AS $$ + BEGIN + NEW.created_at = now(); + RETURN NEW; + END; + $$; + +-- create a trigger which runs the tg__updated_at() function when a /segment is updated +CREATE TRIGGER segment_updated_at + AFTER UPDATE ON api.segments + FOR EACH ROW + EXECUTE PROCEDURE public.tg__updated_at(); + + +-- create a trigger which runs the tg__created_at() function when a /segment is created +CREATE TRIGGER segment_created_at + AFTER INSERT ON api.segments + FOR EACH ROW + EXECUTE PROCEDURE public.tg__created_at(); + +-- create a trigger which runs the tg__created_at() function when a /stream is created +CREATE TRIGGER stream_created_at + AFTER INSERT ON api.streams + FOR EACH ROW + EXECUTE PROCEDURE public.tg__created_at(); diff --git a/services/migrations/migrations/00025_add-is_recording_aborted-to-streams.sql b/services/migrations/migrations/00025_add-is_recording_aborted-to-streams.sql new file mode 100644 index 0000000..ce10bc3 --- /dev/null +++ b/services/migrations/migrations/00025_add-is_recording_aborted-to-streams.sql @@ -0,0 +1,2 @@ +ALTER TABLE api.streams + ADD COLUMN is_recording_aborted BOOLEAN DEFAULT FALSE; diff --git a/services/migrations/migrations/00026_use-moddatetime.sql b/services/migrations/migrations/00026_use-moddatetime.sql new file mode 100644 index 0000000..f2929f9 --- /dev/null +++ b/services/migrations/migrations/00026_use-moddatetime.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION moddatetime; + diff --git a/services/migrations/migrations/00027_create-triggers-for-moddatetime.sql b/services/migrations/migrations/00027_create-triggers-for-moddatetime.sql new file mode 100644 index 0000000..74a4eb3 --- /dev/null +++ b/services/migrations/migrations/00027_create-triggers-for-moddatetime.sql @@ -0,0 +1,49 @@ + + +-- now we set up the triggers + +-- streams created_at +ALTER TABLE api.streams + ALTER created_at SET DEFAULT now(); + +DROP TRIGGER stream_created_at ON api.streams; + +CREATE TRIGGER stream_created_at + BEFORE INSERT ON api.streams + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (created_at); + + +-- streams updated_at +ALTER TABLE api.streams + ALTER updated_at SET DEFAULT now(); + +CREATE TRIGGER stream_updated_at + BEFORE UPDATE ON api.streams + FOR EACH ROW + EXECUTE PROCEDURE moddatetime (updated_at); + + +-- segments created_at +ALTER TABLE api.segments + ALTER created_at SET DEFAULT now(); + +DROP TRIGGER segment_created_at ON api.segments; + +CREATE TRIGGER segment_created_at + BEFORE INSERT ON api.segments + FOR EACH ROW + EXECUTE PROCEDURE moddatetime(created_at); + + +-- segments updated_at +ALTER TABLE api.segments + ALTER updated_at SET DEFAULT now(); + +DROP TRIGGER segment_updated_at ON api.segments; + +CREATE TRIGGER segment_updated_at + BEFORE UPDATE ON api.segments + FOR EACH ROW + EXECUTE PROCEDURE moddatetime(updated_at); + diff --git a/services/migrations/migrations/00028_remove-moddate-on-insert.sql b/services/migrations/migrations/00028_remove-moddate-on-insert.sql new file mode 100644 index 0000000..3cb45ac --- /dev/null +++ b/services/migrations/migrations/00028_remove-moddate-on-insert.sql @@ -0,0 +1,9 @@ +-- A fix for the following error +-- moddatetime: cannot process INSERT events +-- +-- We don't need moddatetime for INSERT events because we have column defaults set the time when the row is created. + + +DROP TRIGGER segment_created_at ON api.segments; +DROP TRIGGER stream_created_at ON api.streams; + diff --git a/services/migrations/migrations/00029_add-discord-message-id.sql b/services/migrations/migrations/00029_add-discord-message-id.sql new file mode 100644 index 0000000..7a7fb4e --- /dev/null +++ b/services/migrations/migrations/00029_add-discord-message-id.sql @@ -0,0 +1,5 @@ + +-- streams needs discord_message_id for chatops +ALTER TABLE api.streams + ADD COLUMN discord_message_id TEXT; + diff --git a/services/migrations/package.json b/services/migrations/package.json index c3024fc..38c2b95 100644 --- a/services/migrations/package.json +++ b/services/migrations/package.json @@ -8,6 +8,7 @@ "test": "echo \"Error: no test specified\" && exit 0", "start": "node index.js" }, + "packageManager": "pnpm@9.6.0", "keywords": [], "author": "@CJ_Clippy", "license": "Unlicense",