add file_size and recording_state to db

README.md

@@ -60,3 +60,5 @@ https://uppy.fp.sbtp.xyz/metrics
 ### Code is run more than it is read
 
 ### The computer doesn't care
+
+### [ONE SHOT. ONE LIFE](https://www.youtube.com/watch?v=Rh-ohspuCmE)

Tiltfile

@@ -145,6 +145,7 @@ docker_build(
         './pnpm-lock.yaml',
         './pnpm-workspace.yaml',
         './services/bot',
+        './packages/types',
     ],
     dockerfile='./d.bot.dockerfile',
     target='dev',

charts/fp/templates/bot.yaml

@@ -20,6 +20,8 @@ spec:
         - name: bot
           image: "{{ .Values.bot.imageName }}"
           env:
+            - name: POSTGREST_URL 
+              value: "{{ .Values.postgrest.url }}"
             - name: AUTOMATION_USER_JWT
               valueFrom:
                 secretKeyRef:
@@ -53,6 +55,6 @@ spec:
           resources:
             limits:
               cpu: 150m
-              memory: 256Mi
+              memory: 512Mi
       restartPolicy: Always
 

charts/fp/templates/capture.yaml

@@ -47,16 +47,16 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: postgrest
-                  key: jwtSecret
+                  key: automationUserJwt
             - name: POSTGREST_URL 
-              value: http://postgrest.futureporn.svc.cluster.local:9000
+              value: "{{ .Values.postgrest.url }}"
             - name: PORT
               value: "{{ .Values.capture.api.port }}"
             - name: S3_ENDPOINT
               value: "{{ .Values.s3.endpoint }}"
             - name: S3_REGION
               value: "{{ .Values.s3.region }}"
-            - name: S3_BUCKET_NAME
+            - name: S3_BUCKET
               value: "{{ .Values.s3.buckets.usc }}"
             - name: S3_ACCESS_KEY_ID
               valueFrom:
@@ -112,5 +112,5 @@ spec:
           resources:
             limits:
               cpu: 100m
-              memory: 128Mi
+              memory: 256Mi
       restartPolicy: Always

charts/fp/values.yaml

@@ -67,6 +67,7 @@ bot:
   imageName: fp/bot
   replicas: 1
 postgrest:
+  url: http://postgrest.futureporn.svc.cluster.local:9000
   image: postgrest/postgrest
   replicas: 1
   port: 9000

d.bot.dockerfile

@@ -8,6 +8,7 @@ ENTRYPOINT ["pnpm"]
 FROM base AS install
 COPY pnpm-lock.yaml .npmrc package.json .
 COPY ./services/bot/ ./services/bot/
+COPY ./packages/types/ ./packages/types/
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm fetch
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline

packages/scout/src/ytdlp.ts

@@ -3,23 +3,27 @@ import 'dotenv/config'
 
 const maxRetries = 3
 
-export class ExhaustedRetries extends Error {
+export class ExhaustedRetriesError extends Error {
   constructor(message?: string) {
     super(message)
-    Object.setPrototypeOf(this, ExhaustedRetries.prototype)
+    Object.setPrototypeOf(this, ExhaustedRetriesError.prototype)
+    this.name = this.constructor.name
+    this.message = `ExhaustedRetries: We retried the request the maximum amount of times. maxRetries of ${maxRetries} was reached.`
   }
   getErrorMessage() {
-    return `ExhaustedRetries: We retried the request the maximum amount of times. maxRetries of ${maxRetries} was reached.`
+    return this.message
   }
 }
 
-export class RoomOffline extends Error {
+export class RoomOfflineError extends Error {
   constructor(message?: string) {
     super(message)
-    Object.setPrototypeOf(this, ExhaustedRetries.prototype)
+    Object.setPrototypeOf(this, RoomOfflineError.prototype)
+    this.name = this.constructor.name
+    this.message = `RoomOffline. ${this.message}`
   }
   getErrorMessage() {
-    return `RoomOffline. ${this.message}`
+    return this.message
   }
 }
 
@@ -37,16 +41,16 @@ export async function getPlaylistUrl (roomUrl: string, proxy = false, retries =
     // we were likely blocked by Cloudflare
     // we make the request a second time, this time via proxy
     if (retries < maxRetries) return getPlaylistUrl(roomUrl, true, retries+=1);
-    else throw new ExhaustedRetries();
+    else throw new ExhaustedRetriesError();
   } else if (output.match(/Unable to find stream URL/)) {
     // sometimes this happens. a retry is in order.
     if (retries < maxRetries) return getPlaylistUrl(roomUrl, proxy, retries+=1);
-    else throw new ExhaustedRetries()
+    else throw new ExhaustedRetriesError()
   } else if (code === 0 && output.match(/https:\/\/.*\.m3u8/)) {
     // this must be an OK result with a playlist
     return output
   } else if (code === 1 && output.match(/Room is currently offline/)) {
-    throw new RoomOffline()
+    throw new RoomOfflineError()
   } else {
     console.error('exotic scenario')
     const msg = `We encountered an exotic scenario where code=${code} and output=${output}. Admin: please patch the code to handle this scenario.`

packages/types/index.d.ts

@@ -4,6 +4,8 @@ export as namespace Futureporn;
 
 declare namespace Futureporn {
 
+    type RecordingState = 'pending' | 'recording' | 'aborted' | 'ended' 
+
 
     interface IMuxAsset {
         id: number;

packages/types/package.json

@@ -5,7 +5,6 @@
   "description": "",
   "scripts": {
     "test": "echo \"Warn: no test specified\" && exit 0",
-    "build": "tsc --build",
     "clean": "rm -rf dist",
     "superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist"
   },

scripts/k8s-secrets.sh

@@ -50,7 +50,8 @@ kubectl --namespace futureporn create secret generic pgadmin4 \
 kubectl --namespace futureporn delete secret postgrest --ignore-not-found
 kubectl --namespace futureporn create secret generic postgrest \
 --from-literal=dbUri=${PGRST_DB_URI} \
---from-literal=jwtSecret=${PGRST_JWT_SECRET}
+--from-literal=jwtSecret=${PGRST_JWT_SECRET} \
+--from-literal=automationUserJwt=${AUTOMATION_USER_JWT}
 
 kubectl --namespace futureporn delete secret capture --ignore-not-found
 kubectl --namespace futureporn create secret generic capture \

scripts/postgres-create.sh

@@ -70,9 +70,9 @@ kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PAS
     IS_TEMPLATE = False;"
 
 
-## Create PgBoss db (for backend tasks)
+## Create graphile_worker db (for backend tasks)
 kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
-    CREATE DATABASE pgboss \
+    CREATE DATABASE graphile_worker \
     WITH \
     OWNER = postgres \
     ENCODING = 'UTF8' \
@@ -97,10 +97,11 @@ kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PAS
 
 
 ## grant futureporn user all privs
-kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
-    GRANT ALL PRIVILEGES ON DATABASE pgboss TO futureporn;"
 kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
     GRANT ALL PRIVILEGES ON DATABASE postgrest TO futureporn;"
+kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
+    GRANT ALL PRIVILEGES ON DATABASE graphile_worker TO futureporn;"
+
 
 
 ## import schema

services/bot/package.json

@@ -7,7 +7,7 @@
   "scripts": {
     "test": "echo \"Warn: no test specified\" && exit 0",
     "start": "node ./dist/index.js",
-    "dev.nodemon": "nodemon --ext js,ts,json,yaml --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"",
+    "dev.nodemon": "nodemon --legacy-watch --ext js,ts --watch ./src --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"",
     "dev": "tsx --watch ./src/index.ts",
     "build": "tsc --build",
     "clean": "rm -rf dist",
@@ -24,6 +24,7 @@
     "graphile-worker": "^0.16.6"
   },
   "devDependencies": {
+    "@futureporn/types": "workspace:^",
     "nodemon": "^3.1.4",
     "ts-node": "^10.9.2",
     "tsx": "^4.16.2",

services/bot/pnpm-lock.yaml

@@ -21,12 +21,15 @@ importers:
         specifier: ^0.16.6
         version: 0.16.6(typescript@5.5.4)
     devDependencies:
+      '@futureporn/types':
+        specifier: workspace:^
+        version: link:../../packages/types
       nodemon:
         specifier: ^3.1.4
         version: 3.1.4
       ts-node:
         specifier: ^10.9.2
-        version: 10.9.2(@types/node@22.0.0)(typescript@5.5.4)
+        version: 10.9.2(@types/node@22.1.0)(typescript@5.5.4)
       tsx:
         specifier: ^4.16.2
         version: 4.16.2
@@ -306,17 +309,20 @@ packages:
   '@types/node@22.0.0':
     resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==}
 
+  '@types/node@22.1.0':
+    resolution: {integrity: sha512-AOmuRF0R2/5j1knA3c6G3HOk523Ga+l+ZXltX8SF1+5oqcXijjfTd8fY3XRZqSihEu9XhtQnKYLmkFaoxgsJHw==}
+
   '@types/pg@8.11.6':
     resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==}
 
   '@types/semver@7.5.8':
     resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==}
 
-  '@types/ws@8.5.11':
+  '@types/ws@8.5.12':
-    resolution: {integrity: sha512-4+q7P5h3SpJxaBft0Dzpbr6lmMaqh0Jr2tbhJZ/luAwvD7ohSCniYkwz/pLxuT2h0EOa6QADgJj1Ko+TzRfZ+w==}
+    resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==}
 
-  '@vladfrangu/async_event_emitter@2.4.4':
+  '@vladfrangu/async_event_emitter@2.4.5':
-    resolution: {integrity: sha512-ZL62PFXEIeGUI8btfJ5S8Flc286eU1ZUSjwyFQtIGXfRUDPZKO+CDJMYb1R71LjGWRZ4n202O+a6FGjsgTw58g==}
+    resolution: {integrity: sha512-J7T3gUr3Wz0l7Ni1f9upgBZ7+J22/Q1B7dl0X6fG+fTsD+H+31DIosMHj4Um1dWQwqbcQ3oQf+YS2foYkDc9cQ==}
     engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
 
   acorn-walk@8.3.3:
@@ -767,6 +773,9 @@ packages:
   undici-types@6.11.1:
     resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==}
 
+  undici-types@6.13.0:
+    resolution: {integrity: sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg==}
+
   undici@6.13.0:
     resolution: {integrity: sha512-Q2rtqmZWrbP8nePMq7mOJIN98M0fYvSgV89vwl/BQRT4mDOeY2GXZngfGpcBBhtky3woM7G24wZV3Q304Bv6cw==}
     engines: {node: '>=18.0'}
@@ -854,7 +863,7 @@ snapshots:
       '@discordjs/util': 1.1.0
       '@sapphire/async-queue': 1.5.3
       '@sapphire/snowflake': 3.5.3
-      '@vladfrangu/async_event_emitter': 2.4.4
+      '@vladfrangu/async_event_emitter': 2.4.5
       discord-api-types: 0.37.83
       magic-bytes.js: 1.10.0
       tslib: 2.6.2
@@ -868,8 +877,8 @@ snapshots:
       '@discordjs/rest': 2.3.0
       '@discordjs/util': 1.1.0
       '@sapphire/async-queue': 1.5.3
-      '@types/ws': 8.5.11
+      '@types/ws': 8.5.12
-      '@vladfrangu/async_event_emitter': 2.4.4
+      '@vladfrangu/async_event_emitter': 2.4.5
       discord-api-types: 0.37.83
       tslib: 2.6.2
       ws: 8.18.0
@@ -992,6 +1001,10 @@ snapshots:
     dependencies:
       undici-types: 6.11.1
 
+  '@types/node@22.1.0':
+    dependencies:
+      undici-types: 6.13.0
+
   '@types/pg@8.11.6':
     dependencies:
       '@types/node': 22.0.0
@@ -1000,11 +1013,11 @@ snapshots:
 
   '@types/semver@7.5.8': {}
 
-  '@types/ws@8.5.11':
+  '@types/ws@8.5.12':
     dependencies:
-      '@types/node': 22.0.0
+      '@types/node': 22.1.0
 
-  '@vladfrangu/async_event_emitter@2.4.4': {}
+  '@vladfrangu/async_event_emitter@2.4.5': {}
 
   acorn-walk@8.3.3:
     dependencies:
@@ -1419,14 +1432,14 @@ snapshots:
 
   ts-mixer@6.0.4: {}
 
-  ts-node@10.9.2(@types/node@22.0.0)(typescript@5.5.4):
+  ts-node@10.9.2(@types/node@22.1.0)(typescript@5.5.4):
     dependencies:
       '@cspotcode/source-map-support': 0.8.1
       '@tsconfig/node10': 1.0.11
       '@tsconfig/node12': 1.0.11
       '@tsconfig/node14': 1.0.3
       '@tsconfig/node16': 1.0.4
-      '@types/node': 22.0.0
+      '@types/node': 22.1.0
       acorn: 8.12.1
       acorn-walk: 8.3.3
       arg: 4.1.3
@@ -1454,6 +1467,8 @@ snapshots:
 
   undici-types@6.11.1: {}
 
+  undici-types@6.13.0: {}
+
   undici@6.13.0: {}
 
   v8-compile-cache-lib@3.0.1: {}

services/bot/src/commands/utilities/record.ts

@@ -73,7 +73,8 @@ export default {
 
     const message = await idk.fetch()
     const discordMessageId = message.id
-    await workerUtils.addJob('startRecording', { url, discordMessageId })
+    await workerUtils.addJob('startRecording', { url, discordMessageId }, { maxAttempts: 3 })
+    
 
 	},
 };

services/bot/src/events/clientReady.ts

@@ -0,0 +1,9 @@
+import { Client, Events, type Interaction } from 'discord.js';
+
+export default {
+  name: Events.ClientReady,
+  once: true,
+  execute(client: Client) {
+    console.log(`Ready! Logged in as ${client?.user?.tag}`);
+  }
+}

services/bot/src/events/interactionCreate.ts

@@ -0,0 +1,71 @@
+import { Events, type Interaction, Client, Collection } from 'discord.js';
+import type { WorkerUtils } from 'graphile-worker';
+
+interface ExtendedClient extends Client {
+  commands: Collection<string, any>
+}
+
+export default {
+	name: Events.InteractionCreate,
+  once: false,
+	async execute(interaction: Interaction, workerUtils: WorkerUtils) {
+		// if (!interaction.isChatInputCommand()) return;
+    // console.log(interaction.client)
+    // const command = interaction.client.commands.get(interaction.commandName);
+    if (interaction.isButton()) {
+      console.log(`the interaction is a button type with customId=${interaction.customId}, message.id=${interaction.message.id}, user=${interaction.user.id} (${interaction.user.globalName})`)
+      if (interaction.customId === 'stop') {
+        interaction.reply('[stop] IDK IDK IDK ??? @todo')
+        workerUtils.addJob('stopRecording', { discordMessageId: interaction.message.id, userId: interaction.user.id })
+      } else if (interaction.customId === 'retry') {
+        interaction.reply('[retry] IDK IDK IDK ??? @todo')
+        workerUtils.addJob('startRecording', { discordMessageId: interaction.message.id, userId: interaction.user.id })
+      } else {
+        console.error(`this button's customId=${interaction.customId} did not match one of the known customIds`)
+      }
+
+    } else if (interaction.isChatInputCommand()) {
+      console.log(`the interaction is a ChatInputCommandInteraction with commandName=${interaction.commandName}, user=${interaction.user.id} (${interaction.user.globalName})`)
+      const client = interaction.client as ExtendedClient
+      const command = client.commands.get(interaction.commandName);
+      
+      if (!command) {
+      	console.error(`No command matching ${interaction.commandName} was found.`);
+      	return;
+      }
+
+      command.execute({ interaction, workerUtils })
+    }
+    
+	},
+};
+
+// const { Events } = require('discord.js');
+
+// module.exports = {
+// 	name: Events.ClientReady,
+// 	once: true,
+// 	execute(client) {
+// 		console.log(`Ready! Logged in as ${client.user.tag}`);
+// 	},
+// };
+
+
+
+// client.on(Events.InteractionCreate, interaction => {
+//   if (interaction.isChatInputCommand()) {
+//     const { commandName } = interaction;
+//     console.log(`Received interaction with commandName=${commandName}`)
+//     const cmd = commands.find((c) => c.data.name === commandName)
+//     if (!cmd) {
+//       console.log(`no command handler matches commandName=${commandName}`)
+//       return;
+//     }
+//     cmd.execute({ interaction, workerUtils })
+//   } else {
+//     // probably a ButtonInteraction
+//     console.log(interaction)
+//   }
+// });
+
+

services/bot/src/events/messageReactionAdd.ts

@@ -0,0 +1,24 @@
+import { Client, Events, MessageReaction, User, type Interaction } from 'discord.js';
+
+export default {
+  name: Events.MessageReactionAdd,
+  once: false,
+  async execute(reaction: MessageReaction, user: User) {
+    // When a reaction is received, check if the structure is partial
+    if (reaction.partial) {
+      // If the message this reaction belongs to was removed, the fetching might result in an API error which should be handled
+      try {
+        await reaction.fetch();
+      } catch (error) {
+        console.error('Something went wrong when fetching the message:', error);
+        // Return as `reaction.message.author` may be undefined/null
+        return;
+      }
+    }
+
+    // Now the message has been cached and is fully available
+    console.log(`${reaction.message.author}'s message "${reaction.message.content}" gained a reaction!`);
+    // The reaction is now also fully available and the properties will be reflected accurately:
+    console.log(`${reaction.count} user(s) have given the same reaction to this message!`);
+  }
+}

services/bot/src/index.ts

@@ -1,9 +1,10 @@
 import 'dotenv/config'
-import { type ChatInputCommandInteraction, Client, Events, GatewayIntentBits, Partials } from 'discord.js'
+import { type ChatInputCommandInteraction, Client, GatewayIntentBits, Partials, Collection } from 'discord.js'
 import loadCommands from './loadCommands.js'
 import deployCommands from './deployCommands.js'
 import discordMessageUpdate from './tasks/discordMessageUpdate.js'
-import { makeWorkerUtils, type WorkerUtils } from 'graphile-worker'
+import { makeWorkerUtils, type WorkerUtils, type RunnerOptions, run } from 'graphile-worker'
+import loadEvents from './loadEvents.js'
 
 export interface ExecuteArguments {
   interaction: ChatInputCommandInteraction;
@@ -15,9 +16,31 @@ if (!process.env.DISCORD_TOKEN) throw new Error("DISCORD_TOKEN was missing from
 if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
 if (!process.env.WORKER_CONNECTION_STRING) throw new Error("WORKER_CONNECTION_STRING was missing from env");
 
+const preset: GraphileConfig.Preset = {
+  worker: {
+    connectionString: process.env.WORKER_CONNECTION_STRING,
+    concurrentJobs: 1,
+    fileExtensions: [".js", ".ts"]
+  },
+};
+
 async function setupGraphileWorker() {
+  const runnerOptions: RunnerOptions = {
+    preset,
+    taskList: {
+      'discordMessageUpdate': discordMessageUpdate
+    }
+  }
+  
+  const runner = await run(runnerOptions)
+  if (!runner) throw new Error('failed to initialize graphile worker');
+  await runner.promise
+}
+
+
+async function setupWorkerUtils() {
   const workerUtils = await makeWorkerUtils({
-    connectionString: process.env.WORKER_CONNECTION_STRING!,
+    preset
   });
   await workerUtils.migrate()
   return workerUtils
@@ -30,7 +53,7 @@ async function setupDiscordBot(commands: any[], workerUtils: WorkerUtils) {
   
 
   console.log(`Create a new client instance`)
-  const client = new Client({
+  let client: any = new Client({
     intents: [
       GatewayIntentBits.Guilds,
       GatewayIntentBits.GuildMessages,
@@ -42,67 +65,14 @@ async function setupDiscordBot(commands: any[], workerUtils: WorkerUtils) {
       Partials.Reaction,
     ]
   });
-
+  client.commands = new Collection();
-
+  commands.forEach((c) => client.commands.set(c.data.name, c))
-  client.on(Events.InteractionCreate, interaction => {
-    if (!interaction.isChatInputCommand()) return;
-    const { commandName } = interaction;
-    console.log(`Received interaction with commandName=${commandName}`)
-    const cmd = commands.find((c) => c.data.name === commandName)
-    if (!cmd) {
-      console.log(`no command handler matches commandName=${commandName}`)
-      return;
-    }
-    cmd.execute({ interaction, workerUtils })
-  });
-  // When the client is ready, run this code (only once).
-  // The distinction between `client: Client<boolean>` and `readyClient: Client<true>` is important for TypeScript developers.
-  // It makes some properties non-nullable.
-  client.once(Events.ClientReady, readyClient => {
-    console.log(`Ready! Logged in as ${readyClient.user.tag}`);
-    // client.channels.cache.get(process.env.DISCORD_CHANNEL_ID).send('testing 123');
-    // readyClient.channels.fetch(channelId).then(channel => {
-    //     channel.send('generic welcome message!')
-    // });
-    // console.log(readyClient.channels)
-    // const channel = readyClient.channels.cache.get(process.env.DISCORD_CHANNEL_ID);
-    // channel.send('testing 135');
-  });
-
-
-  client.on(Events.InteractionCreate, async interaction => {
-    if (!interaction.isChatInputCommand()) return;
-
-    const { commandName } = interaction;
-
-    if (commandName === 'react') {
-      const message = await interaction.reply({ content: 'You can react with Unicode emojis!', fetchReply: true });
-      message.react('😄');
-    }
-  })
-
-  client.on(Events.MessageReactionAdd, async (reaction, user) => {
-    // When a reaction is received, check if the structure is partial
-    if (reaction.partial) {
-      // If the message this reaction belongs to was removed, the fetching might result in an API error which should be handled
-      try {
-        await reaction.fetch();
-      } catch (error) {
-        console.error('Something went wrong when fetching the message:', error);
-        // Return as `reaction.message.author` may be undefined/null
-        return;
-      }
-    }
-
-    // Now the message has been cached and is fully available
-    console.log(`${reaction.message.author}'s message "${reaction.message.content}" gained a reaction!`);
-    // The reaction is now also fully available and the properties will be reflected accurately:
-    console.log(`${reaction.count} user(s) have given the same reaction to this message!`);
-  });
 
 
   // Log in to Discord with your client's token
   client.login(process.env.DISCORD_TOKEN);
+  await loadEvents(client, workerUtils)
+  
 
 }
 
@@ -112,8 +82,9 @@ async function main() {
   if (!commands) throw new Error('there were no commands available to be loaded.');
   await deployCommands(commands.map((c) => c.data.toJSON()))
   console.log(`${commands.length} commands deployed: ${commands.map((c) => c.data.name).join(', ')}`)
-  const workerUtils = await setupGraphileWorker()
+  const workerUtils = await setupWorkerUtils()
   setupDiscordBot(commands, workerUtils)
+  setupGraphileWorker()
 }
 
 main().catch((e) => {

services/bot/src/loadCommands.ts

@@ -14,13 +14,13 @@ export default async function loadCommands(): Promise<any[]> {
   for (const folder of commandFolders) {
     const commandsPath = path.join(foldersPath, folder);
     const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('.ts') || file.endsWith('.js'));
-    console.log(`commandFiles=${commandFiles}`)
+    console.log(`commandFiles=${commandFiles}`);
     // console.log(`Grab the SlashCommandBuilder#toJSON() output of each command's data for deployment`)
     for (const file of commandFiles) {
       const filePath = path.join(commandsPath, file);
       const command = (await import(filePath)).default;
       // console.log(command)
-      if ('data' in command && 'execute' in command) {
+      if (command?.data && command?.execute) {
         commands.push(command);
       } else {
         console.log(`[WARNING] The command at ${filePath} is missing a required "data" or "execute" property.`);

services/bot/src/loadEvents.ts

@@ -0,0 +1,23 @@
+import * as path from 'node:path';
+import * as fs from 'node:fs';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'url';
+import type { Client } from 'discord.js';
+import type { WorkerUtils } from 'graphile-worker';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export default async function loadEvents(client: Client, workerUtils: WorkerUtils) {
+  console.log(`loading events`);
+  const eventsPath = path.join(__dirname, 'events');
+  const eventFiles = fs.readdirSync(eventsPath).filter(file => file.endsWith('.ts') || file.endsWith('.js'));
+  console.log(`eventFiles=${eventFiles}`);
+  for (const file of eventFiles) {
+    const filePath = path.join(eventsPath, file);
+    const event = (await import(filePath)).default;
+    if (event.once) {
+      client.once(event.name, (...args) => event.execute(...args, workerUtils));
+    } else {
+      client.on(event.name, (...args) => event.execute(...args, workerUtils));
+    }
+  }
+}

services/bot/src/tasks/discordMessageUpdate.ts

@@ -1,106 +0,0 @@
-import 'dotenv/config'
-import { type Task, type WorkerUtils } from 'graphile-worker';
-import { type ChatInputCommandInteraction, Client, Events, GatewayIntentBits, Partials, Message, TextChannel } from 'discord.js'
-
-
-// export interface DiscordMessageUpdateJob extends Job {
-//   data: {
-//     captureJobId: string;
-//   }
-// }
-
-interface Payload {
-  discord_message_id: string;
-  capture_job_id: string;
-}
-
-function assertPayload(payload: any): asserts payload is Payload {
-  if (typeof payload !== "object" || !payload) throw new Error("invalid payload");  
-  if (typeof payload.discord_message_id !== "string") throw new Error("invalid discord_message_id");
-  if (typeof payload.capture_job_id.to !== "string") throw new Error("invalid capture_job_id");
-}
-
-
-if (!process.env.AUTOMATION_USER_JWT) throw new Error(`AUTOMATION_USER_JWT was missing from env`);
-if (!process.env.DISCORD_TOKEN) throw new Error("DISCORD_TOKEN was missing from env");
-if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
-if (!process.env.DISCORD_GUILD_ID) throw new Error("DISCORD_GUILD_ID was missing from env");
-
-/**
- * discordMessageUpdate is the task where we edit a previously sent discord message to display new status information sent to us from @futureporn/capture
- * 
- * Sometimes the update is changing the state, one of Pending|Recording|Aborted|Ended.
- * Sometimes the update is updating the Filesize of the recording in-progress
- * Sometimes the update is adding a thumbnail image to the message
- */
-export default async function discordMessageUpdate <Task> (payload: Payload) {
-  assertPayload(payload)
-  // const { captureJobId } = job.data
-  console.log(`discordMessageUpdate job has begun with captureJobId=${payload.capture_job_id}`)
-  
-
-  // // find the discord_interactions record containing the captureJobId
-  // const res = await fetch(`http://postgrest.futureporn.svc.cluster.local:9000/discord_interactions?capture_job_id=eq.${captureJobId}`)
-  // if (!res.ok) throw new Error('failed to fetch the discord_interactions');
-  // const body = await res.json() as DiscordInteraction
-  // console.log('discord_interactions as follows')
-  // console.log(body)
-
-  // // create a discord.js client
-  // const client = new Client({
-  //   intents: [
-  //     GatewayIntentBits.Guilds,
-  //     GatewayIntentBits.GuildMessages
-  //   ],
-  //   partials: [
-  //     Partials.Message,
-  //     Partials.Channel,
-  //   ]
-  // });
-
-  // // const messageManager = client.
-  // // const guild = client.guilds.cache.get(process.env.DISCORD_GUILD_ID!);
-  // // if (!guild) throw new Error('guild was undefined')
-  
-
-  // const channel = await client.channels.fetch(process.env.DISCORD_CHANNEL_ID!) as TextChannel
-  // if (!channel) throw new Error(`discord channel was undefined`);
-  
-  // // console.log('we got the following channel')
-  // // console.log(channel)
-
-  // const message = await channel.messages.fetch(body.discord_message_id)
-
-  // console.log('we got the following message')
-  // console.log(message)
-
-  // get the message
-  // client.channel.messages.get()
-  // client.rest.
-
-  
-
-  // const message: Message = {
-  //   id: body.discord_message_id
-  // };
-  // await message.fetchReference()
-  // message.edit('test message update payload thingy')
-  
-  
-  // client.rest.updateMessage(message, "My new content");
-  // TextChannel.fetchMessage(msgId).then(console.log);
-
-  
-
-  // TextChannel
-
-  // channel.messages.fetch(`Your Message ID`).then(message => {
-  //   message.edit("New message Text");
-  // }).catch(err => {
-  //   console.error(err);
-  // });
-
-  // using the discord_interaction's discord_message_id, use discord.js to update the discord message.
-  
-
-}

services/bot/src/tasks/updateDiscordMessage.ts

@@ -0,0 +1,191 @@
+import 'dotenv/config'
+import type { RecordingState } from '@futureporn/types'
+import { type Task, type Helpers } from 'graphile-worker';
+import { 
+  Client, 
+  GatewayIntentBits, 
+  TextChannel,
+  ActionRowBuilder, 
+  ButtonBuilder,
+  type MessageActionRowComponentBuilder,
+  ButtonStyle,
+  EmbedBuilder,
+  Guild
+} from 'discord.js';
+
+// export interface DiscordMessageUpdateJob extends Job {
+//   data: {
+//     captureJobId: string;
+//   }
+// }
+
+interface Payload {
+  recordId: number;
+}
+
+
+
+function assertPayload(payload: any): asserts payload is Payload {
+  if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
+  if (!payload.recordId) throw new Error(`recordId was absent in the payload`);
+}
+
+
+if (!process.env.AUTOMATION_USER_JWT) throw new Error(`AUTOMATION_USER_JWT was missing from env`);
+if (!process.env.DISCORD_TOKEN) throw new Error("DISCORD_TOKEN was missing from env");
+if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
+if (!process.env.DISCORD_GUILD_ID) throw new Error("DISCORD_GUILD_ID was missing from env");
+
+
+
+async function editDiscordMessage({ helpers, state, discordMessageId }: { helpers: Helpers, state: RecordingState, discordMessageId: string }) {
+  
+  // const { captureJobId } = job.data
+  helpers.logger.info(`discordMessageUpdate job has begun with discordMessageId=${discordMessageId}, state=${state}`)
+  
+
+  // create a discord.js client
+  const client = new Client({
+    intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildMessages],
+  });
+
+  // Log in to Discord with your client's token
+  client.login(process.env.DISCORD_TOKEN);
+
+  
+  const guild = await client.guilds.fetch(process.env.DISCORD_GUILD_ID!) as Guild
+  if (!guild) throw new Error('guild was undefined');
+  
+  helpers.logger.info('here is the guild as follows')
+  helpers.logger.info(guild.toString())
+  helpers.logger.info(`fetching discord channel id=${process.env.DISCORD_CHANNEL_ID} from discord guild`)
+  const channel = await client.channels.fetch(process.env.DISCORD_CHANNEL_ID!) as TextChannel
+  if (!channel) throw new Error(`discord channel was undefined`);
+
+  const message = await channel.messages.fetch(discordMessageId)
+
+  helpers.logger.info(`the following is the message taht we have fetched`)
+  helpers.logger.info(message.toString())
+
+  const statusEmbed = getStatusEmbed(state)
+  const buttonRow = getButtonRow(state)
+
+
+  // const embed = new EmbedBuilder().setTitle('Attachments');
+
+
+  const updatedMessage = {
+    embeds: [
+      statusEmbed
+    ],
+    components: [
+      buttonRow
+    ]
+  };
+
+  message.edit(updatedMessage)
+
+
+}
+
+
+async function getRecordFromDatabase(recordId: number) {
+  const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`)
+  if (!res.ok) {
+    throw new Error(`failed fetching record ${recordId}. status=${res.status}, statusText=${res.statusText}`)
+  }
+  const body = await res.json() as any      
+  return body[0];
+}
+
+
+
+/**
+ * updateDiscordMessage is the task where we edit a previously sent discord message to display 
+ * the most up-to-date status information from the database
+ * 
+ * Sometimes the update is changing the state, one of Pending|Recording|Aborted|Ended.
+ * Sometimes the update is updating the Filesize of the recording in-progress
+ * Sometimes the update is adding a thumbnail image to the message
+ */
+export const updateDiscordMessage: Task = async function (payload, helpers: Helpers) {
+  try {
+    assertPayload(payload)
+    const record = await getRecordFromDatabase(payload.recordId)
+    const { discordMessageId, state } = record
+    editDiscordMessage({ helpers, state, discordMessageId })
+  } catch (e) {
+    helpers.logger.error(`caught an error during updateDiscordMessage. e=${e}`)
+  }
+}
+
+function getStatusEmbed(state: RecordingState) {
+  let title, description, color;
+  if (state === 'pending') {
+    title = "Pending"
+    description = "Waiting for a worker to accept the job."
+    color = 2326507
+  } else if (state === 'recording') {
+    title = 'Recording'
+    description = 'The stream is being recorded.'
+    color = 392960
+  } else if (state === 'aborted') {
+    title = "Aborted"
+    description = "The recording was stopped by the user."
+    color = 8289651
+  } else if (state === 'ended') {
+    title = "Ended"
+    description = "The recording has stopped."
+    color = 10855845
+  } else {
+    title = 'Unknown'
+    description = 'The recording is in an unknown state? (this is a bug.)'
+    color = 10855845
+  }
+  return new EmbedBuilder().setTitle(title).setDescription(description).setColor(color)
+}
+
+
+
+function getButtonRow(state: RecordingState) {
+  let id, label, emoji, style;
+
+  if (state === 'pending') {
+    id = 'stop';
+    label = 'Cancel';
+    emoji = '❌';
+    style = ButtonStyle.Danger;
+  } else if (state === 'recording') {
+    id = 'stop';
+    label = 'Stop Recording';
+    emoji = '🛑';
+    style = ButtonStyle.Danger;
+  } else if (state === 'aborted') {
+    id = 'retry';
+    label = 'Retry Recording';
+    emoji = '🔄';
+    style = ButtonStyle.Success;
+  } else if (state === 'ended') {
+    id = 'download';
+    label = 'Download Recording';
+    emoji = '📥';
+    style = ButtonStyle.Primary;
+  } else {
+    id = 'unknown';
+    label = 'Unknown State';
+    emoji = '🤔';
+    style = ButtonStyle.Secondary;
+  }
+
+  return new ActionRowBuilder<MessageActionRowComponentBuilder>()
+    .addComponents([
+      new ButtonBuilder()
+        .setCustomId(id)
+        .setLabel(label)
+        .setEmoji(emoji)
+        .setStyle(style),
+    ]);
+}
+
+
+export default updateDiscordMessage

services/capture/package.json

@@ -21,6 +21,7 @@
     "@aws-sdk/lib-storage": "^3.588.0",
     "@aws-sdk/types": "^3.609.0",
     "@futureporn/scout": "workspace:^",
+    "@futureporn/types": "workspace:^",
     "@futureporn/utils": "workspace:^",
     "@paralleldrive/cuid2": "^2.2.2",
     "@types/chai": "^4.3.16",

services/capture/pnpm-lock.yaml

@@ -20,6 +20,9 @@ importers:
       '@futureporn/scout':
         specifier: workspace:^
         version: link:../../packages/scout
+      '@futureporn/types':
+        specifier: workspace:^
+        version: link:../../packages/types
       '@futureporn/utils':
         specifier: workspace:^
         version: link:../../packages/utils

services/capture/src/app.ts

@@ -13,6 +13,10 @@ interface RecordBodyType {
   url: string;
   discordMessageId: string;
 }
+interface MessageBodyType {
+  state: 'pending' | 'recording' | 'aborted' | 'ended';
+  discordMessageId: string;
+}
 
 const build = function (opts: Record<string, any>={}, connectionString: string) {
   const app: ExtendedFastifyInstance = fastify(opts)
@@ -21,6 +25,17 @@ const build = function (opts: Record<string, any>={}, connectionString: string)
   app.get('/', async function (request, reply) {
     return { app: '@futureporn/capture', version }
   })
+  app.put('/api/message', async function (request: FastifyRequest<{ Body: MessageBodyType }>, reply) {
+    const { state, discordMessageId } = request.body
+    if (app?.graphile) {
+      const jobId = await app.graphile.addJob('discordMessageUpdate', {
+        discordMessageId,
+        state
+      }, { maxAttempts: 3 })
+    } else {
+      console.error('app.graphile was missing')
+    }
+  })
   app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) {
     const { url, discordMessageId } = request.body
     console.log(`POST /api/record with url=${url}`)
@@ -29,7 +44,7 @@ const build = function (opts: Record<string, any>={}, connectionString: string)
       const jobId = await app.graphile.addJob('startRecording', {
         url, 
         discordMessageId
-      })
+      }, { maxAttempts: 3 })
       return { jobId }
     } else {
       console.error(`app.graphile was missing! Is the graphile worker plugin registered to the fastify instance?`)

services/capture/src/tasks/record.ts

@@ -2,7 +2,7 @@
 import { Helpers, type Task } from 'graphile-worker'
 import Record from '../Record.ts'
 import { getPlaylistUrl } from '@futureporn/scout/ytdlp.ts'
-
+import type { RecordingState } from '@futureporn/types'
 
 /**
  * url is the URL to be recorded. Ex: chaturbate.com/projektmelody
@@ -22,7 +22,7 @@ interface RecordingRecord {
 function assertPayload(payload: any): asserts payload is Payload {
   if (typeof payload !== "object" || !payload) throw new Error("invalid payload");  
   if (typeof payload.url !== "string") throw new Error("invalid url");
-  if (typeof payload.recordId !== "number") throw new Error("invalid recordId");
+  if (typeof payload.recordId !== "number") throw new Error(`invalid recordId=${payload.recordId}`);
 }
 
 function assertEnv() {
@@ -44,13 +44,12 @@ async function getRecording(url: string, recordId: number, abortSignal: AbortSig
   const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
   const inputStream = Record.getFFmpegStream({ url: playlistUrl })
   
-  const record = new Record({ inputStream, bucket, s3Client, jobId: ''+recordId, abortSignal })
+  const record = new Record({ inputStream, bucket, s3Client, jobId: ''+recordId }) // @todo add abortsignal
-  record.start()
   return record
 }
 
 async function checkIfAborted(recordId: number): Promise<boolean> {
-  const res = await fetch(`${process.env.POSTGREST_URL}/records?id.eq=${recordId}`, {
+  const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`, {
     headers: {
       'Content-Type': 'application/json',
       'Accepts': 'application/json'
@@ -64,8 +63,26 @@ async function checkIfAborted(recordId: number): Promise<boolean> {
   return body[0].isAborted
 }
 
+async function updateDatabaseRecord({recordId, state, filesize}: { recordId: number, state: RecordingState, filesize: number }): Promise<void> {
+  const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`, {
+    method: 'PATCH',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accepts': 'application/json',
+      'Prefer': 'return=representation'
+    },
+    body: JSON.stringify({ state, filesize })
+  })
+  if (!res.ok) {
+    throw new Error(`failed to checkIfAborted. status=${res.status}, statusText=${res.statusText}`);
+  }
+  const body = await res.json() as RecordingRecord[];
+  if (!body[0]) throw new Error(`failed to get a record that matched recordId=${recordId}`)
+  return body[0].isAborted
+}
 
 export const record: Task = async function (payload, helpers) {
+  console.log(payload)
   assertPayload(payload)
   assertEnv()
   const { url, recordId } = payload
@@ -73,27 +90,39 @@ export const record: Task = async function (payload, helpers) {
   let interval
   try {
     const record = await getRecording(url, recordId, abortController.signal)
-    // every 30s, poll db to see if our job has been aborted by the user
+    // every 30s, we
+    // 1. poll db to see if our job has been aborted by the user
+    // 2. update the db record with the RecordingState and filesize
     interval = setInterval(async () => {
+      try {
+        helpers.logger.info(`checkIfAborted()`)
         const isAborted = await checkIfAborted(recordId)
         if (isAborted) {
           abortController.abort()
         }
+        let state: RecordingState = 'recording'
+      } catch (e) {
+        helpers.logger.error(`error while checking if this job was aborted. For sake of the recording in progress we are ignoring the following error. ${e}`)
+      }
     }, 30000)
+
+    // start recording and await the S3 upload being finished
+    await record.start() 
+
   } finally {
     clearInterval(interval)
   }
 
+
+  
+
   // const recordId = await createRecordingRecord(payload, helpers)
   // const { url } = payload;
   // console.log(`@todo simulated start_recording with url=${url}, recordId=${recordId}`)
   // await helpers.addJob('record', { url, recordId })
 }
 
-// // export default record
+
-// export default function (payload: Payload, helpers: Helpers) {
-//   helpers.logger.info('WHEEEEEEEEEEEEEEEE (record.ts task executor)')
-// }
 
 
 export default record

services/capture/src/tasks/startRecording.ts

@@ -15,7 +15,7 @@ interface Payload {
 function assertPayload(payload: any): asserts payload is Payload {
   if (typeof payload !== "object" || !payload) throw new Error("invalid payload");  
   if (typeof payload.url !== "string") throw new Error("invalid url");
-  if (typeof payload.discordMessageId !== "string") throw new Error("invalid discordMessageId");
+  if (typeof payload.discordMessageId !== "string") throw new Error(`invalid discordMessageId=${payload.discordMessageId}`);
 }
 
 function assertEnv() {
@@ -26,8 +26,8 @@ async function createRecordingRecord(payload: Payload, helpers: Helpers): Promis
   const { url, discordMessageId } = payload
   const record = {
     url,
-    discordMessageId,
+    discord_message_id: discordMessageId,
-    isAborted: false
+    is_aborted: false
   }
   const res = await fetch('http://postgrest.futureporn.svc.cluster.local:9000/records', {
     method: 'POST',
@@ -43,9 +43,11 @@ async function createRecordingRecord(payload: Payload, helpers: Helpers): Promis
     const statusText = res.statusText
     throw new Error(`fetch failed to create recording record in database. status=${status}, statusText=${statusText}`)
   }
-  helpers.logger.info('res.headers as follows.')
+  helpers.logger.info('res.headers.location as follows.')
-  helpers.logger.info(res.headers)
+  helpers.logger.info(res.headers.get('location')!)
-  return res.headers.Location.split('.').at(-1)
+  const id = res.headers.get('location')?.split('.').at(-1)
+  if (!id) throw new Error('id could not be parsed from location header');
+  return parseInt(id)
 }
 
 export const startRecording: Task = async function (payload, helpers) {
@@ -54,7 +56,7 @@ export const startRecording: Task = async function (payload, helpers) {
   const recordId = await createRecordingRecord(payload, helpers)
   const { url } = payload;
   console.log(`@todo simulated start_recording with url=${url}, recordId=${recordId}`)
-  await helpers.addJob('record', { url, recordId })
+  await helpers.addJob('record', { url, recordId }, { maxAttempts: 3 })
 }
 
 export default startRecording

services/mailbox/src/index.ts

@@ -27,7 +27,7 @@ async function handleMessage({ workerUtils, email, msg }: { workerUtils: WorkerU
 
       if (isMatch) {
           console.log('    ✏️✏️ adding process_notif_email job to queue')
-          workerUtils.addJob('process_notif_email', { isMatch, url, platform, channel, displayName, date, userId, avatar })
+          workerUtils.addJob('process_notif_email', { isMatch, url, platform, channel, displayName, date, userId, avatar }, { maxAttempts: 3 })
       }
 
       console.log('    ✏️ archiving e-mail')
@@ -50,8 +50,8 @@ async function main() {
   })
 
   // demonstrate that we are connected @todo remove this
-  workerUtils.addJob('hello', { name: 'worker' })
+  workerUtils.addJob('hello', { name: 'worker' }, { maxAttempts: 3 })
-  workerUtils.addJob('identify_image_color', { url: 'https://futureporn-b2.b-cdn.net/ti8ht9bgwj6k783j7hglfg8j_projektmelody-chaturbate-2024-07-18.png' })
+  workerUtils.addJob('identify_image_color', { url: 'https://futureporn-b2.b-cdn.net/ti8ht9bgwj6k783j7hglfg8j_projektmelody-chaturbate-2024-07-18.png' }, { maxAttempts: 3 })
 
   // connect to IMAP inbox and wait for new e-mails
   const email = new Email()

services/migrations/migrations/00002_add-records-table.sql

@@ -3,7 +3,9 @@ CREATE TABLE api.records (
   id INT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
   url TEXT NOT NULL,
   discord_message_id TEXT NOT NULL,
-  is_aborted BOOLEAN DEFAULT FALSE
+  is_aborted BOOLEAN DEFAULT FALSE,
+  recording_state TEXT NOT NULL,
+  file_size BIGINT NOT NULL
 );
 
 -- roles & permissions for our backend automation user

services/migrations/migrations/00003_add-filesize-and-state.sql

@@ -0,0 +1,9 @@
+ALTER TABLE api.records
+  ADD COLUMN recording_state TEXT NOT NULL DEFAULT 'pending' CHECK(recording_state IN ('pending', 'recording', 'aborted', 'ended')),
+  ADD COLUMN file_size BIGINT NOT NULL DEFAULT 0;
+
+-- Grant permissions to the new columns for our backend automation user
+GRANT UPDATE (recording_state, file_size) ON api.records TO automation;
+
+-- Grant read-only access to the new columns for our web anon user
+GRANT SELECT (recording_state, file_size) ON api.records TO web_anon;

services/migrations/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@futureporn/migrations",
   "type": "module",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "",
   "main": "index.js",
   "scripts": {