add file_size and recording_state to db
ci / build (push) Has been cancelled Details

This commit is contained in:
CJ_Clippy 2024-08-02 14:12:56 -08:00
parent 8faa5e2195
commit 5bb2296a00
31 changed files with 511 additions and 236 deletions

View File

@ -60,3 +60,5 @@ https://uppy.fp.sbtp.xyz/metrics
### Code is run more than it is read
### The computer doesn't care
### [ONE SHOT. ONE LIFE](https://www.youtube.com/watch?v=Rh-ohspuCmE)

View File

@ -145,6 +145,7 @@ docker_build(
'./pnpm-lock.yaml',
'./pnpm-workspace.yaml',
'./services/bot',
'./packages/types',
],
dockerfile='./d.bot.dockerfile',
target='dev',

View File

@ -20,6 +20,8 @@ spec:
- name: bot
image: "{{ .Values.bot.imageName }}"
env:
- name: POSTGREST_URL
value: "{{ .Values.postgrest.url }}"
- name: AUTOMATION_USER_JWT
valueFrom:
secretKeyRef:
@ -53,6 +55,6 @@ spec:
resources:
limits:
cpu: 150m
memory: 256Mi
memory: 512Mi
restartPolicy: Always

View File

@ -47,16 +47,16 @@ spec:
valueFrom:
secretKeyRef:
name: postgrest
key: jwtSecret
key: automationUserJwt
- name: POSTGREST_URL
value: http://postgrest.futureporn.svc.cluster.local:9000
value: "{{ .Values.postgrest.url }}"
- name: PORT
value: "{{ .Values.capture.api.port }}"
- name: S3_ENDPOINT
value: "{{ .Values.s3.endpoint }}"
- name: S3_REGION
value: "{{ .Values.s3.region }}"
- name: S3_BUCKET_NAME
- name: S3_BUCKET
value: "{{ .Values.s3.buckets.usc }}"
- name: S3_ACCESS_KEY_ID
valueFrom:
@ -112,5 +112,5 @@ spec:
resources:
limits:
cpu: 100m
memory: 128Mi
memory: 256Mi
restartPolicy: Always

View File

@ -67,6 +67,7 @@ bot:
imageName: fp/bot
replicas: 1
postgrest:
url: http://postgrest.futureporn.svc.cluster.local:9000
image: postgrest/postgrest
replicas: 1
port: 9000

View File

@ -8,6 +8,7 @@ ENTRYPOINT ["pnpm"]
FROM base AS install
COPY pnpm-lock.yaml .npmrc package.json .
COPY ./services/bot/ ./services/bot/
COPY ./packages/types/ ./packages/types/
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm fetch
RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile --prefer-offline

View File

@ -3,23 +3,27 @@ import 'dotenv/config'
const maxRetries = 3
export class ExhaustedRetries extends Error {
export class ExhaustedRetriesError extends Error {
constructor(message?: string) {
super(message)
Object.setPrototypeOf(this, ExhaustedRetries.prototype)
Object.setPrototypeOf(this, ExhaustedRetriesError.prototype)
this.name = this.constructor.name
this.message = `ExhaustedRetries: We retried the request the maximum amount of times. maxRetries of ${maxRetries} was reached.`
}
getErrorMessage() {
return `ExhaustedRetries: We retried the request the maximum amount of times. maxRetries of ${maxRetries} was reached.`
return this.message
}
}
export class RoomOffline extends Error {
export class RoomOfflineError extends Error {
constructor(message?: string) {
super(message)
Object.setPrototypeOf(this, ExhaustedRetries.prototype)
Object.setPrototypeOf(this, RoomOfflineError.prototype)
this.name = this.constructor.name
this.message = `RoomOffline. ${this.message}`
}
getErrorMessage() {
return `RoomOffline. ${this.message}`
return this.message
}
}
@ -37,16 +41,16 @@ export async function getPlaylistUrl (roomUrl: string, proxy = false, retries =
// we were likely blocked by Cloudflare
// we make the request a second time, this time via proxy
if (retries < maxRetries) return getPlaylistUrl(roomUrl, true, retries+=1);
else throw new ExhaustedRetries();
else throw new ExhaustedRetriesError();
} else if (output.match(/Unable to find stream URL/)) {
// sometimes this happens. a retry is in order.
if (retries < maxRetries) return getPlaylistUrl(roomUrl, proxy, retries+=1);
else throw new ExhaustedRetries()
else throw new ExhaustedRetriesError()
} else if (code === 0 && output.match(/https:\/\/.*\.m3u8/)) {
// this must be an OK result with a playlist
return output
} else if (code === 1 && output.match(/Room is currently offline/)) {
throw new RoomOffline()
throw new RoomOfflineError()
} else {
console.error('exotic scenario')
const msg = `We encountered an exotic scenario where code=${code} and output=${output}. Admin: please patch the code to handle this scenario.`

View File

@ -4,6 +4,8 @@ export as namespace Futureporn;
declare namespace Futureporn {
type RecordingState = 'pending' | 'recording' | 'aborted' | 'ended'
interface IMuxAsset {
id: number;

View File

@ -5,7 +5,6 @@
"description": "",
"scripts": {
"test": "echo \"Warn: no test specified\" && exit 0",
"build": "tsc --build",
"clean": "rm -rf dist",
"superclean": "rm -rf node_modules && rm -rf pnpm-lock.yaml && rm -rf dist"
},

View File

@ -50,7 +50,8 @@ kubectl --namespace futureporn create secret generic pgadmin4 \
kubectl --namespace futureporn delete secret postgrest --ignore-not-found
kubectl --namespace futureporn create secret generic postgrest \
--from-literal=dbUri=${PGRST_DB_URI} \
--from-literal=jwtSecret=${PGRST_JWT_SECRET}
--from-literal=jwtSecret=${PGRST_JWT_SECRET} \
--from-literal=automationUserJwt=${AUTOMATION_USER_JWT}
kubectl --namespace futureporn delete secret capture --ignore-not-found
kubectl --namespace futureporn create secret generic capture \

View File

@ -70,9 +70,9 @@ kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PAS
IS_TEMPLATE = False;"
## Create PgBoss db (for backend tasks)
## Create graphile_worker db (for backend tasks)
kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
CREATE DATABASE pgboss \
CREATE DATABASE graphile_worker \
WITH \
OWNER = postgres \
ENCODING = 'UTF8' \
@ -97,10 +97,11 @@ kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PAS
## grant futureporn user all privs
kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
GRANT ALL PRIVILEGES ON DATABASE pgboss TO futureporn;"
kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
GRANT ALL PRIVILEGES ON DATABASE postgrest TO futureporn;"
kubectl -n futureporn exec ${postgres_pod_name} -- env PGPASSWORD=${POSTGRES_PASSWORD} psql -U postgres --command "\
GRANT ALL PRIVILEGES ON DATABASE graphile_worker TO futureporn;"
## import schema

View File

@ -7,7 +7,7 @@
"scripts": {
"test": "echo \"Warn: no test specified\" && exit 0",
"start": "node ./dist/index.js",
"dev.nodemon": "nodemon --ext js,ts,json,yaml --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"",
"dev.nodemon": "nodemon --legacy-watch --ext js,ts --watch ./src --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"",
"dev": "tsx --watch ./src/index.ts",
"build": "tsc --build",
"clean": "rm -rf dist",
@ -24,6 +24,7 @@
"graphile-worker": "^0.16.6"
},
"devDependencies": {
"@futureporn/types": "workspace:^",
"nodemon": "^3.1.4",
"ts-node": "^10.9.2",
"tsx": "^4.16.2",

View File

@ -21,12 +21,15 @@ importers:
specifier: ^0.16.6
version: 0.16.6(typescript@5.5.4)
devDependencies:
'@futureporn/types':
specifier: workspace:^
version: link:../../packages/types
nodemon:
specifier: ^3.1.4
version: 3.1.4
ts-node:
specifier: ^10.9.2
version: 10.9.2(@types/node@22.0.0)(typescript@5.5.4)
version: 10.9.2(@types/node@22.1.0)(typescript@5.5.4)
tsx:
specifier: ^4.16.2
version: 4.16.2
@ -306,17 +309,20 @@ packages:
'@types/node@22.0.0':
resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==}
'@types/node@22.1.0':
resolution: {integrity: sha512-AOmuRF0R2/5j1knA3c6G3HOk523Ga+l+ZXltX8SF1+5oqcXijjfTd8fY3XRZqSihEu9XhtQnKYLmkFaoxgsJHw==}
'@types/pg@8.11.6':
resolution: {integrity: sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==}
'@types/semver@7.5.8':
resolution: {integrity: sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==}
'@types/ws@8.5.11':
resolution: {integrity: sha512-4+q7P5h3SpJxaBft0Dzpbr6lmMaqh0Jr2tbhJZ/luAwvD7ohSCniYkwz/pLxuT2h0EOa6QADgJj1Ko+TzRfZ+w==}
'@types/ws@8.5.12':
resolution: {integrity: sha512-3tPRkv1EtkDpzlgyKyI8pGsGZAGPEaXeu0DOj5DI25Ja91bdAYddYHbADRYVrZMRbfW+1l5YwXVDKohDJNQxkQ==}
'@vladfrangu/async_event_emitter@2.4.4':
resolution: {integrity: sha512-ZL62PFXEIeGUI8btfJ5S8Flc286eU1ZUSjwyFQtIGXfRUDPZKO+CDJMYb1R71LjGWRZ4n202O+a6FGjsgTw58g==}
'@vladfrangu/async_event_emitter@2.4.5':
resolution: {integrity: sha512-J7T3gUr3Wz0l7Ni1f9upgBZ7+J22/Q1B7dl0X6fG+fTsD+H+31DIosMHj4Um1dWQwqbcQ3oQf+YS2foYkDc9cQ==}
engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
acorn-walk@8.3.3:
@ -767,6 +773,9 @@ packages:
undici-types@6.11.1:
resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==}
undici-types@6.13.0:
resolution: {integrity: sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg==}
undici@6.13.0:
resolution: {integrity: sha512-Q2rtqmZWrbP8nePMq7mOJIN98M0fYvSgV89vwl/BQRT4mDOeY2GXZngfGpcBBhtky3woM7G24wZV3Q304Bv6cw==}
engines: {node: '>=18.0'}
@ -854,7 +863,7 @@ snapshots:
'@discordjs/util': 1.1.0
'@sapphire/async-queue': 1.5.3
'@sapphire/snowflake': 3.5.3
'@vladfrangu/async_event_emitter': 2.4.4
'@vladfrangu/async_event_emitter': 2.4.5
discord-api-types: 0.37.83
magic-bytes.js: 1.10.0
tslib: 2.6.2
@ -868,8 +877,8 @@ snapshots:
'@discordjs/rest': 2.3.0
'@discordjs/util': 1.1.0
'@sapphire/async-queue': 1.5.3
'@types/ws': 8.5.11
'@vladfrangu/async_event_emitter': 2.4.4
'@types/ws': 8.5.12
'@vladfrangu/async_event_emitter': 2.4.5
discord-api-types: 0.37.83
tslib: 2.6.2
ws: 8.18.0
@ -992,6 +1001,10 @@ snapshots:
dependencies:
undici-types: 6.11.1
'@types/node@22.1.0':
dependencies:
undici-types: 6.13.0
'@types/pg@8.11.6':
dependencies:
'@types/node': 22.0.0
@ -1000,11 +1013,11 @@ snapshots:
'@types/semver@7.5.8': {}
'@types/ws@8.5.11':
'@types/ws@8.5.12':
dependencies:
'@types/node': 22.0.0
'@types/node': 22.1.0
'@vladfrangu/async_event_emitter@2.4.4': {}
'@vladfrangu/async_event_emitter@2.4.5': {}
acorn-walk@8.3.3:
dependencies:
@ -1419,14 +1432,14 @@ snapshots:
ts-mixer@6.0.4: {}
ts-node@10.9.2(@types/node@22.0.0)(typescript@5.5.4):
ts-node@10.9.2(@types/node@22.1.0)(typescript@5.5.4):
dependencies:
'@cspotcode/source-map-support': 0.8.1
'@tsconfig/node10': 1.0.11
'@tsconfig/node12': 1.0.11
'@tsconfig/node14': 1.0.3
'@tsconfig/node16': 1.0.4
'@types/node': 22.0.0
'@types/node': 22.1.0
acorn: 8.12.1
acorn-walk: 8.3.3
arg: 4.1.3
@ -1454,6 +1467,8 @@ snapshots:
undici-types@6.11.1: {}
undici-types@6.13.0: {}
undici@6.13.0: {}
v8-compile-cache-lib@3.0.1: {}

View File

@ -73,7 +73,8 @@ export default {
const message = await idk.fetch()
const discordMessageId = message.id
await workerUtils.addJob('startRecording', { url, discordMessageId })
await workerUtils.addJob('startRecording', { url, discordMessageId }, { maxAttempts: 3 })
},
};

View File

@ -0,0 +1,9 @@
import { Client, Events, type Interaction } from 'discord.js';
export default {
name: Events.ClientReady,
once: true,
execute(client: Client) {
console.log(`Ready! Logged in as ${client?.user?.tag}`);
}
}

View File

@ -0,0 +1,71 @@
import { Events, type Interaction, Client, Collection } from 'discord.js';
import type { WorkerUtils } from 'graphile-worker';
interface ExtendedClient extends Client {
commands: Collection<string, any>
}
export default {
name: Events.InteractionCreate,
once: false,
async execute(interaction: Interaction, workerUtils: WorkerUtils) {
// if (!interaction.isChatInputCommand()) return;
// console.log(interaction.client)
// const command = interaction.client.commands.get(interaction.commandName);
if (interaction.isButton()) {
console.log(`the interaction is a button type with customId=${interaction.customId}, message.id=${interaction.message.id}, user=${interaction.user.id} (${interaction.user.globalName})`)
if (interaction.customId === 'stop') {
interaction.reply('[stop] IDK IDK IDK ??? @todo')
workerUtils.addJob('stopRecording', { discordMessageId: interaction.message.id, userId: interaction.user.id })
} else if (interaction.customId === 'retry') {
interaction.reply('[retry] IDK IDK IDK ??? @todo')
workerUtils.addJob('startRecording', { discordMessageId: interaction.message.id, userId: interaction.user.id })
} else {
console.error(`this button's customId=${interaction.customId} did not match one of the known customIds`)
}
} else if (interaction.isChatInputCommand()) {
console.log(`the interaction is a ChatInputCommandInteraction with commandName=${interaction.commandName}, user=${interaction.user.id} (${interaction.user.globalName})`)
const client = interaction.client as ExtendedClient
const command = client.commands.get(interaction.commandName);
if (!command) {
console.error(`No command matching ${interaction.commandName} was found.`);
return;
}
command.execute({ interaction, workerUtils })
}
},
};
// const { Events } = require('discord.js');
// module.exports = {
// name: Events.ClientReady,
// once: true,
// execute(client) {
// console.log(`Ready! Logged in as ${client.user.tag}`);
// },
// };
// client.on(Events.InteractionCreate, interaction => {
// if (interaction.isChatInputCommand()) {
// const { commandName } = interaction;
// console.log(`Received interaction with commandName=${commandName}`)
// const cmd = commands.find((c) => c.data.name === commandName)
// if (!cmd) {
// console.log(`no command handler matches commandName=${commandName}`)
// return;
// }
// cmd.execute({ interaction, workerUtils })
// } else {
// // probably a ButtonInteraction
// console.log(interaction)
// }
// });

View File

@ -0,0 +1,24 @@
import { Client, Events, MessageReaction, User, type Interaction } from 'discord.js';
export default {
name: Events.MessageReactionAdd,
once: false,
async execute(reaction: MessageReaction, user: User) {
// When a reaction is received, check if the structure is partial
if (reaction.partial) {
// If the message this reaction belongs to was removed, the fetching might result in an API error which should be handled
try {
await reaction.fetch();
} catch (error) {
console.error('Something went wrong when fetching the message:', error);
// Return as `reaction.message.author` may be undefined/null
return;
}
}
// Now the message has been cached and is fully available
console.log(`${reaction.message.author}'s message "${reaction.message.content}" gained a reaction!`);
// The reaction is now also fully available and the properties will be reflected accurately:
console.log(`${reaction.count} user(s) have given the same reaction to this message!`);
}
}

View File

@ -1,9 +1,10 @@
import 'dotenv/config'
import { type ChatInputCommandInteraction, Client, Events, GatewayIntentBits, Partials } from 'discord.js'
import { type ChatInputCommandInteraction, Client, GatewayIntentBits, Partials, Collection } from 'discord.js'
import loadCommands from './loadCommands.js'
import deployCommands from './deployCommands.js'
import discordMessageUpdate from './tasks/discordMessageUpdate.js'
import { makeWorkerUtils, type WorkerUtils } from 'graphile-worker'
import { makeWorkerUtils, type WorkerUtils, type RunnerOptions, run } from 'graphile-worker'
import loadEvents from './loadEvents.js'
export interface ExecuteArguments {
interaction: ChatInputCommandInteraction;
@ -15,9 +16,31 @@ if (!process.env.DISCORD_TOKEN) throw new Error("DISCORD_TOKEN was missing from
if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
if (!process.env.WORKER_CONNECTION_STRING) throw new Error("WORKER_CONNECTION_STRING was missing from env");
const preset: GraphileConfig.Preset = {
worker: {
connectionString: process.env.WORKER_CONNECTION_STRING,
concurrentJobs: 1,
fileExtensions: [".js", ".ts"]
},
};
async function setupGraphileWorker() {
const runnerOptions: RunnerOptions = {
preset,
taskList: {
'discordMessageUpdate': discordMessageUpdate
}
}
const runner = await run(runnerOptions)
if (!runner) throw new Error('failed to initialize graphile worker');
await runner.promise
}
async function setupWorkerUtils() {
const workerUtils = await makeWorkerUtils({
connectionString: process.env.WORKER_CONNECTION_STRING!,
preset
});
await workerUtils.migrate()
return workerUtils
@ -30,7 +53,7 @@ async function setupDiscordBot(commands: any[], workerUtils: WorkerUtils) {
console.log(`Create a new client instance`)
const client = new Client({
let client: any = new Client({
intents: [
GatewayIntentBits.Guilds,
GatewayIntentBits.GuildMessages,
@ -42,67 +65,14 @@ async function setupDiscordBot(commands: any[], workerUtils: WorkerUtils) {
Partials.Reaction,
]
});
client.on(Events.InteractionCreate, interaction => {
if (!interaction.isChatInputCommand()) return;
const { commandName } = interaction;
console.log(`Received interaction with commandName=${commandName}`)
const cmd = commands.find((c) => c.data.name === commandName)
if (!cmd) {
console.log(`no command handler matches commandName=${commandName}`)
return;
}
cmd.execute({ interaction, workerUtils })
});
// When the client is ready, run this code (only once).
// The distinction between `client: Client<boolean>` and `readyClient: Client<true>` is important for TypeScript developers.
// It makes some properties non-nullable.
client.once(Events.ClientReady, readyClient => {
console.log(`Ready! Logged in as ${readyClient.user.tag}`);
// client.channels.cache.get(process.env.DISCORD_CHANNEL_ID).send('testing 123');
// readyClient.channels.fetch(channelId).then(channel => {
// channel.send('generic welcome message!')
// });
// console.log(readyClient.channels)
// const channel = readyClient.channels.cache.get(process.env.DISCORD_CHANNEL_ID);
// channel.send('testing 135');
});
client.on(Events.InteractionCreate, async interaction => {
if (!interaction.isChatInputCommand()) return;
const { commandName } = interaction;
if (commandName === 'react') {
const message = await interaction.reply({ content: 'You can react with Unicode emojis!', fetchReply: true });
message.react('😄');
}
})
client.on(Events.MessageReactionAdd, async (reaction, user) => {
// When a reaction is received, check if the structure is partial
if (reaction.partial) {
// If the message this reaction belongs to was removed, the fetching might result in an API error which should be handled
try {
await reaction.fetch();
} catch (error) {
console.error('Something went wrong when fetching the message:', error);
// Return as `reaction.message.author` may be undefined/null
return;
}
}
// Now the message has been cached and is fully available
console.log(`${reaction.message.author}'s message "${reaction.message.content}" gained a reaction!`);
// The reaction is now also fully available and the properties will be reflected accurately:
console.log(`${reaction.count} user(s) have given the same reaction to this message!`);
});
client.commands = new Collection();
commands.forEach((c) => client.commands.set(c.data.name, c))
// Log in to Discord with your client's token
client.login(process.env.DISCORD_TOKEN);
await loadEvents(client, workerUtils)
}
@ -112,8 +82,9 @@ async function main() {
if (!commands) throw new Error('there were no commands available to be loaded.');
await deployCommands(commands.map((c) => c.data.toJSON()))
console.log(`${commands.length} commands deployed: ${commands.map((c) => c.data.name).join(', ')}`)
const workerUtils = await setupGraphileWorker()
const workerUtils = await setupWorkerUtils()
setupDiscordBot(commands, workerUtils)
setupGraphileWorker()
}
main().catch((e) => {

View File

@ -14,13 +14,13 @@ export default async function loadCommands(): Promise<any[]> {
for (const folder of commandFolders) {
const commandsPath = path.join(foldersPath, folder);
const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('.ts') || file.endsWith('.js'));
console.log(`commandFiles=${commandFiles}`)
console.log(`commandFiles=${commandFiles}`);
// console.log(`Grab the SlashCommandBuilder#toJSON() output of each command's data for deployment`)
for (const file of commandFiles) {
const filePath = path.join(commandsPath, file);
const command = (await import(filePath)).default;
// console.log(command)
if ('data' in command && 'execute' in command) {
if (command?.data && command?.execute) {
commands.push(command);
} else {
console.log(`[WARNING] The command at ${filePath} is missing a required "data" or "execute" property.`);

View File

@ -0,0 +1,23 @@
import * as path from 'node:path';
import * as fs from 'node:fs';
import { dirname } from 'node:path';
import { fileURLToPath } from 'url';
import type { Client } from 'discord.js';
import type { WorkerUtils } from 'graphile-worker';
const __dirname = dirname(fileURLToPath(import.meta.url));
export default async function loadEvents(client: Client, workerUtils: WorkerUtils) {
console.log(`loading events`);
const eventsPath = path.join(__dirname, 'events');
const eventFiles = fs.readdirSync(eventsPath).filter(file => file.endsWith('.ts') || file.endsWith('.js'));
console.log(`eventFiles=${eventFiles}`);
for (const file of eventFiles) {
const filePath = path.join(eventsPath, file);
const event = (await import(filePath)).default;
if (event.once) {
client.once(event.name, (...args) => event.execute(...args, workerUtils));
} else {
client.on(event.name, (...args) => event.execute(...args, workerUtils));
}
}
}

View File

@ -1,106 +0,0 @@
import 'dotenv/config'
import { type Task, type WorkerUtils } from 'graphile-worker';
import { type ChatInputCommandInteraction, Client, Events, GatewayIntentBits, Partials, Message, TextChannel } from 'discord.js'
// export interface DiscordMessageUpdateJob extends Job {
// data: {
// captureJobId: string;
// }
// }
interface Payload {
discord_message_id: string;
capture_job_id: string;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.discord_message_id !== "string") throw new Error("invalid discord_message_id");
if (typeof payload.capture_job_id.to !== "string") throw new Error("invalid capture_job_id");
}
if (!process.env.AUTOMATION_USER_JWT) throw new Error(`AUTOMATION_USER_JWT was missing from env`);
if (!process.env.DISCORD_TOKEN) throw new Error("DISCORD_TOKEN was missing from env");
if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
if (!process.env.DISCORD_GUILD_ID) throw new Error("DISCORD_GUILD_ID was missing from env");
/**
* discordMessageUpdate is the task where we edit a previously sent discord message to display new status information sent to us from @futureporn/capture
*
* Sometimes the update is changing the state, one of Pending|Recording|Aborted|Ended.
* Sometimes the update is updating the Filesize of the recording in-progress
* Sometimes the update is adding a thumbnail image to the message
*/
export default async function discordMessageUpdate <Task> (payload: Payload) {
assertPayload(payload)
// const { captureJobId } = job.data
console.log(`discordMessageUpdate job has begun with captureJobId=${payload.capture_job_id}`)
// // find the discord_interactions record containing the captureJobId
// const res = await fetch(`http://postgrest.futureporn.svc.cluster.local:9000/discord_interactions?capture_job_id=eq.${captureJobId}`)
// if (!res.ok) throw new Error('failed to fetch the discord_interactions');
// const body = await res.json() as DiscordInteraction
// console.log('discord_interactions as follows')
// console.log(body)
// // create a discord.js client
// const client = new Client({
// intents: [
// GatewayIntentBits.Guilds,
// GatewayIntentBits.GuildMessages
// ],
// partials: [
// Partials.Message,
// Partials.Channel,
// ]
// });
// // const messageManager = client.
// // const guild = client.guilds.cache.get(process.env.DISCORD_GUILD_ID!);
// // if (!guild) throw new Error('guild was undefined')
// const channel = await client.channels.fetch(process.env.DISCORD_CHANNEL_ID!) as TextChannel
// if (!channel) throw new Error(`discord channel was undefined`);
// // console.log('we got the following channel')
// // console.log(channel)
// const message = await channel.messages.fetch(body.discord_message_id)
// console.log('we got the following message')
// console.log(message)
// get the message
// client.channel.messages.get()
// client.rest.
// const message: Message = {
// id: body.discord_message_id
// };
// await message.fetchReference()
// message.edit('test message update payload thingy')
// client.rest.updateMessage(message, "My new content");
// TextChannel.fetchMessage(msgId).then(console.log);
// TextChannel
// channel.messages.fetch(`Your Message ID`).then(message => {
// message.edit("New message Text");
// }).catch(err => {
// console.error(err);
// });
// using the discord_interaction's discord_message_id, use discord.js to update the discord message.
}

View File

@ -0,0 +1,191 @@
import 'dotenv/config'
import type { RecordingState } from '@futureporn/types'
import { type Task, type Helpers } from 'graphile-worker';
import {
Client,
GatewayIntentBits,
TextChannel,
ActionRowBuilder,
ButtonBuilder,
type MessageActionRowComponentBuilder,
ButtonStyle,
EmbedBuilder,
Guild
} from 'discord.js';
// export interface DiscordMessageUpdateJob extends Job {
// data: {
// captureJobId: string;
// }
// }
interface Payload {
recordId: number;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (!payload.recordId) throw new Error(`recordId was absent in the payload`);
}
if (!process.env.AUTOMATION_USER_JWT) throw new Error(`AUTOMATION_USER_JWT was missing from env`);
if (!process.env.DISCORD_TOKEN) throw new Error("DISCORD_TOKEN was missing from env");
if (!process.env.DISCORD_CHANNEL_ID) throw new Error("DISCORD_CHANNEL_ID was missing from env");
if (!process.env.DISCORD_GUILD_ID) throw new Error("DISCORD_GUILD_ID was missing from env");
async function editDiscordMessage({ helpers, state, discordMessageId }: { helpers: Helpers, state: RecordingState, discordMessageId: string }) {
// const { captureJobId } = job.data
helpers.logger.info(`discordMessageUpdate job has begun with discordMessageId=${discordMessageId}, state=${state}`)
// create a discord.js client
const client = new Client({
intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildMessages],
});
// Log in to Discord with your client's token
client.login(process.env.DISCORD_TOKEN);
const guild = await client.guilds.fetch(process.env.DISCORD_GUILD_ID!) as Guild
if (!guild) throw new Error('guild was undefined');
helpers.logger.info('here is the guild as follows')
helpers.logger.info(guild.toString())
helpers.logger.info(`fetching discord channel id=${process.env.DISCORD_CHANNEL_ID} from discord guild`)
const channel = await client.channels.fetch(process.env.DISCORD_CHANNEL_ID!) as TextChannel
if (!channel) throw new Error(`discord channel was undefined`);
const message = await channel.messages.fetch(discordMessageId)
helpers.logger.info(`the following is the message taht we have fetched`)
helpers.logger.info(message.toString())
const statusEmbed = getStatusEmbed(state)
const buttonRow = getButtonRow(state)
// const embed = new EmbedBuilder().setTitle('Attachments');
const updatedMessage = {
embeds: [
statusEmbed
],
components: [
buttonRow
]
};
message.edit(updatedMessage)
}
async function getRecordFromDatabase(recordId: number) {
const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`)
if (!res.ok) {
throw new Error(`failed fetching record ${recordId}. status=${res.status}, statusText=${res.statusText}`)
}
const body = await res.json() as any
return body[0];
}
/**
* updateDiscordMessage is the task where we edit a previously sent discord message to display
* the most up-to-date status information from the database
*
* Sometimes the update is changing the state, one of Pending|Recording|Aborted|Ended.
* Sometimes the update is updating the Filesize of the recording in-progress
* Sometimes the update is adding a thumbnail image to the message
*/
export const updateDiscordMessage: Task = async function (payload, helpers: Helpers) {
try {
assertPayload(payload)
const record = await getRecordFromDatabase(payload.recordId)
const { discordMessageId, state } = record
editDiscordMessage({ helpers, state, discordMessageId })
} catch (e) {
helpers.logger.error(`caught an error during updateDiscordMessage. e=${e}`)
}
}
function getStatusEmbed(state: RecordingState) {
let title, description, color;
if (state === 'pending') {
title = "Pending"
description = "Waiting for a worker to accept the job."
color = 2326507
} else if (state === 'recording') {
title = 'Recording'
description = 'The stream is being recorded.'
color = 392960
} else if (state === 'aborted') {
title = "Aborted"
description = "The recording was stopped by the user."
color = 8289651
} else if (state === 'ended') {
title = "Ended"
description = "The recording has stopped."
color = 10855845
} else {
title = 'Unknown'
description = 'The recording is in an unknown state? (this is a bug.)'
color = 10855845
}
return new EmbedBuilder().setTitle(title).setDescription(description).setColor(color)
}
function getButtonRow(state: RecordingState) {
let id, label, emoji, style;
if (state === 'pending') {
id = 'stop';
label = 'Cancel';
emoji = '❌';
style = ButtonStyle.Danger;
} else if (state === 'recording') {
id = 'stop';
label = 'Stop Recording';
emoji = '🛑';
style = ButtonStyle.Danger;
} else if (state === 'aborted') {
id = 'retry';
label = 'Retry Recording';
emoji = '🔄';
style = ButtonStyle.Success;
} else if (state === 'ended') {
id = 'download';
label = 'Download Recording';
emoji = '📥';
style = ButtonStyle.Primary;
} else {
id = 'unknown';
label = 'Unknown State';
emoji = '🤔';
style = ButtonStyle.Secondary;
}
return new ActionRowBuilder<MessageActionRowComponentBuilder>()
.addComponents([
new ButtonBuilder()
.setCustomId(id)
.setLabel(label)
.setEmoji(emoji)
.setStyle(style),
]);
}
export default updateDiscordMessage

View File

@ -21,6 +21,7 @@
"@aws-sdk/lib-storage": "^3.588.0",
"@aws-sdk/types": "^3.609.0",
"@futureporn/scout": "workspace:^",
"@futureporn/types": "workspace:^",
"@futureporn/utils": "workspace:^",
"@paralleldrive/cuid2": "^2.2.2",
"@types/chai": "^4.3.16",

View File

@ -20,6 +20,9 @@ importers:
'@futureporn/scout':
specifier: workspace:^
version: link:../../packages/scout
'@futureporn/types':
specifier: workspace:^
version: link:../../packages/types
'@futureporn/utils':
specifier: workspace:^
version: link:../../packages/utils

View File

@ -13,6 +13,10 @@ interface RecordBodyType {
url: string;
discordMessageId: string;
}
interface MessageBodyType {
state: 'pending' | 'recording' | 'aborted' | 'ended';
discordMessageId: string;
}
const build = function (opts: Record<string, any>={}, connectionString: string) {
const app: ExtendedFastifyInstance = fastify(opts)
@ -21,6 +25,17 @@ const build = function (opts: Record<string, any>={}, connectionString: string)
app.get('/', async function (request, reply) {
return { app: '@futureporn/capture', version }
})
app.put('/api/message', async function (request: FastifyRequest<{ Body: MessageBodyType }>, reply) {
const { state, discordMessageId } = request.body
if (app?.graphile) {
const jobId = await app.graphile.addJob('discordMessageUpdate', {
discordMessageId,
state
}, { maxAttempts: 3 })
} else {
console.error('app.graphile was missing')
}
})
app.post('/api/record', async function (request: FastifyRequest<{ Body: RecordBodyType }>, reply) {
const { url, discordMessageId } = request.body
console.log(`POST /api/record with url=${url}`)
@ -29,7 +44,7 @@ const build = function (opts: Record<string, any>={}, connectionString: string)
const jobId = await app.graphile.addJob('startRecording', {
url,
discordMessageId
})
}, { maxAttempts: 3 })
return { jobId }
} else {
console.error(`app.graphile was missing! Is the graphile worker plugin registered to the fastify instance?`)

View File

@ -2,7 +2,7 @@
import { Helpers, type Task } from 'graphile-worker'
import Record from '../Record.ts'
import { getPlaylistUrl } from '@futureporn/scout/ytdlp.ts'
import type { RecordingState } from '@futureporn/types'
/**
* url is the URL to be recorded. Ex: chaturbate.com/projektmelody
@ -22,7 +22,7 @@ interface RecordingRecord {
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.url !== "string") throw new Error("invalid url");
if (typeof payload.recordId !== "number") throw new Error("invalid recordId");
if (typeof payload.recordId !== "number") throw new Error(`invalid recordId=${payload.recordId}`);
}
function assertEnv() {
@ -44,13 +44,12 @@ async function getRecording(url: string, recordId: number, abortSignal: AbortSig
const s3Client = Record.makeS3Client({ accessKeyId, secretAccessKey, region, endpoint })
const inputStream = Record.getFFmpegStream({ url: playlistUrl })
const record = new Record({ inputStream, bucket, s3Client, jobId: ''+recordId, abortSignal })
record.start()
const record = new Record({ inputStream, bucket, s3Client, jobId: ''+recordId }) // @todo add abortsignal
return record
}
async function checkIfAborted(recordId: number): Promise<boolean> {
const res = await fetch(`${process.env.POSTGREST_URL}/records?id.eq=${recordId}`, {
const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`, {
headers: {
'Content-Type': 'application/json',
'Accepts': 'application/json'
@ -64,8 +63,26 @@ async function checkIfAborted(recordId: number): Promise<boolean> {
return body[0].isAborted
}
async function updateDatabaseRecord({recordId, state, filesize}: { recordId: number, state: RecordingState, filesize: number }): Promise<void> {
const res = await fetch(`${process.env.POSTGREST_URL}/records?id=eq.${recordId}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json',
'Accepts': 'application/json',
'Prefer': 'return=representation'
},
body: JSON.stringify({ state, filesize })
})
if (!res.ok) {
throw new Error(`failed to checkIfAborted. status=${res.status}, statusText=${res.statusText}`);
}
const body = await res.json() as RecordingRecord[];
if (!body[0]) throw new Error(`failed to get a record that matched recordId=${recordId}`)
return body[0].isAborted
}
export const record: Task = async function (payload, helpers) {
console.log(payload)
assertPayload(payload)
assertEnv()
const { url, recordId } = payload
@ -73,27 +90,39 @@ export const record: Task = async function (payload, helpers) {
let interval
try {
const record = await getRecording(url, recordId, abortController.signal)
// every 30s, poll db to see if our job has been aborted by the user
// every 30s, we
// 1. poll db to see if our job has been aborted by the user
// 2. update the db record with the RecordingState and filesize
interval = setInterval(async () => {
try {
helpers.logger.info(`checkIfAborted()`)
const isAborted = await checkIfAborted(recordId)
if (isAborted) {
abortController.abort()
}
let state: RecordingState = 'recording'
} catch (e) {
helpers.logger.error(`error while checking if this job was aborted. For sake of the recording in progress we are ignoring the following error. ${e}`)
}
}, 30000)
// start recording and await the S3 upload being finished
await record.start()
} finally {
clearInterval(interval)
}
// const recordId = await createRecordingRecord(payload, helpers)
// const { url } = payload;
// console.log(`@todo simulated start_recording with url=${url}, recordId=${recordId}`)
// await helpers.addJob('record', { url, recordId })
}
// // export default record
// export default function (payload: Payload, helpers: Helpers) {
// helpers.logger.info('WHEEEEEEEEEEEEEEEE (record.ts task executor)')
// }
export default record

View File

@ -15,7 +15,7 @@ interface Payload {
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.url !== "string") throw new Error("invalid url");
if (typeof payload.discordMessageId !== "string") throw new Error("invalid discordMessageId");
if (typeof payload.discordMessageId !== "string") throw new Error(`invalid discordMessageId=${payload.discordMessageId}`);
}
function assertEnv() {
@ -26,8 +26,8 @@ async function createRecordingRecord(payload: Payload, helpers: Helpers): Promis
const { url, discordMessageId } = payload
const record = {
url,
discordMessageId,
isAborted: false
discord_message_id: discordMessageId,
is_aborted: false
}
const res = await fetch('http://postgrest.futureporn.svc.cluster.local:9000/records', {
method: 'POST',
@ -43,9 +43,11 @@ async function createRecordingRecord(payload: Payload, helpers: Helpers): Promis
const statusText = res.statusText
throw new Error(`fetch failed to create recording record in database. status=${status}, statusText=${statusText}`)
}
helpers.logger.info('res.headers as follows.')
helpers.logger.info(res.headers)
return res.headers.Location.split('.').at(-1)
helpers.logger.info('res.headers.location as follows.')
helpers.logger.info(res.headers.get('location')!)
const id = res.headers.get('location')?.split('.').at(-1)
if (!id) throw new Error('id could not be parsed from location header');
return parseInt(id)
}
export const startRecording: Task = async function (payload, helpers) {
@ -54,7 +56,7 @@ export const startRecording: Task = async function (payload, helpers) {
const recordId = await createRecordingRecord(payload, helpers)
const { url } = payload;
console.log(`@todo simulated start_recording with url=${url}, recordId=${recordId}`)
await helpers.addJob('record', { url, recordId })
await helpers.addJob('record', { url, recordId }, { maxAttempts: 3 })
}
export default startRecording

View File

@ -27,7 +27,7 @@ async function handleMessage({ workerUtils, email, msg }: { workerUtils: WorkerU
if (isMatch) {
console.log(' ✏️✏️ adding process_notif_email job to queue')
workerUtils.addJob('process_notif_email', { isMatch, url, platform, channel, displayName, date, userId, avatar })
workerUtils.addJob('process_notif_email', { isMatch, url, platform, channel, displayName, date, userId, avatar }, { maxAttempts: 3 })
}
console.log(' ✏️ archiving e-mail')
@ -50,8 +50,8 @@ async function main() {
})
// demonstrate that we are connected @todo remove this
workerUtils.addJob('hello', { name: 'worker' })
workerUtils.addJob('identify_image_color', { url: 'https://futureporn-b2.b-cdn.net/ti8ht9bgwj6k783j7hglfg8j_projektmelody-chaturbate-2024-07-18.png' })
workerUtils.addJob('hello', { name: 'worker' }, { maxAttempts: 3 })
workerUtils.addJob('identify_image_color', { url: 'https://futureporn-b2.b-cdn.net/ti8ht9bgwj6k783j7hglfg8j_projektmelody-chaturbate-2024-07-18.png' }, { maxAttempts: 3 })
// connect to IMAP inbox and wait for new e-mails
const email = new Email()

View File

@ -3,7 +3,9 @@ CREATE TABLE api.records (
id INT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
url TEXT NOT NULL,
discord_message_id TEXT NOT NULL,
is_aborted BOOLEAN DEFAULT FALSE
is_aborted BOOLEAN DEFAULT FALSE,
recording_state TEXT NOT NULL,
file_size BIGINT NOT NULL
);
-- roles & permissions for our backend automation user

View File

@ -0,0 +1,9 @@
ALTER TABLE api.records
ADD COLUMN recording_state TEXT NOT NULL DEFAULT 'pending' CHECK(recording_state IN ('pending', 'recording', 'aborted', 'ended')),
ADD COLUMN file_size BIGINT NOT NULL DEFAULT 0;
-- Grant permissions to the new columns for our backend automation user
GRANT UPDATE (recording_state, file_size) ON api.records TO automation;
-- Grant read-only access to the new columns for our web anon user
GRANT SELECT (recording_state, file_size) ON api.records TO web_anon;

View File

@ -1,7 +1,7 @@
{
"name": "@futureporn/migrations",
"type": "module",
"version": "0.2.1",
"version": "0.3.0",
"description": "",
"main": "index.js",
"scripts": {