From e810fd3322aea6d9ce1449ee1f130ec2adc718ba Mon Sep 17 00:00:00 2001 From: CJ_Clippy Date: Wed, 13 Aug 2025 04:31:07 -0800 Subject: [PATCH] add b2-cli --- services/our/Dockerfile | 3 + services/our/src/app.ts | 3 +- services/our/src/plugins/admin.ts | 162 ++++++++++++++---- services/our/src/plugins/vods.ts | 10 +- services/our/src/tasks/copyV1S3ToV2.ts | 133 ++++++++++++++ services/our/src/tasks/findWork.ts | 5 +- .../our/src/tasks/scheduleVodProcessing.ts | 1 + services/our/src/views/admin.hbs | 79 ++++++++- services/our/src/views/partials/navbar.hbs | 5 + services/our/src/views/profile.hbs | 10 ++ services/our/src/views/stream.hbs | 15 +- services/our/src/views/uploads/show.hbs | 116 ++++++++----- services/our/src/views/vod.hbs | 2 +- services/our/src/views/vods.hbs | 2 +- 14 files changed, 457 insertions(+), 89 deletions(-) create mode 100644 services/our/src/tasks/copyV1S3ToV2.ts diff --git a/services/our/Dockerfile b/services/our/Dockerfile index 733a569..36e2705 100644 --- a/services/our/Dockerfile +++ b/services/our/Dockerfile @@ -36,6 +36,9 @@ COPY --from=ghcr.io/ggml-org/whisper.cpp:main-e7bf0294ec9099b5fc21f5ba969805dfb2 ENV PATH="$PATH:/app/whisper.cpp/build/bin" ENV LD_LIBRARY_PATH="/app/whisper.cpp/build/src:/app/whisper.cpp/build/ggml/src:/usr/local/lib:/usr/lib" +# Install b2-cli +RUN wget https://github.com/Backblaze/B2_Command_Line_Tool/releases/download/v4.4.1/b2-linux -o /usr/local/bin/b2 && chmod +x /usr/local/bin/b2 + # Copy and install dependencies COPY package.json package-lock.json ./ RUN npm install --ignore-scripts=false --foreground-scripts --verbose diff --git a/services/our/src/app.ts b/services/our/src/app.ts index d4cb06a..43675d9 100644 --- a/services/our/src/app.ts +++ b/services/our/src/app.ts @@ -6,6 +6,7 @@ import vtubersRoutes from './plugins/vtubers' import usersRoutes from './plugins/users' import indexRoutes from './plugins/index' import streamsRoutes from './plugins/streams' +import adminRoutes from './plugins/admin' import hls from './plugins/hls.ts' import fastifyStatic from '@fastify/static' import fastifySecureSession from '@fastify/secure-session' @@ -32,7 +33,6 @@ import { icons } from './utils/icons.ts' import logger from './utils/logger.ts' import fastifyCaching from '@fastify/caching' - export function buildApp() { const app = Fastify() @@ -197,6 +197,7 @@ export function buildApp() { app.register(uploadsRoutes) app.register(usersRoutes) app.register(indexRoutes) + app.register(adminRoutes) app.register(authRoutes) return app diff --git a/services/our/src/plugins/admin.ts b/services/our/src/plugins/admin.ts index 66a55cf..973a8ba 100644 --- a/services/our/src/plugins/admin.ts +++ b/services/our/src/plugins/admin.ts @@ -1,41 +1,141 @@ -import { PrismaClient } from '../../generated/prisma' +import { PrismaClient, Prisma } from '../../generated/prisma' import { withAccelerate } from "@prisma/extension-accelerate" import { type FastifyInstance, type FastifyReply, type FastifyRequest } from 'fastify' import { env } from '../config/env' import { constants } from '../config/constants' -import { isModerator, isAdmin } from '../utils/privs' -import type { pool } +import { isAdmin } from '../utils/privs' +import { makeWorkerUtils, WorkerUtils } from 'graphile-worker' +import logger from '../utils/logger' +import { } from '../../generated/prisma' -// const prisma = new PrismaClient().$extends(withAccelerate()) +const prisma = new PrismaClient().$extends(withAccelerate()) + +interface JobRow { + id: number; + queue_name: string | null; + task_identifier: string; + priority: number; + run_at: Date; + attempts: number; + max_attempts: number; + last_error: string | null; + created_at: Date; + updated_at: Date; + key: string | null; + locked_at: Date | null; + locked_by: string | null; + revision: number; + flags: string | null; +} + +async function getJobs() { + const jobs = await prisma.$queryRaw( + Prisma.sql` + SELECT + id, + queue_name, + task_identifier, + priority, + run_at, + attempts, + max_attempts, + last_error, + created_at, + updated_at, + key, + locked_at, + locked_by, + revision, + flags + FROM graphile_worker.jobs + ORDER BY id DESC + LIMIT 2500 + ` + ); + + return jobs; +} -async function indexRoutes(fastify: FastifyInstance): Promise { - fastify.delete('/admin/queue', async function (request, reply) { - // const userId = request.session.get('userId') - // const user = await prisma.user.findUnique({ - // where: { id: userId }, - // include: { - // roles: true - // } - // }) - // if (isAdmin(user)) { - // // @todo delete the queue +export default async function adminRoutes(fastify: FastifyInstance): Promise { + + fastify.get('/admin', async function (request, reply) { + const userId = request.session.get('userId'); + if (!userId) { + throw new Error('No user is logged in.'); + } + + const user = await prisma.user.findUnique({ + where: { id: userId }, + include: { roles: true } + }); + if (!user) throw new Error('User not found.'); + if (!isAdmin(user)) { + return reply.status(403).send("you are not a jedi, yet"); + } else { + + const jobs = await getJobs(); + logger.debug(jobs[0]) + + return reply.viewAsync("admin.hbs", { + jobs, + user, + site: constants.site, + }, { layout: 'layouts/main.hbs' }); + } + }) + + fastify.delete('/admin/jobs', async function (request, reply) { - // // const pool = new Pool({ - // // connectionString: process.env.DATABASE_URL, - // // }); - - // // const workerUtils = await WorkerUtils.make({ pgPool: pool }); - - // // await permanentlyFailAllJobs(); - // // await workerUtils.release(); - // // await pool.end(); - - // } - // return reply.viewAsync("admin.hbs", { - // user, - // site: constants.site - // }); + const userId = request.session.get('userId') + const user = await prisma.user.findUnique({ + where: { id: userId }, + include: { + roles: true + } }) - } + if (!user) throw new Error('User not found.'); + + if (isAdmin(user)) { + + let workerUtils: WorkerUtils | undefined; + try { + + const workerUtils = await makeWorkerUtils({ connectionString: env.DATABASE_URL }); + + + + const jobIdsResult = await prisma.$queryRaw<{ id: number }[]>( + Prisma.sql`SELECT id FROM graphile_worker.jobs` + ); + + + + + const jobIds: string[] = jobIdsResult.map(j => '' + j.id); + logger.warn(`Permanently failing all graphile-worker jobs on behalf of ${user.id}.`); + + if (jobIds.length > 0) { + await workerUtils.permanentlyFailJobs(jobIds, 'Job deleted by admin'); + } + + + + } + finally { + if (workerUtils) { + await workerUtils.release(); + } + } + + + } + return reply.viewAsync("admin.hbs", { + user, + site: constants.site, + notification: 'Jobs beleted!', + + }); + }) +} diff --git a/services/our/src/plugins/vods.ts b/services/our/src/plugins/vods.ts index 7dc8d9b..cfff124 100644 --- a/services/our/src/plugins/vods.ts +++ b/services/our/src/plugins/vods.ts @@ -57,9 +57,13 @@ export default async function vodsRoutes( const vods = await prisma.vod.findMany({ where: { - status: { - in: ['approved', 'processed', 'processing'], - }, + AND: [ + { + status: { + in: ['approved', 'processed', 'processing'], + } + } + ] }, orderBy: { createdAt: 'desc' }, include: { diff --git a/services/our/src/tasks/copyV1S3ToV2.ts b/services/our/src/tasks/copyV1S3ToV2.ts new file mode 100644 index 0000000..8d2707a --- /dev/null +++ b/services/our/src/tasks/copyV1S3ToV2.ts @@ -0,0 +1,133 @@ +// Copy futureporn.net s3 asset to future.porn s3 bucket. +// ex: https://futureporn-b2.b-cdn.net/(...) -> https://fp-usc.b-cdn.net/(...) + +import logger from "../utils/logger"; +import { Task } from "graphile-worker"; +import { PrismaClient } from "../../generated/prisma"; +import { withAccelerate } from "@prisma/extension-accelerate"; +import { generateS3Path } from '../utils/formatters'; +import { getOrDownloadAsset } from "../utils/cache"; +import { getNanoSpawn } from "../utils/nanoSpawn"; +import { env } from "../config/env"; +import type { default as NanoSpawn } from 'nano-spawn'; + +const prisma = new PrismaClient().$extends(withAccelerate()); + +interface Payload { + vodId: string; +} + +function assertPayload(payload: any): asserts payload is Payload { + if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object."); + if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId"); +} + +function isV1Url(url: string): Boolean { + return url.includes('futureporn-b2.b-cdn.net'); +} + + +/** + * Copy a file from V1 bucket to V2 bucket + * @param v1Url The V1 file URL (b2://futureporn/...) + * @param v2Key Desired key in V2 bucket + * @param contentType Optional override for content type + */ +async function copyFromBucketToBucket(spawn: typeof NanoSpawn, v1Url: string, v2Key: string, contentType?: string) { + // Get file info from V1 + logger.info(`v1Url=${v1Url}`); + const infoResult = await spawn('b2', ['file', 'info', v1Url]); + + const infoJson = JSON.parse(infoResult.stdout); + + const v2Url = `b2://${process.env.S3_BUCKET}/${v2Key}`; + + logger.info(`Copying ${v1Url} to ${v2Url}...`); + + // Copy file by ID + const copyResult = await spawn('b2', [ + 'file', 'copy-by-id', + infoJson.fileId, + process.env.S3_BUCKET, + v2Key, + ]); + + logger.info('Copying complete.'); + + + return v2Url; +} + +// example v1 https://futureporn-b2.b-cdn.net/projektmelody-chaturbate-2023-01-01.mp4 +// example v2 https://fp-usc.b-cdn.net/projektmelody-chaturbate-2023-01-01.mp4 +const copyV1S3ToV2: Task = async (payload: any) => { + + const spawn = await getNanoSpawn(); + + assertPayload(payload) + const { vodId } = payload + const vod = await prisma.vod.findFirstOrThrow({ + where: { + id: vodId, + AND: [ + { thumbnail: { not: '' } }, + { thumbnail: { not: null } } + ] + }, + select: { + thumbnail: true, + sourceVideo: true, + streamDate: true, + vtubers: true, + } + }) + + + + // find what we need to copy over. + // potentially vod.thumbnail and vod.sourceVideo + + + + + const thumbnail = vod.thumbnail; + const sourceVideo = vod.sourceVideo; + + + if (!thumbnail && !sourceVideo) { + logger.info(`thumbnail and sourceVideo for ${vodId} are missing. nothing to do.`); + return; + } + + const slug = vod.vtubers[0].slug + if (!slug) throw new Error(`vtuber ${vod.vtubers[0].id} is missing a slug`); + + + let v2ThumbnailKey: string | undefined = undefined; + let v2SourceVideoKey: string | undefined = undefined; + + if (thumbnail && isV1Url(thumbnail)) { + v2ThumbnailKey = generateS3Path(slug, vod.streamDate, vodId, 'thumbnail.png'); + await copyFromBucketToBucket(spawn, thumbnail.replace('https://futureporn-b2.b-cdn.net', 'b2://futureporn'), v2ThumbnailKey, 'application/png'); + } + + if (sourceVideo && isV1Url(sourceVideo)) { + v2SourceVideoKey = generateS3Path(slug, vod.streamDate, vodId, 'source.mp4'); + await copyFromBucketToBucket(spawn, sourceVideo.replace('https://futureporn-b2.b-cdn.net', 'b2://futureporn'), v2SourceVideoKey, 'video/mp4'); + } + + + + logger.debug(`updating vod record`); + await prisma.vod.update({ + where: { id: vodId }, + data: { + ...(v2ThumbnailKey ? { thumbnail: v2ThumbnailKey } : {}), // Variable 'v2ThumbnailKey' is used before being assigned.ts(2454) + + ...(v2SourceVideoKey ? { sourceVideo: v2SourceVideoKey } : {}), + } + }); +} + + +export default copyV1S3ToV2; diff --git a/services/our/src/tasks/findWork.ts b/services/our/src/tasks/findWork.ts index e6b176d..bf24c9e 100644 --- a/services/our/src/tasks/findWork.ts +++ b/services/our/src/tasks/findWork.ts @@ -9,7 +9,10 @@ const findWork: Task = async (_payload, helpers: Helpers) => { const approvedUploads = await prisma.vod.findMany({ where: { - status: "approved", + OR: [ + { status: "approved" }, + { status: "pending" }, + ] }, }); diff --git a/services/our/src/tasks/scheduleVodProcessing.ts b/services/our/src/tasks/scheduleVodProcessing.ts index b65b29e..6aebb4c 100644 --- a/services/our/src/tasks/scheduleVodProcessing.ts +++ b/services/our/src/tasks/scheduleVodProcessing.ts @@ -47,6 +47,7 @@ const scheduleVodProcessing: Task = async (payload: unknown, helpers) => { if (!vod.asrVttKey) jobs.push(helpers.addJob("createTranscription", { vodId })); if (!vod.slvttVTTKey) jobs.push(helpers.addJob("createStoryboard", { vodId })); if (!vod.magnetLink) jobs.push(helpers.addJob("createTorrent", { vodId })); + if (vod.thumbnail && vod.sourceVideo) jobs.push(helpers.addJob("copyV1S3ToV2", { vodId })); const changes = jobs.length; if (changes > 0) { diff --git a/services/our/src/views/admin.hbs b/services/our/src/views/admin.hbs index d8df832..b6751aa 100644 --- a/services/our/src/views/admin.hbs +++ b/services/our/src/views/admin.hbs @@ -1,7 +1,78 @@ -
-

Admin page

+
+ {{> navbar}} +
+
+
+

Admin page

- +
+ {{#if notification}} +
{{notification}}
+ {{/if}} +
-
\ No newline at end of file +
+
+

Graphile Worker Jobs

+

+ {{#if jobs.length}} + + + + + + + + + + + + + + + + + + + + + + {{#each jobs}} + + + + + + + + + + + + + + + + + + {{/each}} + +
IdQueueTaskPriorityRun AtAttemptsMax AttemptsLast ErrorCreatedUpdatedKeyLocked AtLocked ByRevisionFlags
{{this.id}}{{this.queue_name}}{{this.task_identifier}}{{this.priority}}{{this.run_at}}{{this.attempts}}{{this.max_attempts}}{{this.last_error}}{{this.created_at}}{{this.updated_at}}{{this.key}}{{this.locked_at}}{{this.locked_by}}{{this.revision}}{{this.flags}}
+ {{else}} +

No jobs found.

+ {{/if}} +

+

+ +

+

+ +

+
+
+
+ \ No newline at end of file diff --git a/services/our/src/views/partials/navbar.hbs b/services/our/src/views/partials/navbar.hbs index d3c4a70..a57d2ad 100644 --- a/services/our/src/views/partials/navbar.hbs +++ b/services/our/src/views/partials/navbar.hbs @@ -50,6 +50,11 @@ {{else}} Log in via Patreon {{/if}} + + + {{#if (hasRole "admin" user)}} + Admin + {{/if}} \ No newline at end of file diff --git a/services/our/src/views/profile.hbs b/services/our/src/views/profile.hbs index 5ee0644..5c7c228 100644 --- a/services/our/src/views/profile.hbs +++ b/services/our/src/views/profile.hbs @@ -20,6 +20,16 @@

--}} + + {{#if (hasRole "admin" user)}} +
+
+

Admin section

+ Admin page +
+
+ {{/if}} + Logout {{>footer}} diff --git a/services/our/src/views/stream.hbs b/services/our/src/views/stream.hbs index 1f8d839..7bc3438 100644 --- a/services/our/src/views/stream.hbs +++ b/services/our/src/views/stream.hbs @@ -1,22 +1,21 @@ {{#> main}} - -
+ +
{{> navbar}}
- - +
-

+

{{#each vod.vtubers}} {{this.displayName}}{{#unless @last}}, {{/unless}} {{/each}} – {{formatDate vod.stream.date}}

-

Details

+

Details

{{!-- - vtuber - datetime (formatted) @@ -24,11 +23,11 @@ - announcementUrl - platforms --}} -

Vods

+

Vods

- +
diff --git a/services/our/src/views/uploads/show.hbs b/services/our/src/views/uploads/show.hbs index 2bea886..3bd31b1 100644 --- a/services/our/src/views/uploads/show.hbs +++ b/services/our/src/views/uploads/show.hbs @@ -2,7 +2,7 @@ -
+
{{> navbar}}
@@ -19,69 +19,105 @@
-

Upload {{upload.id}}

+

Upload {{upload.id}}

-

Stream Date

-

{{upload.streamDate}}

+
+

Stream Date

+

{{upload.streamDate}}

+
{{#if upload.notes}} -

Notes

-

{{upload.notes}}

+
+

Notes

+

{{upload.notes}}

+
{{/if}} -

Vtubers

-
    - {{#each upload.vtubers}} -
  • {{this.displayName}}
    • - {{/each}} -
+
+

Vtubers

+
    + {{#each upload.vtubers}} +
  • {{this.displayName}}
    • + {{/each}} +
+
-

Files

-

File segments listed here will be concatenated together in the top-down order listed.

+

Files

+ {{#if upload.sourceVideo}} +
+

Source Video

+

{{upload.sourceVideo}}

+
+ {{/if}} + + + +

File Segments

+ {{#if upload.segmentKeys.length}} +

+ File segments listed here will be concatenated together in the top-down order listed. +

Drag & drop to reorder

-
+
{{#each upload.segmentKeys}} {{/each}}
+ {{else}} +
+ There are no uploaded file segments. +
+ {{/if}} +

Status

+
+ + {{upload.status}} + +
-

Status

- {{upload.status}}

{{#if (isModerator user)}}
-
@@ -93,13 +129,15 @@ -

+
{{!-- {{#if (isEqual upload.status "pending")}} --}} - {{!-- {{/if}} --}} diff --git a/services/our/src/views/vod.hbs b/services/our/src/views/vod.hbs index 746f0f1..6798e5a 100644 --- a/services/our/src/views/vod.hbs +++ b/services/our/src/views/vod.hbs @@ -62,7 +62,7 @@
-
+
{{icon "processing" 24}} HTTP Live Streaming is processing.
diff --git a/services/our/src/views/vods.hbs b/services/our/src/views/vods.hbs index c710de9..d2c7e1b 100644 --- a/services/our/src/views/vods.hbs +++ b/services/our/src/views/vods.hbs @@ -31,7 +31,7 @@
VOD ID {{this.id}} {{#each this.vtubers}} - {{this.displayName}} + {{this.displayName}} {{/each}} {{formatDate this.stream.date}}