add b2-cli

This commit is contained in:
CJ_Clippy 2025-08-13 04:31:07 -08:00
parent 9d96b4d235
commit e810fd3322
14 changed files with 457 additions and 89 deletions

View File

@ -36,6 +36,9 @@ COPY --from=ghcr.io/ggml-org/whisper.cpp:main-e7bf0294ec9099b5fc21f5ba969805dfb2
ENV PATH="$PATH:/app/whisper.cpp/build/bin"
ENV LD_LIBRARY_PATH="/app/whisper.cpp/build/src:/app/whisper.cpp/build/ggml/src:/usr/local/lib:/usr/lib"
# Install b2-cli
RUN wget https://github.com/Backblaze/B2_Command_Line_Tool/releases/download/v4.4.1/b2-linux -o /usr/local/bin/b2 && chmod +x /usr/local/bin/b2
# Copy and install dependencies
COPY package.json package-lock.json ./
RUN npm install --ignore-scripts=false --foreground-scripts --verbose

View File

@ -6,6 +6,7 @@ import vtubersRoutes from './plugins/vtubers'
import usersRoutes from './plugins/users'
import indexRoutes from './plugins/index'
import streamsRoutes from './plugins/streams'
import adminRoutes from './plugins/admin'
import hls from './plugins/hls.ts'
import fastifyStatic from '@fastify/static'
import fastifySecureSession from '@fastify/secure-session'
@ -32,7 +33,6 @@ import { icons } from './utils/icons.ts'
import logger from './utils/logger.ts'
import fastifyCaching from '@fastify/caching'
export function buildApp() {
const app = Fastify()
@ -197,6 +197,7 @@ export function buildApp() {
app.register(uploadsRoutes)
app.register(usersRoutes)
app.register(indexRoutes)
app.register(adminRoutes)
app.register(authRoutes)
return app

View File

@ -1,41 +1,141 @@
import { PrismaClient } from '../../generated/prisma'
import { PrismaClient, Prisma } from '../../generated/prisma'
import { withAccelerate } from "@prisma/extension-accelerate"
import { type FastifyInstance, type FastifyReply, type FastifyRequest } from 'fastify'
import { env } from '../config/env'
import { constants } from '../config/constants'
import { isModerator, isAdmin } from '../utils/privs'
import type { pool }
import { isAdmin } from '../utils/privs'
import { makeWorkerUtils, WorkerUtils } from 'graphile-worker'
import logger from '../utils/logger'
import { } from '../../generated/prisma'
// const prisma = new PrismaClient().$extends(withAccelerate())
const prisma = new PrismaClient().$extends(withAccelerate())
interface JobRow {
id: number;
queue_name: string | null;
task_identifier: string;
priority: number;
run_at: Date;
attempts: number;
max_attempts: number;
last_error: string | null;
created_at: Date;
updated_at: Date;
key: string | null;
locked_at: Date | null;
locked_by: string | null;
revision: number;
flags: string | null;
}
async function getJobs() {
const jobs = await prisma.$queryRaw<JobRow[]>(
Prisma.sql`
SELECT
id,
queue_name,
task_identifier,
priority,
run_at,
attempts,
max_attempts,
last_error,
created_at,
updated_at,
key,
locked_at,
locked_by,
revision,
flags
FROM graphile_worker.jobs
ORDER BY id DESC
LIMIT 2500
`
);
return jobs;
}
async function indexRoutes(fastify: FastifyInstance): Promise<void> {
fastify.delete('/admin/queue', async function (request, reply) {
// const userId = request.session.get('userId')
// const user = await prisma.user.findUnique({
// where: { id: userId },
// include: {
// roles: true
// }
// })
// if (isAdmin(user)) {
// // @todo delete the queue
export default async function adminRoutes(fastify: FastifyInstance): Promise<void> {
fastify.get('/admin', async function (request, reply) {
const userId = request.session.get('userId');
if (!userId) {
throw new Error('No user is logged in.');
}
const user = await prisma.user.findUnique({
where: { id: userId },
include: { roles: true }
});
if (!user) throw new Error('User not found.');
if (!isAdmin(user)) {
return reply.status(403).send("you are not a jedi, yet");
} else {
const jobs = await getJobs();
logger.debug(jobs[0])
return reply.viewAsync("admin.hbs", {
jobs,
user,
site: constants.site,
}, { layout: 'layouts/main.hbs' });
}
})
fastify.delete('/admin/jobs', async function (request, reply) {
// // const pool = new Pool({
// // connectionString: process.env.DATABASE_URL,
// // });
// // const workerUtils = await WorkerUtils.make({ pgPool: pool });
// // await permanentlyFailAllJobs();
// // await workerUtils.release();
// // await pool.end();
// }
// return reply.viewAsync("admin.hbs", {
// user,
// site: constants.site
// });
const userId = request.session.get('userId')
const user = await prisma.user.findUnique({
where: { id: userId },
include: {
roles: true
}
})
}
if (!user) throw new Error('User not found.');
if (isAdmin(user)) {
let workerUtils: WorkerUtils | undefined;
try {
const workerUtils = await makeWorkerUtils({ connectionString: env.DATABASE_URL });
const jobIdsResult = await prisma.$queryRaw<{ id: number }[]>(
Prisma.sql`SELECT id FROM graphile_worker.jobs`
);
const jobIds: string[] = jobIdsResult.map(j => '' + j.id);
logger.warn(`Permanently failing all graphile-worker jobs on behalf of ${user.id}.`);
if (jobIds.length > 0) {
await workerUtils.permanentlyFailJobs(jobIds, 'Job deleted by admin');
}
}
finally {
if (workerUtils) {
await workerUtils.release();
}
}
}
return reply.viewAsync("admin.hbs", {
user,
site: constants.site,
notification: 'Jobs beleted!',
});
})
}

View File

@ -57,9 +57,13 @@ export default async function vodsRoutes(
const vods = await prisma.vod.findMany({
where: {
status: {
in: ['approved', 'processed', 'processing'],
},
AND: [
{
status: {
in: ['approved', 'processed', 'processing'],
}
}
]
},
orderBy: { createdAt: 'desc' },
include: {

View File

@ -0,0 +1,133 @@
// Copy futureporn.net s3 asset to future.porn s3 bucket.
// ex: https://futureporn-b2.b-cdn.net/(...) -> https://fp-usc.b-cdn.net/(...)
import logger from "../utils/logger";
import { Task } from "graphile-worker";
import { PrismaClient } from "../../generated/prisma";
import { withAccelerate } from "@prisma/extension-accelerate";
import { generateS3Path } from '../utils/formatters';
import { getOrDownloadAsset } from "../utils/cache";
import { getNanoSpawn } from "../utils/nanoSpawn";
import { env } from "../config/env";
import type { default as NanoSpawn } from 'nano-spawn';
const prisma = new PrismaClient().$extends(withAccelerate());
interface Payload {
vodId: string;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
}
function isV1Url(url: string): Boolean {
return url.includes('futureporn-b2.b-cdn.net');
}
/**
* Copy a file from V1 bucket to V2 bucket
* @param v1Url The V1 file URL (b2://futureporn/...)
* @param v2Key Desired key in V2 bucket
* @param contentType Optional override for content type
*/
async function copyFromBucketToBucket(spawn: typeof NanoSpawn, v1Url: string, v2Key: string, contentType?: string) {
// Get file info from V1
logger.info(`v1Url=${v1Url}`);
const infoResult = await spawn('b2', ['file', 'info', v1Url]);
const infoJson = JSON.parse(infoResult.stdout);
const v2Url = `b2://${process.env.S3_BUCKET}/${v2Key}`;
logger.info(`Copying ${v1Url} to ${v2Url}...`);
// Copy file by ID
const copyResult = await spawn('b2', [
'file', 'copy-by-id',
infoJson.fileId,
process.env.S3_BUCKET,
v2Key,
]);
logger.info('Copying complete.');
return v2Url;
}
// example v1 https://futureporn-b2.b-cdn.net/projektmelody-chaturbate-2023-01-01.mp4
// example v2 https://fp-usc.b-cdn.net/projektmelody-chaturbate-2023-01-01.mp4
const copyV1S3ToV2: Task = async (payload: any) => {
const spawn = await getNanoSpawn();
assertPayload(payload)
const { vodId } = payload
const vod = await prisma.vod.findFirstOrThrow({
where: {
id: vodId,
AND: [
{ thumbnail: { not: '' } },
{ thumbnail: { not: null } }
]
},
select: {
thumbnail: true,
sourceVideo: true,
streamDate: true,
vtubers: true,
}
})
// find what we need to copy over.
// potentially vod.thumbnail and vod.sourceVideo
const thumbnail = vod.thumbnail;
const sourceVideo = vod.sourceVideo;
if (!thumbnail && !sourceVideo) {
logger.info(`thumbnail and sourceVideo for ${vodId} are missing. nothing to do.`);
return;
}
const slug = vod.vtubers[0].slug
if (!slug) throw new Error(`vtuber ${vod.vtubers[0].id} is missing a slug`);
let v2ThumbnailKey: string | undefined = undefined;
let v2SourceVideoKey: string | undefined = undefined;
if (thumbnail && isV1Url(thumbnail)) {
v2ThumbnailKey = generateS3Path(slug, vod.streamDate, vodId, 'thumbnail.png');
await copyFromBucketToBucket(spawn, thumbnail.replace('https://futureporn-b2.b-cdn.net', 'b2://futureporn'), v2ThumbnailKey, 'application/png');
}
if (sourceVideo && isV1Url(sourceVideo)) {
v2SourceVideoKey = generateS3Path(slug, vod.streamDate, vodId, 'source.mp4');
await copyFromBucketToBucket(spawn, sourceVideo.replace('https://futureporn-b2.b-cdn.net', 'b2://futureporn'), v2SourceVideoKey, 'video/mp4');
}
logger.debug(`updating vod record`);
await prisma.vod.update({
where: { id: vodId },
data: {
...(v2ThumbnailKey ? { thumbnail: v2ThumbnailKey } : {}), // Variable 'v2ThumbnailKey' is used before being assigned.ts(2454)
...(v2SourceVideoKey ? { sourceVideo: v2SourceVideoKey } : {}),
}
});
}
export default copyV1S3ToV2;

View File

@ -9,7 +9,10 @@ const findWork: Task = async (_payload, helpers: Helpers) => {
const approvedUploads = await prisma.vod.findMany({
where: {
status: "approved",
OR: [
{ status: "approved" },
{ status: "pending" },
]
},
});

View File

@ -47,6 +47,7 @@ const scheduleVodProcessing: Task = async (payload: unknown, helpers) => {
if (!vod.asrVttKey) jobs.push(helpers.addJob("createTranscription", { vodId }));
if (!vod.slvttVTTKey) jobs.push(helpers.addJob("createStoryboard", { vodId }));
if (!vod.magnetLink) jobs.push(helpers.addJob("createTorrent", { vodId }));
if (vod.thumbnail && vod.sourceVideo) jobs.push(helpers.addJob("copyV1S3ToV2", { vodId }));
const changes = jobs.length;
if (changes > 0) {

View File

@ -1,7 +1,78 @@
<main class="container">
<h1>Admin page</h1>
<header>
{{> navbar}}
</header>
<section class="section">
<main class="">
<h1 class="title is-1">Admin page</h1>
<button hx-delete="/admin/queue">Clear graphile worker queue</button>
<article id="notification">
{{#if notification}}
<div class="notification is-success">{{notification}}</div>
{{/if}}
</article>
</main>
<section>
<div class="">
<h1 class="title">Graphile Worker Jobs</h1>
<p>
{{#if jobs.length}}
<table class="table is-fullwidth is-striped is-hoverable">
<thead>
<tr>
<th>Id</th>
<th>Queue</th>
<th>Task</th>
<th>Priority</th>
<th>Run At</th>
<th>Attempts</th>
<th>Max Attempts</th>
<th>Last Error</th>
<th>Created</th>
<th>Updated</th>
<th>Key</th>
<th>Locked At</th>
<th>Locked By</th>
<th>Revision</th>
<th>Flags</th>
</tr>
</thead>
<tbody>
{{#each jobs}}
<tr>
<td>{{this.id}}</td>
<td>{{this.queue_name}}</td>
<td>{{this.task_identifier}}</td>
<td>{{this.priority}}</td>
<td>{{this.run_at}}</td>
<td>{{this.attempts}}</td>
<td>{{this.max_attempts}}</td>
<td>{{this.last_error}}</td>
<td>{{this.created_at}}</td>
<td>{{this.updated_at}}</td>
<td>{{this.key}}</td>
<td>{{this.locked_at}}</td>
<td>{{this.locked_by}}</td>
<td>{{this.revision}}</td>
<td>{{this.flags}}</td>
</tr>
{{/each}}
</tbody>
</table>
{{else}}
<p>No jobs found.</p>
{{/if}}
</p>
<p>
<button class="button is-danger" hx-delete="/admin/jobs" hx-target="#notification" hx-swap="outerHTML"
hx-select="#notification">
Permanently Fail all Graphile Worker jobs.
</button>
</p>
<p>
</p>
</div>
</section>
</main>
</section>

View File

@ -50,6 +50,11 @@
{{else}}
<a class="navbar-item" href="/auth/patreon">Log in via Patreon</a>
{{/if}}
{{#if (hasRole "admin" user)}}
<a class="navbar-item" href="/admin">Admin</a>
{{/if}}
</div>
</div>
</nav>

View File

@ -20,6 +20,16 @@
</p> --}}
</section>
{{#if (hasRole "admin" user)}}
<section class="section">
<div class="box">
<h2 class="title is-3">Admin section</h2>
<a href="/admin">Admin page</a>
</div>
</section>
{{/if}}
<a href="/logout">Logout</a>
{{>footer}}

View File

@ -1,22 +1,21 @@
{{#> main}}
<!-- Header -->
<header class="container">
<header>
{{> navbar}}
</header>
<!-- ./ Header -->
<!-- Main -->
<main class="container">
<section>
<h1>
<h1 class="title is-1">
{{#each vod.vtubers}}
{{this.displayName}}{{#unless @last}}, {{/unless}}
{{/each}}
{{formatDate vod.stream.date}}
</h1>
<h2>Details</h2>
<h2 class="title is-2">Details</h2>
{{!-- - vtuber
- datetime (formatted)
@ -24,11 +23,11 @@
- announcementUrl
- platforms
--}}
<h2>Vods</h2>
<h2 class="title is-2">Vods</h2>
<div class="overflow-auto">
<table class="striped">
<table class="table striped">
<thead>
<tr>
<th>VOD ID</th>

View File

@ -2,7 +2,7 @@
<!-- Header -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/video.js@8.22.0/dist/video-js.min.css">
<header class="container">
<header>
{{> navbar}}
</header>
<!-- ./ Header -->
@ -19,69 +19,105 @@
<section>
<h1>Upload {{upload.id}}</h1>
<h1 class="title is-1">Upload {{upload.id}}</h1>
<h2>Stream Date</h2>
<p>{{upload.streamDate}}</p>
<div class="mb-5">
<h2 class="title">Stream Date</h2>
<p>{{upload.streamDate}}</p>
</div>
{{#if upload.notes}}
<h2>Notes</h2>
<p class="breaklines">{{upload.notes}}</p>
<div class="mb-5 notification">
<h2 class="title">Notes</h2>
<p class="breaklines">{{upload.notes}}</p>
</div>
{{/if}}
<h2>Vtubers</h2>
<ul>
{{#each upload.vtubers}}
<li>{{this.displayName}}</li>
{{/each}}
</ul>
<div class="mb-5">
<h2 class="title">Vtubers</h2>
<ul>
{{#each upload.vtubers}}
<li><span class="tag is-large">{{this.displayName}}</span></li>
{{/each}}
</ul>
</div>
<h2>Files</h2>
<p>File segments listed here will be concatenated together in the top-down order listed.</p>
<h2 class="title">Files</h2>
{{#if upload.sourceVideo}}
<div class="mb-5">
<h3 class="title is-4">Source Video</h3>
<p class="notification">{{upload.sourceVideo}}</p>
</div>
{{/if}}
<h3 class="title is-4">File Segments</h3>
{{#if upload.segmentKeys.length}}
<p class="notification is-info">
File segments listed here will be concatenated together in the top-down order listed.
</p>
<p><i>Drag &amp; drop to reorder</i></p>
<div id="segmentKeys">
<div id="segmentKeys" class="mb-5">
{{#each upload.segmentKeys}}
<div data-key="{{this.key}}" data-name="{{this.name}}">
<svg class="handle" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24">
<path fill="currentColor" d="M4 15v-2h16v2zm0-4V9h16v2z" />
</svg><a href="{{getCdnUrl this.key}}" target="_blank">{{this.name}}</a>
</svg>
<a href="{{getCdnUrl this.key}}" target="_blank">{{this.name}}</a>
</div>
{{/each}}
</div>
{{else}}
<div class="notification is-warning is-light">
There are no uploaded file segments.
</div>
{{/if}}
<h2 class="title">Status</h2>
<div class="mb-5">
<span class="tag is-primary is-light is-medium">
{{upload.status}}
</span>
</div>
<h2>Status</h2>
{{upload.status}}
<h3></h3>
<form method="PATCH" hx-patch="/uploads/{{upload.id}}" hx-params="*" hx-target="body">
{{#if (isModerator user)}}
<fieldset>
<label for="radio-pending">
<label class="radio" for="radio-pending">
<input type="radio" id="radio-pending" name="status" value="pending" {{#if (isEqual upload.status 'pending')}}
checked {{/if}}>
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24">
<path fill="currentColor"
d="M7 13.5q.625 0 1.063-.437T8.5 12t-.437-1.062T7 10.5t-1.062.438T5.5 12t.438 1.063T7 13.5m5 0q.625 0 1.063-.437T13.5 12t-.437-1.062T12 10.5t-1.062.438T10.5 12t.438 1.063T12 13.5m5 0q.625 0 1.063-.437T18.5 12t-.437-1.062T17 10.5t-1.062.438T15.5 12t.438 1.063T17 13.5M12 22q-2.075 0-3.9-.788t-3.175-2.137T2.788 15.9T2 12t.788-3.9t2.137-3.175T8.1 2.788T12 2t3.9.788t3.175 2.137T21.213 8.1T22 12t-.788 3.9t-2.137 3.175t-3.175 2.138T12 22m0-2q3.35 0 5.675-2.325T20 12t-2.325-5.675T12 4T6.325 6.325T4 12t2.325 5.675T12 20m0-8" />
</svg> pending
<div class="icon">
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24">
<path fill="currentColor"
d="M7 13.5q.625 0 1.063-.437T8.5 12t-.437-1.062T7 10.5t-1.062.438T5.5 12t.438 1.063T7 13.5m5 0q.625 0 1.063-.437T13.5 12t-.437-1.062T12 10.5t-1.062.438T10.5 12t.438 1.063T12 13.5m5 0q.625 0 1.063-.437T18.5 12t-.437-1.062T17 10.5t-1.062.438T15.5 12t.438 1.063T17 13.5M12 22q-2.075 0-3.9-.788t-3.175-2.137T2.788 15.9T2 12t.788-3.9t2.137-3.175T8.1 2.788T12 2t3.9.788t3.175 2.137T21.213 8.1T22 12t-.788 3.9t-2.137 3.175t-3.175 2.138T12 22m0-2q3.35 0 5.675-2.325T20 12t-2.325-5.675T12 4T6.325 6.325T4 12t2.325 5.675T12 20m0-8" />
</svg>
</div>pending
</label>
<label for="radio-approved">
<label class="radio" for="radio-approved">
<input type="radio" id="radio-approved" name="status" value="approved"
{{#if (isEqual upload.status 'approved')}} checked {{/if}}>
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
<path fill="currentColor" d="m14 21.414l-5-5.001L10.413 15L14 18.586L21.585 11L23 12.415z" />
<path fill="currentColor"
d="M16 2a14 14 0 1 0 14 14A14 14 0 0 0 16 2m0 26a12 12 0 1 1 12-12a12 12 0 0 1-12 12" />
</svg> approved
<div class="icon">
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
<path fill="currentColor" d="m14 21.414l-5-5.001L10.413 15L14 18.586L21.585 11L23 12.415z" />
<path fill="currentColor"
d="M16 2a14 14 0 1 0 14 14A14 14 0 0 0 16 2m0 26a12 12 0 1 1 12-12a12 12 0 0 1-12 12" />
</svg>
</div> approved
</label>
<label for="radio-rejected">
<label class="radio" for="radio-rejected">
<input type="radio" id="radio-rejected" name="status" value="rejected"
{{#if (isEqual upload.status 'rejected')}} checked {{/if}}>
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
<path fill="currentColor"
d="M17.414 16L26 7.414L24.586 6L16 14.586L7.414 6L6 7.414L14.586 16L6 24.586L7.414 26L16 17.414L24.586 26L26 24.586z" />
<div class="icon">
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
<path fill="currentColor"
d="M17.414 16L26 7.414L24.586 6L16 14.586L7.414 6L6 7.414L14.586 16L6 24.586L7.414 26L16 17.414L24.586 26L26 24.586z" />
</div>
</svg> rejected
</label>
</fieldset>
@ -93,13 +129,15 @@
<input type="hidden" name="segmentKeys" value="{{json upload.segmentKeys}}">
<h3></h3>
<div class="mb-5"></div>
{{!-- {{#if (isEqual upload.status "pending")}} --}}
<button name="save">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 1536 1536">
<path fill="currentColor"
d="M384 1408h768v-384H384zm896 0h128V512q0-14-10-38.5t-20-34.5l-281-281q-10-10-34-20t-39-10v416q0 40-28 68t-68 28H352q-40 0-68-28t-28-68V128H128v1280h128V992q0-40 28-68t68-28h832q40 0 68 28t28 68zM896 480V160q0-13-9.5-22.5T864 128H672q-13 0-22.5 9.5T640 160v320q0 13 9.5 22.5T672 512h192q13 0 22.5-9.5T896 480m640 32v928q0 40-28 68t-68 28H96q-40 0-68-28t-28-68V96q0-40 28-68T96 0h928q40 0 88 20t76 48l280 280q28 28 48 76t20 88" />
</svg>
<button class="button" name="save">
<icon class="mr-2">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 1536 1536">
<path fill="currentColor"
d="M384 1408h768v-384H384zm896 0h128V512q0-14-10-38.5t-20-34.5l-281-281q-10-10-34-20t-39-10v416q0 40-28 68t-68 28H352q-40 0-68-28t-28-68V128H128v1280h128V992q0-40 28-68t68-28h832q40 0 68 28t28 68zM896 480V160q0-13-9.5-22.5T864 128H672q-13 0-22.5 9.5T640 160v320q0 13 9.5 22.5T672 512h192q13 0 22.5-9.5T896 480m640 32v928q0 40-28 68t-68 28H96q-40 0-68-28t-28-68V96q0-40 28-68T96 0h928q40 0 88 20t76 48l280 280q28 28 48 76t20 88" />
</svg>
</icon>
Save
</button>
{{!-- {{/if}} --}}

View File

@ -62,7 +62,7 @@
<video id="player" class="hidden"></video>
<div class="section">
<div class="notification">
<div class="notification pt-6 pb-6">
{{icon "processing" 24}} HTTP Live Streaming is processing.
</div>

View File

@ -31,7 +31,7 @@
<td><a href="/vods/{{this.id}}">{{this.id}}</a></td>
<td>
{{#each this.vtubers}}
{{this.displayName}}
<a href="/vt/{{this.slug}}">{{this.displayName}}</a>
{{/each}}
</td>
<td>{{formatDate this.stream.date}}</td>