From bdc4894e3e535999f019de9ffe2d70169e79273c Mon Sep 17 00:00:00 2001 From: CJ_Clippy Date: Mon, 1 Sep 2025 22:49:27 -0800 Subject: [PATCH] add audio analysis --- .../20250902050219_add_lufs/migration.sql | 4 + services/our/prisma/schema.prisma | 18 +- services/our/scripts/analyze.ts | 86 ++++ services/our/src/tasks/analyzeAudio.ts | 135 ++++++ .../consolidate_twitch_channel_rewards.ts | 184 ------- .../src/tasks/createFunscript copy.ts.noexec | 451 ------------------ .../src/tasks/createFunscript.ts.ai.noexec | 75 --- .../our/src/tasks/getSourceVideoMetadata.ts | 14 +- services/our/src/tasks/hello.ts | 21 - .../our/src/tasks/scheduleVodProcessing.ts | 1 + services/our/src/utils/formatters.ts | 16 +- services/our/src/utils/hbsHelpers.ts | 5 +- services/our/src/views/vod.hbs | 43 +- 13 files changed, 306 insertions(+), 747 deletions(-) create mode 100644 services/our/prisma/migrations/20250902050219_add_lufs/migration.sql create mode 100644 services/our/scripts/analyze.ts create mode 100644 services/our/src/tasks/analyzeAudio.ts delete mode 100644 services/our/src/tasks/consolidate_twitch_channel_rewards.ts delete mode 100644 services/our/src/tasks/createFunscript copy.ts.noexec delete mode 100644 services/our/src/tasks/createFunscript.ts.ai.noexec delete mode 100644 services/our/src/tasks/hello.ts diff --git a/services/our/prisma/migrations/20250902050219_add_lufs/migration.sql b/services/our/prisma/migrations/20250902050219_add_lufs/migration.sql new file mode 100644 index 0000000..902941b --- /dev/null +++ b/services/our/prisma/migrations/20250902050219_add_lufs/migration.sql @@ -0,0 +1,4 @@ +-- AlterTable +ALTER TABLE "Vod" ADD COLUMN "audioIntegratedLufs" DOUBLE PRECISION, +ADD COLUMN "audioLoudnessRange" DOUBLE PRECISION, +ADD COLUMN "audioTruePeak" DOUBLE PRECISION; diff --git a/services/our/prisma/schema.prisma b/services/our/prisma/schema.prisma index 58412da..d15a4b9 100644 --- a/services/our/prisma/schema.prisma +++ b/services/our/prisma/schema.prisma @@ -79,12 +79,18 @@ model Vod { sourceVideoCodec String? sourceAudioCodec String? sourceVideoFps Float? - hlsPlaylist String? - thumbnail String? - asrVttKey String? - slvttSheetKeys Json? - slvttVTTKey String? - magnetLink String? + + // audio analysis + audioIntegratedLufs Float? // Integrated loudness (LUFS-I) + audioLoudnessRange Float? // Loudness Range (LRA) + audioTruePeak Float? // True Peak (dBTP) + + hlsPlaylist String? + thumbnail String? + asrVttKey String? + slvttSheetKeys Json? + slvttVTTKey String? + magnetLink String? status VodStatus @default(pending) sha256sum String? diff --git a/services/our/scripts/analyze.ts b/services/our/scripts/analyze.ts new file mode 100644 index 0000000..16724b9 --- /dev/null +++ b/services/our/scripts/analyze.ts @@ -0,0 +1,86 @@ +import { spawn } from "child_process"; + +export interface AudioStats { + peak: number; // True peak in dBTP + lufsIntegrated: number; // LUFS-I + lra: number; // Loudness range +} + +async function runFFmpeg(args: string[]): Promise { + return new Promise((resolve, reject) => { + console.log("Running ffmpeg:", args.join(" ")); + const ff = spawn("ffmpeg", args); + + let stdout = ""; + let stderr = ""; + + ff.stdout.on("data", (data) => { + const str = data.toString(); + stdout += str; + process.stdout.write(str); // debug passthrough + }); + + ff.stderr.on("data", (data) => { + const str = data.toString(); + stderr += str; + process.stdout.write(str); // debug passthrough + }); + + ff.on("close", (code) => { + console.log("ffmpeg exited with code:", code); + if (code === 0) resolve(stdout + "\n" + stderr); + else reject(new Error(stderr)); + }); + }); +} + +function parseEbur128(output: string): AudioStats { + const lines = output.split("\n").map(l => l.trim()); + + let lufsIntegrated = NaN; + let lra = NaN; + let peak = NaN; + + let inTruePeak = false; + + for (const line of lines) { + // Integrated loudness (LUFS-I) + const iMatch = line.match(/^I:\s*(-?\d+(\.\d+)?) LUFS/); + if (iMatch) lufsIntegrated = parseFloat(iMatch[1]); + + // Loudness range (LRA) + const lraMatch = line.match(/^LRA:\s*(-?\d+(\.\d+)?)/); + if (lraMatch) lra = parseFloat(lraMatch[1]); + + // True peak (dBTP) + if (line === "True peak:") inTruePeak = true; + if (inTruePeak) { + const pMatch = line.match(/Peak:\s*(-?\d+(\.\d+)?) dBFS/); + if (pMatch) { + peak = parseFloat(pMatch[1]); + inTruePeak = false; + } + } + } + + return { lufsIntegrated, lra, peak }; +} + +export async function analyzeAudio(inputFile: string): Promise { + const args = [ + "-hide_banner", + "-i", inputFile, + "-filter_complex", "ebur128=peak=true:metadata=1:framelog=verbose", + "-f", "null", + "-", + ]; + + const output = await runFFmpeg(args); + return parseEbur128(output); +} + +// Test +(async () => { + const stats = await analyzeAudio("/home/cj/Videos/28years/voiceover-compressed2.wav"); + console.log("FINAL STATS:", stats); +})(); diff --git a/services/our/src/tasks/analyzeAudio.ts b/services/our/src/tasks/analyzeAudio.ts new file mode 100644 index 0000000..2516ec3 --- /dev/null +++ b/services/our/src/tasks/analyzeAudio.ts @@ -0,0 +1,135 @@ +/** + * Analyze the video's audio track using ITU-R BS.1770/EBU R128 + * to extract Integrated Loudness (LUFS-I), Loudness Range (LRA), + * and True Peak (dBTP). This helps identify which VODs/streamers + * maintain consistent, listener-friendly audio setups. + * + * ## LUFS Reference + * Apple Music -16 LUFS + * YouTube -13 LUFS + * Spotify -14 LUFS + * Tidal -14 LUFS + */ + +import type { Task, Helpers } from "graphile-worker"; +import { PrismaClient } from "../../generated/prisma"; +import { withAccelerate } from "@prisma/extension-accelerate"; +import { getOrDownloadAsset } from "../utils/cache"; +import { env } from "../config/env"; +import { getS3Client } from "../utils/s3"; +import { getNanoSpawn } from "../utils/nanoSpawn"; +import logger from "../utils/logger"; + +const prisma = new PrismaClient().$extends(withAccelerate()); +const ffmpegBin = "ffmpeg"; + +interface Payload { + vodId: string; +} + +interface AudioStats { + lufsIntegrated: number; // LUFS-I + lra: number; // Loudness Range + peak: number; // True Peak (expressed as dBTP ceiling, e.g. -0.1) +} + +export async function runFFmpeg(args: string[]): Promise { + const spawn = await getNanoSpawn(); + + try { + const result = await spawn(ffmpegBin, args, { + stdout: 'pipe', + stderr: 'pipe', + }); + + // success → return combined output + return result.output; + } catch (err: any) { + // failure → throw with stderr + throw new Error(err.output || err.message); + } +} + +function parseEbur128(output: string): AudioStats { + const lines = output.split("\n").map((l) => l.trim()); + + let lufsIntegrated = NaN; + let lra = NaN; + let peak = NaN; + let inTruePeak = false; + + for (const line of lines) { + // Integrated loudness + const iMatch = line.match(/^I:\s*(-?\d+(\.\d+)?) LUFS/); + if (iMatch) lufsIntegrated = parseFloat(iMatch[1]); + + // Loudness range + const lraMatch = line.match(/^LRA:\s*(-?\d+(\.\d+)?)/); + if (lraMatch) lra = parseFloat(lraMatch[1]); + + // True peak + if (line === "True peak:") inTruePeak = true; + if (inTruePeak) { + const pMatch = line.match(/Peak:\s*(-?\d+\.?\d*)\s*dBFS/); + if (pMatch) { + peak = parseFloat(pMatch[1]); // no negation + inTruePeak = false; + } + } + } + + return { lufsIntegrated, lra, peak }; +} + + +async function analyzeAudio(inputFile: string): Promise { + const args = [ + "-hide_banner", + "-i", inputFile, + "-filter_complex", "ebur128=peak=true:metadata=1:framelog=verbose", + "-f", "null", + "-", + ]; + + const output = await runFFmpeg(args); + return parseEbur128(output); +} + +function assertPayload(payload: any): asserts payload is Payload { + if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object."); + if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId"); +} + +export default async function main(payload: any, helpers: Helpers) { + assertPayload(payload); + const { vodId } = payload; + + const vod = await prisma.vod.findFirstOrThrow({ + where: { id: vodId }, + }); + + if (vod.audioIntegratedLufs && vod.audioLoudnessRange && vod.audioTruePeak) { + logger.info(`Skipping analysis — vod ${vodId} already complete.`); + return; + } + + if (!vod.sourceVideo) { + throw new Error(`vod ${vodId} is missing a sourceVideo.`); + } + + const s3Client = getS3Client(); + const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo); + logger.info(`videoFilePath=${videoFilePath}`); + + const results = await analyzeAudio(videoFilePath); + logger.info(`results=${JSON.stringify(results)}`); + + await prisma.vod.update({ + where: { id: vodId }, + data: { + audioIntegratedLufs: results.lufsIntegrated, + audioLoudnessRange: results.lra, + audioTruePeak: results.peak, + }, + }); +} diff --git a/services/our/src/tasks/consolidate_twitch_channel_rewards.ts b/services/our/src/tasks/consolidate_twitch_channel_rewards.ts deleted file mode 100644 index 037a7e2..0000000 --- a/services/our/src/tasks/consolidate_twitch_channel_rewards.ts +++ /dev/null @@ -1,184 +0,0 @@ -// src/tasks/consolidate_twitch_channel_rewards.ts - -import type { Task, Helpers } from "graphile-worker"; -import { PrismaClient, User, type Pick } from '../../generated/prisma'; -import { withAccelerate } from "@prisma/extension-accelerate"; -import { env } from "../config/env"; -import { constants } from "../config/constants"; -import { getRateLimiter } from "../utils/rateLimiter"; -import logger from '../utils/logger'; - -const prisma = new PrismaClient().$extends(withAccelerate()); -const cprPath = env.TWITCH_MOCK ? constants.twitch.dev.paths.channelPointRewards : constants.twitch.prod.paths.channelPointRewards; - -interface Payload { - userId: number; -} - -export interface TwitchChannelPointReward { - id: string; - broadcaster_id: string; - cost: number; - title: string; - is_in_stock: boolean; - [key: string]: any; -} - -const getAuthToken = (user: User) => env.TWITCH_MOCK ? env.TWITCH_MOCK_USER_ACCESS_TOKEN : user.twitchToken?.accessToken; - - -function assertPayload(payload: any): asserts payload is Payload { - if (typeof payload !== "object" || !payload) throw new Error("invalid payload"); - if (typeof payload.userId !== "number") throw new Error("invalid payload.userId"); -} - - -const getTwitchChannelPointRewards = async (user: User) => { - if (!user?.twitchToken) throw new Error("Missing Twitch token"); - - const authToken = getAuthToken(user); - const limiter = getRateLimiter(); - await limiter.consume('twitch', 1); - - const query = new URLSearchParams({ broadcaster_id: user.twitchId }); - const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, { - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Client-Id': env.TWITCH_CLIENT_ID - } - }); - - if (!res.ok) throw new Error(`Failed to fetch rewards: ${res.statusText}`); - return res.json(); -}; - -const createTwitchReward = async (user: User, pick: Pick) => { - const authToken = getAuthToken(user); - const limiter = getRateLimiter(); - await limiter.consume('twitch', 1); - logger.debug('pick as follows') - logger.debug(pick) - - logger.debug(`pick?.waifu?.name=${pick?.waifu?.name}`) - - const query = new URLSearchParams({ broadcaster_id: user.twitchId }); - const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, { - method: 'POST', - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Client-Id': env.TWITCH_CLIENT_ID - }, - body: JSON.stringify({ cost: user.redeemCost, title: pick.waifu.name }) - }); - - if (!res.ok) throw new Error(`Failed to create reward: ${res.statusText}`); - - const data = await res.json(); - const rewardId = data.data?.[0]?.id; - if (!rewardId) throw new Error("No reward ID returned"); - - await prisma.pick.update({ - where: { id: pick.id }, - data: { twitchChannelPointRewardId: rewardId } - }); -}; - -const deleteTwitchReward = async (user: User, rewardId: string) => { - const authToken = getAuthToken(user); - const limiter = getRateLimiter(); - await limiter.consume('twitch', 1); - - const query = new URLSearchParams({ broadcaster_id: user.twitchId, id: rewardId }); - const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, { - method: 'DELETE', - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Client-Id': env.TWITCH_CLIENT_ID - } - }); - - if (!res.ok) throw new Error(`Failed to delete reward ${rewardId}`); -}; - -const updateTwitchReward = async (user: User, rewardId: string, newCost: number) => { - const authToken = getAuthToken(user); - const limiter = getRateLimiter(); - await limiter.consume('twitch', 1); - - const query = new URLSearchParams({ broadcaster_id: user.twitchId, id: rewardId }); - const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, { - method: 'PATCH', - headers: { - 'Authorization': `Bearer ${authToken}`, - 'Client-Id': env.TWITCH_CLIENT_ID - }, - body: JSON.stringify({ cost: newCost }) - }); - - if (!res.ok) throw new Error(`Failed to update reward ${rewardId}`); -}; - -const consolidateTwitchRewards = async (userId: number) => { - const user = await prisma.user.findFirstOrThrow({ - where: { id: userId }, - include: { twitchToken: true, Waifu: true } - }); - - // Fetch picks (most recent N) - const picks = await prisma.pick.findMany({ - where: { userId }, - orderBy: { createdAt: 'desc' }, - take: constants.twitch.maxChannelPointRewards, - include: { - waifu: true - } - }); - - // Ensure every pick has a reward before processing Twitch side - for (const pick of picks) { - if (!pick.twitchChannelPointRewardId) { - logger.debug(`Creating new reward for pick: ${pick.id}`); - await createTwitchReward(user, pick); - } - } - - // Refresh picks after reward creation - const updatedPicks = await prisma.pick.findMany({ - where: { userId }, - orderBy: { createdAt: 'desc' }, - take: constants.twitch.maxChannelPointRewards - }); - - // Get the most recent N reward IDs - const currentPickIds = new Set( - updatedPicks.slice(0, user.waifuChoicePoolSize).map(p => p.twitchChannelPointRewardId) - ); - - logger.debug('currentPickIds as follows'); - logger.debug(currentPickIds); - - // Fetch Twitch-side rewards - const twitchData = await getTwitchChannelPointRewards(user); - const twitchRewards: TwitchChannelPointReward[] = twitchData.data; - - // Delete or update Twitch rewards not in current pick set - for (const reward of twitchRewards) { - if (!updatedPicks.some(p => p.twitchChannelPointRewardId === reward.id)) continue; - - if (!currentPickIds.has(reward.id)) { - logger.debug(`Deleting out-of-date reward: ${reward.id}`); - await deleteTwitchReward(user, reward.id); - } else if (reward.cost !== user.redeemCost) { - logger.debug(`Updating reward cost for: ${reward.id}`); - await updateTwitchReward(user, reward.id, user.redeemCost); - } - } -}; - - -const task: Task = async (payload: any, helpers) => { - assertPayload(payload); - await consolidateTwitchRewards(payload.userId); -}; - -export default task; \ No newline at end of file diff --git a/services/our/src/tasks/createFunscript copy.ts.noexec b/services/our/src/tasks/createFunscript copy.ts.noexec deleted file mode 100644 index c1ba3f5..0000000 --- a/services/our/src/tasks/createFunscript copy.ts.noexec +++ /dev/null @@ -1,451 +0,0 @@ -import type { Task, Helpers } from "graphile-worker"; -import { PrismaClient } from "../../generated/prisma"; -import { withAccelerate } from "@prisma/extension-accelerate"; -import { getOrDownloadAsset } from "../utils/cache"; -import { env } from "../config/env"; -import { getS3Client, uploadFile } from "../utils/s3"; -import { nanoid } from "nanoid"; -import { existsSync, rmSync } from "node:fs"; -import { join, basename, extname } from "node:path"; -import { readFile, writeFile, readdir } from 'node:fs/promises'; -import yaml from 'js-yaml'; -import { getNanoSpawn } from "../utils/nanoSpawn"; -import which from "which"; -import * as ort from 'onnxruntime-node'; - -interface Payload { - vodId: string; -} - -interface Detection { - startFrame: number; - endFrame: number; - className: string; -} - -interface DataYaml { - path: string; - train: string; - val: string; - names: Record; -} - -interface FunscriptAction { - at: number; - pos: number; -} - -interface Funscript { - version: string; - actions: FunscriptAction[]; -} - - -interface ClassPositionMap { - [className: string]: number | 'pattern'; -} - - -const prisma = new PrismaClient().$extends(withAccelerate()); - - -function assertPayload(payload: any): asserts payload is Payload { - if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object."); - if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId"); -} - -async function loadDataYaml(yamlPath: string): Promise { - const yamlContent = await readFile(yamlPath, 'utf8'); - return yaml.load(yamlContent) as DataYaml; -} - - -async function preparePython(helpers) { - const spawn = await getNanoSpawn(); - const venvPath = join(env.VIBEUI_DIR, "venv"); - - // Determine Python executable - let pythonCmd; - try { - pythonCmd = which.sync("python3"); - } catch { - logger.error("Python is not installed or not in PATH."); - throw new Error("Python not found in PATH."); - } - - // If venv doesn't exist, create it - if (!existsSync(venvPath)) { - logger.info("Python venv not found. Creating one..."); - - try { - await spawn(pythonCmd, ["-m", "venv", "venv"], { - cwd: env.VIBEUI_DIR, - }); - - logger.info("Python venv successfully created."); - } catch (err) { - logger.error("Failed to create Python venv:", err); - - // Clean up partially created venv if needed - try { - if (existsSync(venvPath)) { - rmSync(venvPath, { recursive: true, force: true }); - logger.warn("Removed broken venv directory."); - } - } catch (cleanupErr) { - logger.error("Error while cleaning up broken venv:", cleanupErr); - } - - throw new Error("Python venv creation failed. Check if python3 and python3-venv are installed."); - } - } else { - logger.info("Using existing Python venv."); - } -} - - -async function ffprobe(videoPath: string): Promise<{ fps: number; frames: number }> { - const spawn = await getNanoSpawn() - const { stdout } = await spawn('ffprobe', [ - '-v', 'error', - '-select_streams', 'v:0', - '-count_frames', - '-show_entries', 'stream=nb_read_frames,r_frame_rate', - '-of', 'default=nokey=1:noprint_wrappers=1', - videoPath, - ]) - - const [frameRateStr, frameCountStr] = stdout.trim().split('\n') - const [num, denom] = frameRateStr.trim().split('/').map(Number) - const fps = num / denom - const frames = parseInt(frameCountStr.trim(), 10) - - return { fps, frames } -} - - - - - - - -export async function buildFunscript( - helpers: Helpers, - predictionOutput: string, - videoPath: string -): Promise { - const labelDir = join(predictionOutput, 'labels'); - const yamlPath = join(predictionOutput, 'data.yaml'); - const outputPath = join(process.env.CACHE_ROOT ?? '/tmp', `${nanoid()}.funscript`); - logger.info('Starting Funscript generation'); - - try { - - const data = await loadDataYaml(join(env.VIBEUI_DIR, 'data.yaml')) - const classPositionMap = await loadClassPositionMap(data, helpers); - const { fps, totalFrames } = await loadVideoMetadata(videoPath, helpers); - const detectionSegments = await processLabelFiles(labelDir, helpers, data); - const totalDurationMs = Math.floor((totalFrames / fps) * 1000); - const actions = generateActions(totalDurationMs, fps, detectionSegments, classPositionMap); - await writeFunscript(outputPath, actions, helpers); - - return outputPath; - } catch (error) { - logger.error(`Error generating Funscript: ${error instanceof Error ? error.message : 'Unknown error'}`); - throw error; - } -} - -export async function inference(helpers: Helpers, videoFilePath: string): Promise { - const spawn = await getNanoSpawn() - - const modelPath = join(env.VIBEUI_DIR, 'runs/detect/vibeui/weights/best.pt') - - // Generate a unique name based on video name + UUID - const videoExt = extname(videoFilePath) // e.g. '.mp4' - const videoName = basename(videoFilePath, videoExt) // removes the extension - const uniqueName = `${videoName}-${nanoid()}` - const customProjectDir = 'runs' // or any custom folder - const outputPath = join(env.VIBEUI_DIR, customProjectDir, uniqueName) - - await spawn('./venv/bin/yolo', [ - 'predict', - `model=${modelPath}`, - `source=${videoFilePath}`, - 'save=False', - 'save_txt=True', - 'save_conf=True', - `project=${customProjectDir}`, - `name=${uniqueName}`, - ], { - cwd: env.VIBEUI_DIR, - stdio: 'inherit', - }) - - return outputPath // contains labels/ folder and predictions -} - - -async function loadClassPositionMap(data: DataYaml, helpers: Helpers): Promise { - try { - - if ( - !data || - typeof data !== 'object' || - !('names' in data) || - typeof data.names !== 'object' || - data.names === null || - Object.keys(data.names).length === 0 - ) { - throw new Error('Invalid data.yaml: "names" field is missing, not an object, or empty'); - } - - const positionMap: ClassPositionMap = { - ControlledByTipper: 50, - ControlledByTipperHigh: 80, - ControlledByTipperLow: 20, - ControlledByTipperMedium: 50, - ControlledByTipperUltrahigh: 95, - Ring1: 30, - Ring2: 40, - Ring3: 50, - Ring4: 60, - Earthquake: 'pattern', - Fireworks: 'pattern', - Pulse: 'pattern', - Wave: 'pattern', - Pause: 0, - RandomTime: 70, - HighLevel: 80, - LowLevel: 20, - MediumLevel: 50, - UltraHighLevel: 95 - }; - - const names = Object.values(data.names); - for (const name of names) { - if (typeof name !== 'string' || name.trim() === '') { - logger.info(`Skipping invalid class name: ${name}`); - continue; - } - if (!(name in positionMap)) { - logger.info(`No position mapping for class "${name}", defaulting to 0`); - positionMap[name] = 0; - } - } - - logger.info(`Loaded class position map: ${JSON.stringify(positionMap)}`); - return positionMap; - } catch (error) { - logger.error(`Error loading data.yaml: ${error instanceof Error ? error.message : 'Unknown error'}`); - throw error; - } -} - -function generatePatternPositions(startMs: number, durationMs: number, className: string, fps: number): FunscriptAction[] { - const actions: FunscriptAction[] = []; - const frameDurationMs = 1000 / fps; - const totalFrames = Math.floor(durationMs / frameDurationMs); - const intervalMs = 100; - - for (let timeMs = 0; timeMs < durationMs; timeMs += intervalMs) { - const progress = timeMs / durationMs; - let pos = 0; - - switch (className) { - case 'Pulse': - pos = Math.round(50 * Math.sin(progress * 2 * Math.PI)); - break; - case 'Wave': - pos = Math.round(50 + 50 * Math.sin(progress * 2 * Math.PI)); - break; - case 'Fireworks': - pos = Math.random() > 0.5 ? 80 : 0; - break; - case 'Earthquake': - pos = Math.round(90 * Math.sin(progress * 4 * Math.PI) + (Math.random() - 0.5) * 10); - pos = Math.max(0, Math.min(90, pos)); - break; - } - - actions.push({ at: startMs + timeMs, pos }); - } - - return actions; -} - -async function loadVideoMetadata(videoPath: string, helpers: Helpers) { - const { fps, frames: totalFrames } = await ffprobe(videoPath); - logger.info(`Video metadata: fps=${fps}, frames=${totalFrames}`); - return { fps, totalFrames }; -} - -async function processLabelFiles(labelDir: string, helpers: Helpers, data: DataYaml): Promise { - const labelFiles = (await readdir(labelDir)).filter(file => file.endsWith('.txt')); - const detections: Map = new Map(); - const names = data.names; - - for (const file of labelFiles) { - const match = file.match(/(\d+)\.txt$/); - if (!match) { - logger.info(`Skipping invalid filename: ${file}`); - continue; - } - const frameIndex = parseInt(match[1], 10); - if (isNaN(frameIndex)) { - logger.info(`Skipping invalid frame index from filename: ${file}`); - continue; - } - - const content = await readFile(join(labelDir, file), 'utf8'); - const lines = content.trim().split('\n'); - const frameDetections: Detection[] = []; - let maxConfidence = 0; - let selectedClassIndex = -1; - - for (const line of lines) { - const parts = line.trim().split(/\s+/); - if (parts.length < 6) continue; - - const classIndex = parseInt(parts[0], 10); - const confidence = parseFloat(parts[5]); - if (isNaN(classIndex) || isNaN(confidence)) continue; - - if (confidence >= 0.7 && confidence > maxConfidence) { - maxConfidence = confidence; - selectedClassIndex = classIndex; - } - } - - if (maxConfidence > 0) { - const className = (data.names as Record)[selectedClassIndex.toString()]; - if (className) { - frameDetections.push({ startFrame: frameIndex, endFrame: frameIndex, className }); - } - } - - if (frameDetections.length > 0) { - detections.set(frameIndex, frameDetections); - } - } - - // Merge overlapping detections into continuous segments - const detectionSegments: Detection[] = []; - let currentDetection: Detection | null = null; - - for (const [frameIndex, frameDetections] of detections.entries()) { - for (const detection of frameDetections) { - if (!currentDetection || currentDetection.className !== detection.className) { - if (currentDetection) detectionSegments.push(currentDetection); - currentDetection = { ...detection, endFrame: frameIndex }; - } else { - currentDetection.endFrame = frameIndex; - } - } - } - if (currentDetection) detectionSegments.push(currentDetection); - - return detectionSegments; -} - -function generateActions(totalDurationMs: number, fps: number, detectionSegments: Detection[], classPositionMap: ClassPositionMap): FunscriptAction[] { - const intervalMs = 100; - const actions: FunscriptAction[] = []; - - // Generate static position actions - for (let timeMs = 0; timeMs <= totalDurationMs; timeMs += intervalMs) { - const frameIndex = Math.floor((timeMs / 1000) * fps); - let position = 0; - - for (const segment of detectionSegments) { - if (frameIndex >= segment.startFrame && frameIndex <= segment.endFrame) { - const className = segment.className; - if (typeof classPositionMap[className] === 'number') { - position = classPositionMap[className]; - break; - } - } - } - actions.push({ at: timeMs, pos: position }); - } - - // Overlay pattern-based actions - for (const segment of detectionSegments) { - const className = segment.className; - if (classPositionMap[className] === 'pattern') { - const startMs = Math.floor((segment.startFrame / fps) * 1000); - const durationMs = Math.floor(((segment.endFrame - segment.startFrame + 1) / fps) * 1000); - const patternActions = generatePatternPositions(startMs, durationMs, className, fps); - actions.push(...patternActions); - } - } - - // Sort actions by time and remove duplicates - actions.sort((a, b) => a.at - b.at); - const uniqueActions: FunscriptAction[] = []; - let lastTime = -1; - for (const action of actions) { - if (action.at !== lastTime) { - uniqueActions.push(action); - lastTime = action.at; - } - } - - return uniqueActions; -} - -async function writeFunscript(outputPath: string, actions: FunscriptAction[], helpers: Helpers) { - const funscript: Funscript = { version: '1.0', actions }; - await writeFile(outputPath, JSON.stringify(funscript, null, 2)); - logger.info(`Funscript generated: ${outputPath} (${actions.length} actions)`); -} - - -const createFunscript: Task = async (payload: any, helpers: Helpers) => { - assertPayload(payload); - const { vodId } = payload; - - - - const vod = await prisma.vod.findFirstOrThrow({ where: { id: vodId } }); - - if (vod.funscript) { - logger.info(`Doing nothing-- vod ${vodId} already has a funscript.`); - return; - } - - if (!vod.sourceVideo) { - const msg = `Cannot create funscript: Vod ${vodId} is missing a source video.`; - logger.warn(msg); - throw new Error(msg); - } - - - - const s3Client = getS3Client(); - const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo); - logger.info(`Downloaded video to ${videoFilePath}`); - - logger.info(`Creating funscript for vod ${vodId}...`); - - const predictionOutput = await inference(helpers, videoFilePath); - logger.info(`prediction output ${predictionOutput}`); - - - const funscriptFilePath = await buildFunscript(helpers, predictionOutput, videoFilePath) - - - const s3Key = `funscripts/${vodId}.funscript`; - const s3Url = await uploadFile(s3Client, env.S3_BUCKET, s3Key, funscriptFilePath, "application/json"); - - logger.info(`Uploaded funscript to S3: ${s3Url}`); - - await prisma.vod.update({ - where: { id: vodId }, - data: { funscript: s3Key } - }); - - logger.info(`Funscript saved to database for vod ${vodId}`); -}; - -export default createFunscript; diff --git a/services/our/src/tasks/createFunscript.ts.ai.noexec b/services/our/src/tasks/createFunscript.ts.ai.noexec deleted file mode 100644 index bd78269..0000000 --- a/services/our/src/tasks/createFunscript.ts.ai.noexec +++ /dev/null @@ -1,75 +0,0 @@ - -import type { Task, Helpers } from "graphile-worker"; -import { PrismaClient } from "../../generated/prisma"; -import { withAccelerate } from "@prisma/extension-accelerate"; -import { getOrDownloadAsset } from "../utils/cache"; -import { env } from "../config/env"; -import { getS3Client, uploadFile } from "../utils/s3"; -import { buildFunscript } from '../utils/funscripts'; -import { getModelClasses, vibeuiInference } from "../utils/vibeui"; -import { join } from "node:path"; - -interface Payload { - vodId: string; -} - - - -const prisma = new PrismaClient().$extends(withAccelerate()); - - -function assertPayload(payload: any): asserts payload is Payload { - if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object."); - if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId"); -} - - -const createFunscript: Task = async (payload: any, helpers: Helpers) => { - assertPayload(payload); - const { vodId } = payload; - - - - const vod = await prisma.vod.findFirstOrThrow({ where: { id: vodId } }); - - if (vod.funscript) { - logger.info(`Doing nothing-- vod ${vodId} already has a funscript.`); - return; - } - - if (!vod.sourceVideo) { - const msg = `Cannot create funscript: Vod ${vodId} is missing a source video.`; - logger.warn(msg); - throw new Error(msg); - } - - - - const s3Client = getS3Client(); - const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo); - logger.info(`Downloaded video to ${videoFilePath}`); - - logger.info(`Creating funscript for vod ${vodId}...`); - - const modelPath = join(env.VIBEUI_DIR, 'vibeui.onnx') - const predictionOutput = await vibeuiInference(modelPath, videoFilePath); - logger.info(`prediction output ${predictionOutput}`); - const classes = await getModelClasses(modelPath) - - const funscriptFilePath = await buildFunscript(classes, predictionOutput, videoFilePath) - - - const s3Key = `funscripts/${vodId}.funscript`; - const s3Url = await uploadFile(s3Client, env.S3_BUCKET, s3Key, funscriptFilePath, "application/json"); - - logger.info(`Uploaded funscript to S3: ${s3Url}`); - - await prisma.vod.update({ - where: { id: vodId }, - data: { funscript: s3Key } - }); - - logger.info(`Funscript saved to database for vod ${vodId}`); -}; - -export default createFunscript; diff --git a/services/our/src/tasks/getSourceVideoMetadata.ts b/services/our/src/tasks/getSourceVideoMetadata.ts index bc3233a..4f3af80 100644 --- a/services/our/src/tasks/getSourceVideoMetadata.ts +++ b/services/our/src/tasks/getSourceVideoMetadata.ts @@ -4,6 +4,8 @@ import { env } from "../config/env"; import { S3Client } from "@aws-sdk/client-s3"; import logger from "../utils/logger"; import { getNanoSpawn } from "../utils/nanoSpawn"; +import { getOrDownloadAsset } from "../utils/cache"; +import { getS3Client } from "../utils/s3"; const prisma = new PrismaClient(); const client = new S3Client({ @@ -90,7 +92,9 @@ const getSourceVideoMetadata: Task = async (payload: unknown, helpers) => { }); // Skip if already processed - if (vod.sourceVideoDuration) { + if ( + vod.sourceVideoDuration + ) { logger.debug(`VOD ${vodId} already has metadata`); return; } @@ -99,8 +103,14 @@ const getSourceVideoMetadata: Task = async (payload: unknown, helpers) => { throw new Error(`VOD ${vodId} has no sourceVideo`); } + // download vod + const s3Client = getS3Client(); + const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo); + logger.info(`videoFilePath=${videoFilePath}`); + + // Run ffprobe - const meta = await getVideoMetadata(vod.sourceVideo); + const meta = await getVideoMetadata(videoFilePath); // Update DB await prisma.vod.update({ diff --git a/services/our/src/tasks/hello.ts b/services/our/src/tasks/hello.ts deleted file mode 100644 index 67e4bd4..0000000 --- a/services/our/src/tasks/hello.ts +++ /dev/null @@ -1,21 +0,0 @@ -// src/tasks/hello.ts - -import type { Task, Helpers } from "graphile-worker"; -import logger from "../utils/logger"; - -interface Payload { - name: string; -} - -function assertPayload(payload: any): asserts payload is Payload { - if (typeof payload !== "object" || !payload) throw new Error("invalid"); - if (typeof payload.name !== "string") throw new Error("invalid"); -} - - -export default async function hello(payload: any, helpers: Helpers) { - assertPayload(payload); - const { name } = payload; - logger.info(`Helloooooo, ${name}`); -}; - diff --git a/services/our/src/tasks/scheduleVodProcessing.ts b/services/our/src/tasks/scheduleVodProcessing.ts index edf19c2..428e3b1 100644 --- a/services/our/src/tasks/scheduleVodProcessing.ts +++ b/services/our/src/tasks/scheduleVodProcessing.ts @@ -42,6 +42,7 @@ const scheduleVodProcessing: Task = async (payload: unknown, helpers) => { jobs.push(helpers.addJob("copyV1S3ToV2", { vodId })); if (!vod.sourceVideo) jobs.push(helpers.addJob("getSourceVideo", { vodId })); if (!vod.sourceVideoDuration) jobs.push(helpers.addJob("getSourceVideoMetadata", { vodId })) + if (!vod.audioIntegratedLufs || !vod.audioLoudnessRange || !vod.audioTruePeak) jobs.push(helpers.addJob("analyzeAudio", { vodId })) if (!vod.sha256sum) jobs.push(helpers.addJob("generateVideoChecksum", { vodId })); if (!vod.thumbnail) jobs.push(helpers.addJob("createVideoThumbnail", { vodId })); if (!vod.hlsPlaylist) jobs.push(helpers.addJob("createHlsPlaylist", { vodId })); diff --git a/services/our/src/utils/formatters.ts b/services/our/src/utils/formatters.ts index 3761d2c..2db9fdc 100644 --- a/services/our/src/utils/formatters.ts +++ b/services/our/src/utils/formatters.ts @@ -37,4 +37,18 @@ export function generateS3Path(slug: string, date: Date, vodId: string, filename const day = pad(getDate(date)); return `fp/${slug}/${year}/${month}/${day}/${vodId}/${filename}`; -} \ No newline at end of file +} + + +export function formatDuration(ms: number): string { + const totalSeconds = Math.floor(ms / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + + const hh = hours.toString().padStart(2, '0'); + const mm = minutes.toString().padStart(2, '0'); + const ss = seconds.toString().padStart(2, '0'); + + return hours > 0 ? `${hh}:${mm}:${ss}` : `${mm}:${ss}`; +} diff --git a/services/our/src/utils/hbsHelpers.ts b/services/our/src/utils/hbsHelpers.ts index d6a5bc8..b943adb 100644 --- a/services/our/src/utils/hbsHelpers.ts +++ b/services/our/src/utils/hbsHelpers.ts @@ -5,7 +5,7 @@ import { Role } from '../../generated/prisma' import { isModerator, hasRole } from './privs' import { signUrl } from './cdn' import { extractBasePath } from './filesystem' -import { truncate } from './formatters.ts' +import { formatDuration, truncate } from './formatters.ts' import { icons } from './icons.ts' import logger from './logger.ts' import HandlebarsLib, { HelperOptions, SafeString } from 'handlebars'; @@ -19,6 +19,9 @@ export function registerHbsHelpers(Handlebars: typeof HandlebarsLib) { if (!dateString) return '' return format(new Date(dateString), 'yyyy-MM-dd') }) + Handlebars.registerHelper('formatDuration', function (duration) { + return formatDuration(duration) + }) Handlebars.registerHelper('identicon', function (str, size = 48) { return jdenticon.toSvg(str, size) }) diff --git a/services/our/src/views/vod.hbs b/services/our/src/views/vod.hbs index 45e251b..88d1672 100644 --- a/services/our/src/views/vod.hbs +++ b/services/our/src/views/vod.hbs @@ -304,15 +304,46 @@

VOD Metadata

-
+ +
{{#if vod.sourceVideoDuration }} -

Duration: {{vod.sourceVideoDuration}}

-

FPS: {{vod.sourceVideoFPS}}

-

Video Codec: {{vod.sourceVideoCodec}}

-

Audio Codec: {{vod.sourceAudioCodec}}

+
+

Duration {{formatDuration vod.sourceVideoDuration}}

+

FPS {{vod.sourceVideoFps}}

+

Video Codec {{vod.sourceVideoCodec}}

+

Audio Codec {{vod.sourceAudioCodec}}

+
+ {{else}} +
+ VOD metadata is processing +
+ {{/if}} +
+
+ + +
+

Audio Analysis

+
+ {{#if vod.audioIntegratedLufs }} +
+

+ LUFS-I + {{vod.audioIntegratedLufs}} +

+

+ LRA + {{vod.audioLoudnessRange}} +

+

+ TP + {{vod.audioTruePeak}} +

+
{{else}}
- Vod metadata is processing + Audio analysis is processing
{{/if}}