add audio analysis

services/our/prisma/migrations/20250902050219_add_lufs/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "Vod" ADD COLUMN     "audioIntegratedLufs" DOUBLE PRECISION,
+ADD COLUMN     "audioLoudnessRange" DOUBLE PRECISION,
+ADD COLUMN     "audioTruePeak" DOUBLE PRECISION;

services/our/prisma/schema.prisma

@@ -79,12 +79,18 @@ model Vod {
   sourceVideoCodec    String?
   sourceAudioCodec    String?
   sourceVideoFps      Float?
-  hlsPlaylist         String?
+
-  thumbnail           String?
+  // audio analysis
-  asrVttKey           String?
+  audioIntegratedLufs Float? // Integrated loudness (LUFS-I)
-  slvttSheetKeys      Json?
+  audioLoudnessRange  Float? // Loudness Range (LRA)
-  slvttVTTKey         String?
+  audioTruePeak       Float? // True Peak (dBTP)
-  magnetLink          String?
+
+  hlsPlaylist    String?
+  thumbnail      String?
+  asrVttKey      String?
+  slvttSheetKeys Json?
+  slvttVTTKey    String?
+  magnetLink     String?
 
   status           VodStatus @default(pending)
   sha256sum        String?

services/our/scripts/analyze.ts

@@ -0,0 +1,86 @@
+import { spawn } from "child_process";
+
+export interface AudioStats {
+  peak: number;           // True peak in dBTP
+  lufsIntegrated: number; // LUFS-I
+  lra: number;            // Loudness range
+}
+
+async function runFFmpeg(args: string[]): Promise<string> {
+  return new Promise((resolve, reject) => {
+    console.log("Running ffmpeg:", args.join(" "));
+    const ff = spawn("ffmpeg", args);
+
+    let stdout = "";
+    let stderr = "";
+
+    ff.stdout.on("data", (data) => {
+      const str = data.toString();
+      stdout += str;
+      process.stdout.write(str); // debug passthrough
+    });
+
+    ff.stderr.on("data", (data) => {
+      const str = data.toString();
+      stderr += str;
+      process.stdout.write(str); // debug passthrough
+    });
+
+    ff.on("close", (code) => {
+      console.log("ffmpeg exited with code:", code);
+      if (code === 0) resolve(stdout + "\n" + stderr);
+      else reject(new Error(stderr));
+    });
+  });
+}
+
+function parseEbur128(output: string): AudioStats {
+  const lines = output.split("\n").map(l => l.trim());
+
+  let lufsIntegrated = NaN;
+  let lra = NaN;
+  let peak = NaN;
+
+  let inTruePeak = false;
+
+  for (const line of lines) {
+    // Integrated loudness (LUFS-I)
+    const iMatch = line.match(/^I:\s*(-?\d+(\.\d+)?) LUFS/);
+    if (iMatch) lufsIntegrated = parseFloat(iMatch[1]);
+
+    // Loudness range (LRA)
+    const lraMatch = line.match(/^LRA:\s*(-?\d+(\.\d+)?)/);
+    if (lraMatch) lra = parseFloat(lraMatch[1]);
+
+    // True peak (dBTP)
+    if (line === "True peak:") inTruePeak = true;
+    if (inTruePeak) {
+      const pMatch = line.match(/Peak:\s*(-?\d+(\.\d+)?) dBFS/);
+      if (pMatch) {
+        peak = parseFloat(pMatch[1]);
+        inTruePeak = false;
+      }
+    }
+  }
+
+  return { lufsIntegrated, lra, peak };
+}
+
+export async function analyzeAudio(inputFile: string): Promise<AudioStats> {
+  const args = [
+    "-hide_banner",
+    "-i", inputFile,
+    "-filter_complex", "ebur128=peak=true:metadata=1:framelog=verbose",
+    "-f", "null",
+    "-",
+  ];
+
+  const output = await runFFmpeg(args);
+  return parseEbur128(output);
+}
+
+// Test
+(async () => {
+  const stats = await analyzeAudio("/home/cj/Videos/28years/voiceover-compressed2.wav");
+  console.log("FINAL STATS:", stats);
+})();

services/our/src/tasks/analyzeAudio.ts

@@ -0,0 +1,135 @@
+/**
+ * Analyze the video's audio track using ITU-R BS.1770/EBU R128
+ * to extract Integrated Loudness (LUFS-I), Loudness Range (LRA),
+ * and True Peak (dBTP). This helps identify which VODs/streamers
+ * maintain consistent, listener-friendly audio setups.
+ *
+ * ## LUFS Reference
+ * Apple Music -16 LUFS
+ * YouTube     -13 LUFS
+ * Spotify     -14 LUFS
+ * Tidal       -14 LUFS
+ */
+
+import type { Task, Helpers } from "graphile-worker";
+import { PrismaClient } from "../../generated/prisma";
+import { withAccelerate } from "@prisma/extension-accelerate";
+import { getOrDownloadAsset } from "../utils/cache";
+import { env } from "../config/env";
+import { getS3Client } from "../utils/s3";
+import { getNanoSpawn } from "../utils/nanoSpawn";
+import logger from "../utils/logger";
+
+const prisma = new PrismaClient().$extends(withAccelerate());
+const ffmpegBin = "ffmpeg";
+
+interface Payload {
+  vodId: string;
+}
+
+interface AudioStats {
+  lufsIntegrated: number; // LUFS-I
+  lra: number;            // Loudness Range
+  peak: number;           // True Peak (expressed as dBTP ceiling, e.g. -0.1)
+}
+
+export async function runFFmpeg(args: string[]): Promise<string> {
+  const spawn = await getNanoSpawn();
+
+  try {
+    const result = await spawn(ffmpegBin, args, {
+      stdout: 'pipe',
+      stderr: 'pipe',
+    });
+
+    // success → return combined output
+    return result.output;
+  } catch (err: any) {
+    // failure → throw with stderr
+    throw new Error(err.output || err.message);
+  }
+}
+
+function parseEbur128(output: string): AudioStats {
+  const lines = output.split("\n").map((l) => l.trim());
+
+  let lufsIntegrated = NaN;
+  let lra = NaN;
+  let peak = NaN;
+  let inTruePeak = false;
+
+  for (const line of lines) {
+    // Integrated loudness
+    const iMatch = line.match(/^I:\s*(-?\d+(\.\d+)?) LUFS/);
+    if (iMatch) lufsIntegrated = parseFloat(iMatch[1]);
+
+    // Loudness range
+    const lraMatch = line.match(/^LRA:\s*(-?\d+(\.\d+)?)/);
+    if (lraMatch) lra = parseFloat(lraMatch[1]);
+
+    // True peak
+    if (line === "True peak:") inTruePeak = true;
+    if (inTruePeak) {
+      const pMatch = line.match(/Peak:\s*(-?\d+\.?\d*)\s*dBFS/);
+      if (pMatch) {
+        peak = parseFloat(pMatch[1]); // no negation
+        inTruePeak = false;
+      }
+    }
+  }
+
+  return { lufsIntegrated, lra, peak };
+}
+
+
+async function analyzeAudio(inputFile: string): Promise<AudioStats> {
+  const args = [
+    "-hide_banner",
+    "-i", inputFile,
+    "-filter_complex", "ebur128=peak=true:metadata=1:framelog=verbose",
+    "-f", "null",
+    "-",
+  ];
+
+  const output = await runFFmpeg(args);
+  return parseEbur128(output);
+}
+
+function assertPayload(payload: any): asserts payload is Payload {
+  if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
+  if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
+}
+
+export default async function main(payload: any, helpers: Helpers) {
+  assertPayload(payload);
+  const { vodId } = payload;
+
+  const vod = await prisma.vod.findFirstOrThrow({
+    where: { id: vodId },
+  });
+
+  if (vod.audioIntegratedLufs && vod.audioLoudnessRange && vod.audioTruePeak) {
+    logger.info(`Skipping analysis — vod ${vodId} already complete.`);
+    return;
+  }
+
+  if (!vod.sourceVideo) {
+    throw new Error(`vod ${vodId} is missing a sourceVideo.`);
+  }
+
+  const s3Client = getS3Client();
+  const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
+  logger.info(`videoFilePath=${videoFilePath}`);
+
+  const results = await analyzeAudio(videoFilePath);
+  logger.info(`results=${JSON.stringify(results)}`);
+
+  await prisma.vod.update({
+    where: { id: vodId },
+    data: {
+      audioIntegratedLufs: results.lufsIntegrated,
+      audioLoudnessRange: results.lra,
+      audioTruePeak: results.peak,
+    },
+  });
+}

services/our/src/tasks/consolidate_twitch_channel_rewards.ts

@@ -1,184 +0,0 @@
-// src/tasks/consolidate_twitch_channel_rewards.ts
-
-import type { Task, Helpers } from "graphile-worker";
-import { PrismaClient, User, type Pick } from '../../generated/prisma';
-import { withAccelerate } from "@prisma/extension-accelerate";
-import { env } from "../config/env";
-import { constants } from "../config/constants";
-import { getRateLimiter } from "../utils/rateLimiter";
-import logger from '../utils/logger';
-
-const prisma = new PrismaClient().$extends(withAccelerate());
-const cprPath = env.TWITCH_MOCK ? constants.twitch.dev.paths.channelPointRewards : constants.twitch.prod.paths.channelPointRewards;
-
-interface Payload {
-    userId: number;
-}
-
-export interface TwitchChannelPointReward {
-    id: string;
-    broadcaster_id: string;
-    cost: number;
-    title: string;
-    is_in_stock: boolean;
-    [key: string]: any;
-}
-
-const getAuthToken = (user: User) => env.TWITCH_MOCK ? env.TWITCH_MOCK_USER_ACCESS_TOKEN : user.twitchToken?.accessToken;
-
-
-function assertPayload(payload: any): asserts payload is Payload {
-    if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
-    if (typeof payload.userId !== "number") throw new Error("invalid payload.userId");
-}
-
-
-const getTwitchChannelPointRewards = async (user: User) => {
-    if (!user?.twitchToken) throw new Error("Missing Twitch token");
-
-    const authToken = getAuthToken(user);
-    const limiter = getRateLimiter();
-    await limiter.consume('twitch', 1);
-
-    const query = new URLSearchParams({ broadcaster_id: user.twitchId });
-    const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
-        headers: {
-            'Authorization': `Bearer ${authToken}`,
-            'Client-Id': env.TWITCH_CLIENT_ID
-        }
-    });
-
-    if (!res.ok) throw new Error(`Failed to fetch rewards: ${res.statusText}`);
-    return res.json();
-};
-
-const createTwitchReward = async (user: User, pick: Pick) => {
-    const authToken = getAuthToken(user);
-    const limiter = getRateLimiter();
-    await limiter.consume('twitch', 1);
-    logger.debug('pick as follows')
-    logger.debug(pick)
-
-    logger.debug(`pick?.waifu?.name=${pick?.waifu?.name}`)
-
-    const query = new URLSearchParams({ broadcaster_id: user.twitchId });
-    const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
-        method: 'POST',
-        headers: {
-            'Authorization': `Bearer ${authToken}`,
-            'Client-Id': env.TWITCH_CLIENT_ID
-        },
-        body: JSON.stringify({ cost: user.redeemCost, title: pick.waifu.name })
-    });
-
-    if (!res.ok) throw new Error(`Failed to create reward: ${res.statusText}`);
-
-    const data = await res.json();
-    const rewardId = data.data?.[0]?.id;
-    if (!rewardId) throw new Error("No reward ID returned");
-
-    await prisma.pick.update({
-        where: { id: pick.id },
-        data: { twitchChannelPointRewardId: rewardId }
-    });
-};
-
-const deleteTwitchReward = async (user: User, rewardId: string) => {
-    const authToken = getAuthToken(user);
-    const limiter = getRateLimiter();
-    await limiter.consume('twitch', 1);
-
-    const query = new URLSearchParams({ broadcaster_id: user.twitchId, id: rewardId });
-    const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
-        method: 'DELETE',
-        headers: {
-            'Authorization': `Bearer ${authToken}`,
-            'Client-Id': env.TWITCH_CLIENT_ID
-        }
-    });
-
-    if (!res.ok) throw new Error(`Failed to delete reward ${rewardId}`);
-};
-
-const updateTwitchReward = async (user: User, rewardId: string, newCost: number) => {
-    const authToken = getAuthToken(user);
-    const limiter = getRateLimiter();
-    await limiter.consume('twitch', 1);
-
-    const query = new URLSearchParams({ broadcaster_id: user.twitchId, id: rewardId });
-    const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
-        method: 'PATCH',
-        headers: {
-            'Authorization': `Bearer ${authToken}`,
-            'Client-Id': env.TWITCH_CLIENT_ID
-        },
-        body: JSON.stringify({ cost: newCost })
-    });
-
-    if (!res.ok) throw new Error(`Failed to update reward ${rewardId}`);
-};
-
-const consolidateTwitchRewards = async (userId: number) => {
-    const user = await prisma.user.findFirstOrThrow({
-        where: { id: userId },
-        include: { twitchToken: true, Waifu: true }
-    });
-
-    // Fetch picks (most recent N)
-    const picks = await prisma.pick.findMany({
-        where: { userId },
-        orderBy: { createdAt: 'desc' },
-        take: constants.twitch.maxChannelPointRewards,
-        include: {
-            waifu: true
-        }
-    });
-
-    // Ensure every pick has a reward before processing Twitch side
-    for (const pick of picks) {
-        if (!pick.twitchChannelPointRewardId) {
-            logger.debug(`Creating new reward for pick: ${pick.id}`);
-            await createTwitchReward(user, pick);
-        }
-    }
-
-    // Refresh picks after reward creation
-    const updatedPicks = await prisma.pick.findMany({
-        where: { userId },
-        orderBy: { createdAt: 'desc' },
-        take: constants.twitch.maxChannelPointRewards
-    });
-
-    // Get the most recent N reward IDs
-    const currentPickIds = new Set(
-        updatedPicks.slice(0, user.waifuChoicePoolSize).map(p => p.twitchChannelPointRewardId)
-    );
-
-    logger.debug('currentPickIds as follows');
-    logger.debug(currentPickIds);
-
-    // Fetch Twitch-side rewards
-    const twitchData = await getTwitchChannelPointRewards(user);
-    const twitchRewards: TwitchChannelPointReward[] = twitchData.data;
-
-    // Delete or update Twitch rewards not in current pick set
-    for (const reward of twitchRewards) {
-        if (!updatedPicks.some(p => p.twitchChannelPointRewardId === reward.id)) continue;
-
-        if (!currentPickIds.has(reward.id)) {
-            logger.debug(`Deleting out-of-date reward: ${reward.id}`);
-            await deleteTwitchReward(user, reward.id);
-        } else if (reward.cost !== user.redeemCost) {
-            logger.debug(`Updating reward cost for: ${reward.id}`);
-            await updateTwitchReward(user, reward.id, user.redeemCost);
-        }
-    }
-};
-
-
-const task: Task = async (payload: any, helpers) => {
-    assertPayload(payload);
-    await consolidateTwitchRewards(payload.userId);
-};
-
-export default task;

services/our/src/tasks/createFunscript copy.ts.noexec

@@ -1,451 +0,0 @@
-import type { Task, Helpers } from "graphile-worker";
-import { PrismaClient } from "../../generated/prisma";
-import { withAccelerate } from "@prisma/extension-accelerate";
-import { getOrDownloadAsset } from "../utils/cache";
-import { env } from "../config/env";
-import { getS3Client, uploadFile } from "../utils/s3";
-import { nanoid } from "nanoid";
-import { existsSync, rmSync } from "node:fs";
-import { join, basename, extname } from "node:path";
-import { readFile, writeFile, readdir } from 'node:fs/promises';
-import yaml from 'js-yaml';
-import { getNanoSpawn } from "../utils/nanoSpawn";
-import which from "which";
-import * as ort from 'onnxruntime-node';
-
-interface Payload {
-  vodId: string;
-}
-
-interface Detection {
-  startFrame: number;
-  endFrame: number;
-  className: string;
-}
-
-interface DataYaml {
-  path: string;
-  train: string;
-  val: string;
-  names: Record<string, string>;
-}
-
-interface FunscriptAction {
-  at: number;
-  pos: number;
-}
-
-interface Funscript {
-  version: string;
-  actions: FunscriptAction[];
-}
-
-
-interface ClassPositionMap {
-  [className: string]: number | 'pattern';
-}
-
-
-const prisma = new PrismaClient().$extends(withAccelerate());
-
-
-function assertPayload(payload: any): asserts payload is Payload {
-  if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
-  if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
-}
-
-async function loadDataYaml(yamlPath: string): Promise<DataYaml> {
-  const yamlContent = await readFile(yamlPath, 'utf8');
-  return yaml.load(yamlContent) as DataYaml;
-}
-
-
-async function preparePython(helpers) {
-  const spawn = await getNanoSpawn();
-  const venvPath = join(env.VIBEUI_DIR, "venv");
-
-  // Determine Python executable
-  let pythonCmd;
-  try {
-    pythonCmd = which.sync("python3");
-  } catch {
-    logger.error("Python is not installed or not in PATH.");
-    throw new Error("Python not found in PATH.");
-  }
-
-  // If venv doesn't exist, create it
-  if (!existsSync(venvPath)) {
-    logger.info("Python venv not found. Creating one...");
-
-    try {
-      await spawn(pythonCmd, ["-m", "venv", "venv"], {
-        cwd: env.VIBEUI_DIR,
-      });
-
-      logger.info("Python venv successfully created.");
-    } catch (err) {
-      logger.error("Failed to create Python venv:", err);
-
-      // Clean up partially created venv if needed
-      try {
-        if (existsSync(venvPath)) {
-          rmSync(venvPath, { recursive: true, force: true });
-          logger.warn("Removed broken venv directory.");
-        }
-      } catch (cleanupErr) {
-        logger.error("Error while cleaning up broken venv:", cleanupErr);
-      }
-
-      throw new Error("Python venv creation failed. Check if python3 and python3-venv are installed.");
-    }
-  } else {
-    logger.info("Using existing Python venv.");
-  }
-}
-
-
-async function ffprobe(videoPath: string): Promise<{ fps: number; frames: number }> {
-  const spawn = await getNanoSpawn()
-  const { stdout } = await spawn('ffprobe', [
-    '-v', 'error',
-    '-select_streams', 'v:0',
-    '-count_frames',
-    '-show_entries', 'stream=nb_read_frames,r_frame_rate',
-    '-of', 'default=nokey=1:noprint_wrappers=1',
-    videoPath,
-  ])
-
-  const [frameRateStr, frameCountStr] = stdout.trim().split('\n')
-  const [num, denom] = frameRateStr.trim().split('/').map(Number)
-  const fps = num / denom
-  const frames = parseInt(frameCountStr.trim(), 10)
-
-  return { fps, frames }
-}
-
-
-
-
-
-
-
-export async function buildFunscript(
-  helpers: Helpers,
-  predictionOutput: string,
-  videoPath: string
-): Promise<string> {
-  const labelDir = join(predictionOutput, 'labels');
-  const yamlPath = join(predictionOutput, 'data.yaml');
-  const outputPath = join(process.env.CACHE_ROOT ?? '/tmp', `${nanoid()}.funscript`);
-  logger.info('Starting Funscript generation');
-
-  try {
-
-    const data = await loadDataYaml(join(env.VIBEUI_DIR, 'data.yaml'))
-    const classPositionMap = await loadClassPositionMap(data, helpers);
-    const { fps, totalFrames } = await loadVideoMetadata(videoPath, helpers);
-    const detectionSegments = await processLabelFiles(labelDir, helpers, data);
-    const totalDurationMs = Math.floor((totalFrames / fps) * 1000);
-    const actions = generateActions(totalDurationMs, fps, detectionSegments, classPositionMap);
-    await writeFunscript(outputPath, actions, helpers);
-
-    return outputPath;
-  } catch (error) {
-    logger.error(`Error generating Funscript: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    throw error;
-  }
-}
-
-export async function inference(helpers: Helpers, videoFilePath: string): Promise<string> {
-  const spawn = await getNanoSpawn()
-
-  const modelPath = join(env.VIBEUI_DIR, 'runs/detect/vibeui/weights/best.pt')
-
-  // Generate a unique name based on video name + UUID
-  const videoExt = extname(videoFilePath)             // e.g. '.mp4'
-  const videoName = basename(videoFilePath, videoExt) // removes the extension
-  const uniqueName = `${videoName}-${nanoid()}`
-  const customProjectDir = 'runs' // or any custom folder
-  const outputPath = join(env.VIBEUI_DIR, customProjectDir, uniqueName)
-
-  await spawn('./venv/bin/yolo', [
-    'predict',
-    `model=${modelPath}`,
-    `source=${videoFilePath}`,
-    'save=False',
-    'save_txt=True',
-    'save_conf=True',
-    `project=${customProjectDir}`,
-    `name=${uniqueName}`,
-  ], {
-    cwd: env.VIBEUI_DIR,
-    stdio: 'inherit',
-  })
-
-  return outputPath // contains labels/ folder and predictions
-}
-
-
-async function loadClassPositionMap(data: DataYaml, helpers: Helpers): Promise<ClassPositionMap> {
-  try {
-
-    if (
-      !data ||
-      typeof data !== 'object' ||
-      !('names' in data) ||
-      typeof data.names !== 'object' ||
-      data.names === null ||
-      Object.keys(data.names).length === 0
-    ) {
-      throw new Error('Invalid data.yaml: "names" field is missing, not an object, or empty');
-    }
-
-    const positionMap: ClassPositionMap = {
-      ControlledByTipper: 50,
-      ControlledByTipperHigh: 80,
-      ControlledByTipperLow: 20,
-      ControlledByTipperMedium: 50,
-      ControlledByTipperUltrahigh: 95,
-      Ring1: 30,
-      Ring2: 40,
-      Ring3: 50,
-      Ring4: 60,
-      Earthquake: 'pattern',
-      Fireworks: 'pattern',
-      Pulse: 'pattern',
-      Wave: 'pattern',
-      Pause: 0,
-      RandomTime: 70,
-      HighLevel: 80,
-      LowLevel: 20,
-      MediumLevel: 50,
-      UltraHighLevel: 95
-    };
-
-    const names = Object.values(data.names);
-    for (const name of names) {
-      if (typeof name !== 'string' || name.trim() === '') {
-        logger.info(`Skipping invalid class name: ${name}`);
-        continue;
-      }
-      if (!(name in positionMap)) {
-        logger.info(`No position mapping for class "${name}", defaulting to 0`);
-        positionMap[name] = 0;
-      }
-    }
-
-    logger.info(`Loaded class position map: ${JSON.stringify(positionMap)}`);
-    return positionMap;
-  } catch (error) {
-    logger.error(`Error loading data.yaml: ${error instanceof Error ? error.message : 'Unknown error'}`);
-    throw error;
-  }
-}
-
-function generatePatternPositions(startMs: number, durationMs: number, className: string, fps: number): FunscriptAction[] {
-  const actions: FunscriptAction[] = [];
-  const frameDurationMs = 1000 / fps;
-  const totalFrames = Math.floor(durationMs / frameDurationMs);
-  const intervalMs = 100;
-
-  for (let timeMs = 0; timeMs < durationMs; timeMs += intervalMs) {
-    const progress = timeMs / durationMs;
-    let pos = 0;
-
-    switch (className) {
-      case 'Pulse':
-        pos = Math.round(50 * Math.sin(progress * 2 * Math.PI));
-        break;
-      case 'Wave':
-        pos = Math.round(50 + 50 * Math.sin(progress * 2 * Math.PI));
-        break;
-      case 'Fireworks':
-        pos = Math.random() > 0.5 ? 80 : 0;
-        break;
-      case 'Earthquake':
-        pos = Math.round(90 * Math.sin(progress * 4 * Math.PI) + (Math.random() - 0.5) * 10);
-        pos = Math.max(0, Math.min(90, pos));
-        break;
-    }
-
-    actions.push({ at: startMs + timeMs, pos });
-  }
-
-  return actions;
-}
-
-async function loadVideoMetadata(videoPath: string, helpers: Helpers) {
-  const { fps, frames: totalFrames } = await ffprobe(videoPath);
-  logger.info(`Video metadata: fps=${fps}, frames=${totalFrames}`);
-  return { fps, totalFrames };
-}
-
-async function processLabelFiles(labelDir: string, helpers: Helpers, data: DataYaml): Promise<Detection[]> {
-  const labelFiles = (await readdir(labelDir)).filter(file => file.endsWith('.txt'));
-  const detections: Map<number, Detection[]> = new Map();
-  const names = data.names;
-
-  for (const file of labelFiles) {
-    const match = file.match(/(\d+)\.txt$/);
-    if (!match) {
-      logger.info(`Skipping invalid filename: ${file}`);
-      continue;
-    }
-    const frameIndex = parseInt(match[1], 10);
-    if (isNaN(frameIndex)) {
-      logger.info(`Skipping invalid frame index from filename: ${file}`);
-      continue;
-    }
-
-    const content = await readFile(join(labelDir, file), 'utf8');
-    const lines = content.trim().split('\n');
-    const frameDetections: Detection[] = [];
-    let maxConfidence = 0;
-    let selectedClassIndex = -1;
-
-    for (const line of lines) {
-      const parts = line.trim().split(/\s+/);
-      if (parts.length < 6) continue;
-
-      const classIndex = parseInt(parts[0], 10);
-      const confidence = parseFloat(parts[5]);
-      if (isNaN(classIndex) || isNaN(confidence)) continue;
-
-      if (confidence >= 0.7 && confidence > maxConfidence) {
-        maxConfidence = confidence;
-        selectedClassIndex = classIndex;
-      }
-    }
-
-    if (maxConfidence > 0) {
-      const className = (data.names as Record<string, string>)[selectedClassIndex.toString()];
-      if (className) {
-        frameDetections.push({ startFrame: frameIndex, endFrame: frameIndex, className });
-      }
-    }
-
-    if (frameDetections.length > 0) {
-      detections.set(frameIndex, frameDetections);
-    }
-  }
-
-  // Merge overlapping detections into continuous segments
-  const detectionSegments: Detection[] = [];
-  let currentDetection: Detection | null = null;
-
-  for (const [frameIndex, frameDetections] of detections.entries()) {
-    for (const detection of frameDetections) {
-      if (!currentDetection || currentDetection.className !== detection.className) {
-        if (currentDetection) detectionSegments.push(currentDetection);
-        currentDetection = { ...detection, endFrame: frameIndex };
-      } else {
-        currentDetection.endFrame = frameIndex;
-      }
-    }
-  }
-  if (currentDetection) detectionSegments.push(currentDetection);
-
-  return detectionSegments;
-}
-
-function generateActions(totalDurationMs: number, fps: number, detectionSegments: Detection[], classPositionMap: ClassPositionMap): FunscriptAction[] {
-  const intervalMs = 100;
-  const actions: FunscriptAction[] = [];
-
-  // Generate static position actions
-  for (let timeMs = 0; timeMs <= totalDurationMs; timeMs += intervalMs) {
-    const frameIndex = Math.floor((timeMs / 1000) * fps);
-    let position = 0;
-
-    for (const segment of detectionSegments) {
-      if (frameIndex >= segment.startFrame && frameIndex <= segment.endFrame) {
-        const className = segment.className;
-        if (typeof classPositionMap[className] === 'number') {
-          position = classPositionMap[className];
-          break;
-        }
-      }
-    }
-    actions.push({ at: timeMs, pos: position });
-  }
-
-  // Overlay pattern-based actions
-  for (const segment of detectionSegments) {
-    const className = segment.className;
-    if (classPositionMap[className] === 'pattern') {
-      const startMs = Math.floor((segment.startFrame / fps) * 1000);
-      const durationMs = Math.floor(((segment.endFrame - segment.startFrame + 1) / fps) * 1000);
-      const patternActions = generatePatternPositions(startMs, durationMs, className, fps);
-      actions.push(...patternActions);
-    }
-  }
-
-  // Sort actions by time and remove duplicates
-  actions.sort((a, b) => a.at - b.at);
-  const uniqueActions: FunscriptAction[] = [];
-  let lastTime = -1;
-  for (const action of actions) {
-    if (action.at !== lastTime) {
-      uniqueActions.push(action);
-      lastTime = action.at;
-    }
-  }
-
-  return uniqueActions;
-}
-
-async function writeFunscript(outputPath: string, actions: FunscriptAction[], helpers: Helpers) {
-  const funscript: Funscript = { version: '1.0', actions };
-  await writeFile(outputPath, JSON.stringify(funscript, null, 2));
-  logger.info(`Funscript generated: ${outputPath} (${actions.length} actions)`);
-}
-
-
-const createFunscript: Task = async (payload: any, helpers: Helpers) => {
-  assertPayload(payload);
-  const { vodId } = payload;
-
-
-
-  const vod = await prisma.vod.findFirstOrThrow({ where: { id: vodId } });
-
-  if (vod.funscript) {
-    logger.info(`Doing nothing-- vod ${vodId} already has a funscript.`);
-    return;
-  }
-
-  if (!vod.sourceVideo) {
-    const msg = `Cannot create funscript: Vod ${vodId} is missing a source video.`;
-    logger.warn(msg);
-    throw new Error(msg);
-  }
-
-
-
-  const s3Client = getS3Client();
-  const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
-  logger.info(`Downloaded video to ${videoFilePath}`);
-
-  logger.info(`Creating funscript for vod ${vodId}...`);
-
-  const predictionOutput = await inference(helpers, videoFilePath);
-  logger.info(`prediction output ${predictionOutput}`);
-
-
-  const funscriptFilePath = await buildFunscript(helpers, predictionOutput, videoFilePath)
-
-
-  const s3Key = `funscripts/${vodId}.funscript`;
-  const s3Url = await uploadFile(s3Client, env.S3_BUCKET, s3Key, funscriptFilePath, "application/json");
-
-  logger.info(`Uploaded funscript to S3: ${s3Url}`);
-
-  await prisma.vod.update({
-    where: { id: vodId },
-    data: { funscript: s3Key }
-  });
-
-  logger.info(`Funscript saved to database for vod ${vodId}`);
-};
-
-export default createFunscript;

services/our/src/tasks/createFunscript.ts.ai.noexec

@@ -1,75 +0,0 @@
-
-import type { Task, Helpers } from "graphile-worker";
-import { PrismaClient } from "../../generated/prisma";
-import { withAccelerate } from "@prisma/extension-accelerate";
-import { getOrDownloadAsset } from "../utils/cache";
-import { env } from "../config/env";
-import { getS3Client, uploadFile } from "../utils/s3";
-import { buildFunscript } from '../utils/funscripts';
-import { getModelClasses, vibeuiInference } from "../utils/vibeui";
-import { join } from "node:path";
-
-interface Payload {
-  vodId: string;
-}
-
-
-
-const prisma = new PrismaClient().$extends(withAccelerate());
-
-
-function assertPayload(payload: any): asserts payload is Payload {
-  if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
-  if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
-}
-
-
-const createFunscript: Task = async (payload: any, helpers: Helpers) => {
-  assertPayload(payload);
-  const { vodId } = payload;
-
-
-
-  const vod = await prisma.vod.findFirstOrThrow({ where: { id: vodId } });
-
-  if (vod.funscript) {
-    logger.info(`Doing nothing-- vod ${vodId} already has a funscript.`);
-    return;
-  }
-
-  if (!vod.sourceVideo) {
-    const msg = `Cannot create funscript: Vod ${vodId} is missing a source video.`;
-    logger.warn(msg);
-    throw new Error(msg);
-  }
-
-
-
-  const s3Client = getS3Client();
-  const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
-  logger.info(`Downloaded video to ${videoFilePath}`);
-
-  logger.info(`Creating funscript for vod ${vodId}...`);
-
-  const modelPath = join(env.VIBEUI_DIR, 'vibeui.onnx')
-  const predictionOutput = await vibeuiInference(modelPath, videoFilePath);
-  logger.info(`prediction output ${predictionOutput}`);
-  const classes = await getModelClasses(modelPath)
-
-  const funscriptFilePath = await buildFunscript(classes, predictionOutput, videoFilePath)
-
-
-  const s3Key = `funscripts/${vodId}.funscript`;
-  const s3Url = await uploadFile(s3Client, env.S3_BUCKET, s3Key, funscriptFilePath, "application/json");
-
-  logger.info(`Uploaded funscript to S3: ${s3Url}`);
-
-  await prisma.vod.update({
-    where: { id: vodId },
-    data: { funscript: s3Key }
-  });
-
-  logger.info(`Funscript saved to database for vod ${vodId}`);
-};
-
-export default createFunscript;

services/our/src/tasks/getSourceVideoMetadata.ts

@@ -4,6 +4,8 @@ import { env } from "../config/env";
 import { S3Client } from "@aws-sdk/client-s3";
 import logger from "../utils/logger";
 import { getNanoSpawn } from "../utils/nanoSpawn";
+import { getOrDownloadAsset } from "../utils/cache";
+import { getS3Client } from "../utils/s3";
 
 const prisma = new PrismaClient();
 const client = new S3Client({
@@ -90,7 +92,9 @@ const getSourceVideoMetadata: Task = async (payload: unknown, helpers) => {
     });
 
     // Skip if already processed
-    if (vod.sourceVideoDuration) {
+    if (
+      vod.sourceVideoDuration
+    ) {
       logger.debug(`VOD ${vodId} already has metadata`);
       return;
     }
@@ -99,8 +103,14 @@ const getSourceVideoMetadata: Task = async (payload: unknown, helpers) => {
       throw new Error(`VOD ${vodId} has no sourceVideo`);
     }
 
+    // download vod
+    const s3Client = getS3Client();
+    const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
+    logger.info(`videoFilePath=${videoFilePath}`);
+
+
     // Run ffprobe
-    const meta = await getVideoMetadata(vod.sourceVideo);
+    const meta = await getVideoMetadata(videoFilePath);
 
     // Update DB
     await prisma.vod.update({

services/our/src/tasks/hello.ts

@@ -1,21 +0,0 @@
-// src/tasks/hello.ts
-
-import type { Task, Helpers } from "graphile-worker";
-import logger from "../utils/logger";
-
-interface Payload {
-    name: string;
-}
-
-function assertPayload(payload: any): asserts payload is Payload {
-    if (typeof payload !== "object" || !payload) throw new Error("invalid");
-    if (typeof payload.name !== "string") throw new Error("invalid");
-}
-
-
-export default async function hello(payload: any, helpers: Helpers) {
-    assertPayload(payload);
-    const { name } = payload;
-    logger.info(`Helloooooo, ${name}`);
-};
-

services/our/src/tasks/scheduleVodProcessing.ts

@@ -42,6 +42,7 @@ const scheduleVodProcessing: Task = async (payload: unknown, helpers) => {
   jobs.push(helpers.addJob("copyV1S3ToV2", { vodId }));
   if (!vod.sourceVideo) jobs.push(helpers.addJob("getSourceVideo", { vodId }));
   if (!vod.sourceVideoDuration) jobs.push(helpers.addJob("getSourceVideoMetadata", { vodId }))
+  if (!vod.audioIntegratedLufs || !vod.audioLoudnessRange || !vod.audioTruePeak) jobs.push(helpers.addJob("analyzeAudio", { vodId }))
   if (!vod.sha256sum) jobs.push(helpers.addJob("generateVideoChecksum", { vodId }));
   if (!vod.thumbnail) jobs.push(helpers.addJob("createVideoThumbnail", { vodId }));
   if (!vod.hlsPlaylist) jobs.push(helpers.addJob("createHlsPlaylist", { vodId }));

services/our/src/utils/formatters.ts

@@ -37,4 +37,18 @@ export function generateS3Path(slug: string, date: Date, vodId: string, filename
     const day = pad(getDate(date));
 
     return `fp/${slug}/${year}/${month}/${day}/${vodId}/${filename}`;
-}
+}
+
+
+export function formatDuration(ms: number): string {
+    const totalSeconds = Math.floor(ms / 1000);
+    const hours = Math.floor(totalSeconds / 3600);
+    const minutes = Math.floor((totalSeconds % 3600) / 60);
+    const seconds = totalSeconds % 60;
+
+    const hh = hours.toString().padStart(2, '0');
+    const mm = minutes.toString().padStart(2, '0');
+    const ss = seconds.toString().padStart(2, '0');
+
+    return hours > 0 ? `${hh}:${mm}:${ss}` : `${mm}:${ss}`;
+}

services/our/src/utils/hbsHelpers.ts

@@ -5,7 +5,7 @@ import { Role } from '../../generated/prisma'
 import { isModerator, hasRole } from './privs'
 import { signUrl } from './cdn'
 import { extractBasePath } from './filesystem'
-import { truncate } from './formatters.ts'
+import { formatDuration, truncate } from './formatters.ts'
 import { icons } from './icons.ts'
 import logger from './logger.ts'
 import HandlebarsLib, { HelperOptions, SafeString } from 'handlebars';
@@ -19,6 +19,9 @@ export function registerHbsHelpers(Handlebars: typeof HandlebarsLib) {
     if (!dateString) return ''
     return format(new Date(dateString), 'yyyy-MM-dd')
   })
+  Handlebars.registerHelper('formatDuration', function (duration) {
+    return formatDuration(duration)
+  })
   Handlebars.registerHelper('identicon', function (str, size = 48) {
     return jdenticon.toSvg(str, size)
   })

services/our/src/views/vod.hbs

@@ -304,15 +304,46 @@
 
       <div class="mb-5">
         <h2 class="title is-4">VOD Metadata</h2>
-        <div class="box">
+
+        <div class="ml-5 mt-4">
           {{#if vod.sourceVideoDuration }}
-          <p>Duration: {{vod.sourceVideoDuration}}</p>
+          <div class="content">
-          <p>FPS: {{vod.sourceVideoFPS}}</p>
+            <p><strong>Duration</strong> {{formatDuration vod.sourceVideoDuration}}</p>
-          <p>Video Codec: {{vod.sourceVideoCodec}}</p>
+            <p><strong>FPS</strong> {{vod.sourceVideoFps}}</p>
-          <p>Audio Codec: {{vod.sourceAudioCodec}}</p>
+            <p><strong>Video Codec</strong> {{vod.sourceVideoCodec}}</p>
+            <p><strong>Audio Codec</strong> {{vod.sourceAudioCodec}}</p>
+          </div>
+          {{else}}
+          <article class="notification is-light">
+            VOD metadata is processing
+          </article>
+          {{/if}}
+        </div>
+      </div>
+
+
+      <div class="mb-5">
+        <h2 class="title is-4">Audio Analysis</h2>
+        <div class="ml-5 mt-5 mb-5">
+          {{#if vod.audioIntegratedLufs }}
+          <div class="content">
+            <p>
+              <abbr
+                title="Loudness Units relative to Full Scale - the overall perceived loudness of the audio">LUFS-I</abbr>
+              <strong>{{vod.audioIntegratedLufs}}</strong>
+            </p>
+            <p>
+              <abbr title="Loudness Range - the dynamic range of the audio in LUFS">LRA</abbr>
+              <strong>{{vod.audioLoudnessRange}}</strong>
+            </p>
+            <p>
+              <abbr title="True Peak: the maximum instantaneous signal level in dBTP">TP</abbr>
+              <strong>{{vod.audioTruePeak}}</strong>
+            </p>
+          </div>
           {{else}}
           <article class="notification">
-            Vod metadata is processing
+            Audio analysis is processing
           </article>
           {{/if}}
         </div>