futureporn/fp
1
0
Fork 0
Code Issues 33 Pull Requests Actions 3 Packages Projects Releases Wiki Activity

add audio analysis
Some checks are pending
ci / build (push) Waiting to run
Details
ci / test (push) Waiting to run
Details

Browse Source
This commit is contained in:
CJ_Clippy 2025-09-01 22:49:27 -08:00
parent 28f8b7e94e
commit bdc4894e3e
13 changed files with 306 additions and 747 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
services/our/prisma/migrations/20250902050219_add_lufs/migration.sql Normal file
View File

@ -0,0 +1,4 @@
-- AlterTable
ALTER TABLE "Vod" ADD COLUMN "audioIntegratedLufs" DOUBLE PRECISION,
ADD COLUMN "audioLoudnessRange" DOUBLE PRECISION,
ADD COLUMN "audioTruePeak" DOUBLE PRECISION;

18
services/our/prisma/schema.prisma
View File

@ -79,12 +79,18 @@ model Vod {
sourceVideoCodec String?
sourceAudioCodec String?
sourceVideoFps Float?
hlsPlaylist String?
thumbnail String?
asrVttKey String?
slvttSheetKeys Json?
slvttVTTKey String?
magnetLink String?
// audio analysis
audioIntegratedLufs Float? // Integrated loudness (LUFS-I)
audioLoudnessRange Float? // Loudness Range (LRA)
audioTruePeak Float? // True Peak (dBTP)
hlsPlaylist String?
thumbnail String?
asrVttKey String?
slvttSheetKeys Json?
slvttVTTKey String?
magnetLink String?
status VodStatus @default(pending)
sha256sum String?

86
services/our/scripts/analyze.ts Normal file
View File

@ -0,0 +1,86 @@
import { spawn } from "child_process";
export interface AudioStats {
peak: number; // True peak in dBTP
lufsIntegrated: number; // LUFS-I
lra: number; // Loudness range
}
async function runFFmpeg(args: string[]): Promise<string> {
return new Promise((resolve, reject) => {
console.log("Running ffmpeg:", args.join(" "));
const ff = spawn("ffmpeg", args);
let stdout = "";
let stderr = "";
ff.stdout.on("data", (data) => {
const str = data.toString();
stdout += str;
process.stdout.write(str); // debug passthrough
});
ff.stderr.on("data", (data) => {
const str = data.toString();
stderr += str;
process.stdout.write(str); // debug passthrough
});
ff.on("close", (code) => {
console.log("ffmpeg exited with code:", code);
if (code === 0) resolve(stdout + "\n" + stderr);
else reject(new Error(stderr));
});
});
}
function parseEbur128(output: string): AudioStats {
const lines = output.split("\n").map(l => l.trim());
let lufsIntegrated = NaN;
let lra = NaN;
let peak = NaN;
let inTruePeak = false;
for (const line of lines) {
// Integrated loudness (LUFS-I)
const iMatch = line.match(/^I:\s*(-?\d+(\.\d+)?) LUFS/);
if (iMatch) lufsIntegrated = parseFloat(iMatch[1]);
// Loudness range (LRA)
const lraMatch = line.match(/^LRA:\s*(-?\d+(\.\d+)?)/);
if (lraMatch) lra = parseFloat(lraMatch[1]);
// True peak (dBTP)
if (line === "True peak:") inTruePeak = true;
if (inTruePeak) {
const pMatch = line.match(/Peak:\s*(-?\d+(\.\d+)?) dBFS/);
if (pMatch) {
peak = parseFloat(pMatch[1]);
inTruePeak = false;
}
}
}
return { lufsIntegrated, lra, peak };
}
export async function analyzeAudio(inputFile: string): Promise<AudioStats> {
const args = [
"-hide_banner",
"-i", inputFile,
"-filter_complex", "ebur128=peak=true:metadata=1:framelog=verbose",
"-f", "null",
"-",
];
const output = await runFFmpeg(args);
return parseEbur128(output);
}
// Test
(async () => {
const stats = await analyzeAudio("/home/cj/Videos/28years/voiceover-compressed2.wav");
console.log("FINAL STATS:", stats);
})();

135
services/our/src/tasks/analyzeAudio.ts Normal file
View File

@ -0,0 +1,135 @@
/**
* Analyze the video's audio track using ITU-R BS.1770/EBU R128
* to extract Integrated Loudness (LUFS-I), Loudness Range (LRA),
* and True Peak (dBTP). This helps identify which VODs/streamers
* maintain consistent, listener-friendly audio setups.
*
* ## LUFS Reference
* Apple Music -16 LUFS
* YouTube -13 LUFS
* Spotify -14 LUFS
* Tidal -14 LUFS
*/
import type { Task, Helpers } from "graphile-worker";
import { PrismaClient } from "../../generated/prisma";
import { withAccelerate } from "@prisma/extension-accelerate";
import { getOrDownloadAsset } from "../utils/cache";
import { env } from "../config/env";
import { getS3Client } from "../utils/s3";
import { getNanoSpawn } from "../utils/nanoSpawn";
import logger from "../utils/logger";
const prisma = new PrismaClient().$extends(withAccelerate());
const ffmpegBin = "ffmpeg";
interface Payload {
vodId: string;
}
interface AudioStats {
lufsIntegrated: number; // LUFS-I
lra: number; // Loudness Range
peak: number; // True Peak (expressed as dBTP ceiling, e.g. -0.1)
}
export async function runFFmpeg(args: string[]): Promise<string> {
const spawn = await getNanoSpawn();
try {
const result = await spawn(ffmpegBin, args, {
stdout: 'pipe',
stderr: 'pipe',
});
// success → return combined output
return result.output;
} catch (err: any) {
// failure → throw with stderr
throw new Error(err.output || err.message);
}
}
function parseEbur128(output: string): AudioStats {
const lines = output.split("\n").map((l) => l.trim());
let lufsIntegrated = NaN;
let lra = NaN;
let peak = NaN;
let inTruePeak = false;
for (const line of lines) {
// Integrated loudness
const iMatch = line.match(/^I:\s*(-?\d+(\.\d+)?) LUFS/);
if (iMatch) lufsIntegrated = parseFloat(iMatch[1]);
// Loudness range
const lraMatch = line.match(/^LRA:\s*(-?\d+(\.\d+)?)/);
if (lraMatch) lra = parseFloat(lraMatch[1]);
// True peak
if (line === "True peak:") inTruePeak = true;
if (inTruePeak) {
const pMatch = line.match(/Peak:\s*(-?\d+\.?\d*)\s*dBFS/);
if (pMatch) {
peak = parseFloat(pMatch[1]); // no negation
inTruePeak = false;
}
}
}
return { lufsIntegrated, lra, peak };
}
async function analyzeAudio(inputFile: string): Promise<AudioStats> {
const args = [
"-hide_banner",
"-i", inputFile,
"-filter_complex", "ebur128=peak=true:metadata=1:framelog=verbose",
"-f", "null",
"-",
];
const output = await runFFmpeg(args);
return parseEbur128(output);
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
}
export default async function main(payload: any, helpers: Helpers) {
assertPayload(payload);
const { vodId } = payload;
const vod = await prisma.vod.findFirstOrThrow({
where: { id: vodId },
});
if (vod.audioIntegratedLufs && vod.audioLoudnessRange && vod.audioTruePeak) {
logger.info(`Skipping analysis — vod ${vodId} already complete.`);
return;
}
if (!vod.sourceVideo) {
throw new Error(`vod ${vodId} is missing a sourceVideo.`);
}
const s3Client = getS3Client();
const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
logger.info(`videoFilePath=${videoFilePath}`);
const results = await analyzeAudio(videoFilePath);
logger.info(`results=${JSON.stringify(results)}`);
await prisma.vod.update({
where: { id: vodId },
data: {
audioIntegratedLufs: results.lufsIntegrated,
audioLoudnessRange: results.lra,
audioTruePeak: results.peak,
},
});
}

184
services/our/src/tasks/consolidate_twitch_channel_rewards.ts
View File

@ -1,184 +0,0 @@
// src/tasks/consolidate_twitch_channel_rewards.ts
import type { Task, Helpers } from "graphile-worker";
import { PrismaClient, User, type Pick } from '../../generated/prisma';
import { withAccelerate } from "@prisma/extension-accelerate";
import { env } from "../config/env";
import { constants } from "../config/constants";
import { getRateLimiter } from "../utils/rateLimiter";
import logger from '../utils/logger';
const prisma = new PrismaClient().$extends(withAccelerate());
const cprPath = env.TWITCH_MOCK ? constants.twitch.dev.paths.channelPointRewards : constants.twitch.prod.paths.channelPointRewards;
interface Payload {
userId: number;
}
export interface TwitchChannelPointReward {
id: string;
broadcaster_id: string;
cost: number;
title: string;
is_in_stock: boolean;
[key: string]: any;
}
const getAuthToken = (user: User) => env.TWITCH_MOCK ? env.TWITCH_MOCK_USER_ACCESS_TOKEN : user.twitchToken?.accessToken;
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload");
if (typeof payload.userId !== "number") throw new Error("invalid payload.userId");
}
const getTwitchChannelPointRewards = async (user: User) => {
if (!user?.twitchToken) throw new Error("Missing Twitch token");
const authToken = getAuthToken(user);
const limiter = getRateLimiter();
await limiter.consume('twitch', 1);
const query = new URLSearchParams({ broadcaster_id: user.twitchId });
const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
headers: {
'Authorization': `Bearer ${authToken}`,
'Client-Id': env.TWITCH_CLIENT_ID
}
});
if (!res.ok) throw new Error(`Failed to fetch rewards: ${res.statusText}`);
return res.json();
};
const createTwitchReward = async (user: User, pick: Pick) => {
const authToken = getAuthToken(user);
const limiter = getRateLimiter();
await limiter.consume('twitch', 1);
logger.debug('pick as follows')
logger.debug(pick)
logger.debug(`pick?.waifu?.name=${pick?.waifu?.name}`)
const query = new URLSearchParams({ broadcaster_id: user.twitchId });
const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${authToken}`,
'Client-Id': env.TWITCH_CLIENT_ID
},
body: JSON.stringify({ cost: user.redeemCost, title: pick.waifu.name })
});
if (!res.ok) throw new Error(`Failed to create reward: ${res.statusText}`);
const data = await res.json();
const rewardId = data.data?.[0]?.id;
if (!rewardId) throw new Error("No reward ID returned");
await prisma.pick.update({
where: { id: pick.id },
data: { twitchChannelPointRewardId: rewardId }
});
};
const deleteTwitchReward = async (user: User, rewardId: string) => {
const authToken = getAuthToken(user);
const limiter = getRateLimiter();
await limiter.consume('twitch', 1);
const query = new URLSearchParams({ broadcaster_id: user.twitchId, id: rewardId });
const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
method: 'DELETE',
headers: {
'Authorization': `Bearer ${authToken}`,
'Client-Id': env.TWITCH_CLIENT_ID
}
});
if (!res.ok) throw new Error(`Failed to delete reward ${rewardId}`);
};
const updateTwitchReward = async (user: User, rewardId: string, newCost: number) => {
const authToken = getAuthToken(user);
const limiter = getRateLimiter();
await limiter.consume('twitch', 1);
const query = new URLSearchParams({ broadcaster_id: user.twitchId, id: rewardId });
const res = await fetch(`${env.TWITCH_API_ORIGIN}${cprPath}?${query}`, {
method: 'PATCH',
headers: {
'Authorization': `Bearer ${authToken}`,
'Client-Id': env.TWITCH_CLIENT_ID
},
body: JSON.stringify({ cost: newCost })
});
if (!res.ok) throw new Error(`Failed to update reward ${rewardId}`);
};
const consolidateTwitchRewards = async (userId: number) => {
const user = await prisma.user.findFirstOrThrow({
where: { id: userId },
include: { twitchToken: true, Waifu: true }
});
// Fetch picks (most recent N)
const picks = await prisma.pick.findMany({
where: { userId },
orderBy: { createdAt: 'desc' },
take: constants.twitch.maxChannelPointRewards,
include: {
waifu: true
}
});
// Ensure every pick has a reward before processing Twitch side
for (const pick of picks) {
if (!pick.twitchChannelPointRewardId) {
logger.debug(`Creating new reward for pick: ${pick.id}`);
await createTwitchReward(user, pick);
}
}
// Refresh picks after reward creation
const updatedPicks = await prisma.pick.findMany({
where: { userId },
orderBy: { createdAt: 'desc' },
take: constants.twitch.maxChannelPointRewards
});
// Get the most recent N reward IDs
const currentPickIds = new Set(
updatedPicks.slice(0, user.waifuChoicePoolSize).map(p => p.twitchChannelPointRewardId)
);
logger.debug('currentPickIds as follows');
logger.debug(currentPickIds);
// Fetch Twitch-side rewards
const twitchData = await getTwitchChannelPointRewards(user);
const twitchRewards: TwitchChannelPointReward[] = twitchData.data;
// Delete or update Twitch rewards not in current pick set
for (const reward of twitchRewards) {
if (!updatedPicks.some(p => p.twitchChannelPointRewardId === reward.id)) continue;
if (!currentPickIds.has(reward.id)) {
logger.debug(`Deleting out-of-date reward: ${reward.id}`);
await deleteTwitchReward(user, reward.id);
} else if (reward.cost !== user.redeemCost) {
logger.debug(`Updating reward cost for: ${reward.id}`);
await updateTwitchReward(user, reward.id, user.redeemCost);
}
}
};
const task: Task = async (payload: any, helpers) => {
assertPayload(payload);
await consolidateTwitchRewards(payload.userId);
};
export default task;

451
services/our/src/tasks/createFunscript copy.ts.noexec
View File

@ -1,451 +0,0 @@
import type { Task, Helpers } from "graphile-worker";
import { PrismaClient } from "../../generated/prisma";
import { withAccelerate } from "@prisma/extension-accelerate";
import { getOrDownloadAsset } from "../utils/cache";
import { env } from "../config/env";
import { getS3Client, uploadFile } from "../utils/s3";
import { nanoid } from "nanoid";
import { existsSync, rmSync } from "node:fs";
import { join, basename, extname } from "node:path";
import { readFile, writeFile, readdir } from 'node:fs/promises';
import yaml from 'js-yaml';
import { getNanoSpawn } from "../utils/nanoSpawn";
import which from "which";
import * as ort from 'onnxruntime-node';
interface Payload {
vodId: string;
}
interface Detection {
startFrame: number;
endFrame: number;
className: string;
}
interface DataYaml {
path: string;
train: string;
val: string;
names: Record<string, string>;
}
interface FunscriptAction {
at: number;
pos: number;
}
interface Funscript {
version: string;
actions: FunscriptAction[];
}
interface ClassPositionMap {
[className: string]: number | 'pattern';
}
const prisma = new PrismaClient().$extends(withAccelerate());
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
}
async function loadDataYaml(yamlPath: string): Promise<DataYaml> {
const yamlContent = await readFile(yamlPath, 'utf8');
return yaml.load(yamlContent) as DataYaml;
}
async function preparePython(helpers) {
const spawn = await getNanoSpawn();
const venvPath = join(env.VIBEUI_DIR, "venv");
// Determine Python executable
let pythonCmd;
try {
pythonCmd = which.sync("python3");
} catch {
logger.error("Python is not installed or not in PATH.");
throw new Error("Python not found in PATH.");
}
// If venv doesn't exist, create it
if (!existsSync(venvPath)) {
logger.info("Python venv not found. Creating one...");
try {
await spawn(pythonCmd, ["-m", "venv", "venv"], {
cwd: env.VIBEUI_DIR,
});
logger.info("Python venv successfully created.");
} catch (err) {
logger.error("Failed to create Python venv:", err);
// Clean up partially created venv if needed
try {
if (existsSync(venvPath)) {
rmSync(venvPath, { recursive: true, force: true });
logger.warn("Removed broken venv directory.");
}
} catch (cleanupErr) {
logger.error("Error while cleaning up broken venv:", cleanupErr);
}
throw new Error("Python venv creation failed. Check if python3 and python3-venv are installed.");
}
} else {
logger.info("Using existing Python venv.");
}
}
async function ffprobe(videoPath: string): Promise<{ fps: number; frames: number }> {
const spawn = await getNanoSpawn()
const { stdout } = await spawn('ffprobe', [
'-v', 'error',
'-select_streams', 'v:0',
'-count_frames',
'-show_entries', 'stream=nb_read_frames,r_frame_rate',
'-of', 'default=nokey=1:noprint_wrappers=1',
videoPath,
])
const [frameRateStr, frameCountStr] = stdout.trim().split('\n')
const [num, denom] = frameRateStr.trim().split('/').map(Number)
const fps = num / denom
const frames = parseInt(frameCountStr.trim(), 10)
return { fps, frames }
}
export async function buildFunscript(
helpers: Helpers,
predictionOutput: string,
videoPath: string
): Promise<string> {
const labelDir = join(predictionOutput, 'labels');
const yamlPath = join(predictionOutput, 'data.yaml');
const outputPath = join(process.env.CACHE_ROOT ?? '/tmp', `${nanoid()}.funscript`);
logger.info('Starting Funscript generation');
try {
const data = await loadDataYaml(join(env.VIBEUI_DIR, 'data.yaml'))
const classPositionMap = await loadClassPositionMap(data, helpers);
const { fps, totalFrames } = await loadVideoMetadata(videoPath, helpers);
const detectionSegments = await processLabelFiles(labelDir, helpers, data);
const totalDurationMs = Math.floor((totalFrames / fps) * 1000);
const actions = generateActions(totalDurationMs, fps, detectionSegments, classPositionMap);
await writeFunscript(outputPath, actions, helpers);
return outputPath;
} catch (error) {
logger.error(`Error generating Funscript: ${error instanceof Error ? error.message : 'Unknown error'}`);
throw error;
}
}
export async function inference(helpers: Helpers, videoFilePath: string): Promise<string> {
const spawn = await getNanoSpawn()
const modelPath = join(env.VIBEUI_DIR, 'runs/detect/vibeui/weights/best.pt')
// Generate a unique name based on video name + UUID
const videoExt = extname(videoFilePath) // e.g. '.mp4'
const videoName = basename(videoFilePath, videoExt) // removes the extension
const uniqueName = `${videoName}-${nanoid()}`
const customProjectDir = 'runs' // or any custom folder
const outputPath = join(env.VIBEUI_DIR, customProjectDir, uniqueName)
await spawn('./venv/bin/yolo', [
'predict',
`model=${modelPath}`,
`source=${videoFilePath}`,
'save=False',
'save_txt=True',
'save_conf=True',
`project=${customProjectDir}`,
`name=${uniqueName}`,
], {
cwd: env.VIBEUI_DIR,
stdio: 'inherit',
})
return outputPath // contains labels/ folder and predictions
}
async function loadClassPositionMap(data: DataYaml, helpers: Helpers): Promise<ClassPositionMap> {
try {
if (
!data ||
typeof data !== 'object' ||
!('names' in data) ||
typeof data.names !== 'object' ||
data.names === null ||
Object.keys(data.names).length === 0
) {
throw new Error('Invalid data.yaml: "names" field is missing, not an object, or empty');
}
const positionMap: ClassPositionMap = {
ControlledByTipper: 50,
ControlledByTipperHigh: 80,
ControlledByTipperLow: 20,
ControlledByTipperMedium: 50,
ControlledByTipperUltrahigh: 95,
Ring1: 30,
Ring2: 40,
Ring3: 50,
Ring4: 60,
Earthquake: 'pattern',
Fireworks: 'pattern',
Pulse: 'pattern',
Wave: 'pattern',
Pause: 0,
RandomTime: 70,
HighLevel: 80,
LowLevel: 20,
MediumLevel: 50,
UltraHighLevel: 95
};
const names = Object.values(data.names);
for (const name of names) {
if (typeof name !== 'string' || name.trim() === '') {
logger.info(`Skipping invalid class name: ${name}`);
continue;
}
if (!(name in positionMap)) {
logger.info(`No position mapping for class "${name}", defaulting to 0`);
positionMap[name] = 0;
}
}
logger.info(`Loaded class position map: ${JSON.stringify(positionMap)}`);
return positionMap;
} catch (error) {
logger.error(`Error loading data.yaml: ${error instanceof Error ? error.message : 'Unknown error'}`);
throw error;
}
}
function generatePatternPositions(startMs: number, durationMs: number, className: string, fps: number): FunscriptAction[] {
const actions: FunscriptAction[] = [];
const frameDurationMs = 1000 / fps;
const totalFrames = Math.floor(durationMs / frameDurationMs);
const intervalMs = 100;
for (let timeMs = 0; timeMs < durationMs; timeMs += intervalMs) {
const progress = timeMs / durationMs;
let pos = 0;
switch (className) {
case 'Pulse':
pos = Math.round(50 * Math.sin(progress * 2 * Math.PI));
break;
case 'Wave':
pos = Math.round(50 + 50 * Math.sin(progress * 2 * Math.PI));
break;
case 'Fireworks':
pos = Math.random() > 0.5 ? 80 : 0;
break;
case 'Earthquake':
pos = Math.round(90 * Math.sin(progress * 4 * Math.PI) + (Math.random() - 0.5) * 10);
pos = Math.max(0, Math.min(90, pos));
break;
}
actions.push({ at: startMs + timeMs, pos });
}
return actions;
}
async function loadVideoMetadata(videoPath: string, helpers: Helpers) {
const { fps, frames: totalFrames } = await ffprobe(videoPath);
logger.info(`Video metadata: fps=${fps}, frames=${totalFrames}`);
return { fps, totalFrames };
}
async function processLabelFiles(labelDir: string, helpers: Helpers, data: DataYaml): Promise<Detection[]> {
const labelFiles = (await readdir(labelDir)).filter(file => file.endsWith('.txt'));
const detections: Map<number, Detection[]> = new Map();
const names = data.names;
for (const file of labelFiles) {
const match = file.match(/(\d+)\.txt$/);
if (!match) {
logger.info(`Skipping invalid filename: ${file}`);
continue;
}
const frameIndex = parseInt(match[1], 10);
if (isNaN(frameIndex)) {
logger.info(`Skipping invalid frame index from filename: ${file}`);
continue;
}
const content = await readFile(join(labelDir, file), 'utf8');
const lines = content.trim().split('\n');
const frameDetections: Detection[] = [];
let maxConfidence = 0;
let selectedClassIndex = -1;
for (const line of lines) {
const parts = line.trim().split(/\s+/);
if (parts.length < 6) continue;
const classIndex = parseInt(parts[0], 10);
const confidence = parseFloat(parts[5]);
if (isNaN(classIndex) || isNaN(confidence)) continue;
if (confidence >= 0.7 && confidence > maxConfidence) {
maxConfidence = confidence;
selectedClassIndex = classIndex;
}
}
if (maxConfidence > 0) {
const className = (data.names as Record<string, string>)[selectedClassIndex.toString()];
if (className) {
frameDetections.push({ startFrame: frameIndex, endFrame: frameIndex, className });
}
}
if (frameDetections.length > 0) {
detections.set(frameIndex, frameDetections);
}
}
// Merge overlapping detections into continuous segments
const detectionSegments: Detection[] = [];
let currentDetection: Detection | null = null;
for (const [frameIndex, frameDetections] of detections.entries()) {
for (const detection of frameDetections) {
if (!currentDetection || currentDetection.className !== detection.className) {
if (currentDetection) detectionSegments.push(currentDetection);
currentDetection = { ...detection, endFrame: frameIndex };
} else {
currentDetection.endFrame = frameIndex;
}
}
}
if (currentDetection) detectionSegments.push(currentDetection);
return detectionSegments;
}
function generateActions(totalDurationMs: number, fps: number, detectionSegments: Detection[], classPositionMap: ClassPositionMap): FunscriptAction[] {
const intervalMs = 100;
const actions: FunscriptAction[] = [];
// Generate static position actions
for (let timeMs = 0; timeMs <= totalDurationMs; timeMs += intervalMs) {
const frameIndex = Math.floor((timeMs / 1000) * fps);
let position = 0;
for (const segment of detectionSegments) {
if (frameIndex >= segment.startFrame && frameIndex <= segment.endFrame) {
const className = segment.className;
if (typeof classPositionMap[className] === 'number') {
position = classPositionMap[className];
break;
}
}
}
actions.push({ at: timeMs, pos: position });
}
// Overlay pattern-based actions
for (const segment of detectionSegments) {
const className = segment.className;
if (classPositionMap[className] === 'pattern') {
const startMs = Math.floor((segment.startFrame / fps) * 1000);
const durationMs = Math.floor(((segment.endFrame - segment.startFrame + 1) / fps) * 1000);
const patternActions = generatePatternPositions(startMs, durationMs, className, fps);
actions.push(...patternActions);
}
}
// Sort actions by time and remove duplicates
actions.sort((a, b) => a.at - b.at);
const uniqueActions: FunscriptAction[] = [];
let lastTime = -1;
for (const action of actions) {
if (action.at !== lastTime) {
uniqueActions.push(action);
lastTime = action.at;
}
}
return uniqueActions;
}
async function writeFunscript(outputPath: string, actions: FunscriptAction[], helpers: Helpers) {
const funscript: Funscript = { version: '1.0', actions };
await writeFile(outputPath, JSON.stringify(funscript, null, 2));
logger.info(`Funscript generated: ${outputPath} (${actions.length} actions)`);
}
const createFunscript: Task = async (payload: any, helpers: Helpers) => {
assertPayload(payload);
const { vodId } = payload;
const vod = await prisma.vod.findFirstOrThrow({ where: { id: vodId } });
if (vod.funscript) {
logger.info(`Doing nothing-- vod ${vodId} already has a funscript.`);
return;
}
if (!vod.sourceVideo) {
const msg = `Cannot create funscript: Vod ${vodId} is missing a source video.`;
logger.warn(msg);
throw new Error(msg);
}
const s3Client = getS3Client();
const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
logger.info(`Downloaded video to ${videoFilePath}`);
logger.info(`Creating funscript for vod ${vodId}...`);
const predictionOutput = await inference(helpers, videoFilePath);
logger.info(`prediction output ${predictionOutput}`);
const funscriptFilePath = await buildFunscript(helpers, predictionOutput, videoFilePath)
const s3Key = `funscripts/${vodId}.funscript`;
const s3Url = await uploadFile(s3Client, env.S3_BUCKET, s3Key, funscriptFilePath, "application/json");
logger.info(`Uploaded funscript to S3: ${s3Url}`);
await prisma.vod.update({
where: { id: vodId },
data: { funscript: s3Key }
});
logger.info(`Funscript saved to database for vod ${vodId}`);
};
export default createFunscript;

75
services/our/src/tasks/createFunscript.ts.ai.noexec
View File

@ -1,75 +0,0 @@
import type { Task, Helpers } from "graphile-worker";
import { PrismaClient } from "../../generated/prisma";
import { withAccelerate } from "@prisma/extension-accelerate";
import { getOrDownloadAsset } from "../utils/cache";
import { env } from "../config/env";
import { getS3Client, uploadFile } from "../utils/s3";
import { buildFunscript } from '../utils/funscripts';
import { getModelClasses, vibeuiInference } from "../utils/vibeui";
import { join } from "node:path";
interface Payload {
vodId: string;
}
const prisma = new PrismaClient().$extends(withAccelerate());
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object.");
if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId");
}
const createFunscript: Task = async (payload: any, helpers: Helpers) => {
assertPayload(payload);
const { vodId } = payload;
const vod = await prisma.vod.findFirstOrThrow({ where: { id: vodId } });
if (vod.funscript) {
logger.info(`Doing nothing-- vod ${vodId} already has a funscript.`);
return;
}
if (!vod.sourceVideo) {
const msg = `Cannot create funscript: Vod ${vodId} is missing a source video.`;
logger.warn(msg);
throw new Error(msg);
}
const s3Client = getS3Client();
const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
logger.info(`Downloaded video to ${videoFilePath}`);
logger.info(`Creating funscript for vod ${vodId}...`);
const modelPath = join(env.VIBEUI_DIR, 'vibeui.onnx')
const predictionOutput = await vibeuiInference(modelPath, videoFilePath);
logger.info(`prediction output ${predictionOutput}`);
const classes = await getModelClasses(modelPath)
const funscriptFilePath = await buildFunscript(classes, predictionOutput, videoFilePath)
const s3Key = `funscripts/${vodId}.funscript`;
const s3Url = await uploadFile(s3Client, env.S3_BUCKET, s3Key, funscriptFilePath, "application/json");
logger.info(`Uploaded funscript to S3: ${s3Url}`);
await prisma.vod.update({
where: { id: vodId },
data: { funscript: s3Key }
});
logger.info(`Funscript saved to database for vod ${vodId}`);
};
export default createFunscript;

14
services/our/src/tasks/getSourceVideoMetadata.ts
View File

@ -4,6 +4,8 @@ import { env } from "../config/env";
import { S3Client } from "@aws-sdk/client-s3";
import logger from "../utils/logger";
import { getNanoSpawn } from "../utils/nanoSpawn";
import { getOrDownloadAsset } from "../utils/cache";
import { getS3Client } from "../utils/s3";
const prisma = new PrismaClient();
const client = new S3Client({
@ -90,7 +92,9 @@ const getSourceVideoMetadata: Task = async (payload: unknown, helpers) => {
});
// Skip if already processed
if (vod.sourceVideoDuration) {
if (
vod.sourceVideoDuration
) {
logger.debug(`VOD ${vodId} already has metadata`);
return;
}
@ -99,8 +103,14 @@ const getSourceVideoMetadata: Task = async (payload: unknown, helpers) => {
throw new Error(`VOD ${vodId} has no sourceVideo`);
}
// download vod
const s3Client = getS3Client();
const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo);
logger.info(`videoFilePath=${videoFilePath}`);
// Run ffprobe
const meta = await getVideoMetadata(vod.sourceVideo);
const meta = await getVideoMetadata(videoFilePath);
// Update DB
await prisma.vod.update({

21
services/our/src/tasks/hello.ts
View File

@ -1,21 +0,0 @@
// src/tasks/hello.ts
import type { Task, Helpers } from "graphile-worker";
import logger from "../utils/logger";
interface Payload {
name: string;
}
function assertPayload(payload: any): asserts payload is Payload {
if (typeof payload !== "object" || !payload) throw new Error("invalid");
if (typeof payload.name !== "string") throw new Error("invalid");
}
export default async function hello(payload: any, helpers: Helpers) {
assertPayload(payload);
const { name } = payload;
logger.info(`Helloooooo, ${name}`);
};

1
services/our/src/tasks/scheduleVodProcessing.ts
View File

@ -42,6 +42,7 @@ const scheduleVodProcessing: Task = async (payload: unknown, helpers) => {
jobs.push(helpers.addJob("copyV1S3ToV2", { vodId }));
if (!vod.sourceVideo) jobs.push(helpers.addJob("getSourceVideo", { vodId }));
if (!vod.sourceVideoDuration) jobs.push(helpers.addJob("getSourceVideoMetadata", { vodId }))
if (!vod.audioIntegratedLufs || !vod.audioLoudnessRange || !vod.audioTruePeak) jobs.push(helpers.addJob("analyzeAudio", { vodId }))
if (!vod.sha256sum) jobs.push(helpers.addJob("generateVideoChecksum", { vodId }));
if (!vod.thumbnail) jobs.push(helpers.addJob("createVideoThumbnail", { vodId }));
if (!vod.hlsPlaylist) jobs.push(helpers.addJob("createHlsPlaylist", { vodId }));

16
services/our/src/utils/formatters.ts
View File

@ -37,4 +37,18 @@ export function generateS3Path(slug: string, date: Date, vodId: string, filename
const day = pad(getDate(date));
return `fp/${slug}/${year}/${month}/${day}/${vodId}/${filename}`;
}
}
export function formatDuration(ms: number): string {
const totalSeconds = Math.floor(ms / 1000);
const hours = Math.floor(totalSeconds / 3600);
const minutes = Math.floor((totalSeconds % 3600) / 60);
const seconds = totalSeconds % 60;
const hh = hours.toString().padStart(2, '0');
const mm = minutes.toString().padStart(2, '0');
const ss = seconds.toString().padStart(2, '0');
return hours > 0 ? `${hh}:${mm}:${ss}` : `${mm}:${ss}`;
}

5
services/our/src/utils/hbsHelpers.ts
View File

@ -5,7 +5,7 @@ import { Role } from '../../generated/prisma'
import { isModerator, hasRole } from './privs'
import { signUrl } from './cdn'
import { extractBasePath } from './filesystem'
import { truncate } from './formatters.ts'
import { formatDuration, truncate } from './formatters.ts'
import { icons } from './icons.ts'
import logger from './logger.ts'
import HandlebarsLib, { HelperOptions, SafeString } from 'handlebars';
@ -19,6 +19,9 @@ export function registerHbsHelpers(Handlebars: typeof HandlebarsLib) {
if (!dateString) return ''
return format(new Date(dateString), 'yyyy-MM-dd')
})
Handlebars.registerHelper('formatDuration', function (duration) {
return formatDuration(duration)
})
Handlebars.registerHelper('identicon', function (str, size = 48) {
return jdenticon.toSvg(str, size)
})

43
services/our/src/views/vod.hbs
View File

@ -304,15 +304,46 @@
<div class="mb-5">
<h2 class="title is-4">VOD Metadata</h2>
<div class="box">
<div class="ml-5 mt-4">
{{#if vod.sourceVideoDuration }}
<p>Duration: {{vod.sourceVideoDuration}}</p>
<p>FPS: {{vod.sourceVideoFPS}}</p>
<p>Video Codec: {{vod.sourceVideoCodec}}</p>
<p>Audio Codec: {{vod.sourceAudioCodec}}</p>
<div class="content">
<p><strong>Duration</strong> {{formatDuration vod.sourceVideoDuration}}</p>
<p><strong>FPS</strong> {{vod.sourceVideoFps}}</p>
<p><strong>Video Codec</strong> {{vod.sourceVideoCodec}}</p>
<p><strong>Audio Codec</strong> {{vod.sourceAudioCodec}}</p>
</div>
{{else}}
<article class="notification is-light">
VOD metadata is processing
</article>
{{/if}}
</div>
</div>
<div class="mb-5">
<h2 class="title is-4">Audio Analysis</h2>
<div class="ml-5 mt-5 mb-5">
{{#if vod.audioIntegratedLufs }}
<div class="content">
<p>
<abbr
title="Loudness Units relative to Full Scale - the overall perceived loudness of the audio">LUFS-I</abbr>
<strong>{{vod.audioIntegratedLufs}}</strong>
</p>
<p>
<abbr title="Loudness Range - the dynamic range of the audio in LUFS">LRA</abbr>
<strong>{{vod.audioLoudnessRange}}</strong>
</p>
<p>
<abbr title="True Peak: the maximum instantaneous signal level in dBTP">TP</abbr>
<strong>{{vod.audioTruePeak}}</strong>
</p>
</div>
{{else}}
<article class="notification">
Vod metadata is processing
Audio analysis is processing
</article>
{{/if}}
</div>