diff --git a/services/our/prisma/migrations/20250811120353_add_magnet_link/migration.sql b/services/our/prisma/migrations/20250811120353_add_magnet_link/migration.sql new file mode 100644 index 0000000..868daa1 --- /dev/null +++ b/services/our/prisma/migrations/20250811120353_add_magnet_link/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Vod" ADD COLUMN "magetLink" TEXT; diff --git a/services/our/prisma/migrations/20250811121442_fix_magnet_typo/migration.sql b/services/our/prisma/migrations/20250811121442_fix_magnet_typo/migration.sql new file mode 100644 index 0000000..4747371 --- /dev/null +++ b/services/our/prisma/migrations/20250811121442_fix_magnet_typo/migration.sql @@ -0,0 +1,9 @@ +/* + Warnings: + + - You are about to drop the column `magetLink` on the `Vod` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Vod" DROP COLUMN "magetLink", +ADD COLUMN "magnetLink" TEXT; diff --git a/services/our/prisma/schema.prisma b/services/our/prisma/schema.prisma index 8a97668..e7d9e77 100644 --- a/services/our/prisma/schema.prisma +++ b/services/our/prisma/schema.prisma @@ -80,6 +80,7 @@ model Vod { asrVttKey String? slvttSheetKeys Json? slvttVTTKey String? + magnetLink String? status VodStatus @default(pending) sha256sum String? diff --git a/services/our/src/app.ts b/services/our/src/app.ts index 6f3b688..7c6ca97 100644 --- a/services/our/src/app.ts +++ b/services/our/src/app.ts @@ -168,7 +168,6 @@ export function buildApp() { handlebars: Handlebars, }, templates: join(__dirname, '..', 'src', 'views'), - layout: 'layouts/main', viewExt: 'hbs', options: { partials: { diff --git a/services/our/src/assets/svg/magnet.svg b/services/our/src/assets/svg/magnet.svg new file mode 100644 index 0000000..41204aa --- /dev/null +++ b/services/our/src/assets/svg/magnet.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/our/src/assets/svg/patreon.svg b/services/our/src/assets/svg/patreon.svg new file mode 100644 index 0000000..70f4a26 --- /dev/null +++ b/services/our/src/assets/svg/patreon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/our/src/assets/svg/rss.svg b/services/our/src/assets/svg/rss.svg new file mode 100644 index 0000000..ef74403 --- /dev/null +++ b/services/our/src/assets/svg/rss.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/services/our/src/config/env.ts b/services/our/src/config/env.ts index 3ca1b81..76abb23 100644 --- a/services/our/src/config/env.ts +++ b/services/our/src/config/env.ts @@ -31,6 +31,9 @@ const EnvSchema = z.object({ APP_DIR: z.string().default('/app'), WHISPER_DIR: z.string(), LOG_LEVEL: z.string().default('info'), + SEEDBOX_SFTP_URL: z.string(), + SEEDBOX_SFTP_USERNAME: z.string(), + SEEDBOX_SFTP_PASSWORD: z.string(), }); const parsed = EnvSchema.safeParse(process.env); diff --git a/services/our/src/plugins/README.md b/services/our/src/plugins/README.md new file mode 100644 index 0000000..81b2184 --- /dev/null +++ b/services/our/src/plugins/README.md @@ -0,0 +1,3 @@ +Everything in fastify is a plugin. Routes are plugins too. + +@see https://fastify.dev/docs/latest/Guides/Plugins-Guide/#register \ No newline at end of file diff --git a/services/our/src/plugins/hls.ts b/services/our/src/plugins/hls.ts index dd5b1a1..945b0e0 100644 --- a/services/our/src/plugins/hls.ts +++ b/services/our/src/plugins/hls.ts @@ -9,6 +9,7 @@ import { constants } from '../config/constants' import { readFile } from 'fs/promises' import { extractBasePath } from '../utils/filesystem' import { dirname } from 'node:path' +import logger from '../utils/logger' const prisma = new PrismaClient().$extends(withAccelerate()) const s3 = getS3Client() @@ -61,7 +62,7 @@ function rewriteMp4ReferencesWithSignedUrls( signFn: (path: string) => string ): string { - console.log(`rewriteMp4ReferencesWithSignedUrls called with ${playlistContent} ${cdnBasePath} ${signFn}`) + logger.debug(`rewriteMp4ReferencesWithSignedUrls called with ${playlistContent} ${cdnBasePath} ${signFn}`) const cleanBase = cdnBasePath.replace(/^\/|\/$/g, '') // remove leading/trailing slash @@ -101,8 +102,8 @@ export default async function registerHlsRoute(app: FastifyInstance) { select: { hlsPlaylist: true }, }) - console.log(`vod as follows`) - console.log(vod) + logger.debug(`vod as follows`) + logger.debug(vod) if (!vod.hlsPlaylist) return reply.status(404).send('') const s3Key = `${dirname(vod.hlsPlaylist)}/${manifest}` @@ -119,7 +120,7 @@ export default async function registerHlsRoute(app: FastifyInstance) { // Otherwise, rewrite .mp4 references with signed URLs const tokenPath = `/${dirname(vod.hlsPlaylist)}/` - console.log(`tokenPath=${tokenPath} hlsPlaylist=${vod.hlsPlaylist}`) + logger.debug(`tokenPath=${tokenPath} hlsPlaylist=${vod.hlsPlaylist}`) if (!tokenPath.startsWith('/')) { throw new Error('tokenPath did not start with a forward slash'); diff --git a/services/our/src/plugins/index.ts b/services/our/src/plugins/index.ts index a3667d5..57cd79f 100644 --- a/services/our/src/plugins/index.ts +++ b/services/our/src/plugins/index.ts @@ -20,7 +20,7 @@ export default async function indexRoutes(fastify: FastifyInstance): Promise ({ + title: `Stream on ${vod.streamDate.toDateString()}`, + link: `${env.ORIGIN}/vod/${vod.id}`, + guid: `${env.ORIGIN}/vod/${vod.id}`, + pubDate: vod.streamDate.toUTCString(), + description: vod.notes || 'No description available', + })); + + const title = 'future.porn - ' + vtuber.displayName || vtuber.slug + + return reply + .type('application/rss+xml') + .view('/feed.hbs', { + title, + description: vtuber.description || title, + link: `${env.ORIGIN}/vtuber/${vtuber.slug || vtuber.id}`, + items, + }, { layout: 'layouts/xml.hbs' }); }); } \ No newline at end of file diff --git a/services/our/src/tasks/createTorrent.ts b/services/our/src/tasks/createTorrent.ts new file mode 100644 index 0000000..f4a45f8 --- /dev/null +++ b/services/our/src/tasks/createTorrent.ts @@ -0,0 +1,154 @@ +import type { Helpers } from "graphile-worker"; +import { PrismaClient } from "../../generated/prisma"; +import { withAccelerate } from "@prisma/extension-accelerate"; +import { getOrDownloadAsset } from "../utils/cache"; +import { env } from "../config/env"; +import { getS3Client, uploadFile } from "../utils/s3"; +import { nanoid } from "nanoid"; +import { getNanoSpawn } from "../utils/nanoSpawn"; +import { preparePython } from "../utils/python"; +import logger from "../utils/logger"; +import { basename, join } from "node:path"; +import SftpClient from 'ssh2-sftp-client'; + + +const prisma = new PrismaClient().$extends(withAccelerate()); + + +interface Payload { + vodId: string; +} + + +// async function createTorrent(payload: any, helpers: Helpers) { +// helpers.logger.debug(`createTorrent`) + + +// if (!inputFilePath) { +// throw new Error("inputFilePath is missing"); +// } + +// await preparePython() +// const outputFilePath = inputFilePath.replace(/\.[^/.]+$/, '') + '-thumb.png'; +// const spawn = await getNanoSpawn(); + + +// helpers.logger.debug('result as follows') +// helpers.logger.debug(JSON.stringify(result, null, 2)) + +// helpers.logger.info(`✅ Thumbnail saved to: ${outputFilePath}`); +// return outputFilePath + +// } +function assertPayload(payload: any): asserts payload is Payload { + if (typeof payload !== "object" || !payload) throw new Error("invalid payload-- was not an object."); + if (typeof payload.vodId !== "string") throw new Error("invalid payload-- was missing vodId"); +} + + +export default async function createTorrent(payload: any, helpers: Helpers) { + assertPayload(payload) + const { vodId } = payload + const vod = await prisma.vod.findFirstOrThrow({ + where: { + id: vodId + } + }) + + const spawn = await getNanoSpawn(); + + // * [x] load vod + + // * [x] exit if video.thumbnail already defined + if (vod.magnetLink) { + logger.info(`Doing nothing-- vod ${vodId} already has a magnet link.`) + return; // Exit the function early + } + + if (!vod.sourceVideo) { + throw new Error(`Failed to create magnet link-- vod ${vodId} is missing a sourceVideo.`); + } + + + logger.info('Creating magnet link.') + const s3Client = getS3Client() + + // * [x] download video segments from pull-thru cache + const videoFilePath = await getOrDownloadAsset(s3Client, env.S3_BUCKET, vod.sourceVideo) + logger.debug(`videoFilePath=${videoFilePath}`) + + // * [x] run torrentfile + + // torrentfile create + // --magnet + // --prog 0 + // --out ./test-fixture.torrent + // --announce udp://tracker.futureporn.net/ + // --source https://futureporn.net/ + // --comment https://futureporn.net/ + // --web-seed https://futureporn-b2.b-cdn.net/test-fixture.ts + // --meta-version 3 + // ~/Downloads/test-fixture.ts + + const torrentOutputFile = join(env.CACHE_ROOT, `${nanoid()}.torrent`); + + const result = await spawn('./venv/bin/torrentfile', [ + 'create', + '--magnet', + '--prog', '0', + '--meta-version', '2', + '--comment', 'https://future.porn', + '--source', `https://future.porn/vod/${vodId}`, + '--out', torrentOutputFile, + videoFilePath, + ], { + cwd: env.APP_DIR, + }); + + + + logger.trace(JSON.stringify(result)); + + + const match = result.stdout.match(/magnet:\?[^\s]+/); + if (!match) { + throw new Error('No magnet link found in torrentfile output:\n' + result.stdout); + } + + const magnetLink = match[0]; + logger.debug(`Magnet link=${magnetLink}`); + + + // upload torrent file to seedbox sftp + // Actually I don't think we need this, because our seedbox can use the RSS feed and get informed about torrents that way + // let sftp = new SftpClient(); + // const torrentBasename = basename(torrentOutputFile); + + // const parsed = new URL(env.SEEDBOX_SFTP_URL); + // logger.debug(`url=${env.SEEDBOX_SFTP_URL} hostname=${parsed.hostname} port=${parsed.port} username=${env.SEEDBOX_SFTP_USERNAME} password=${env.SEEDBOX_SFTP_PASSWORD}`); + // await sftp.connect({ + // host: parsed.hostname, + // port: parsed.port, + // username: env.SEEDBOX_SFTP_USERNAME, + // password: env.SEEDBOX_SFTP_PASSWORD + // }) + + // const remoteFilePath = join(parsed.pathname, torrentBasename) + + // const data = await sftp.list(parsed.pathname); + // logger.debug(`the data=${JSON.stringify(data)}`); + + // logger.debug(`uploading ${torrentOutputFile} to ${remoteFilePath}`) + // await sftp.put(torrentOutputFile, remoteFilePath); + + + logger.debug(`updating vod record`); + await prisma.vod.update({ + where: { id: vodId }, + data: { magnetLink } + }); + + logger.debug(`all done.`) + + +} \ No newline at end of file diff --git a/services/our/src/tasks/getSourceVideo.ts b/services/our/src/tasks/getSourceVideo.ts index dce72d1..88f5ae2 100644 --- a/services/our/src/tasks/getSourceVideo.ts +++ b/services/our/src/tasks/getSourceVideo.ts @@ -1,8 +1,6 @@ import type { Task, Helpers } from "graphile-worker"; import { PrismaClient } from "../../generated/prisma"; import { access, stat, writeFile } from "node:fs/promises"; -import { createReadStream, createWriteStream } from "node:fs"; -import path from "node:path"; import { join } from "node:path"; import { getOrDownloadAsset } from "../utils/cache"; import { env } from "../config/env"; @@ -11,6 +9,8 @@ import { uploadFile } from "../utils/s3"; import { S3Client } from "@aws-sdk/client-s3"; import { VodSegment } from "../types"; import { getNanoSpawn } from "../utils/nanoSpawn"; +import { parseISO, getYear, getMonth, getDate } from 'date-fns'; +import logger from "../utils/logger"; const prisma = new PrismaClient(); const client = new S3Client({ @@ -46,7 +46,7 @@ async function validateSegments(segments: VodSegment[], helpers: Helpers) { throw new Error("No VOD segments provided"); } - helpers.logger.info(`Processing ${segments.length} video segments`); + logger.info(`Processing ${segments.length} video segments`); } async function downloadSegments( @@ -57,14 +57,14 @@ async function downloadSegments( for (const [index, segment] of segmentKeys.entries()) { try { - helpers.logger.debug(`Downloading segment ${index + 1}/${segmentKeys.length}`); + logger.debug(`Downloading segment ${index + 1}/${segmentKeys.length}`); const path = await getOrDownloadAsset(client, env.S3_BUCKET, segment.key); downloadedPaths.push(path); // Verify the segment exists and is accessible await access(path); const size = await getFileSize(path); - helpers.logger.debug(`Segment ${index + 1} size: ${(size / 1024 / 1024).toFixed(2)}MB`); + logger.debug(`Segment ${index + 1} size: ${(size / 1024 / 1024).toFixed(2)}MB`); } catch (error) { throw new Error(`Failed to download segment ${segment.key}: ${error instanceof Error ? error.message : String(error)}`); } @@ -89,7 +89,7 @@ async function concatenateSegments( concatSpec: FFmpegConcatSpec, helpers: Helpers ): Promise { - helpers.logger.info(`Concatenating ${concatSpec.files.length} segments`); + logger.info(`Concatenating ${concatSpec.files.length} segments`); try { const spawn = await getNanoSpawn(); @@ -105,13 +105,13 @@ async function concatenateSegments( cwd: env.CACHE_ROOT, }); - helpers.logger.debug(`FFmpeg output: ${proc.stdout}`); - helpers.logger.debug(`FFmpeg stderr: ${proc.stderr}`); + logger.debug(`FFmpeg output: ${proc.stdout}`); + logger.debug(`FFmpeg stderr: ${proc.stderr}`); // Verify output file await access(concatSpec.outputPath); const outputSize = await getFileSize(concatSpec.outputPath); - helpers.logger.info(`Concatenated file size: ${(outputSize / 1024 / 1024).toFixed(2)}MB`); + logger.info(`Concatenated file size: ${(outputSize / 1024 / 1024).toFixed(2)}MB`); return concatSpec.outputPath; } catch (error) { @@ -124,7 +124,7 @@ async function updateVodWithSourceVideo( sourcePath: string, helpers: Helpers ): Promise { - helpers.logger.info(`Updating VOD ${vodId} with source video ${sourcePath}`); + logger.info(`Updating VOD ${vodId} with source video ${sourcePath}`); await prisma.vod.update({ where: { id: vodId }, @@ -158,7 +158,7 @@ const getSourceVideo: Task = async (payload: unknown, helpers) => { assertPayload(payload); const { vodId } = payload; - helpers.logger.info(`Processing source video for VOD ${vodId}`); + logger.info(`Processing source video for VOD ${vodId}`); try { // Get VOD info from database @@ -168,12 +168,14 @@ const getSourceVideo: Task = async (payload: unknown, helpers) => { sourceVideo: true, segmentKeys: true, status: true, + vtubers: true, + streamDate: true, }, }); // Skip if already processed if (vod.sourceVideo) { - helpers.logger.debug(`VOD ${vodId} already has a source video`); + logger.debug(`VOD ${vodId} already has a source video`); return; } @@ -196,7 +198,7 @@ const getSourceVideo: Task = async (payload: unknown, helpers) => { if (downloadedPaths.length === 1) { // Single segment - no concatenation needed sourceVideoPath = downloadedPaths[0]; - helpers.logger.info(`Using single segment as source video`); + logger.info(`Using single segment as source video`); } else { // Multiple segments - concatenate const concatSpec: FFmpegConcatSpec = { @@ -211,14 +213,17 @@ const getSourceVideo: Task = async (payload: unknown, helpers) => { // upload the concatenated video - const key = await uploadFile(client, env.S3_BUCKET, `source/${nanoid()}.mp4`, sourceVideoPath, 'video/mp4'); + const year = getYear(vod.streamDate); + const month = getMonth(vod.streamDate) + 1; + const day = getDate(vod.streamDate); + const key = await uploadFile(client, env.S3_BUCKET, `fp/${vod.vtubers[0].slug}/${year}/${month}/${day}/${nanoid()}/source.mp4`, sourceVideoPath, 'video/mp4'); // Update database with source video path await updateVodWithSourceVideo(vodId, key, helpers); - helpers.logger.info(`Successfully processed source video for VOD ${vodId}`); + logger.info(`Successfully processed source video for VOD ${vodId}`); } catch (error) { - helpers.logger.error(`Failed to process source video for VOD ${vodId}: ${error instanceof Error ? error.message : String(error)}`); + logger.error(`Failed to process source video for VOD ${vodId}: ${error instanceof Error ? error.message : String(error)}`); // await prisma.vod.update({ // where: { id: vodId }, // data: { status: "failed" }, diff --git a/services/our/src/tasks/scheduleVodProcessing.ts b/services/our/src/tasks/scheduleVodProcessing.ts index 8313e6c..b65b29e 100644 --- a/services/our/src/tasks/scheduleVodProcessing.ts +++ b/services/our/src/tasks/scheduleVodProcessing.ts @@ -1,7 +1,6 @@ -import type { Helpers, Task, Job } from "graphile-worker"; +import type { Task, Job } from "graphile-worker"; import { PrismaClient } from "../../generated/prisma"; import { withAccelerate } from "@prisma/extension-accelerate"; -import { addMinutes, addSeconds } from 'date-fns'; interface Payload { @@ -47,6 +46,7 @@ const scheduleVodProcessing: Task = async (payload: unknown, helpers) => { if (!vod.funscript) jobs.push(helpers.addJob("createFunscript", { vodId })); if (!vod.asrVttKey) jobs.push(helpers.addJob("createTranscription", { vodId })); if (!vod.slvttVTTKey) jobs.push(helpers.addJob("createStoryboard", { vodId })); + if (!vod.magnetLink) jobs.push(helpers.addJob("createTorrent", { vodId })); const changes = jobs.length; if (changes > 0) { diff --git a/services/our/src/utils/privs.ts b/services/our/src/utils/privs.ts index d77c4ef..cb1fdc4 100644 --- a/services/our/src/utils/privs.ts +++ b/services/our/src/utils/privs.ts @@ -1,3 +1,5 @@ +import logger from "./logger"; + type UserWithRoles = { roles: { name: string }[] }; @@ -20,9 +22,11 @@ export function isUnprivilegedUser(user: { roles: { name: string }[] }): boolean export function hasRole(...args: any[]) { - const options = args.pop(); // handlebars injects this - const roles: string[] = args.slice(0, -1); - const user = args[args.length - 1]; + const options = args.pop(); // handlebars options object + const user = args.pop(); // user is second-to-last arg + const roles: string[] = args; // everything else is role names + + logger.trace(`roles=${roles.join(',')} user=${JSON.stringify(user)}`); if (!user?.roles) return false; return user.roles.some((r: any) => roles.includes(r.name)); diff --git a/services/our/src/utils/python.ts b/services/our/src/utils/python.ts index 80cd851..516de28 100644 --- a/services/our/src/utils/python.ts +++ b/services/our/src/utils/python.ts @@ -23,7 +23,7 @@ export async function preparePython() { await spawn(pythonCmd, ["-m", "venv", venvPath], { cwd: env.APP_DIR, }); - console.log("✅ Python venv created."); + console.log("Python venv created."); } else { console.log("Using existing Python venv."); } @@ -34,15 +34,15 @@ export async function preparePython() { await spawn(pipCmd, ["install", "-r", "requirements.txt"], { cwd: env.APP_DIR, }); - console.log("✅ requirements.txt installed."); + console.log("requirements.txt installed."); // 4. Confirm vcsi CLI binary exists const vcsiBinary = join(venvBin, "vcsi"); if (!existsSync(vcsiBinary)) { - console.error("❌ vcsi binary not found in venv after installing requirements."); + console.error("vcsi binary not found in venv after installing requirements."); console.error("Make sure 'vcsi' is listed in requirements.txt and that it installs a CLI."); throw new Error("vcsi installation failed or did not expose CLI."); } - console.log("✅ vcsi CLI is available at", vcsiBinary); + console.log("vcsi CLI is available at", vcsiBinary); } diff --git a/services/our/src/views/feed.hbs b/services/our/src/views/feed.hbs new file mode 100644 index 0000000..5531818 --- /dev/null +++ b/services/our/src/views/feed.hbs @@ -0,0 +1,16 @@ + + + {{title}} + {{link}} + {{description}} + {{#each items}} + + {{this.title}} + {{this.link}} + {{this.guid}} + {{this.pubDate}} + + + {{/each}} + + \ No newline at end of file diff --git a/services/our/src/views/layouts/xml.hbs b/services/our/src/views/layouts/xml.hbs new file mode 100644 index 0000000..4e1a4b0 --- /dev/null +++ b/services/our/src/views/layouts/xml.hbs @@ -0,0 +1,2 @@ + +{{{body}}} \ No newline at end of file diff --git a/services/our/src/views/perks.hbs b/services/our/src/views/perks.hbs index 36462a8..02b88a3 100644 --- a/services/our/src/views/perks.hbs +++ b/services/our/src/views/perks.hbs @@ -8,9 +8,10 @@

Perks

-

future.porn is free to use, but to keep the site running, we need your help! In return, we offer extra perks +

future.porn is free to use, but to keep the site running we need your help! In return, we offer extra perks to supporters.

+ @@ -63,6 +64,8 @@ + {{!-- + @todo add these things @@ -87,9 +90,15 @@ + --}}
✔️ ✔️
CC Search ✔️
+ + +
{{> footer}} diff --git a/services/our/src/views/vod.hbs b/services/our/src/views/vod.hbs index 6810099..a11c47a 100644 --- a/services/our/src/views/vod.hbs +++ b/services/our/src/views/vod.hbs @@ -119,7 +119,7 @@

Downloads

-

Raw Recorded File Segments

+

Raw Segments

{{#if vod.segmentKeys}}
    {{#each vod.segmentKeys}} @@ -135,13 +135,24 @@ {{/if}} -

    Concatenated Video

    +

    VOD

    {{#if vod.sourceVideo}} + + {{#if (hasRole "supporterTier1" user)}}

    {{icon "download" 24}} Download

    + {{else}} +

    + {{icon "patreon" 24}} + + CDN Download + +

    + {{/if}} +

    {{#if vod.sha256sum}}sha256sum {{vod.sha256sum}}{{/if}}

    {{#if vod.cidv1}}

    IPFS cidv1 {{vod.cidv1}}

    @@ -150,6 +161,13 @@ IPFS CID is processing. {{/if}} + {{#if vod.magnetLink}} +

    {{icon "magnet" 24}} Magnet Link

    + {{else}} +
    + Magnet Link is processing. +
    + {{/if}} {{else}}
    Video Source is processing. diff --git a/services/our/src/views/vtubers/list.hbs b/services/our/src/views/vtubers/list.hbs index 5c5f9be..2d13643 100644 --- a/services/our/src/views/vtubers/list.hbs +++ b/services/our/src/views/vtubers/list.hbs @@ -1,4 +1,5 @@ {{#> main}} +
    {{> navbar}}
    diff --git a/services/our/src/views/vtubers/show.hbs b/services/our/src/views/vtubers/show.hbs index 19e900c..c3f39a0 100644 --- a/services/our/src/views/vtubers/show.hbs +++ b/services/our/src/views/vtubers/show.hbs @@ -1,4 +1,6 @@ {{#> main}} + +
    @@ -38,7 +40,8 @@ --}}

    - {{vtuber.displayName}} + {{vtuber.displayName}} {{icon "rss" 32}}