signed urls allow querystrings

services/pocketbase/package.json

@@ -1,6 +1,6 @@
 {
   "name": "futureporn",
-  "version": "3.1.3",
+  "version": "3.3.0",
   "private": true,
   "description": "Dedication to the preservation of lewdtuber history",
   "license": "Unlicense",

services/pocketbase/pb_hooks/cdn.pb.js

@@ -10,22 +10,10 @@
  * @see https://github.com/pocketbase/pocketbase/discussions/5995
  * 
  */
-onFileDownloadRequest((e) => {
+onFileDownloadRequest((event) => {
 
 
-
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // console.log('onFileDownloadRequest hook has been triggered ~~~');
-  // e.next()
-
+  // console.log('event', JSON.stringify(event))
   // e.app
   // e.collection
   // e.record
@@ -36,72 +24,89 @@ onFileDownloadRequest((e) => {
   const securityKey = process.env?.BUNNY_TOKEN_KEY;
   const baseUrl = process.env?.BUNNY_ZONE_URL;
 
-  console.log(`securityKey=${securityKey}, baseUrl=${baseUrl}`)
+  // console.log(`securityKey=${securityKey}, baseUrl=${baseUrl}`)
 
   if (!securityKey) {
     console.error('BUNNY_TOKEN_KEY was missing from env');
-    return e.next();
+    return event.next();
   }
   if (!baseUrl) {
     console.error('BUNNY_ZONE_URL was missing from env');
-    return e.next();
+    return event.next();
   }
 
-
-
-
   /**
-   * Generates a BunnyCDN-style signed URL using directory tokens.
-  *
-  * We sign URLs to make hotlinking difficult
-  * @see https://support.bunny.net/hc/en-us/articles/360016055099-How-to-sign-URLs-for-BunnyCDN-Token-Authentication
-  * @see https://github.com/pocketbase/pocketbase/discussions/5983#discussioncomment-11426659 // HMAC in pocketbase
-  * @see https://github.com/pocketbase/pocketbase/discussions/6772 // base64 encode the hex
+   * Generate a signed BunnyCDN URL.
+   * @param {string} securityKey - Your BunnyCDN security token
+   * @param {string} baseUrl - The base URL (protocol + host)
+   * @param {string} path - Path to the file (starting with /)
+   * @param {string} rawQuery - Raw query string, e.g., "width=500&quality=5"
+   * @param {number} expires - Unix timestamp for expiration
    */
-  function signUrl(securityKey, baseUrl, path, expires) {
+  function signUrlCool(securityKey, baseUrl, path, rawQuery = "", expires) {
 
     if (!path.startsWith('/')) path = '/' + path;
     if (baseUrl.endsWith('/')) throw new Error(`baseUrl must not end with a slash. got baseUrl=${baseUrl}`);
 
+    // Build parameter string (sort keys alphabetically)
+    let parameterData = "";
+    if (rawQuery) {
+      const params = rawQuery
+        .split("&")
+        .map(p => p.split("="))
+        .filter(([key]) => key && key !== "token" && key !== "expires")
+        .sort(([a], [b]) => a.localeCompare(b));
 
-    const hashableBase = securityKey + path + expires;
+      if (params.length) {
+        parameterData = params.map(([k, v]) => `${k}=${v}`).join("&");
+      }
+    }
 
-    // Generate and encode the token 
+    // Build hashable base
+    const hashableBase = securityKey + path + expires + parameterData;
+    // console.log(`hashableBase`, hashableBase)
+
+    // Compute token using your $security.sha256 workflow
     const tokenH = $security.sha256(hashableBase);
-
     const token = Buffer.from(tokenH, "hex")
       .toString("base64")
-      .replace(/\n/g, "") // Remove newlines
-      .replace(/\+/g, "-") // Replace + with -
-      .replace(/\//g, "_") // Replace / with _
-      .replace(/=/g, ""); // Remove =
+      .replace(/\n/g, "")
+      .replace(/\+/g, "-")
+      .replace(/\//g, "_")
+      .replace(/=/g, "");
 
+    // Build final signed URL
+    let tokenUrl = baseUrl + path + "?token=" + token;
+    if (parameterData) tokenUrl += "&" + parameterData;
+    tokenUrl += "&expires=" + expires;
 
-    // Generate the URL
-    const signedUrl = baseUrl + path + '?token=' + token + '&expires=' + expires;
-
-    return signedUrl;
+    return tokenUrl;
   }
 
-  console.log(`record: ${JSON.stringify(e.record)}`)
-  console.log(`collection: ${JSON.stringify(e.collection)}`)
-  console.log(`app: ${JSON.stringify(e.app)}`)
-  console.log(`fileField: ${JSON.stringify(e.fileField)}`)
-  console.log(`servedPath: ${JSON.stringify(e.servedPath)}`)
-  console.log(`servedName: ${JSON.stringify(e.servedName)}`)
+
+
+  const rawQuery = event.requestEvent.request.url.rawQuery;
+
+  // console.log(`record: ${JSON.stringify(event.record)}`)
+  // // console.log(`collection: ${JSON.stringify(event.collection)}`)
+  // console.log(`app: ${JSON.stringify(event.app)}`)
+  // console.log(`fileField: ${JSON.stringify(event.fileField)}`)
+  // console.log(`servedPath: ${JSON.stringify(event.servedPath)}`)
+  // console.log(`servedName: ${JSON.stringify(event.servedName)}`)
 
   // Our job here is to take the servedPath, and sign it using bunnycdn method
   // Then serve a 302 redirect instead of serving the file proxied thru PB
 
-  const path = e.servedPath;
+  const path = event.servedPath;
   const expires = Math.round(Date.now() / 1000) + 3600;
-  const signedUrl = signUrl(securityKey, baseUrl, path, expires);
-  console.log(`signedUrl=${signedUrl}`);
+  const signedUrl = signUrlCool(securityKey, baseUrl, path, rawQuery, expires);
+  // console.log(`rawQUery`, rawQuery, 'path', path);
+  // console.log(`signedUrl=${signedUrl}`);
 
   // This redirect is a tricky thing. We do this to avoid proxying file requests via our pocketbase origin server. 
   // The idea is to reduce load. 
   // HOWEVER, this redirect slows down image loading because it now takes 2 requests per image.
-  e.redirect(302, signedUrl);
+  event.redirect(302, signedUrl);
 
-  e.next()
+  event.next()
 })

services/pocketbase/pb_hooks/pages/_private/vod-list.ejs

@@ -29,7 +29,7 @@
         <td style="width: 160px;">
           <% if (vod.thumbnail) { %>
           <figure class="image is-3by2">
-            <img src="/api/files/<%= vod.collectionId %>/<%= vod.id %>/<%= vod.thumbnail %>" alt="Thumbnail" style="width: 120px; border-radius: 8px;">
+            <img src="/api/files/<%= vod.collectionId %>/<%= vod.id %>/<%= vod.thumbnail %>?quality=5&width=12" alt="Thumbnail" style="width: 120px; border-radius: 8px;">
           </figure>
           <% } else { %>
           <span>No thumbnail</span>

services/pocketbase/utils/data_migrations/2025-11-14-copy-thumbnails.ts

@@ -0,0 +1,149 @@
+// 2025-11-07-import-thumbnails.js
+
+import PocketBase from 'pocketbase';
+import { readFileSync } from 'node:fs';
+import { basename, join } from 'node:path';
+import spawn from 'nano-spawn';
+import { tmpdir } from 'node:os';
+import mime from 'mime';
+
+const pb = new PocketBase(process.env.POCKETBASE_URL || 'http://127.0.0.1:8090');
+if (!process.env.POCKETBASE_USERNAME) throw new Error('POCKETBASE_USERNAME missing');
+if (!process.env.POCKETBASE_PASSWORD) throw new Error('POCKETBASE_PASSWORD missing');
+if (!process.env.B2_BUCKET_FROM) throw new Error('B2_BUCKET_FROM missing');
+if (!process.env.V1_DATA_FILE) throw new Error('V1_DATA_FILE missing');
+
+
+interface ManifestItem {
+  thumbnail_url: string;
+  thumbnail_key: string;
+  tmp_file: string;
+  vod_id: string;
+}
+
+type Manifest = ManifestItem[];
+
+interface Vod {
+  id: string;
+  thumbnail: string;
+  streamDate: string;
+}
+
+const vodsCollectionName = 'pbc_144770472';
+// pbc_144770472 is the vods collection
+// cv6m31vj98gmtsx is a sample vod id
+
+
+async function main() {
+  console.log('Authenticating with PocketBase...');
+  await pb
+    .collection("_superusers")
+    .authWithPassword(process.env.POCKETBASE_USERNAME!, process.env.POCKETBASE_PASSWORD!);
+
+
+  //   Use vod.streamDate to find the correct entry
+  // load v1 datafile
+  const v1VodDataRaw = readFileSync(process.env.V1_DATA_FILE!, { encoding: 'utf-8' });
+  const v1VodData = JSON.parse(v1VodDataRaw).map((vod: any) => vod.data);
+  console.log('v1VodData sample', v1VodData[0])
+
+
+
+  // # BUILD A MANIFEST
+  let manifest: Manifest = [];
+  const vods = await pb.collection('vods').getFullList();
+  for (const vod of vods) {
+    // console.log('v2VodData sample', vod)
+    // console.log(`for this vod`, vod)
+    const v1Vod = v1VodData.find((vod1: Vod) => new Date(vod1.streamDate).getTime() === new Date(vod.streamDate).getTime());
+    console.log(`v1vod`, v1Vod);
+    if (!v1Vod) {
+      console.warn(`failed to find matching v1 data vod for vod ${vod.id} ${vod.streamDate}`);
+      continue;
+    }
+
+    // skip if there is no thumbnail in the v1 vod
+    if (!v1Vod.thumbnail) continue;
+
+    // get a temporary file path to which we will DL
+    const tmpFile = join(tmpdir(), basename(v1Vod.thumbnail));
+
+    // we will take the thumbnail url, download it, then upload it to the specified vod record.
+    const entry: ManifestItem = {
+      thumbnail_url: v1Vod.thumbnail,
+      thumbnail_key: basename(v1Vod.thumbnail),
+      tmp_file: tmpFile,
+      vod_id: vod.id,
+    };
+    manifest.push(entry);
+  }
+
+
+  // sanity check
+  const invalidVods = manifest.filter((m) => m.thumbnail_url.includes('mp4'))
+  if (invalidVods.length > 0) {
+    console.warn('invalid thumbnails found', invalidVods)
+    // throw new Error('invalid. mp4s found in thumbnails');
+  }
+
+  const validManifest = manifest.filter((m) => !m.thumbnail_url.includes('mp4'));
+
+
+  // console.log('manifest', manifest);
+
+  // # ACTUAL WORK
+  // Download the thumbnail to tmp file
+  // upload the tmp file to pocketbase
+
+  function sleep(ms: number) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async function retry(fn: any, retries = 6, delayMs = 500) {
+    let lastError;
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        return await fn();
+      } catch (err) {
+        lastError = err;
+        console.warn(`Attempt ${attempt} failed: ${err}. Retrying in ${delayMs}ms...`);
+        if (attempt < retries) await sleep(delayMs);
+      }
+    }
+    throw lastError;
+  }
+
+
+  for (const [i, m] of validManifest.entries()) {
+
+
+    var from = "b2://" + process.env.B2_BUCKET_FROM + "/" + m.thumbnail_key;
+    var to = m.tmp_file;
+    var vodId = m.vod_id;
+
+    console.log("processing thumbnail " + i + ". " + from + " -> " + to + " (vod_id=" + m.vod_id + ")");
+
+    // Retry the download
+    await retry(function () {
+      return spawn('b2', ['file', 'download', from, to]);
+    });
+
+    // Retry the PocketBase upload
+    await retry(async function () {
+      var fileData = readFileSync(m.tmp_file);
+      var mimetype = mime.getType(m.tmp_file) || undefined;
+      var file = new File([fileData], basename(m.tmp_file), { type: mimetype });
+      var form = new FormData();
+      form.set("thumbnail", file);
+      return pb.collection('vods').update(vodId, form);
+    });
+
+  }
+
+  console.log("All done.");
+
+
+
+}
+
+main()

services/pocketbase/utils/deploy.js

@@ -8,8 +8,8 @@ async function main() {
   // upload the site
   await spawn('rsync', [
     '-avz',
-    '--exclude',
-    'pb_data',
+    '--exclude=pb_data',
+    '--exclude=*.local',
     '-e',
     'ssh',
     '.',

services/worker/.config/env.ts

@@ -16,6 +16,8 @@ const env = (() => {
   if (!process.env.AWS_SECRET_ACCESS_KEY) throw new Error('AWS_SECRET_ACCESS_KEY missing in env');
   if (!process.env.AWS_REGION) throw new Error('AWS_REGION missing in env');
   if (!process.env.AWS_ENDPOINT) throw new Error('AWS_ENDPOINT missing in env');
+  if (!process.env.FANSLY_USERNAME) throw new Error('FANSLY_USERNAME missing in env');
+  if (!process.env.FANSLY_PASSWORD) throw new Error('FANSLY_PASSWORD missing in env');
 
   const {
     PORT,
@@ -34,6 +36,8 @@ const env = (() => {
     AWS_SECRET_ACCESS_KEY,
     AWS_REGION,
     AWS_ENDPOINT,
+    FANSLY_USERNAME,
+    FANSLY_PASSWORD,
   } = process.env
   return {
     PORT,
@@ -52,6 +56,8 @@ const env = (() => {
     AWS_SECRET_ACCESS_KEY,
     AWS_REGION,
     AWS_ENDPOINT,
+    FANSLY_PASSWORD,
+    FANSLY_USERNAME,
   }
 })()
 

services/worker/package.json

@@ -20,14 +20,18 @@
     "nanoid": "^5.1.6",
     "onnxruntime-web": "^1.23.2",
     "pocketbase": "^0.26.3",
+    "puppeteer": "^24.30.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
     "sharp": "^0.34.5",
     "slugify": "^1.6.6",
     "which": "^5.0.0"
   },
   "devDependencies": {
-    "tsx": "^4.20.6"
+    "tsx": "^4.20.6",
+    "vitest": "^4.0.8"
   },
   "scripts": {
     "start": "tsx src/index.ts"
   }
-}
+}

services/worker/src/queues/generalQueue.ts

@@ -13,3 +13,15 @@ await generalQueue.upsertJobScheduler(
   },
 );
 
+// await generalQueue.upsertJobScheduler(
+//   'copy-v2',
+//   {
+//     pattern: '* * * * *', // Runs at 07:03 every day
+//   },
+//   {
+//     name: 'copyV2ThumbToV3',
+//     data: {},
+//     opts: {}, // Optional additional job options
+//   },
+// );
+

services/worker/src/workers/generalWorker.ts

@@ -3,7 +3,7 @@
 import { Worker } from 'bullmq';
 import { connection } from '../../.config/bullmq.config.ts';
 import { presignMuxAssets } from '../processors/presignMuxAssets.ts';
-
+import { copyV2ThumbToV3 } from '../processors/copyV2ThumbToV3.ts';
 
 new Worker(
   'generalQueue',
@@ -13,6 +13,9 @@ new Worker(
       case 'presignMuxAssets':
         return await presignMuxAssets(job);
 
+      case 'copyV2ThumbToV3':
+        return await copyV2ThumbToV3(job);
+
       // case 'analyzeAudio':
       //   return await analyzeAudio(job);