init

.gitignore

@@ -0,0 +1,144 @@
+# Created by https://www.toptal.com/developers/gitignore/api/node
+# Edit at https://www.toptal.com/developers/gitignore?templates=node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+### Node Patch ###
+# Serverless Webpack directories
+.webpack/
+
+# Optional stylelint cache
+
+# SvelteKit build / generate output
+.svelte-kit
+
+# End of https://www.toptal.com/developers/gitignore/api/node

README.md

@@ -0,0 +1,161 @@
+# scout
+
+## Installation
+
+
+### Dependencies
+
+    ffplay
+
+## Usage
+
+Daemon mode. Log chat and room events
+
+    ./index.js -D
+
+Export mode. Export sql chat messages as IndexedDB
+
+
+
+
+## Dev notes
+
+### Ably realtime event formats
+
+
+#### room:notice:<roomId>
+
+```json
+{
+  "name": "room:notice:6DDY7ZC",
+  "id": "u-Y6UQ8JV3:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755044232:9830",
+    "ts": 1675504423.225136,
+    "messages": [
+      " %%%[emoticon stwad2|https://static-pub.highwebmedia.com/uploads/avatar/2022/11/26/01/46/adec68059ed40d9dbb78260c9a46871c8c2070c4.jpg|76|78|/emoticon_report_abuse/stwad2/]%%% skinny_sis's Spin the Wheel ",
+      " Tip 33 tokens and you can win one of the 12 prizes ",
+      " Chance of winning: 67%. Type /wheel for the prizes "
+    ],
+    "to_user": "",
+    "notice_type": "app",
+    "foreground": "rgb(103,77,255)",
+    "background": null,
+    "weight": "bolder",
+    "method": "lazy",
+    "pub_ts": 1675504423.230338
+  }
+}
+```
+
+
+#### room:title_change:<roomId>
+
+```json
+{
+  "name": "room:title_change:6DDY7ZC",
+  "id": "vAj6M1bn9D:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755100793:8346",
+    "ts": 1675510079.3160765,
+    "title": "❤️CUM SHOW❤️ Pvt is open 30per/min [1386 tokens left] #bigboobs #teen #18 #new #asian",
+    "pub_ts": 1675510079.3162484,
+    "method": "single"
+  }
+}
+```
+
+#### room:update:<roomId>
+
+```json
+{
+  "name": "room:update:6DDY7ZC",
+  "id": "meLVS3wTan:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755100794:436",
+    "ts": 1675510079.4469855,
+    "target": "refresh_panel",
+    "target_user": "",
+    "pub_ts": 1675510079.4471552,
+    "method": "single"
+  }
+}
+```
+
+#### room:status:<roomId>:0
+
+```json
+{
+  "name": "room:status:ZF09ZAC:1",
+  "id": "cCs6ThzT-p:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755114015:96114",
+    "ts": 1675511401.588585,
+    "status": "away",
+    "message": "",
+    "hash": "",
+    "method": "lazy",
+    "pub_ts": 1675511401.5939016
+  }
+}
+
+{
+  "name": "room:status:ZF09ZAC:1",
+  "id": "Zk1EF44xrP:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755114435:80861",
+    "ts": 1675511443.533211,
+    "status": "public",
+    "message": "",
+    "hash": "",
+    "pub_ts": 1675511443.5339801,
+    "method": "single"
+  }
+}
+```
+
+
+#### room:update:<roomId>
+
+```json
+{
+  "name": "room:update:ZF09ZAC",
+  "id": "vOqwMouvAB:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755118892:73991",
+    "ts": 1675511889.2104225,
+    "target": "refresh_panel",
+    "target_user": "",
+    "pub_ts": 1675511889.2105591,
+    "method": "single"
+  }
+}
+```
+
+
+#### room:status:<roomId>:0
+
+```json
+{
+  "name": "room:status:ZF09ZAC:1",
+  "id": "gs04uEHZTV:0:0",
+  "encoding": null,
+  "data": {
+    "tid": "16755147455:43279",
+    "ts": 1675514745.5818536,
+    "status": "offline",
+    "message": "",
+    "hash": "",
+    "pub_ts": 1675514745.5831933,
+    "method": "single"
+  }
+}
+```
+
+#### 

ecosystem.config.cjs

@@ -0,0 +1,7 @@
+module.exports = {
+    apps: [{
+        name: 'scout daemon',
+        script: './index.js',
+        args: 'daemon'
+    }]
+}

index.js

@@ -0,0 +1,206 @@
+
+
+// import { containsCBInviteLink } from "./src/tweetProcess.js"
+import Room from './src/Room.js'
+import { loggerFactory } from "./src/logger.js"
+// import Stream from './src/Stream.js'
+import { assertYtdlpExistence } from './src/ytdlp.js'
+import * as faye from './src/faye.js'
+import * as sound from './src/sound.js'
+import * as cb from './src/cb.js'
+// import { extractRoomId } from './src/parsers.js'
+import { appEnv, getAppContext } from './src/appContext.js'
+import { onCbMessage, onCbStart, onCbStop, onCbTitleChange, onCbSilence, onCbTip } from './src/cbCallbacks.js'
+import fs from 'node:fs'
+import Database from 'better-sqlite3';
+import os from 'node:os'
+import path from 'node:path'
+import yargs from 'yargs'
+import { hideBin } from 'yargs/helpers'
+
+const epoch = new Date(0).valueOf()
+const now = new Date().valueOf()
+
+
+async function getDataDir() {
+  // Define dataDir
+  const dataDir = path.join(os.homedir(), '.local', 'share', 'futureporn-scout');
+
+  try {
+    // Check if dataDir exists
+    await fs.promises.access(dataDir);
+  } catch (error) {
+    // Create dataDir if it doesn't exist
+    await fs.promises.mkdir(dataDir, { recursive: true });
+  }
+
+  return dataDir;
+}
+
+
+
+
+
+
+async function init () {
+  const dataDir = await getDataDir()
+  const dbPath = path.join(dataDir, 'scout.db')
+
+  await assertYtdlpExistence()
+
+  const db = new Database(dbPath);
+  db.pragma('journal_mode = WAL');
+
+  const logger = loggerFactory({
+    defaultMeta: { service: 'futureporn/scout' }
+  })
+
+  return getAppContext(appEnv, logger, db, sound, cb)
+}
+
+
+
+
+async function registerRoomListeners(appContext, rooms) {
+  for (const room of rooms) {
+    const roomName = room.name;
+    const roomId = room.id;
+    appContext.logger.log({ level: 'info', message: `monitoring room:${roomName} (${roomId})` })
+    const rm = new Room({
+      roomName: roomName,
+      roomId: roomId,
+      onStart: (m) => onCbStart(appContext, roomName, m),
+      onStop: (m) => onCbStop(appContext, roomName, m),
+      onMessage: (m) => onCbMessage(appContext, roomName, m),
+      onSilence: (m) => onCbSilence(appContext, roomName, m),
+      onTitleChange: (m) => onCbTitleChange(appContext, roomName, m),
+      onTip: (m) => onCbTip(appContext, roomName, m),
+    })
+
+    // retry if errors
+    let retry = true;
+    while (retry) {
+      try {
+        await rm.monitorRealtime();
+        retry = false; // If no error occurs, exit the loop
+        appContext.logger.log({ level: 'debug', message: 'monitorRealtime() completed without error.' })
+      } catch (e) {
+        appContext.logger.log({ level: 'info', message: `Error caught when attempting to monitor ${roomName} realtime. ${e}. Retrying.` });
+        const minTimeout = 5000; // 5 seconds in milliseconds
+        const maxTimeout = 15000; // 15 seconds in milliseconds
+        const timeout = Math.floor(Math.random() * (maxTimeout - minTimeout + 1)) + minTimeout;
+        appContext.logger.log({ level: 'info', message: `${timeout} courtesy timer before retry` })
+        await new Promise((resolve) => setTimeout(() => resolve(), timeout));
+      }
+    }
+
+  }
+}
+
+
+// /**
+//  * These room timers are used to differentiate between stream ending, and temporary outages
+//  * Room timers track how long a room has been offline.
+//  * If the room has been offline for less than 5 minutes and a start event occurs, the offline event must have been a temporary outage.
+//  */
+// function registerRoomTimers(appContext, rooms) {
+//   for (const room of rooms) {
+//     const roomName = room.name
+//     appContext.logger.log({ level: 'debug', message: `registering room timer for ${roomName}` })
+//     appContext.roomTimers[roomName] = {}
+//     appContext.roomTimers[roomName].offline = null
+//   }
+// }
+
+
+async function daemon () {
+  const appContext = await init()
+  const stmt = appContext.db.prepare('SELECT name, id FROM rooms WHERE monitor = TRUE')
+  const rooms = stmt.all()
+  appContext.faye = faye.fayeFactory(appContext)
+  // registerRoomTimers(appContext, rooms)
+  registerRoomListeners(appContext, rooms)
+}
+
+
+async function exportLogs (options) {
+  const appContext = await init()
+
+  // let stmt = appContext.db.prepare('SELECT ...')
+  if (options.auto) {
+    const stream = new Stream(appContext, options.room);
+    const r = stream.getMostRecentStream()
+    console.log(r)
+  }
+
+  // use lifecycles database to find most recent start & stop events.
+  // use a 5 minute threshold to determine stream starts/s
+  //   stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' ORDER BY data_ts ASC;`)
+  // } else if (!!options.since && !!options.until) {
+  //   stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts >= ${options.since} ORDER BY data_ts ASC;`)
+  // } else if (!!options.since && !options.until) {
+  //   stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts >= ${options.since} ORDER BY data_ts ASC;`)
+  // } else if (!options.since && !!options.until) {
+  //   stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts <= ${options.until} ORDER BY data_ts ASC;`)
+  // }
+  // const messages = stmt.all()
+
+  // console.log(messages)
+}
+
+yargs(hideBin(process.argv))
+  .command({
+    command: 'daemon',
+    alias: 'd',
+    desc: 'Listen & log chaturbate events', 
+    builder: () => {},
+    handler: (argv) => {
+      console.info(argv)
+      daemon()
+    }
+  })
+  .command({
+    command: 'export',
+    alias: 'e',
+    desc: 'Export chat logs as JSON',
+    builder: (yargs) => {
+      return yargs
+        .option('output', {
+          alias: 'o',
+          describe: 'output file location',
+          required: true
+        })
+        .option('room', {
+          aliases: ['r', 'n'],
+          describe: 'the name of the chaturbate room',
+          required: true
+        })
+        .option('since', {
+          alias: 's',
+          describe: 'export logs since this date',
+          default: epoch,
+          required: false
+        })
+        .option('until', {
+          alias: 'u',
+          describe: 'export logs until this date',
+          default: now,
+          required: false
+        })
+        // .option('auto', {
+        //   alias: 'a',
+        //   describe: 'gets the most recent stream',
+        //   required: false
+        // })
+    },
+    handler: (argv) => {
+      if (argv.auto && (argv.since !== epoch || argv.until !== now)) {
+        throw new Error('cannot use --auto with --since or --until.')
+      }
+      console.info(argv)
+      exportLogs(argv)
+    }
+  })
+  .demandCommand(1)
+  .help()
+  .parse()

package.json

@@ -0,0 +1,43 @@
+{
+  "name": "scout",
+  "version": "0.0.1",
+  "description": "event emitter that detects start and end of stream",
+  "main": "index.js",
+  "license": "Unlicense",
+  "private": true,
+  "scripts": {
+    "start": "pm2 start ecosystem.config.cjs; pm2 monit",
+    "test": "mocha ./test/unit",
+    "dev": "FUTUREPORN_WORKDIR=/home/chris/Downloads pnpm nodemon index.js daemon"
+  },
+  "type": "module",
+  "dependencies": {
+    "@neutralinojs/neu": "^9.5.1",
+    "ably": "1.2.13",
+    "better-sqlite3": "^8.3.0",
+    "cheerio": "^1.0.0-rc.12",
+    "date-fns": "^2.29.3",
+    "dexie-export-import": "^4.0.7",
+    "dotenv": "^16.0.3",
+    "execa": "^7.1.1",
+    "faye": "^1.4.0",
+    "fetch-blob": "^3.2.0",
+    "formdata-polyfill": "^4.0.10",
+    "node-fetch": "^3.3.0",
+    "p-retry": "^5.1.2",
+    "pm2": "^5.3.0",
+    "seedrandom": "^3.0.5",
+    "tough-cookie": "^4.1.2",
+    "tough-cookie-file-store": "^2.0.3",
+    "twitter-v2": "^1.1.0",
+    "winston": "^3.8.2",
+    "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "chai": "^4.3.7",
+    "chai-events": "^0.0.3",
+    "mocha": "^10.2.0",
+    "nodemon": "^2.0.20",
+    "sinon": "^15.1.0"
+  }
+}

src/Room.js

@@ -0,0 +1,400 @@
+
+import fetch from "node-fetch";
+import cheerio from "cheerio";
+import {
+  Cookie,
+  CookieJar
+} from "tough-cookie";
+import {
+  FileCookieStore
+} from "tough-cookie-file-store";
+import os from 'os';
+import path from 'path';
+import {
+  loggerFactory
+} from './logger.js'
+import {
+  isBefore
+} from 'date-fns'
+import Ably from 'ably';
+import {
+  FormData
+} from 'formdata-polyfill/esm.min.js'
+
+
+const logger = loggerFactory({
+  defaultMeta: {
+    service: "futureporn/scout"
+  }
+})
+
+export default class Room {
+  constructor(opts) {
+    this.roomName = opts.roomName
+    this.roomId = opts.roomId
+    this.roomUrl = 'https://chaturbate.com/' + this.roomName
+    this.csrfToken = null
+    this.tokenRequest = null
+    this.ablyTokenRequest = null
+    this.dossier = null
+    this.realtimeHost = null
+    this.fallbackHosts = null
+    this.pushServiceAuth = null
+    this.datadir = path.join(os.homedir(), '.local/share/futureporn-scout')
+    this.cookieJar = new CookieJar(new FileCookieStore(path.join(this.datadir, "cookie.json")));
+    this.ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36';
+    this.onStart = opts.onStart || null;
+    this.onStop = opts.onStop || null;
+    this.onMessage = opts.onMessage || null;
+    this.onSilence = opts.onSilence || null;
+    this.onTitleChange = opts.onTitleChange || null;
+    this.onTip = opts.onTip || null;
+    this.realtime = null;
+  }
+
+
+
+  // deprecated. fragile because it depends on existing room dossier
+  // async getRoomId () {
+  //   if (this.roomId !== '') {
+  //     return this.roomId
+  //   } else {
+  //     const dossier = await getInitialRoomDossier(room)
+  //     return dossier.room_uid
+  //   }
+  // }
+
+  // deprecated. fragile because it depends on present room dossier
+  // async getInitialRoomDossier() {
+  //   try {
+  //     const res = await fetch(this.roomUrl);
+  //     const body = await res.text();
+  //     const $ = cheerio.load(body);
+  //     let rawScript = $('script:contains(window.initialRoomDossier)').html();
+  //     if (!rawScript) {
+  //       throw new Error('window.initialRoomDossier is null. This could mean the channel is in password mode');
+  //     }
+  //     let rawDossier = rawScript.slice(rawScript.indexOf('"'), rawScript.lastIndexOf('"') + 1);
+  //     let dossier = JSON.parse(JSON.parse(rawDossier));
+  //     return dossier;
+  //   } catch (error) {
+  //     // Handle the error gracefully
+  //     logger.log({ level: 'error', message: `Error fetching initial room dossier: ${error.message}` });
+  //     return null; // Or any other appropriate action you want to take
+  //   }
+  // }
+  
+
+
+
+  static getPermissionsForm(roomId, csrfToken) {
+    logger.log({ level: 'debug', message: `getting permission form with room id ${roomId}` })
+    if (!roomId) throw new Error('roomId is required but it was undefined')
+    let form = new FormData();
+    const topics = `{
+          "RoomTipAlertTopic#RoomTipAlertTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },
+          "RoomPurchaseTopic#RoomPurchaseTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },
+          "RoomFanClubJoinedTopic#RoomFanClubJoinedTopic:${roomId}": {
+              "broadcaster_uid":"${roomId}"
+          },
+          "RoomMessageTopic#RoomMessageTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },
+          "GlobalPushServiceBackendChangeTopic#GlobalPushServiceBackendChangeTopic":{
+          },
+          "RoomAnonPresenceTopic#RoomAnonPresenceTopic:${roomId}":{\
+              "broadcaster_uid":"${roomId}"
+          },
+          "QualityUpdateTopic#QualityUpdateTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomNoticeTopic#RoomNoticeTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomEnterLeaveTopic#RoomEnterLeaveTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomModeratorPromotedTopic#RoomModeratorPromotedTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomModeratorRevokedTopic#RoomModeratorRevokedTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomStatusTopic#RoomStatusTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomTitleChangeTopic#RoomTitleChangeTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomSilenceTopic#RoomSilenceTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomKickTopic#RoomKickTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomUpdateTopic#RoomUpdateTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          },"RoomSettingsTopic#RoomSettingsTopic:${roomId}":{
+              "broadcaster_uid":"${roomId}"
+          }
+      }`
+    form.append('topics', topics)
+    form.append('csrfmiddlewaretoken', csrfToken)
+    return form
+  }
+
+
+  async monitorRealtime () {
+    if (!this.pushServiceAuth) this.pushServiceAuth = await this.getPushServiceAuth();
+    if (!this.roomId) throw new Error('roomId was missing')
+
+
+    const roomMessageTopicString = this.pushServiceAuth.channels[`RoomMessageTopic#RoomMessageTopic:${this.roomId}`]
+    const roomSilenceTopicString = this.pushServiceAuth.channels[`RoomSilenceTopic#RoomSilenceTopic:${this.roomId}`]
+    const roomStatusTopicString = this.pushServiceAuth.channels[`RoomStatusTopic#RoomStatusTopic:${this.roomId}`]
+    const roomTitleChangeTopicString = this.pushServiceAuth.channels[`RoomTitleChangeTopic#RoomTitleChangeTopic:${this.roomId}`]
+    const roomTipAlertTopicString = this.pushServiceAuth.channels[`RoomTipAlertTopic#RoomTipAlertTopic:${this.roomId}`]
+
+    this.realtime = await this.getRealtime()
+    this.realtime.connection.once('connected', (idk) => {
+      logger.log({ level: 'info', message: 'CB Realtime Connected!' })
+    })
+    const messageChannel = this.realtime.channels.get(roomMessageTopicString);
+    messageChannel.subscribe((message) => {
+      this.onMessage(message)
+    })
+
+    const silenceChannel = this.realtime.channels.get(roomSilenceTopicString);
+    silenceChannel.subscribe((message) => {
+      this.onSilence(message)
+    })
+
+    
+    const statusChannel = this.realtime.channels.get(roomStatusTopicString);
+    statusChannel.subscribe((message) => {
+      logger.log({ level: 'debug', message: `got statusChannel message! ${JSON.stringify(message)}`})
+      if (message.data.status === 'public') {
+        this.onStart(message)
+      } else if (message.data.status === 'offline') {
+        this.onStop(message)
+      }
+      logger.log({ level: 'debug', message: `Received room:status:<roomId>:0` })
+      logger.log({ level: 'debug', message: JSON.stringify(message, 0, 2) })
+    });
+
+    const titleChannel = this.realtime.channels.get(roomTitleChangeTopicString);
+    titleChannel.subscribe((message) => {
+      this.onTitleChange(message)
+    })
+
+
+    const tipAlertChannel = this.realtime.channels.get(roomTipAlertTopicString);
+    tipAlertChannel.subscribe((message) => {
+      logger.log({ level: 'debug', message: `got tipAlertChannel message! ${JSON.stringify(message)}`})
+      this.onTip(message)
+    })
+
+    await this.realtime.connect()
+  }
+
+
+  async refreshToken () {
+    const authToken = await getNewToken()
+
+  }
+
+  /**
+   * make a request against CB's server which
+   *   - verifies custom credentials
+   *   - issues signed Ably TokenRequest
+   * 
+   * The response contains the following
+   *   - token
+   *   - channels
+   *   - failures
+   *   - token_request <-- Ably TokenRequest
+   *   - client_id
+   *   - settings
+   *
+   * GET https://chaturbate.com/push_service/auth/
+   *  
+   * more info-- https://ably.com/docs/core-features/authentication#token-authentication
+   */
+  async getPushServiceAuth () {
+
+    // https://faqs.ably.com/40104-timestamp-not-current
+    // this was caused by signed token request re-use after it had expired.
+    // we must not cache the signed token request
+
+
+    // https://help.ably.io/error/40102
+    // caused by mismatching client_id, when using a new client_id issued by CB pushServiceAuth with an old ably connection
+    // (possibly) caused by not sending Cookie header during CB pushServiceAuth
+    //            thus leading to CB thinking we're a new anon and issuing a new client_id
+    //            @todo please verify if this is the cause
+
+    // 09:44:42.182 Ably: Transport.onIdleTimerExpire(): No activity seen from realtime in 25113ms; assuming connection has dropped
+    // saw this when my internet died. unsure of whether or not the connection comes back online.
+    // if it doesn't come back online, this could bite me in the future.
+    // this is a thing we can test by airgapping.
+    // if there's a way to implement an onIdleTimerExpire callback, that would be the solution.
+    // @todo
+  
+    logger.log({
+      level: 'debug',
+      message: `getting pushServiceAuth.`
+    })
+
+    await this.fetchCookiesIfNeeded()
+    this.csrfToken = await this.getCsrfToken()
+    const cookieString = await this.cookieJar.getCookieString(this.roomUrl)
+
+    logger.log({
+      level: 'debug',
+      message: `using cookie string ${cookieString}`
+    })
+
+    const form = Room.getPermissionsForm(this.roomId, this.csrfToken)
+    const res = await fetch("https://chaturbate.com/push_service/auth/", {
+      "credentials": "include",
+      "headers": {
+        "Cookie": cookieString,
+        "User-Agent": this.ua,
+        "Accept": "*/*",
+        "Accept-Language": "en-US,en;q=0.5",
+        "X-Requested-With": "XMLHttpRequest",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-origin",
+        "Sec-GPC": "1"
+      },
+      "referrer": this.roomUrl,
+      "body": form,
+      "method": "POST",
+      "mode": "cors"
+    });
+    this.pushServiceAuth = await res.json()
+
+    logger.log({
+      level: 'debug',
+      message: JSON.stringify(this.pushServiceAuth)
+    })
+    return this.pushServiceAuth
+  }
+
+
+
+  async getTokenRequest() {
+    this.pushServiceAuth = await this.getPushServiceAuth()
+    return this.pushServiceAuth.token_request
+  }
+
+  async getRealtime() {
+    // deps
+    // csrfToken, tokenRequest, realtimeHost, fallbackHosts
+
+    if (!this.realtimeHost || !this.fallbackHosts) {
+      this.pushServiceAuth = await this.getPushServiceAuth()
+      this.realtimeHost = this.pushServiceAuth.settings.realtime_host
+      this.fallbackHosts = this.pushServiceAuth.settings.fallback_hosts
+    }
+
+    const realtime = new Ably.Realtime.Promise({
+      autoConnect: false,
+      closeOnUnload: true,
+      transportParams: {
+        remainPresentFor: '0'
+      },
+      realtimeHost: this.realtimeHost,
+      restHost: this.realtimeHost,
+      fallback_hosts: this.fallbackHosts,
+      authCallback: (async(tokenParams, cb) => {
+
+        // @see https://github.com/insanity54/futureporn/issues/203
+        // @todo get a new signed token from CB's servers
+
+
+
+        logger.log({
+          level: 'debug',
+          message: `Ably authCallback. Getting a fresh new Ably TokenRequest.`
+        })
+
+        // This is where we get a signed TokenRequest from CB!
+        this.tokenRequest = await this.getTokenRequest()
+        logger.log({
+          level: 'debug',
+          message: `Got a new TokenRequest`
+        })
+        logger.log({
+          level: 'debug',
+          message: JSON.stringify(this.tokenRequest)
+        })
+
+        cb(null, this.tokenRequest)
+      })
+    })
+
+    return realtime
+
+  }
+
+    static getTokenCookie(cookies) {
+      return cookies.find((c) => c.key == 'csrftoken');
+    }
+
+    static getSbrCookie(cookies) {
+      return cookies.find((c) => c.key == 'sbr');
+    }
+
+
+    async fetchCookiesIfNeeded () {
+      let cookies = await this.cookieJar.getCookies(this.roomUrl);
+      let tokenCookie = Room.getTokenCookie(cookies)
+      logger.log({
+        level: 'debug',
+        message: `tokenCookie is as follows. ${tokenCookie}`
+      })
+      if (
+        tokenCookie === undefined ||
+        isBefore(tokenCookie.expires, new Date())
+      ) {
+        logger.log({
+          level: 'debug',
+          message: 'tokenCookie is either undefined or expired. Getting new cookie.'
+        })
+        let res = await fetch(this.roomUrl, {
+          "credentials": "omit",
+          "headers": {
+            "User-Agent": this.ua,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1"
+          },
+          "method": "GET",
+          "mode": "cors"
+        })
+
+        const rawHeaders = res.headers.raw()
+        if (Array.isArray(rawHeaders["set-cookie"]))
+          cookies = rawHeaders["set-cookie"].map(Cookie.parse);
+        else cookies = [Cookie.parse(rawHeaders["set-cookie"])];
+
+        for (const cookie of cookies) {
+          logger.log({ level: 'debug', message: `adding cookie ${JSON.stringify(cookie)} to the cookieJar` })
+          await this.cookieJar.setCookie(cookie, this.roomUrl)
+        }
+      } else {
+        logger.log({ level: 'debug', message: `🍪✅ cookies are valid.`})
+      }
+    }
+
+    async getCsrfToken() {
+      let cookies = await this.cookieJar.getCookies(this.roomUrl);
+      let tokenCookie = Room.getTokenCookie(cookies)
+      return tokenCookie.value
+    }
+
+
+  }

src/Stream.js

@@ -0,0 +1,124 @@
+export const createVod = (appContext, roomName) => {
+
+}
+
+function getMostRecentStream() {
+  let output = []
+  let checkSum = 0
+  for (const [index, row] of data.entries()) {
+
+    if (index === 0 && row.data_status === 'offline')
+
+    if (row.delta === null) {
+      // skip first row
+      output.push(row)
+      continue;
+    } else if (row.data_status === 'public' && row.delta < outageThreshold) {
+      // this is an error row!
+      // we dont add the row to output but we incr the duration
+      duration += row.delta
+    } else if (row.data_status === 'offline' && row.delta ) {
+
+    } else {
+      // 
+      duration += row.delta
+    }
+    console.log(`row ${i}`)
+    if (row.delta !== null && row.data_status === 'public' && row.data_ts < outageThreshold) {
+    }
+  }
+}
+
+
+export default class Stream {
+  constructor(appContext, roomName) {
+    this.appContext = appContext
+    this.roomName = roomName
+    this.chat = []
+  }
+
+
+
+
+  getMostRecentStream() {
+    const statement = this.appContext.db.prepare(
+      `SELECT data_status, data_ts FROM lifecycles WHERE _room = ? ORDER BY data_ts DESC;`
+    );
+
+    let sta = null
+    let end = null
+    let sum = 0
+
+    const outageThreshold = 1000*60*5
+    const rows = statement.all(this.roomName)
+
+    for (const [index, row] of data.entries()) {
+      if (index === 0) {
+        if (row.data_status === 'offline') {
+          end = row.data_ts;
+        } else {
+          throw new Error('First row is not an offline event! Not configured to handle this scenario.')
+        }
+      } else {
+        const isUp = (rows[index-1].data_status === 'offline' && row.data_status === 'online')
+      }
+    }
+
+
+    // const deltaRows = rows.map((obj, index) => {
+    //   if (index === 0) {
+    //     // For the first object, set the difference as null or any default value you prefer
+    //     return { ...obj, delta: null };
+    //   } else {
+    //     const previousObj = rows[index - 1];
+    //     const delta = obj.data_ts - previousObj.data_ts;
+    //     return { ...obj, delta };
+    //   }
+    // });
+
+    
+    // const filteredData = removeErrorRows(deltaRows, outageThreshold);
+
+    // return filteredData
+    // find the most recent stop event
+    // find the most recent start event
+    //
+    // return {
+    //   duration: stop.data_ts - start.data_ts,
+    //   start: start.data_ts,
+    //   stop: stop.data_ts
+    // }
+  }
+
+
+
+
+
+  getStreamSegmentsUsingEndTs(ts) {
+
+  }
+
+  getChat() {
+    const statement = this.appContext.db.prepare(``)
+    const chat = statement.all()
+    // @todo convert rows to indexedDB
+  }
+
+  async uploadToStrapi() {
+    this.appContext.logger.log({ level: 'info', message: `creating vod for room ${roomName}` })
+
+  }
+
+}
+
+
+export const createStreamVOD = () => {
+  // find the relevant StreamSegments
+  // group the StreamSegments into a Stream
+  // create a vod using the start/stop timestamps of the Stream, and the messages falling within those timestamps
+
+  const stream = new Stream(appContext, )
+  stream.getStreamSegmentsUsingEndTs(message.data.ts)
+  stream.getChat()
+  stream.uploadToStrapi()
+}

src/StreamSegment.js

@@ -0,0 +1,6 @@
+
+export default class StreamSegment {
+  constructor() {
+    
+  }
+}

src/appContext.js

@@ -0,0 +1,22 @@
+import 'dotenv/config'
+
+
+
+export const appEnv = new Array('PUBSUB_SERVER_URL')
+
+
+export function getAppContext(appEnv, logger, db, sound, cb) {
+  return {
+    env: appEnv.reduce((acc, ev) => {
+      if (typeof process.env[ev] === 'undefined') throw new Error(`${ev} is undefined in env`);
+      acc[ev] = process.env[ev];
+      return acc;
+    }, {}),
+    logger,
+    db,
+    roomTimers: {},
+    sound,
+    cb
+  };
+}
+

src/cb.js

@@ -0,0 +1,42 @@
+import cheerio from 'cheerio'
+import fetch from 'node-fetch'
+import { execa } from 'execa'
+import pRetry from 'p-retry'
+
+export async function getRandomRoom () {
+  const res = await fetch('https://chaturbate.com/')
+  const body = await res.text()
+  const $ = cheerio.load(body)
+  let roomsRaw = $('a[data-room]')
+  let rooms = []
+  $(roomsRaw).each((_, e) => {
+    rooms.push($(e).attr('href'))
+  })
+
+  // greets https://stackoverflow.com/a/4435017/1004931
+  var randomIndex = Math.floor(Math.random() * rooms.length);
+  return rooms[randomIndex].replaceAll('/', '')
+}
+
+
+// helper for testing
+export const getRandomPlaylist = async (appContext) => {
+  const roomName = await getRandomRoom();
+  return getPlaylistUrl(appContext, roomName)
+}
+
+
+
+export const getPlaylistUrl = async (appContext, roomName) => {
+  const operation = async () => {
+    const { stdout: playlistUrl } = await execa('yt-dlp', ['-g', `https://chaturbate.com/${roomName}`]);
+    return playlistUrl.trim();
+  };
+  return await pRetry(operation, { 
+    retries: 6, 
+    minTimeout: 2000,
+    onFailedAttempt(e) {
+      appContext.logger.log({ level: 'debug', message: `error getting playlist url during attempt ${e.attemptNumber}. ${e.retriesLeft} retries remaining.` })
+    }
+  });
+};

src/cbCallbacks.js

@@ -0,0 +1,233 @@
+// import { createVod } from './strapi.js'
+import Stream from './Stream.js'
+import { signalStart, sendSignal } from './faye.js'
+// import { execa } from 'execa'
+
+
+export const onCbTip = async (appContext, roomName, message) => {
+  appContext.logger.log({ level: 'debug', message: `$$$ [TIP] ${JSON.stringify(message)}` })
+  const stmt = appContext.db.prepare(`INSERT INTO tips VALUES ($_room, $name, $encoding, $data_tid, $data_ts, $data_amount, $data_message, $data_history, $data_is_anonymous_tip, $data_to_username, $data_from_username, $data_gender, $data_is_broadcaster, $data_in_fanclub, $data_is_following, $data_is_mod, $data_has_tokens, $data_tipped_recently, $data_tipped_alot_recently, $data_tipped_tons_recently, $data_method, $data_pub_ts)`)
+  stmt.run({
+    '_room': roomName,
+    'name': message?.name || null,
+    'encoding': message?.encoding || null,
+    'data_tid': message?.data?.tid || null,
+    'data_ts': message?.data?.ts || null,
+    'data_amount': message?.data?.amount || null,
+    'data_message': message?.data?.message || null,
+    'data_history': (!!message?.data?.history) ? 1 : 0,
+    'data_is_anonymous_tip': (!!message?.data?.is_anonymous_tip) ? 1 : 0,
+    'data_to_username': message?.data?.to_username || null,
+    'data_from_username': message?.data?.from_username || null,
+    'data_gender': message?.data?.gender || null,
+    'data_is_broadcaster': (!!message?.data?.is_broadcaster) ? 1 : 0,
+    'data_in_fanclub': (!!message?.data?.in_fanclub) ? 1 : 0,
+    'data_is_following': (!!message?.data?.is_following) ? 1 : 0,
+    'data_is_mod': (!!message?.data?.is_mod) ? 1 : 0,
+    'data_has_tokens': (!!message?.data?.has_tokens) ? 1 : 0,
+    'data_tipped_recently': (!!message?.data?.tipped_recently) ? 1 : 0,
+    'data_tipped_alot_recently': (!!message?.data?.tipped_alot_recently) ? 1 : 0,
+    'data_tipped_tons_recently': (!!message?.data?.tipped_tons_recently) ? 1 : 0,
+    'data_method': message?.data?.method || null,
+    'data_pub_ts': message?.data?.pub_ts || null,
+  })
+}
+
+
+export const onCbStart = async (appContext, roomName, message) => {
+  appContext.logger.log({ level: 'debug', message: `🟢🟢🟢 [STREAM START] ${JSON.stringify(message)}` })
+  const stmt = appContext.db.prepare(`INSERT INTO lifecycles VALUES ($_room, $name, $id, $encoding, $data_tid, $data_ts, $data_status, $data_message, $data_hash, $data_method, $data_pub_ts)`)
+  stmt.run({
+    '_room': roomName,
+    'name': message?.name || null,
+    'id': message?.id || null,
+    'encoding': message?.encoding || null,
+    'data_tid': message?.data?.tid || null,
+    'data_ts': message?.data?.status || null,
+    'data_status': message?.data?.status || null,
+    'data_message': message?.data?.message || null,
+    'data_hash': message?.data?.hash || null,
+    'data_method': message?.data?.method || null,
+    'data_pub_ts': message?.data?.pub_ts || null,
+  })
+  appContext.sound.alert()
+
+  if (appContext.cb.getPlaylistUrl === undefined) {
+    console.log('getPlaylistUrl is undefined')
+    console.log(appContext)
+    throw new Error('getPlaylistUrl missing!');
+  } else {
+    console.log(`appContext.cb.getPlaylistUrl is defined and it is ${appContext.cb.getPlaylistUrl}`)
+  }
+
+
+  const playlistUrl = await appContext.cb.getPlaylistUrl(appContext, roomName)
+  signalStart(appContext, roomName, playlistUrl)
+
+
+  // clear any offline timer which might be running. 
+  // this is to prevent temporary outages from being counted as a new stream
+  // @todo this is faulty. more thought needed to implement a timer.
+  // !!appContext.roomTimers[roomName].offline && clearTimeout(appContext.roomTimers[roomName].offline)
+}
+
+
+export const onCbStop = (appContext, roomName, message) => {
+  appContext.logger.log({ level: 'debug', message: `🛑🛑🛑 [STREAM STOP] ${JSON.stringify(message)}` })
+  const stmt = appContext.db.prepare(`INSERT INTO lifecycles VALUES (
+    $_room,
+    $name,
+    $id,
+    $encoding,
+    $data_tid,
+    $data_ts,
+    $data_status,
+    $data_message,
+    $data_hash,
+    $data_method,
+    $data_pub_ts
+  )`)
+  stmt.run({
+    '_room': roomName,
+    'name': message?.name || null,
+    'id': message?.id || null,
+    'encoding': message?.encoding || null,
+    'data_tid': message?.data?.tid || null,
+    'data_ts': message?.data?.ts || null,
+    'data_status': message?.data?.status || null,
+    'data_message': message?.data?.message || null,
+    'data_hash': message?.data?.hash || null,
+    'data_method': message?.data?.method || null,
+    'data_pub_ts': message?.data?.pub_ts || null,
+  })
+
+  sendSignal(appContext, roomName, 'stop')
+
+
+
+  // start offline timer which runs if 5 minutes without a start event elapses
+  // @todo this is FAULTY
+  // @see https://github.com/insanity54/futureporn/issues/206
+  // appContext.roomTimers[roomName].offline = setTimeout(() => createStreamVOD(appContext, message), 1000*60*5) 
+}
+
+
+export const createStreamVOD = (appContext, message) => {
+  // find the relevant StreamSegments
+  // group the StreamSegments into a Stream
+  // create a vod using the start/stop timestamps of the Stream, and the messages falling within those timestamps
+
+  const stream = new Stream()
+  stream.getStreamSegmentsUsingEndTs(message.data.ts)
+  stream.getChat()
+  stream.uploadToStrapi(appContext)
+}
+
+export const onCbTitleChange = (appContext, roomName, message) => {
+  appContext.logger.log({ level: 'debug', message: `title changed! ${JSON.stringify(message)}`})
+  const stmt = appContext.db.prepare(`INSERT INTO titles VALUES (
+    $_room,
+    $name,
+    $id,
+    $encoding,
+    $data_tid,
+    $data_ts,
+    $data_title,
+    $data_method,
+    $data_pub_ts
+  )`)
+  stmt.run({
+    '_room': roomName,
+    'name': message?.name || null,
+    'id': message?.id || null,
+    'encoding': message?.encoding || null,
+    'data_tid': message?.data?.tid || null,
+    'data_ts': message?.data?.ts || null,
+    'data_title': message?.data?.title || null,
+    'data_method': message?.data?.method || null,
+    'data_pub_ts': message?.data?.pub_ts || null,
+  })
+}
+
+export const onCbSilence = (appContext, roomName, message) => {
+  appContext.logger.log({ level: 'debug', message: `Bad boys get busted! ${JSON.stringify(message)}`})
+  const stmt = appContext.db.prepare(`INSERT INTO silences VALUES (
+    $_room,
+    $name,
+    $id,
+    $encoding,
+    $data_tid,
+    $data_ts,
+    $data_username,
+    $data_from_username,
+    $data_method,
+    $data_pub_ts
+  )`)
+  stmt.run({
+    '_room': roomName,
+    'name': message?.name || null,
+    'id': message?.id || null,
+    'encoding': message?.encoding || null,
+    'data_tid': message?.data?.tid || null,
+    'data_ts': message?.data?.ts || null,
+    'data_username': message?.data?.username || null,
+    'data_from_username': message?.data?.from_username || null,
+    'data_method': message?.data?.method || null,
+    'data_pub_ts': message?.data?.pub_ts || null,
+  })
+}
+
+
+export const onCbMessage = (appContext, roomName, message) => {
+  const stmt = appContext.db.prepare(`INSERT INTO messages VALUES (
+    $_room,
+    $name, 
+    $id, 
+    $encoding, 
+    $data_tid, 
+    $data_ts, 
+    $data_message, 
+    $data_font_family,
+    $data_font_color, 
+    $data_id, 
+    $data_background, 
+    $data_from_user_username,
+    $data_from_user_gender,
+    $data_from_user_is_broadcaster,
+    $data_from_user_in_fanclub,
+    $data_from_user_is_following,
+    $data_from_user_is_mod,
+    $data_from_user_has_tokens,
+    $data_from_user_tipped_recently,
+    $data_from_user_tipped_alot_recently,
+    $data_from_user_tipped_tons_recently,
+    $data_method,
+    $data_pub_ts
+  )`);
+  stmt.run({
+    '_room': roomName,
+    'name': message?.name || null,
+    'id': message?.id || null,
+    'encoding': message?.encoding || null,
+    'data_tid': message?.data?.tid || null,
+    'data_ts': message?.data?.ts || null,
+    'data_message': message?.data?.message || null,
+    'data_font_family': message?.data?.font_family || null,
+    'data_font_color': message?.data?.font_color || null,
+    'data_id': message?.data?.id || null,
+    'data_background': message?.data?.background || null,
+    'data_from_user_username': message?.data?.from_user?.username || null,
+    'data_from_user_gender': message?.data?.from_user?.gender || null,
+    'data_from_user_is_broadcaster': (!!message?.data?.from_user?.is_broadcaster) ? 1 : 0,
+    'data_from_user_in_fanclub': (!!message?.data?.from_user?.in_fanclub) ? 1 : 0,
+    'data_from_user_is_following': (!!message?.data?.from_user?.is_following) ? 1 : 0,
+    'data_from_user_is_mod': (!!message?.data?.from_user?.is_mod) ? 1 : 0,
+    'data_from_user_has_tokens': (!!message?.data?.from_user?.has_tokens) ? 1 : 0,
+    'data_from_user_tipped_recently': (!!message?.data?.from_user?.tipped_recently) ? 1 : 0,
+    'data_from_user_tipped_alot_recently': (!!message?.data?.from_user?.tipped_alot_recently) ? 1 : 0,
+    'data_from_user_tipped_tons_recently': (!!message?.data?.from_user?.tipped_tons_recently) ? 1 : 0,
+    'data_method': message?.method || null,
+    'data_pub_ts': message?.pub_ts || null,
+  });
+
+}

src/chaturbate.js.old

@@ -0,0 +1,295 @@
+import fetch from "node-fetch";
+import seedrandom from "seedrandom";
+import { millisecondsToHours } from 'date-fns'
+import { EventEmitter } from "node:events";
+import cheerio from "cheerio";
+import Ably from 'ably';
+import { Cookie, CookieJar } from "tough-cookie";
+import { FileCookieStore } from "tough-cookie-file-store"; 
+import os from 'os';
+import path from 'path';
+import File from 'fetch-blob/file.js'
+import { fileFromSync } from 'fetch-blob/from.js'
+import { FormData } from 'formdata-polyfill/esm.min.js'
+import { loggerFactory } from './logger.js'
+
+
+const logger = loggerFactory({
+    defaultMeta: { service: "futureporn/scout" }
+})
+const datadir = path.join(os.homedir(), '.local/share/futureporn/scout')
+const defaultRoomName = 'projektmelody';
+const defaultRoomUid = 'G0TWFS5';
+const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36';
+var jar = new CookieJar(new FileCookieStore(path.join(datadir, "cookie.json")));
+
+// greets ChatGPT
+export function getRandomNumberString(n) {
+  let num = Math.floor(Math.random() * (10 ** n));
+  return num.toString().padStart(n, '0');
+}
+
+
+// greets ChatGPT
+function generateRandomString(seed) {
+    const possibleCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+    let random = new seedrandom(seed);
+    return [...Array(11)].map(i => possibleCharacters[Math.floor(random() * possibleCharacters.length)]).join('');
+}
+
+export function chat(roomName) {
+    class Chat extends EventEmitter {
+        constructor(roomName = defaultRoomName) {
+            super();
+            this.roomName = roomName;
+            this.socket = new SockJS();
+        }
+        listen() {
+        }
+    }
+    let c = new Chat(roomName);
+    c.listen();
+}
+
+
+export async function monitorRealtimeStatus (roomName, onStart, onStop) {
+    const token = await getCsrfToken()
+    const roomId = await getRoomId(roomName)
+    const auth = await getPushServiceAuth(token, roomName, roomId)
+    const statusChannelString = auth.channels[`RoomStatusTopic#RoomStatusTopic:${roomId}`]
+    const realtime = await getRealtime(token, auth.token_request, auth.settings.realtime_host, auth.settings.fallback_hosts)
+    realtime.connection.once('connected', (idk) => {
+        logger.log({ level: 'info', message: 'CB Realtime Connected!' })
+    })
+    const statusChannel = realtime.channels.get(statusChannelString);
+    statusChannel.subscribe((message) => {
+        if (message.data.status === 'public') {
+            onStart(message)
+        } else if (message.data.status === 'offline') {
+            onStop(message)
+        }
+        logger.log({ level: 'debug', message: `Received room:status:<roomId>:0` })
+        logger.log({ level: 'debug', message: JSON.stringify(message, 0, 2) })
+    });
+    await realtime.connect()
+}
+
+export async function getRoomId (room) {
+    const dossier = await getInitialRoomDossier(room)
+    return dossier.room_uid
+}
+
+export async function getRandomRoom () {
+    const res = await fetch('https://chaturbate.com/')
+    const body = await res.text()
+    const $ = cheerio.load(body)
+    let roomsRaw = $('a[data-room]')
+    let rooms = []
+    $(roomsRaw).each((_, e) => {
+        rooms.push($(e).attr('href'))
+    })
+
+    // greets https://stackoverflow.com/a/4435017/1004931
+    var randomIndex = Math.floor(Math.random() * rooms.length);
+    return rooms[randomIndex].replaceAll('/', '')
+}
+
+// export async function getInitialRoomDossier(roomName = defaultRoomName) {
+//     const res = await fetch(`https://chaturbate.com/${roomName}`);
+//     const body = await res.text();
+//     const $ = cheerio.load(body);
+//     let rawScript = $('script:contains(window.initialRoomDossier)').html();
+//     if (!rawScript) throw new Error('window.initialRoomDossier is null. This could mean the channel is in password mode')
+//     let rawDossier = rawScript.slice(rawScript.indexOf('"'), rawScript.lastIndexOf('"')+1)
+//     let dossier = JSON.parse(JSON.parse(rawDossier));
+//     return dossier
+//     // dossier.wschat_host contains the chat websocket url
+// }
+
+// export async function getViewerCount(roomName = defaultRoomName, presenceId = generateRandomString(millisecondsToHours(Date.now()))) {
+//     const res = await fetch(`https://chaturbate.com/push_service/room_user_count/${roomName}/?presence_id=${presenceId}`, {
+//         "credentials": "include",
+//         "headers": {
+//             "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0",
+//             "Accept": "*/*",
+//             "Accept-Language": "en-US,en;q=0.5",
+//             "Sec-Fetch-Dest": "empty",
+//             "Sec-Fetch-Mode": "no-cors",
+//             "Sec-Fetch-Site": "same-origin",
+//             "Sec-GPC": "1",
+//             "X-Requested-With": "XMLHttpRequest",
+//             "Alt-Used": "chaturbate.com",
+//             "Pragma": "no-cache",
+//             "Cache-Control": "no-cache"
+//         },
+//         "referrer": `https://chaturbate.com/${roomName}/`,
+//         "method": "GET",
+//         "mode": "cors"
+//     });
+//     const json = await res.json();
+//     if (!res.ok)
+//         throw new Error(`HTTP request was not OK! STATUS CODE-- ${res.status}`);
+//     return json?.count;
+// }
+
+
+// /**
+//  * make a request against CB's server which
+//  *   - verifies custom credentials
+//  *   - issues signed Ably TokenRequest
+//  * 
+//  * more info-- https://ably.com/docs/core-features/authentication#token-authentication
+//  */
+// export async function getPushServiceAuth(csrfToken, roomName = defaultRoomName, roomUid = defaultRoomUid) {
+//     let form = new FormData();
+//     const topics = `{
+//         "RoomTipAlertTopic#RoomTipAlertTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },
+//         "RoomPurchaseTopic#RoomPurchaseTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },
+//         "RoomFanClubJoinedTopic#RoomFanClubJoinedTopic:${roomUid}": {
+//             "broadcaster_uid":"${roomUid}"
+//         },
+//         "RoomMessageTopic#RoomMessageTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },
+//         "GlobalPushServiceBackendChangeTopic#GlobalPushServiceBackendChangeTopic":{
+//         },
+//         "RoomAnonPresenceTopic#RoomAnonPresenceTopic:${roomUid}":{\
+//             "broadcaster_uid":"${roomUid}"
+//         },
+//         "QualityUpdateTopic#QualityUpdateTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomNoticeTopic#RoomNoticeTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomEnterLeaveTopic#RoomEnterLeaveTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomModeratorPromotedTopic#RoomModeratorPromotedTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomModeratorRevokedTopic#RoomModeratorRevokedTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomStatusTopic#RoomStatusTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomTitleChangeTopic#RoomTitleChangeTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomSilenceTopic#RoomSilenceTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomKickTopic#RoomKickTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomUpdateTopic#RoomUpdateTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         },"RoomSettingsTopic#RoomSettingsTopic:${roomUid}":{
+//             "broadcaster_uid":"${roomUid}"
+//         }
+//     }`
+
+    
+
+//     form.append('topics', topics)
+//     form.append('csrfmiddlewaretoken', csrfToken)
+//     const res = await fetch("https://chaturbate.com/push_service/auth/", {
+//         "credentials": "include",
+//         "headers": {
+//             "User-Agent": ua,
+//             "Accept": "*/*",
+//             "Accept-Language": "en-US,en;q=0.5",
+//             "X-Requested-With": "XMLHttpRequest",
+//             "Sec-Fetch-Dest": "empty",
+//             "Sec-Fetch-Mode": "cors",
+//             "Sec-Fetch-Site": "same-origin",
+//             "Sec-GPC": "1"
+//         },
+//         "referrer": `https://chaturbate.com/${roomName}/`,
+//         "body": form,
+//         "method": "POST",
+//         "mode": "cors"
+//     });
+
+
+//     const json = await res.json()
+
+//     logger.log({ level: 'debug', message: 'got pushServiceAuth. Following is the json response' })
+//     logger.log({ level: 'debug', message: JSON.stringify(json) })
+
+//     return json
+// }
+// export async function getRealtime(csrfToken, tokenRequest, realtimeHost, fallbackHosts) {
+//     const realtime = new Ably.Realtime.Promise({
+//         autoConnect: false,
+//         closeOnUnload: true,
+//         transportParams: {
+//             remainPresentFor: '0'
+//         },
+//         realtimeHost: realtimeHost,
+//         restHost: realtimeHost,
+//         fallback_hosts: fallbackHosts,
+//         authCallback: (async (tokenParams, cb) => {
+//             logger.log({ level: 'debug', message: `Ably is attempting to authenticate with tokenRequest: ${JSON.stringify(tokenRequest) }` })
+//             if (tokenRequest.timestamp+tokenRequest.ttl < new Date().valueOf()) {
+//                 logger.log({ level: 'debug', message: `tokenRequest is expired! let's get a new one.` })
+//                 const token = await getCsrfToken()
+//                 const roomId = await getRoomId(roomName)
+//                 const auth = await getPushServiceAuth(token, roomName, roomId)
+//                 tokenRequest = await getPushServiceAuth()
+//             }
+//             cb(null, tokenRequest)
+//         })
+//     })
+
+//     return realtime
+
+// }
+
+
+
+// export function getTokenCookie (cookies) {
+//     return cookies.find((c) => c.key == 'csrftoken');
+// }
+
+// export async function getCsrfToken(roomName = defaultRoomName) {
+//     const cbUrl = 'https://chaturbate.com/';
+//     let cookies = await jar.getCookies(cbUrl);
+//     let tokenCookie = getTokenCookie(cookies);
+    
+//     if (typeof tokenCookie === 'undefined') {
+
+//         let res = await fetch(cbUrl, {
+//             "credentials": "omit",
+//             "headers": {
+//                 "User-Agent": ua,
+//                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+//                 "Accept-Language": "en-US,en;q=0.5",
+//                 "Upgrade-Insecure-Requests": "1",
+//                 "Sec-Fetch-Dest": "document",
+//                 "Sec-Fetch-Mode": "navigate",
+//                 "Sec-Fetch-Site": "none",
+//                 "Sec-Fetch-User": "?1"
+//             },
+//             "method": "GET",
+//             "mode": "cors"
+//         });
+
+//         const rawHeaders = res.headers.raw()
+
+//         if (Array.isArray(rawHeaders["set-cookie"]))
+//           cookies = rawHeaders["set-cookie"].map(Cookie.parse);
+//         else cookies = [Cookie.parse(rawHeaders["set-cookie"])];
+    
+//         tokenCookie = getTokenCookie(cookies)
+//         if (typeof tokenCookie === 'undefined') throw new Error(` could not find csrftoken cookie! ${JSON.stringify(cookies)}`)
+
+//         try {
+//             await jar.setCookie(tokenCookie, cbUrl)
+//         } catch (e) {
+//             logger.log({ level: 'error', message: `problem while setting cookie on disk. ${e}` })
+//             logger.log({ level: 'error', message: JSON.stringify(e) })
+//         }
+//     }
+
+//     return tokenCookie.value
+
+// }

src/constants.js

@@ -0,0 +1,2 @@
+export const projektMelodyTwitterId = '1148121315943075841'
+export const projektMelodyCbRoomId = 'G0TWFS5'

src/faye.js

@@ -0,0 +1,19 @@
+import faye from 'faye'
+
+export function fayeFactory(appContext) {
+  return new faye.Client(appContext.env.PUBSUB_SERVER_URL);
+}
+
+export async function sendSignal (appContext, message) {
+  return appContext.faye.publish('/signals', message)
+}
+
+
+export async function signalStart (appContext, roomName, playlistUrl) {
+  if (appContext === undefined) throw new Error('appContext undef');
+  if (roomName === undefined) throw new Error('roomName undef');
+  if (playlistUrl === undefined) throw new Error('playlistUrl undef');
+  await sendSignal(appContext, { signal: 'start', room: roomName, url: playlistUrl })
+}
+
+

src/logger.js

@@ -0,0 +1,25 @@
+import winston from 'winston'
+
+export const loggerFactory = (options) => {
+  const mergedOptions = Object.assign({}, {
+    level: 'info',
+    defaultMeta: { service: 'futureporn' },
+    format: winston.format.timestamp()
+  }, options)
+  const logger = winston.createLogger(mergedOptions);
+
+  if (process.env.NODE_ENV !== 'production') {
+    logger.add(new winston.transports.Console({
+      level: 'debug',
+      format: winston.format.simple()
+    }))
+  } else {
+    logger.add(new winston.transports.Console({
+      level: 'info',
+      format: winston.format.json()
+    }))
+  }
+
+  return logger
+}
+

src/parsers.js

@@ -0,0 +1,13 @@
+
+
+
+export function extractRoomId(input) {
+  const regex = /^(?:\w+):(?:\w+):(\w+)/;
+  const matches = input.match(regex);
+
+  if (matches && matches.length > 1) {
+    return matches[1]; // Return the captured ID from the regex match
+  }
+
+  return null; // Return null if no ID is found
+}

src/scrap.json

@@ -0,0 +1,21 @@
+{
+    "tid": "16873746703:39479",
+    "ts": 1687374670.4058313,
+    "amount": 111,
+    "message": "",
+    "history": true,
+    "is_anonymous_tip": false,
+    "to_username": "_keti_",
+    "from_username": "affose",
+    "gender": "m",
+    "is_broadcaster": false,
+    "in_fanclub": false,
+    "is_following": false,
+    "is_mod": false,
+    "has_tokens": true,
+    "tipped_recently": true,
+    "tipped_alot_recently": true,
+    "tipped_tons_recently": false,
+    "method": "lazy",
+    "pub_ts": 1687374670.4170263
+}

src/sound.js

@@ -0,0 +1,18 @@
+import { execa } from 'execa';
+import path, { dirname } from 'node:path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const alertSound = path.join(__dirname, '..', 'assets', 'alert7.mp3')
+
+
+export function alert () {
+  try {
+    execa('ffplay', ['-i', alertSound, '-autoexit'])
+  } catch (e) {
+    console.error(`unable to play alert sound. ${e}`)
+  }
+}
+
+
+

src/strapi.js

@@ -0,0 +1,4 @@
+export const createVod = (appContext, roomName) => {
+  appContext.logger.log({ level: 'info', message: `creating vod for room ${roomName}` })
+
+}

src/tweetProcess.js

@@ -0,0 +1,73 @@
+// const VOD = require('./VOD.js');
+
+import { projektMelodyTwitterId } from './constants.js'
+import { loggerFactory } from './logger.js'
+
+const logger = loggerFactory({
+    defaultMeta: { service: "futureporn/scout" }
+})
+const cbUrlRegex = /chaturbate\.com.*projektmelody/i;
+const containsCBInviteLink = (tweet) => {
+    try {
+        if (tweet?.entities?.urls === undefined) return false;
+        for (const url of tweet.entities.urls) {
+        	if (url?.unwound_url !== undefined) {
+                if (cbUrlRegex.test(url.unwound_url)) {
+                    return true;
+                } else {
+                    return false;
+                }
+	        } else {
+	        	return false;
+	        }
+        }
+    }
+    catch (e) {
+        logger.log({ level: 'error', message: 'ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR ERROR' });
+        logger.log({ level: 'error', message: e });
+        return false;
+    }
+};
+const deriveTitle = (text) => {
+    // greetz https://www.urlregex.com/
+    const urlRegex = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/g;
+    let title = text
+        .replace(urlRegex, '') // remove urls
+        .replace(/\n/g, ' ') // replace newlines with spaces
+        .replace(/>/g, '>') // gimme dem greater-than brackets
+        .replace(/&lt;/g, '<') // i want them less-thans too
+        .replace(/&amp;/g, '&') // ampersands are sexy
+        .replace(/\s+$/, ''); // remove trailing whitespace
+    return title;
+};
+/**
+ * Does stuff with filtered tweets. (side-effects)
+ */
+const processTweet = async (tweet) => {
+    logger.log({ level: 'debug', message: `processTweet() is as follows \n${JSON.stringify(tweet, 0, 2)}` });
+    logger.log({ level: 'debug', message: '>>> Processing Tweet' });
+    logger.log({ level: 'debug', message: tweet });
+    if (containsCBInviteLink(tweet)) {
+        let tweetId = tweet.id;
+        let tweetText = tweet.text;
+        let date = tweet.created_at;
+        let screenName = (tweet.author_id === projektMelodyTwitterId) ? 'ProjektMelody' : tweet.author_id;
+        let announceUrl = `https://twitter.com/${screenName}/status/${tweetId}`;
+        let announceTitle = deriveTitle(tweetText);
+        logger.log({ level: 'debug', message: `[*] Mel Chaturbate Invite Tweet Detected: ${announceUrl} at ${date}` });
+        const vod = new VOD({
+            date,
+            announceTitle,
+            announceUrl
+        });
+        vod.saveMarkdown();
+    }
+};
+export { deriveTitle };
+export { processTweet };
+export { containsCBInviteLink };
+export default {
+    deriveTitle,
+    processTweet,
+    containsCBInviteLink
+};

src/twitter.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+
+import Twitter from 'twitter-v2';
+import { loggerFactory } from './logger.js'
+
+const logger = loggerFactory({
+    defaultMeta: { service: "futureporn/scout" }
+})
+
+const twitterConsumerKey = process.env.TWITTER_API_KEY;
+const twitterConsumerSecret = process.env.TWITTER_API_KEY_SECRET;
+const projektMelodyTwitterId = '1148121315943075841';
+
+if (typeof twitterConsumerKey === 'undefined')
+    throw new Error('TWITTER_API_KEY is undefined');
+if (typeof twitterConsumerSecret === 'undefined')
+    throw new Error('TWITTER_API_KEY_SECRET is undefined');
+
+async function delay(timeout) {
+    logger.log({ level: 'debug', message: `  [*] delaying for ${timeout}ms` });
+    await new Promise(resolve => setTimeout(resolve, timeout));
+}
+var client = new Twitter({
+    consumer_key: twitterConsumerKey,
+    consumer_secret: twitterConsumerSecret
+});
+async function setup() {
+    const ruleBody = {
+        'add': [
+            {
+                'value': 'from:projektmelody -is:retweet',
+                'tag': 'tweets from melody'
+            }
+        ]
+    };
+    // Delete all rules and add just the ones we want
+    try {
+        const { data: rules } = await client.get('tweets/search/stream/rules');
+        logger.log({ level: 'debug', message: rules });
+        const ruleIds = rules.map((r) => r.id);
+        logger.log({ level: 'debug', message: ruleIds });
+        await client.post('tweets/search/stream/rules', {
+            'delete': {
+                "ids": ruleIds
+            }
+        });
+    }
+    catch (e) {
+        logger.log({ level: 'error', message: e });
+        logger.log({ level: 'error', message: 'no big d.' });
+    }
+    logger.log({ level: 'debug', message: `creating rule.` });
+    const ruleRes = await client.post('tweets/search/stream/rules', ruleBody);
+    logger.log({ level: 'debug', message: `rule created with response ${JSON.stringify(ruleRes)}` });
+}
+async function listenForever(streamFactory, dataConsumer) {
+    try {
+        for await (const { data } of streamFactory()) {
+            dataConsumer(data);
+        }
+        // The stream has been closed by Twitter. It is usually safe to reconnect.
+        logger.log({ level: 'debug', message: 'Stream disconnected healthily. Reconnecting.' });
+        listenForever(streamFactory, dataConsumer);
+        await delay(5000);
+    }
+    catch (error) {
+        // An error occurred so we reconnect to the stream. Note that we should
+        // probably have retry logic here to prevent reconnection after a number of
+        // closely timed failures (may indicate a problem that is not downstream).
+        logger.log({ level: 'warn', message: `Stream disconnected with error. Retrying. ${error}` });
+        listenForever(streamFactory, dataConsumer);
+        await delay(5000);
+    }
+}
+export default async function twitter(dataConsumer) {
+    const parameters = {
+        expansions: [
+            'author_id'
+        ],
+        tweet: {
+            fields: ['created_at', 'entities'],
+        }
+    };
+    await setup();
+    listenForever(() => client.stream('tweets/search/stream', parameters), dataConsumer);
+}

src/ytdlp.js

@@ -0,0 +1,7 @@
+
+import { execa } from 'execa'
+
+export const assertYtdlpExistence = async () => {
+  await execa('yt-dlp', ['--version'])
+}
+

test/integration/Room.test.js

@@ -0,0 +1,59 @@
+
+import chai, { expect } from "chai";
+import { EventEmitter } from "node:events";
+import chaiEvents from 'chai-events'
+import Room from '../../src/Room.js'
+
+import { projektMelodyCbRoomId } from '../../src/constants.js'
+
+chai.use(chaiEvents);
+
+
+describe('Room', function () {
+    beforeEach(function (done) {
+        // courtesy delay
+        setTimeout(function(){
+            done();
+        }, 1000);
+    });
+    it('should get a room id', async function () {
+        const room = new Room({
+            roomName: 'projektmelody'
+        })
+        const roomId = await room.getRoomId()
+        expect(roomId).to.equal(projektMelodyCbRoomId)
+        expect(room).to.have.property('roomId', projektMelodyCbRoomId)
+    })
+
+    describe('getCsrfToken', function () {
+        it('should get a token from CB', async function () {
+            const room = new Room({
+                roomName: 'projektmelody'
+            })
+            const token = await room.getCsrfToken()
+            expect(typeof(token) === 'string').to.be.true
+            expect(token.length).to.equal(64)
+        })
+    })
+
+    describe('getPushServiceAuth', function () {
+        it('should resolve with a valid TokenRequest', async function () {
+            const room = new Room({
+                roomName: 'projektmelody'
+            })
+            const tokenRequest = await room.getPushServiceAuth()
+            expect(tokenRequest).to.have.property('token_request')
+            expect(tokenRequest.token_request).to.have.property('timestamp')
+            expect(tokenRequest.token_request).to.have.property('ttl')
+        })
+        it('should re-use the token as long as its not expired', async function () {
+            const room = new Room({
+                roomName: 'projektmelody'
+            })
+            const tokenRequest1 = await room.getPushServiceAuth()
+            const tokenRequest2 = await room.getPushServiceAuth()
+            expect(tokenRequest1.token).to.equal(tokenRequest2.token)
+        })
+    })
+
+})

test/integration/cbCallbacks.test.js

@@ -0,0 +1,145 @@
+
+import 'dotenv/config'
+import chai from "chai";
+import sinon from 'sinon';
+import { onCbStart } from '../../src/cbCallbacks.js'
+import { getAppContext } from '../../src/appContext.js'
+import * as pubsub from '../../src/pubsub.js'
+
+
+
+
+describe('onCbStart', () => {
+  let appContext;
+  let roomName;
+  let message;
+  let dbMock;
+  let clearTimeoutMock;
+
+  beforeEach(() => {
+    // Create a mock object for appContext
+    appContext = {
+      logger: {
+        log: sinon.stub(),
+      },
+      db: {
+        prepare: sinon.stub(),
+      },
+      roomTimers: {
+        [roomName]: {
+          offline: 'mockTimeout',
+        },
+      },
+      sound: {
+        alert: sinon.stub()
+      },
+      cb: {
+        getPlaylistUrl: sinon.stub()
+      },
+      pubsub: pubsub
+    };
+
+    // Set up test data
+    roomName = 'testRoom';
+    message = {
+      name: 'testName',
+      id: 'testId',
+      encoding: 'testEncoding',
+      data: {
+        tid: 'testTid',
+        ts: 'testTs',
+        status: 'testStatus',
+        message: 'testMessage',
+        hash: 'testHash',
+        method: 'testMethod',
+        pub_ts: 'testPubTs',
+      },
+    };
+
+    // Create mock functions
+    dbMock = {
+      run: sinon.stub(),
+    };
+    appContext.db.prepare.returns(dbMock);
+
+    appContext.cb.getPlaylistUrl.resolves('testPlaylistUrl');
+    console.log('here are the mock')
+    console.log(appContext.cb.getPlaylistUrl)
+
+    // clearTimeoutMock = sinon.stub();
+
+
+    // // Stub global functions
+    // sinon.stub(global, 'clearTimeout').callsFake(clearTimeoutMock);
+
+    // // Stub async function using sinon
+    // sinon.stub(global, 'setTimeout').callsFake((callback) => {
+    //   callback();
+    //   return 'mockTimeout';
+    // });
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('should insert into lifecycles and call start', async () => {
+
+    console.log(appContext)
+    console.log(`>>>>>>>>>>>>>>`)
+    console.log(appContext.pubsub.start)
+    console.log(message)
+
+    // Invoke the function
+    await onCbStart(appContext, roomName, message);
+
+    // Assert the logger was called with the correct arguments
+    sinon.assert.calledWith(
+      appContext.logger.log,
+      { level: 'debug', message: `🟢🟢🟢 [STREAM START] ${JSON.stringify(message)}` }
+    );
+
+    // Assert the database prepared statement was called with the correct SQL query
+    sinon.assert.calledWith(
+      appContext.db.prepare,
+      'INSERT INTO lifecycles VALUES ($_room, $name, $id, $encoding, $data_tid, $data_ts, $data_status, $data_message, $data_hash, $data_method, $data_pub_ts)'
+    );
+
+    // Assert the database statement was run with the correct parameters
+    sinon.assert.calledWith(
+      dbMock.run,
+      sinon.match({
+        _room: roomName,
+        name: message.name || null,
+        id: message.id || null,
+        encoding: message.encoding || null,
+        data_tid: message.data?.tid || null,
+        data_ts: message.data?.status || null,
+        data_status: message.data?.status || null,
+        data_message: message.data?.message || null,
+        data_hash: message.data?.hash || null,
+        data_method: message.data?.method || null,
+        data_pub_ts: message.data?.pub_ts || null,
+      })
+    );
+
+    // Assert the soundAlert function was called
+    sinon.assert.calledOnce(appContext.sound.alert);
+
+
+    // // Assert the getPlaylistUrl function was called with the correct arguments
+    sinon.assert.calledWith(appContext.cb.getPlaylistUrl, appContext, roomName);
+
+    // Assert the start function was called with the correct arguments
+    sinon.assert.calledWith(
+      appContext.pubsub.start,
+      appContext,
+      roomName,
+      'testPlaylistUrl'
+    );
+
+    // // Assert clearTimeout was called with the correct argument
+    // sinon.assert.calledWith(clearTimeoutMock, 'mockTimeout');
+
+    });
+  });

test/integration/chaturbate.test.js

@@ -0,0 +1,146 @@
+import { 
+    getRealtime,
+    getPushServiceAuth,
+    getCsrfToken,
+    getInitialRoomDossier,
+    getViewerCount,
+    getRandomNumberString,
+    getRandomRoom,
+    getRoomId
+} from '../../src/chaturbate.js';
+import chai, { expect } from "chai";
+import { EventEmitter } from "node:events";
+import chaiEvents from 'chai-events'
+
+
+chai.use(chaiEvents);
+
+
+describe('chaturbate', function () {
+    beforeEach(function (done) {
+        // courtesy delay
+        setTimeout(function(){
+            done();
+        }, 1000);
+    });
+
+    describe('getRandomRoom', function () {
+        it('should resolve with a name', async function () {
+            const room = await getRandomRoom()
+            expect(room).to.exist
+            expect(typeof room).to.equal('string')
+        })
+    })
+    describe('getRoomId', function () {
+        it('should accept a room name and resolve with an ID', async function () {
+            const id = await getRoomId('projektmelody')
+            expect(id).to.equal('G0TWFS5')
+        })
+    })
+
+    describe('getInitialDossier', function () {
+        it('should return a js object', async function () {
+            const dossier = await getInitialRoomDossier()
+            expect(dossier).to.have.property('wschat_host')
+        });
+    })
+    describe('getViewerCount', function () {
+        it('should return a number', async function () {
+            const count = await getViewerCount()
+            expect(typeof(count) === 'number').to.be.true
+        })
+    })
+    xdescribe('getRealtime', function () {
+        it('should get an event emitter or something idk', async function () {
+            const rt = await getRealtime();
+            expect(rt).to.be.instanceof(EventEmitter)
+        })
+    })
+    describe('getCsrfToken', function () {
+        it('should get a token from CB', async function () {
+            const token = await getCsrfToken()
+            expect(typeof(token) === 'string').to.be.true
+            expect(token.length).to.equal(64)
+        })
+    })
+    describe('getPushServiceAuth', function () {
+        it('should get auth data', async function () {
+            const token = await getCsrfToken()
+            const auth = await getPushServiceAuth(token)
+            expect(typeof(auth) === 'object').to.be.true
+            expect(auth).to.have.property('channels')
+            expect(auth).to.have.property('token_request')
+        })
+    })
+    describe('getRealtime', function () {
+        this.timeout(60*1000)
+        it('should get token for realtime platform', async function () {
+
+
+            // const roomName = await getRandomRoom()
+            const roomName = 'silvess333'
+            const roomId = await getRoomId(roomName)
+            const token = await getCsrfToken()
+            console.log(`roomName:${roomName}, id:${roomId}`)
+
+            const auth = await getPushServiceAuth(token, roomName, roomId)
+            expect(auth).to.have.property('token_request')
+            expect(auth.token_request).to.have.property('ttl')
+            expect(auth.token_request).to.have.property('capability')
+            expect(auth.token_request.capability).to.have.string(roomId)
+
+            expect(auth.channels).to.have.property(`RoomTipAlertTopic#RoomTipAlertTopic:${roomId}`)
+
+
+            const statusChannelString = auth.channels[`RoomStatusTopic#RoomStatusTopic:${roomId}`]
+            const titleChannelString = auth.channels[`RoomTitleChangeTopic#RoomTitleChangeTopic:${roomId}`]
+            const updateChannelString = auth.channels[`RoomUpdateTopic#RoomUpdateTopic:${roomId}`]
+            const presenseChannelString = auth.channels[`RoomAnonPresenceTopic#RoomAnonPresenceTopic:${roomId}`]
+
+            console.log(auth)
+
+            const realtime = await getRealtime(token, auth.token_request, auth.settings.realtime_host, auth.settings.fallback_hosts)
+
+            await realtime.connect()
+
+            console.log('expecting a connected emission here ---v')
+            expect(realtime.connection).to.emit('connected')
+
+            realtime.connection.once('connected', (idk) => {
+                console.log(`  [*] connected! ${JSON.stringify(idk, 0, 2)}`)
+            })
+
+            // const channel = realtime.channels.get(`room:notice:${roomId}`);
+            // const channel = realtime.channels.get(`room:message:${roomId}:0`);
+            const titleChannel = realtime.channels.get(titleChannelString);
+            const updateChannel = realtime.channels.get(updateChannelString);
+            const presenceChannel = realtime.channels.get(presenseChannelString);
+            const statusChannel = realtime.channels.get(statusChannelString);
+            statusChannel.subscribe((message) => {
+                console.log(`Received room:status:<roomId>:0`)
+                console.log(JSON.stringify(message, 0, 2));
+            });
+            titleChannel.subscribe((message) => {
+                console.log(`Received room:title_change:<roomId>`)
+                console.log(JSON.stringify(message, 0, 2));
+            });
+            updateChannel.subscribe((message) => {
+                console.log(`Received room:update:<roomId>`)
+                console.log(JSON.stringify(message, 0, 2));
+                
+            });
+            presenceChannel.subscribe((message) => {
+                console.log(`Received room_anon:presence:<roomId>:0`)
+                console.log(JSON.stringify(message, 0, 2));
+            })
+        })
+    })
+    describe('getRandomNumberString', function() {
+        it('should return a 16 digit number', function () {
+            expect(getRandomNumberString(16)).to.be.lengthOf(16)
+        })
+        it('should return a 1 digit number', function () {
+            expect(getRandomNumberString(1)).to.be.lengthOf(1)
+        })
+    })
+})

test/integration/pubsub.test.js

@@ -0,0 +1,32 @@
+
+import chai from "chai";
+
+import { sendSignal, start } from '../../src/pubsub.js'
+import { getRandomPlaylist } from '../../src/cb.js'
+import faye from 'faye'
+import dotenv from 'dotenv'
+dotenv.config()
+
+describe('pubsub', function () {
+  describe('sendSignal', function () {
+    it('should send', async function () {
+      if (!process.env.PUBSUB_SERVER_URL) throw new Error('PUBSUB_SERVER_URL missing in env');
+      console.log(`sending pubsub signal to ${process.env.PUBSUB_SERVER_URL}`)
+      let pubsub = new faye.Client(process.env.PUBSUB_SERVER_URL)
+      await sendSignal({ pubsub }, 'IT WORKS!')
+      return pubsub.disconnect()
+    })
+  })
+  describe('start', function () {
+    it('should send a signal to futureporn-capture via futureporn-qa', async function () {
+      this.timeout(1000*15)
+      const pubsub = new faye.Client(process.env.PUBSUB_SERVER_URL)
+      const logger = { log: (txt) => console.log(JSON.stringify(txt)) }
+      const appContext = { pubsub, logger }
+      const playlistUrl = await getRandomPlaylist(appContext)
+      await start(appContext, 'projektmelody', playlistUrl)
+      console.log('signal done')
+      appContext.pubsub.disconnect()
+    });
+  })
+});

test/unit/cbCallbacks.test.js

@@ -0,0 +1,141 @@
+import chai from 'chai';
+import dotenv from 'dotenv';
+
+
+
+describe('onCbStart', () => {
+  let appContext;
+  let roomName;
+  let message;
+  let dbMock;
+  let clearTimeoutMock;
+
+  beforeEach(() => {
+    // Create a mock object for appContext
+    appContext = {
+      logger: {
+        log: sinon.stub(),
+      },
+      db: {
+        prepare: sinon.stub(),
+      },
+      roomTimers: {
+        [roomName]: {
+          offline: 'mockTimeout',
+        },
+      },
+      sound: {
+        alert: sinon.stub()
+      },
+      cb: {
+        getPlaylistUrl: sinon.stub()
+      },
+      pubsub: {
+        start: sinon.stub()
+      }
+    };
+
+    // Set up test data
+    roomName = 'testRoom';
+    message = {
+      name: 'testName',
+      id: 'testId',
+      encoding: 'testEncoding',
+      data: {
+        tid: 'testTid',
+        ts: 'testTs',
+        status: 'testStatus',
+        message: 'testMessage',
+        hash: 'testHash',
+        method: 'testMethod',
+        pub_ts: 'testPubTs',
+      },
+    };
+
+    // Create mock functions
+    dbMock = {
+      run: sinon.stub(),
+    };
+    appContext.db.prepare.returns(dbMock);
+
+    appContext.cb.getPlaylistUrl.resolves('testPlaylistUrl');
+    console.log('here are the mock')
+    console.log(appContext.cb.getPlaylistUrl)
+
+    // clearTimeoutMock = sinon.stub();
+
+
+    // // Stub global functions
+    // sinon.stub(global, 'clearTimeout').callsFake(clearTimeoutMock);
+
+    // // Stub async function using sinon
+    // sinon.stub(global, 'setTimeout').callsFake((callback) => {
+    //   callback();
+    //   return 'mockTimeout';
+    // });
+  });
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('should insert into lifecycles and call start', async () => {
+
+    console.log(appContext)
+    console.log(`>>>>>>>>>>>>>>`)
+    console.log(appContext.pubsub.start)
+    console.log(message)
+
+    // Invoke the function
+    await onCbStart(appContext, roomName, message);
+
+    // Assert the logger was called with the correct arguments
+    sinon.assert.calledWith(
+      appContext.logger.log,
+      { level: 'debug', message: `🟢🟢🟢 [STREAM START] ${JSON.stringify(message)}` }
+    );
+
+    // Assert the database prepared statement was called with the correct SQL query
+    sinon.assert.calledWith(
+      appContext.db.prepare,
+      'INSERT INTO lifecycles VALUES ($_room, $name, $id, $encoding, $data_tid, $data_ts, $data_status, $data_message, $data_hash, $data_method, $data_pub_ts)'
+    );
+
+    // Assert the database statement was run with the correct parameters
+    sinon.assert.calledWith(
+      dbMock.run,
+      sinon.match({
+        _room: roomName,
+        name: message.name || null,
+        id: message.id || null,
+        encoding: message.encoding || null,
+        data_tid: message.data?.tid || null,
+        data_ts: message.data?.status || null,
+        data_status: message.data?.status || null,
+        data_message: message.data?.message || null,
+        data_hash: message.data?.hash || null,
+        data_method: message.data?.method || null,
+        data_pub_ts: message.data?.pub_ts || null,
+      })
+    );
+
+    // Assert the soundAlert function was called
+    sinon.assert.calledOnce(appContext.sound.alert);
+
+
+    // // Assert the getPlaylistUrl function was called with the correct arguments
+    sinon.assert.calledWith(appContext.cb.getPlaylistUrl, appContext, roomName);
+
+    // Assert the start function was called with the correct arguments
+    sinon.assert.calledWith(
+      appContext.pubsub.start,
+      appContext,
+      roomName,
+      'testPlaylistUrl'
+    );
+
+    // // Assert clearTimeout was called with the correct argument
+    // sinon.assert.calledWith(clearTimeoutMock, 'mockTimeout');
+
+    });
+  });

test/unit/parsers.test.js

@@ -0,0 +1,15 @@
+
+import chai from "chai";
+
+import { extractRoomId } from '../../src/parsers.js'
+
+
+describe('parsers', function () {
+  describe('extractRoomId', function () {
+    it('should handle various inputs', function () {
+      chai.expect(extractRoomId('room:message:27WDW5C:3')).to.equal('27WDW5C');
+      chai.expect(extractRoomId('room:message:KGHH38V:0')).to.equal('KGHH38V');
+      chai.expect(extractRoomId('room:title_change:KGHH38V')).to.equal('KGHH38V');
+    });
+  })
+});

test/unit/sound.test.js

@@ -0,0 +1,7 @@
+import { alert } from '../../src/sound.js'
+
+describe('sound', function() {
+  it('alert', function() {
+    alert()
+  })
+})

test/unit/tweetProcess.test.js

@@ -0,0 +1,136 @@
+import chai from "chai";
+// import path from "path";
+// import chaiAsPromised from "chai-as-promised";
+import { containsCBInviteLink } from "../../src/tweetProcess.js";
+// chai.use(chaiAsPromised);
+const sampleTweet6 = {
+    entities: {
+        urls: [
+            {
+                unwound_url: 'https://chaturbate.com/b/ProjektMelody'
+            }
+        ]
+    }
+};
+const sampleTweet7 = {
+    entities: {
+        urls: [
+            {
+                unwound_url: 'https://chaturbate.com/?tour=7Bge&room=ProjektMelody&campaign=wXffl&disable_sound=0'
+            }
+        ]
+    }
+};
+const sampleTweet4 = {
+    entities: {
+        urls: [
+            {
+                unwound_url: 'https://chaturbate.com/b/goldengoddessxx'
+            }
+        ]
+    }
+};
+const sampleTweet5 = {
+    entities: {
+        urls: [
+            {
+                unwound_url: 'https://chaturbate.com/?tour=117e&room=goldengoddessxx&campaign=jflef&disable_sound=0'
+            }
+        ]
+    }
+};
+const sampleTweet3 = {
+    created_at: '2021-12-29T18:32:14.000Z',
+    id: '1476259783791497217',
+    author_id: '1148121315943075841',
+    text: 'https://t.co/9jL6fAgMuj',
+    entities: {
+        urls: [
+            {
+                start: 0,
+                end: 23,
+                url: 'https://t.co/9jL6fAgMuj',
+                expanded_url: 'https://twitter.com/ProjektMelody/status/1476259783791497217/photo/1',
+                display_url: 'pic.twitter.com/9jL6fAgMuj'
+            }
+        ]
+    }
+};
+const sampleTweet2 = {
+    text: "I couldn't resist\n\nhttps://t.co/etfBuD5npl",
+    id: '1478141764741611527',
+    author_id: '1148121315943075841',
+    created_at: '2022-01-03T23:10:33.000Z',
+    entities: {
+        urls: [
+            {
+                start: 19,
+                end: 42,
+                url: 'https://t.co/etfBuD5npl',
+                expanded_url: 'http://bit.ly/3n71Pgv',
+                display_url: 'bit.ly/3n71Pgv',
+                images: [],
+                status: 200,
+                title: 'Chaturbate - Free Adult Live Webcams!',
+                description: 'Enjoy free chat and live webcam broadcasts from amateurs around the world. No registration required!',
+                unwound_url: 'https://chaturbate.com/?tour=7Bge&room=projektmelody&campaign=wXffl&disable_sound=0'
+            }
+        ]
+    }
+};
+const sampleTweet1 = {
+    text: "Hey guys! I'm totally ready and getting online now!!\n" +
+        '\n' +
+        'https://t.co/cxNMAIsNDu https://t.co/QjyJQefF1S',
+    id: '1225922638687752192',
+    created_at: '2020-02-07T23:21:48.000Z',
+    author_id: '1148121315943075841',
+    entities: {
+        urls: [
+            {
+                start: 54,
+                end: 77,
+                url: 'https://t.co/cxNMAIsNDu',
+                expanded_url: 'https://chaturbate.com/b/projektmelody/',
+                display_url: 'chaturbate.com/b/projektmelod…',
+                images: [],
+                status: 200,
+                title: 'Watch Projektmelody live on Chaturbate!',
+                description: 'Deck my halls science team',
+                unwound_url: 'https://chaturbate.com/projektmelody/'
+            },
+            {
+                start: 78,
+                end: 101,
+                url: 'https://t.co/QjyJQefF1S',
+                expanded_url: 'https://twitter.com/ProjektMelody/status/1225922638687752192/photo/1',
+                display_url: 'pic.twitter.com/QjyJQefF1S'
+            }
+        ]
+    }
+};
+describe('tweetProcess', function () {
+    describe('deriveTitle', function () {
+    });
+    describe('containsCBInviteLink', function () {
+        it('should return true with a chaturbate.com/b/ style cb link', function () {
+            chai.expect(containsCBInviteLink(sampleTweet1)).to.be.true;
+        });
+        it('should return true with a chaturbate.com/?tour=7Bge&room=projektmelody&campaign=wXffl&disable_sound=0 style cb link', function () {
+            chai.expect(containsCBInviteLink(sampleTweet2)).to.be.true;
+        });
+        it('should return false with a tweet lacking a chaturbate link', function () {
+            chai.expect(containsCBInviteLink(sampleTweet3)).to.be.false;
+        });
+        it('should return false with a chaturbate.com/b/ style link to a room other than projektmelody', function () {
+            chai.expect(containsCBInviteLink(sampleTweet4)).to.be.false;
+        });
+        it('should return false with a chaturbate.com/?(...) style link to a room other than projektmelody', function () {
+            chai.expect(containsCBInviteLink(sampleTweet5)).to.be.false;
+        });
+        it('should be case insensitive', function () {
+            chai.expect(containsCBInviteLink(sampleTweet6)).to.be.true;
+            chai.expect(containsCBInviteLink(sampleTweet7)).to.be.true;
+        });
+    });
+});