futureporn
/
futureporn-scout
1
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

progress

This commit is contained in:
Chris Grimmett 2023-07-14 06:16:58 -08:00
parent 4dc77b37eb
commit d21250c09e
35 changed files with 2548 additions and 652 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
ISSUE1.md Normal file
View File

@ -0,0 +1,77 @@
error after maybe ~24h of running
OH, it's timestamp errors. might be caused by me switching my system clock to UTC
```
> scout@0.0.1 start /home/chris/Documents/futureporn-scout
> node index daemon
{ _: [ 'daemon' ], '$0': 'index' }
debug: registering room timer for projektmelody {"service":"futureporn/scout"}
info: monitoring room:projektmelody {"service":"futureporn/scout"}
debug: Ably TokenRequest is either absent or expired. Let's get a new one. {"service":"futureporn/scout"}
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
info: CB Realtime Connected! {"service":"futureporn/scout"}
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
20:34:08.456 Ably: Auth.requestToken(): token request API call returned error; err = [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
20:34:08.458 Ably: Transport.onProtocolMessage(): Ably requested re-authentication, but unable to obtain a new token: [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
20:34:08.545 Ably: Auth.requestToken(): token request API call returned error; err = [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
20:34:23.646 Ably: Auth.requestToken(): token request API call returned error; err = [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Ably TokenRequest is either absent or expired. Let's get a new one. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
20:34:54.218 Ably: ConnectionManager.actOnErrorFromAuthorize(): Client configured authentication provider returned 403; failing the connection
20:34:54.218 Ably: Connection state: failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon4f0b537a-a86f-44f0-8ef3-0aafca49aa0b) and current clientId (-anon27e17db6-f2ce-4695-b98c-563529582f67); statusCode=403; code=40102]]
20:34:54.219 Ably: Channel state for channel "room:message:G0TWFS5:9": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon4f0b537a-a86f-44f0-8ef3-0aafca49aa0b) and current clientId (-anon27e17db6-f2ce-4695-b98c-563529582f67); statusCode=403; code=40102]]
20:34:54.219 Ably: Channel state for channel "room:silence:G0TWFS5": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon4f0b537a-a86f-44f0-8ef3-0aafca49aa0b) and current clientId (-anon27e17db6-f2ce-4695-b98c-563529582f67); statusCode=403; code=40102]]
20:34:54.220 Ably: Channel state for channel "room:title_change:G0TWFS5": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon4f0b537a-a86f-44f0-8ef3-0aafca49aa0b) and current clientId (-anon27e17db6-f2ce-4695-b98c-563529582f67); statusCode=403; code=40102]]
20:34:54.220 Ably: Channel state for channel "room:status:G0TWFS5:9": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon4f0b537a-a86f-44f0-8ef3-0aafca49aa0b) and current clientId (-anon27e17db6-f2ce-4695-b98c-563529582f67); statusCode=403; code=40102]]
```
Seems like a reproducable problem. occurs ~24h after running
```
> scout@0.0.1 start /home/chris/Documents/futureporn-scout
> node index daemon
{ _: [ 'daemon' ], '$0': 'index' }
debug: registering room timer for projektmelody {"service":"futureporn/scout"}
info: monitoring room:projektmelody {"service":"futureporn/scout"}
debug: Ably TokenRequest is either absent or expired. Let's get a new one. {"service":"futureporn/scout"}
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
info: CB Realtime Connected! {"service":"futureporn/scout"}
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
00:17:41.516 Ably: Auth.requestToken(): token request API call returned error; err = [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
00:17:41.516 Ably: Transport.onProtocolMessage(): Ably requested re-authentication, but unable to obtain a new token: [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
00:17:41.597 Ably: Auth.requestToken(): token request API call returned error; err = [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
00:17:56.680 Ably: Auth.requestToken(): token request API call returned error; err = [ErrorInfo: Timestamp not current; statusCode=401; code=40104; see https://help.ably.io/error/40104 ]
debug: Ably authCallback. Getting a fresh new Ably TokenRequest. {"service":"futureporn/scout"}
debug: Ably TokenRequest is either absent or expired. Let's get a new one. {"service":"futureporn/scout"}
debug: Got a new TokenRequest {"service":"futureporn/scout"}
00:18:27.276 Ably: ConnectionManager.actOnErrorFromAuthorize(): Client configured authentication provider returned 403; failing the connection
00:18:27.277 Ably: Connection state: failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon89dfcc5b-6805-45e0-94db-bc6cfa0e9618) and current clientId (-anon9118cecf-29db-4bab-9592-38ed446c0617); statusCode=403; code=40102]]
00:18:27.277 Ably: Channel state for channel "room:message:G0TWFS5:9": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon89dfcc5b-6805-45e0-94db-bc6cfa0e9618) and current clientId (-anon9118cecf-29db-4bab-9592-38ed446c0617); statusCode=403; code=40102]]
00:18:27.277 Ably: Channel state for channel "room:silence:G0TWFS5": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon89dfcc5b-6805-45e0-94db-bc6cfa0e9618) and current clientId (-anon9118cecf-29db-4bab-9592-38ed446c0617); statusCode=403; code=40102]]
00:18:27.277 Ably: Channel state for channel "room:title_change:G0TWFS5": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon89dfcc5b-6805-45e0-94db-bc6cfa0e9618) and current clientId (-anon9118cecf-29db-4bab-9592-38ed446c0617); statusCode=403; code=40102]]
00:18:27.278 Ably: Channel state for channel "room:status:G0TWFS5:9": failed; reason: [ErrorInfo: Client configured authentication provider returned 403; failing the connection; statusCode=403; code=80019; cause=[ErrorInfo: Mismatch between clientId in token (-anon89dfcc5b-6805-45e0-94db-bc6cfa0e9618) and current clientId (-anon9118cecf-29db-4bab-9592-38ed446c0617); statusCode=403; code=40102]]
```
I need to know how long after running does the error occur?
starting at Tue May 30 02:30:27 AM GMT 2023

30
README.md
View File

@ -20,6 +20,36 @@ Export mode. Export sql chat messages as IndexedDB
## Dev notes
### TUI
A live table which shows ongoing active subscriptions
| Room | Message | Silence | Status | Title | Tip | Password |
| ---- | ------- | ------- | ------ | ----- | --- | -------- |
| pro. | | | | | | x |
| el_. | | | x | | | |
| sky. | | | x | | | |
We're going to need to refactor the code a bit.
First we need an eventloop of sorts. A thing to ensure that we are always subscribing to the correct rooms and the correct Ably channels.
I think it needs to go like this.
1. Startup
2. Get list of rooms we are watching
3. For each room,
* subscribe to status channel
* if we are unable to subcribe,
* subscribe to password channel (? does this work in pw mode ?)
4. When status channel tells us a room goes live,
* subscribe to message, silence, title, tip, and pw channels.
5. When status channel tells us a room goes offline,
* wait 5 mins
* stop this process if room is online again
* unsubscribe from everything except status ch.
### Ably realtime event formats

206
index.js
View File

@ -1,206 +0,0 @@
// import { containsCBInviteLink } from "./src/tweetProcess.js"
import Room from './src/Room.js'
import { loggerFactory } from "./src/logger.js"
// import Stream from './src/Stream.js'
import { assertYtdlpExistence } from './src/ytdlp.js'
import * as faye from './src/faye.js'
import * as sound from './src/sound.js'
import * as cb from './src/cb.js'
// import { extractRoomId } from './src/parsers.js'
import { appEnv, getAppContext } from './src/appContext.js'
import { onCbMessage, onCbStart, onCbStop, onCbTitleChange, onCbSilence, onCbTip } from './src/cbCallbacks.js'
import fs from 'node:fs'
import Database from 'better-sqlite3';
import os from 'node:os'
import path from 'node:path'
import yargs from 'yargs'
import { hideBin } from 'yargs/helpers'
const epoch = new Date(0).valueOf()
const now = new Date().valueOf()
async function getDataDir() {
// Define dataDir
const dataDir = path.join(os.homedir(), '.local', 'share', 'futureporn-scout');
try {
// Check if dataDir exists
await fs.promises.access(dataDir);
} catch (error) {
// Create dataDir if it doesn't exist
await fs.promises.mkdir(dataDir, { recursive: true });
}
return dataDir;
}
async function init () {
const dataDir = await getDataDir()
const dbPath = path.join(dataDir, 'scout.db')
await assertYtdlpExistence()
const db = new Database(dbPath);
db.pragma('journal_mode = WAL');
const logger = loggerFactory({
defaultMeta: { service: 'futureporn/scout' }
})
return getAppContext(appEnv, logger, db, sound, cb)
}
async function registerRoomListeners(appContext, rooms) {
for (const room of rooms) {
const roomName = room.name;
const roomId = room.id;
appContext.logger.log({ level: 'info', message: `monitoring room:${roomName} (${roomId})` })
const rm = new Room({
roomName: roomName,
roomId: roomId,
onStart: (m) => onCbStart(appContext, roomName, m),
onStop: (m) => onCbStop(appContext, roomName, m),
onMessage: (m) => onCbMessage(appContext, roomName, m),
onSilence: (m) => onCbSilence(appContext, roomName, m),
onTitleChange: (m) => onCbTitleChange(appContext, roomName, m),
onTip: (m) => onCbTip(appContext, roomName, m),
})
// retry if errors
let retry = true;
while (retry) {
try {
await rm.monitorRealtime();
retry = false; // If no error occurs, exit the loop
appContext.logger.log({ level: 'debug', message: 'monitorRealtime() completed without error.' })
} catch (e) {
appContext.logger.log({ level: 'info', message: `Error caught when attempting to monitor ${roomName} realtime. ${e}. Retrying.` });
const minTimeout = 5000; // 5 seconds in milliseconds
const maxTimeout = 15000; // 15 seconds in milliseconds
const timeout = Math.floor(Math.random() * (maxTimeout - minTimeout + 1)) + minTimeout;
appContext.logger.log({ level: 'info', message: `${timeout} courtesy timer before retry` })
await new Promise((resolve) => setTimeout(() => resolve(), timeout));
}
}
}
}
// /**
// * These room timers are used to differentiate between stream ending, and temporary outages
// * Room timers track how long a room has been offline.
// * If the room has been offline for less than 5 minutes and a start event occurs, the offline event must have been a temporary outage.
// */
// function registerRoomTimers(appContext, rooms) {
// for (const room of rooms) {
// const roomName = room.name
// appContext.logger.log({ level: 'debug', message: `registering room timer for ${roomName}` })
// appContext.roomTimers[roomName] = {}
// appContext.roomTimers[roomName].offline = null
// }
// }
async function daemon () {
const appContext = await init()
const stmt = appContext.db.prepare('SELECT name, id FROM rooms WHERE monitor = TRUE')
const rooms = stmt.all()
appContext.faye = faye.fayeFactory(appContext)
// registerRoomTimers(appContext, rooms)
registerRoomListeners(appContext, rooms)
}
async function exportLogs (options) {
const appContext = await init()
// let stmt = appContext.db.prepare('SELECT ...')
if (options.auto) {
const stream = new Stream(appContext, options.room);
const r = stream.getMostRecentStream()
console.log(r)
}
// use lifecycles database to find most recent start & stop events.
// use a 5 minute threshold to determine stream starts/s
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' ORDER BY data_ts ASC;`)
// } else if (!!options.since && !!options.until) {
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts >= ${options.since} ORDER BY data_ts ASC;`)
// } else if (!!options.since && !options.until) {
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts >= ${options.since} ORDER BY data_ts ASC;`)
// } else if (!options.since && !!options.until) {
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts <= ${options.until} ORDER BY data_ts ASC;`)
// }
// const messages = stmt.all()
// console.log(messages)
}
yargs(hideBin(process.argv))
.command({
command: 'daemon',
alias: 'd',
desc: 'Listen & log chaturbate events',
builder: () => {},
handler: (argv) => {
console.info(argv)
daemon()
}
})
.command({
command: 'export',
alias: 'e',
desc: 'Export chat logs as JSON',
builder: (yargs) => {
return yargs
.option('output', {
alias: 'o',
describe: 'output file location',
required: true
})
.option('room', {
aliases: ['r', 'n'],
describe: 'the name of the chaturbate room',
required: true
})
.option('since', {
alias: 's',
describe: 'export logs since this date',
default: epoch,
required: false
})
.option('until', {
alias: 'u',
describe: 'export logs until this date',
default: now,
required: false
})
// .option('auto', {
// alias: 'a',
// describe: 'gets the most recent stream',
// required: false
// })
},
handler: (argv) => {
if (argv.auto && (argv.since !== epoch || argv.until !== now)) {
throw new Error('cannot use --auto with --since or --until.')
}
console.info(argv)
exportLogs(argv)
}
})
.demandCommand(1)
.help()
.parse()

326
index.ts Normal file
View File

@ -0,0 +1,326 @@
import Room from './src/Room.js'
import AblyRealtime from './src/AblyRealtime.js'
import { loggerFactory } from "./src/logger.js"
import { assertYtdlpExistence } from './src/ytdlp.js'
import * as faye from './src/faye.js'
import * as sound from './src/sound.js'
import * as cb from './src/cb.js'
import { appEnv, getAppContext } from './src/appContext.ts'
import {
onCbMessage,
onCbStart,
onCbStop,
onCbTitleChange,
onCbSilence,
onCbTip,
onCbPassword,
} from './src/cbCallbacks.js'
import ChaturbateAuth from './src/ChaturbateAuth.js'
import fs from 'node:fs'
import Database from 'better-sqlite3';
import os from 'node:os'
import path from 'node:path'
import yargs from 'yargs'
import { hideBin } from 'yargs/helpers'
import $fastq from 'fastq';
import gotClient from './src/gotClient.js'
const fastq = $fastq.promise
const epoch = new Date(0).valueOf()
const now = new Date().valueOf()
async function getDataDir() {
// Define dataDir
const dataDir = path.join(os.homedir(), '.local', 'share', 'futureporn-scout');
try {
// Check if dataDir exists
await fs.promises.access(dataDir);
} catch (error) {
// Create dataDir if it doesn't exist
await fs.promises.mkdir(dataDir, { recursive: true });
}
return dataDir;
}
async function init () {
const dataDir = await getDataDir()
const dbPath = path.join(dataDir, 'scout.db')
await assertYtdlpExistence()
const db = new Database(dbPath);
db.pragma('journal_mode = WAL');
const logger = loggerFactory({
defaultMeta: { service: 'futureporn/scout' }
})
return getAppContext(appEnv, logger, db, sound, cb, dataDir, gotClient)
}
/**
*
* listen to room status messages 24/7
*
* @todo [ ] when room goes online, spawn a woerker
* which listens to more of it's events
*
* @todo [ ] when room goes offline
* and remains offline for >5 minutes,
* retire the worker
*
* @todo [ ] 1 TUI row per user
* @todo [ ] TUI columns for subscriptins status
*/
async function registerRoomStatusListeners(appContext, rooms, cb, ably) {
// for each room, render a TUI row
// for (const room of rooms) {
// }
const sampleRoom = rooms.at(0)
console.log(sampleRoom)
// get cookie from cb. We only need one for `chaturbate.com/` (not one per room.)
await cb.fetchCookiesIfNeeded(sampleRoom.url)
const csrfToken = await cb.getCsrfToken(sampleRoom.url) // doesn't really matter which roomUrl for this token, as long as it is on chaturbate.com domain
const cookieString = await cb.cookieJar.getCookieString(sampleRoom.url)
console.log('here is the cookiestring')
console.log(cookieString)
await Promise.allSettled((rooms.map((r) => r.id)))
await Promise.allSettled((rooms.map((r) => r.dossier)))
// get permission form for all the rooms we are interested in
const permissionForm = Room.getPermissionsForm(rooms.map((r) => r.id), ['status'], csrfToken)
for (let pair of permissionForm.entries()) {
console.log(pair[0] + ': ' + pair[1]);
}
// get signed ably tokenrequest from cb
const psa = await cb.getPushServiceAuth(sampleRoom.url, permissionForm, cookieString)
const ablyRealtime = new AblyRealtime(appContext, {
chaturbateAuth: cb,
pushServiceAuth: psa // @todo-- this line has no effect, see below
})
// @todo simplify this by removing need to call getRealtimeClient.
// this could happen as a byproduct of AblyRealtime constructor
const realtimeClient = await ablyRealtime.getRealtimeClient(psa)
const channels = ablyRealtime.tokenRequest.channels
const failures = ablyRealtime.tokenRequest?.failures
realtimeClient.connection.once('connected', (idk) => {
this.logger.log({ level: 'info', message: 'CB Realtime Connected!' })
})
// get the channel topic strings
// const topicStrings =
// const roomStatusTopicString = this.pushServiceAuth.channels[`RoomStatusTopic#RoomStatusTopic:${this.id}`]
// const messageChannel = this.realtime.channels.get(roomMessageTopicString);
// messageChannel.subscribe((message) => {
// this.onMessage(message)
// })
// realtimeClient.subscribe()
}
/**
* @deprecated
*/
// async function registerRoomListeners(appContext, rooms, cb, ably) {
// for (const room of rooms) {
// const roomName = room.name;
// const roomId = room.id;
// appContext.logger.log({ level: 'info', message: `monitoring room:${roomName} (${roomId})` })
// // here we create a room instance
// // this instance has callbacks for each type of
// // channel message.
// // @todo all channels are subscribed to
// // at the time of instanciation.
// // we need to change this so subscriptions happen
// // on-demand.
// // the only channel that is subscribed to at instantiation
// // is status channel.
// // if status channel is unavailable, try password channel.
// // if no channels are available, we need to throw an err.
// const rm = new Room(appContext, {
// roomName: roomName,
// roomId: roomId,
// ably: ably,
// cb: cb,
// onStart: (m) => onCbStart(appContext, roomName, m),
// onStop: (m) => onCbStop(appContext, roomName, m),
// onMessage: (m) => onCbMessage(appContext, roomName, m),
// onSilence: (m) => onCbSilence(appContext, roomName, m),
// onTitleChange: (m) => onCbTitleChange(appContext, roomName, m),
// onTip: (m) => onCbTip(appContext, roomName, m),
// onPassword: (m) => onCbPassword(appContext, roomName, m),
// })
// // retry if errors
// let retry = true;
// while (retry) {
// try {
// await rm.monitorRealtime();
// retry = false; // If no error occurs, exit the loop
// appContext.logger.log({ level: 'debug', message: 'monitorRealtime() completed without error.' })
// } catch (e) {
// appContext.logger.log({ level: 'info', message: `Error caught when attempting to monitor ${roomName} realtime. ${e}. Retrying.` });
// const minTimeout = 5000; // 5 seconds in milliseconds
// const maxTimeout = 15000; // 15 seconds in milliseconds
// const timeout = Math.floor(Math.random() * (maxTimeout - minTimeout + 1)) + minTimeout;
// appContext.logger.log({ level: 'info', message: `${timeout} courtesy timer before retry` })
// await new Promise<void>((resolve) => setTimeout(() => resolve(), timeout));
// }
// }
// }
// }
// /**
// * These room timers are used to differentiate between stream ending, and temporary outages
// * Room timers track how long a room has been offline.
// * If the room has been offline for less than 5 minutes and a start event occurs, the offline event must have been a temporary outage.
// */
// function registerRoomTimers(appContext, rooms) {
// for (const room of rooms) {
// const roomName = room.name
// appContext.logger.log({ level: 'debug', message: `registering room timer for ${roomName}` })
// appContext.roomTimers[roomName] = {}
// appContext.roomTimers[roomName].offline = null
// }
// }
async function daemon () {
const appContext = await init()
const stmt = appContext.db.prepare('SELECT name, id FROM rooms WHERE monitor = TRUE')
const roomsRecords = stmt.all()
const rooms = roomsRecords.map((r) => new Room(appContext, { name: r.name, id: r.id }))
appContext.faye = faye.fayeFactory(appContext)
// registerRoomTimers(appContext, rooms)
const chaturbate = new ChaturbateAuth(appContext)
const ably = new AblyRealtime(appContext, cb)
// registerRoomListeners(appContext, rooms, cb, ably)
// @todo [ ] Listen to interesting rooms 24/7
// @todo [ ] When interesting room comes online, listen to more of it's events
// @todo [ ] When interesting room goes offline, stop listening to more of it's events after 5 minutes
// const concurrency = 12
// const queue = fastq(worker, 12)
registerRoomStatusListeners(appContext, rooms, chaturbate, ably)
}
async function exportLogs (options) {
const appContext = await init()
// let stmt = appContext.db.prepare('SELECT ...')
// if (options.auto) {
// const stream = new Stream(appContext, options.room);
// const r = stream.getMostRecentStream()
// console.log(r)
// }
// use lifecycles database to find most recent start & stop events.
// use a 5 minute threshold to determine stream starts/s
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' ORDER BY data_ts ASC;`)
// } else if (!!options.since && !!options.until) {
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts >= ${options.since} ORDER BY data_ts ASC;`)
// } else if (!!options.since && !options.until) {
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts >= ${options.since} ORDER BY data_ts ASC;`)
// } else if (!options.since && !!options.until) {
// stmt = appContext.db.prepare(`SELECT * FROM messages WHERE _room = '${options.room}' AND data_ts <= ${options.until} ORDER BY data_ts ASC;`)
// }
// const messages = stmt.all()
// console.log(messages)
}
yargs(hideBin(process.argv))
.command({
command: 'daemon',
alias: 'd',
desc: 'Listen & log chaturbate events',
builder: () => {},
handler: (argv) => {
console.info(argv)
daemon()
}
})
.command({
command: 'export',
alias: 'e',
desc: 'Export chat logs as JSON',
builder: (yargs) => {
return yargs
.option('output', {
alias: 'o',
describe: 'output file location',
required: true
})
.option('room', {
aliases: ['r', 'n'],
describe: 'the name of the chaturbate room',
required: true
})
.option('since', {
alias: 's',
describe: 'export logs since this date',
default: epoch,
required: false
})
.option('until', {
alias: 'u',
describe: 'export logs until this date',
default: now,
required: false
})
// .option('auto', {
// alias: 'a',
// describe: 'gets the most recent stream',
// required: false
// })
},
handler: (argv) => {
if (argv.auto && (argv.since !== epoch || argv.until !== now)) {
throw new Error('cannot use --auto with --since or --until.')
}
console.info(argv)
exportLogs(argv)
}
})
.demandCommand(1)
.help()
.parse()

13
package.json
View File

@ -6,9 +6,9 @@
"license": "Unlicense",
"private": true,
"scripts": {
"start": "pm2 start ecosystem.config.cjs; pm2 monit",
"start": "pm2 start ecosystem.config.cjs; pm2 logs",
"test": "mocha ./test/unit",
"dev": "FUTUREPORN_WORKDIR=/home/chris/Downloads pnpm nodemon index.js daemon"
"dev": "FUTUREPORN_WORKDIR=/home/chris/Downloads pnpm nodemon --exec \"node --loader ts-node/esm \" -e js,ts,mjs,json index.ts daemon"
},
"type": "module",
"dependencies": {
@ -17,19 +17,28 @@
"better-sqlite3": "^8.3.0",
"cheerio": "^1.0.0-rc.12",
"date-fns": "^2.29.3",
"debounce": "^1.2.1",
"dexie-export-import": "^4.0.7",
"dotenv": "^16.0.3",
"execa": "^7.1.1",
"fastq": "^1.15.0",
"faye": "^1.4.0",
"fetch-blob": "^3.2.0",
"formdata-polyfill": "^4.0.10",
"got": "^13.0.0",
"got-plugin-debounce": "^1.0.3",
"ky": "^0.33.3",
"node-fetch": "^3.3.0",
"p-retry": "^5.1.2",
"pm2": "^5.3.0",
"seedrandom": "^3.0.5",
"terminal-kit": "^3.0.0",
"tough-cookie": "^4.1.2",
"tough-cookie-file-store": "^2.0.3",
"ts-node": "^10.9.1",
"tty-table": "^4.2.1",
"twitter-v2": "^1.1.0",
"typescript": "^5.1.6",
"winston": "^3.8.2",
"yargs": "^17.7.2"
},

43
package.json~ Normal file
View File

@ -0,0 +1,43 @@
{
"name": "scout",
"version": "0.0.1",
"description": "event emitter that detects start and end of stream",
"main": "index.js",
"license": "Unlicense",
"private": true,
"scripts": {
"start": "pm2 start ecosystem.config.cjs; pm2 monit",
"test": "mocha ./test/unit",
"dev": "FUTUREPORN_WORKDIR=/home/chris/Downloads pnpm nodemon index.js daemon"
},
"type": "module",
"dependencies": {
"@neutralinojs/neu": "^9.5.1",
"ably": "1.2.13",
"better-sqlite3": "^8.3.0",
"cheerio": "^1.0.0-rc.12",
"date-fns": "^2.29.3",
"dexie-export-import": "^4.0.7",
"dotenv": "^16.0.3",
"execa": "^7.1.1",
"faye": "^1.4.0",
"fetch-blob": "^3.2.0",
"formdata-polyfill": "^4.0.10",
"node-fetch": "^3.3.0",
"p-retry": "^5.1.2",
"pm2": "^5.3.0",
"seedrandom": "^3.0.5",
"tough-cookie": "^4.1.2",
"tough-cookie-file-store": "^2.0.3",
"twitter-v2": "^1.1.0",
"winston": "^3.8.2",
"yargs": "^17.7.2"
},
"devDependencies": {
"chai": "^4.3.7",
"chai-events": "^0.0.3",
"mocha": "^10.2.0",
"nodemon": "^2.0.20",
"sinon": "^15.1.0"
}
}

975
pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

1
sample-message.json Normal file
View File

@ -0,0 +1 @@
{"name":"room:message:27WDW5C:3","id":"IY89ONPmXC:0:0","encoding":null,"data":{"tid":"16845232175:70912","ts":1684523217.5373857,"message":"%%%[emoticon pkmn144|https://static-pub.highwebmedia.com/uploads/avatar/2018/06/10/04/48/e1f178a35036903165c2cb0446bc9a2a6f2cb70e.jpg|32|32|/emoticon_report_abuse/pkmn144/]%%% %%%[emoticon moderatorsaraandy|https://static-pub.highwebmedia.com/uploads/avatar/2018/02/14/15/14/b137962db6fa0f12cc2458cb5732c3cead73518a.jpg|148|30|/emoticon_report_abuse/moderatorsaraandy/]%%% :: %%%[emoticon tipsshow|https://static-pub.highwebmedia.com/uploads/avatar/2014/01/13/ya7Rwj4DBOpU0xE.jpg|200|72|/emoticon_report_abuse/tipsshow/]%%%","font_family":"default","font_color":"rgb(0,0,0)","id":"9X0K8LC7DGY7NC","background":"linear-gradient(to right,rgba(17,51,204,0.0) 5.0%,rgba(17,51,204,0.2) 20.0%,rgba(17,51,204,0.4) 73.0%,rgba(17,51,204,0.2))","from_user":{"username":"cookiesecret","gender":"m","is_broadcaster":false,"in_fanclub":false,"is_following":true,"is_mod":true,"has_tokens":true,"tipped_recently":true,"tipped_alot_recently":true,"tipped_tons_recently":true},"method":"lazy","pub_ts":1684523217.5499687}} {"service":"futureporn/scout"}

13
sample-silence.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "room:silence:KGHH38V",
"id": "cNXrpYEXUi:0:0",
"encoding": null,
"data": {
"tid": "16845347513:34561",
"ts": 1684534751.3850138,
"username": "rashel_summer1",
"from_username": "clever_bear",
"method": "lazy",
"pub_ts": 1684534751.3961625
}
}

12
sample-title.json Normal file
View File

@ -0,0 +1,12 @@
{
"name": "room:title_change:KGHH38V",
"id": "TOU400J_Rn:0:0",
"encoding": null,
"data": {
"tid": "16845334971:17629",
"ts": 1684533497.171352,
"title": "My name is Alexa! My third time on site!Squeeze tits close to camera #shy #new #18 #cute #young [857 tokens remaining]",
"method": "lazy",
"pub_ts": 1684533497.177245
}
}

36
scrap-push-service-auth.ts Normal file
View File

@ -0,0 +1,36 @@
{
"channels": {
"RoomUserPresenceTopic#RoomUserPresenceTopic:JQ2YJS5:M79SV5L": "room_user:presence:JQ2YJS5:M79SV5L:0",
"UserMessageTopic#UserMessageTopic:M79SV5L": "user:message:M79SV5L",
"UserChatMediaOpenedTopic#UserChatMediaOpenedTopic:M79SV5L": "user:chatmedia_open:M79SV5L",
"UserChatMediaRemovedTopic#UserChatMediaRemovedTopic:M79SV5L": "user:chatmedia_remove:M79SV5L",
"UserPmReadTopic#UserPmReadTopic:M79SV5L": "user:pm_read:M79SV5L",
"UserIgnoreTopic#UserIgnoreTopic:M79SV5L": "user:ignore:M79SV5L",
"UserTokenUpdateTopic#UserTokenUpdateTopic:M79SV5L": "user:token_update:M79SV5L",
"RoomTipAlertTopic#RoomTipAlertTopic:JQ2YJS5": "room:tip_alert:JQ2YJS5",
"RoomPurchaseTopic#RoomPurchaseTopic:JQ2YJS5": "room:purchase:JQ2YJS5",
"RoomFanClubJoinedTopic#RoomFanClubJoinedTopic:JQ2YJS5": "room:fanclub:JQ2YJS5",
"RoomMessageTopic#RoomMessageTopic:JQ2YJS5": "room:message:JQ2YJS5:0",
"GlobalPushServiceBackendChangeTopic#GlobalPushServiceBackendChangeTopic": "global:push_service",
"QualityUpdateTopic#QualityUpdateTopic:JQ2YJS5": "room:quality_update:JQ2YJS5",
"UserColorUpdateTopic#UserColorUpdateTopic:M79SV5L": "user:color_update:M79SV5L",
"UserAlertTopic#UserAlertTopic:M79SV5L": "user:alert:M79SV5L",
"RoomNoticeTopic#RoomNoticeTopic:JQ2YJS5": "room:notice:JQ2YJS5",
"RoomEnterLeaveTopic#RoomEnterLeaveTopic:JQ2YJS5": "room:enter_leave:JQ2YJS5",
"RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:JQ2YJS5": "room:password_protected:JQ2YJS5:0",
"RoomModeratorPromotedTopic#RoomModeratorPromotedTopic:JQ2YJS5": "room:mod_promoted:JQ2YJS5",
"RoomModeratorRevokedTopic#RoomModeratorRevokedTopic:JQ2YJS5": "room:mod_revoked:JQ2YJS5",
"RoomStatusTopic#RoomStatusTopic:JQ2YJS5": "room:status:JQ2YJS5:0",
"RoomTitleChangeTopic#RoomTitleChangeTopic:JQ2YJS5": "room:title_change:JQ2YJS5",
"RoomSilenceTopic#RoomSilenceTopic:JQ2YJS5": "room:silence:JQ2YJS5",
"RoomKickTopic#RoomKickTopic:JQ2YJS5": "room:kick:JQ2YJS5",
"RoomUpdateTopic#RoomUpdateTopic:JQ2YJS5": "room:update:JQ2YJS5",
"RoomSettingsTopic#RoomSettingsTopic:JQ2YJS5": "room:settings:JQ2YJS5",
"RoomUserNoticeTopic#RoomUserNoticeTopic:JQ2YJS5:M79SV5L": "room_user:notice:JQ2YJS5:M79SV5L",
"RoomUserPrivateStatusTopic#RoomUserPrivateStatusTopic:JQ2YJS5:M79SV5L": "room_user:private_status:JQ2YJS5:M79SV5L",
"RoomUserHiddenCamStatusTopic#RoomUserHiddenCamStatusTopic:JQ2YJS5:M79SV5L": "room_user:status:JQ2YJS5:M79SV5L",
"RoomLightBlueTopic#RoomLightBlueTopic:JQ2YJS5": "lightblue:notice:JQ2YJS5",
"GameUpdateTopic#GameUpdateTopic:JQ2YJS5": "room:game_update:JQ2YJS5",
"UserSMCWatchingTopic#UserSMCWatchingTopic:M79SV5L": "user:smc_watching:M79SV5L"
}
}

69
src/AblyRealtime.js Normal file
View File

@ -0,0 +1,69 @@
/**
* Ably realtime client
*
* This client is shared among all the Room instances
* In order to use as little Ably resources as possible
* And avoid reaching limits https://help.ably.io/error/40114
*/
import Ably from 'ably';
export default class AblyRealtime {
constructor(appContext, opts) {
this.logger = appContext.logger;
this.chaturbateAuth = opts.chaturbateAuth;
this.realtimeHost = opts?.realtimeHost;
this.fallbackHosts = opts?.fallbackHosts;
this.pushServiceAuth;
this.tokenRequest;
this.realtime;
}
async getRealtimeClient(pushServiceAuth) {
if (!pushServiceAuth) throw new Error('getRealtime needs pushServiceAuth passed to it');
this.pushServiceAuth = pushServiceAuth
if (!this.realtimeHost || !this.fallbackHosts) {
this.realtimeHost = this.pushServiceAuth.settings.realtime_host
this.fallbackHosts = this.pushServiceAuth.settings.fallback_hosts
}
this.realtime = new Ably.Realtime.Promise({
autoConnect: false,
closeOnUnload: true,
transportParams: {
remainPresentFor: '0'
},
realtimeHost: this.realtimeHost,
restHost: this.realtimeHost,
fallback_hosts: this.fallbackHosts,
authCallback: (async (tokenParams, callback) => {
this.logger.log({
level: 'debug',
message: `Ably authCallback. Getting a fresh new Ably TokenRequest.`
})
// This is where we get a signed Ably TokenRequest from CB!
this.tokenRequest = await this.cb.getTokenRequest()
this.logger.log({
level: 'debug',
message: `Got a new TokenRequest`
})
this.logger.log({
level: 'debug',
message: JSON.stringify(this.tokenRequest)
})
callback(null, this.tokenRequest)
})
})
return this.realtime
}
}

162
src/ChaturbateAuth.js Normal file
View File

@ -0,0 +1,162 @@
import path from 'node:path';
import {
Cookie,
CookieJar
} from "tough-cookie";
import {
FileCookieStore
} from "tough-cookie-file-store";
import cheerio from "cheerio";
import {
isBefore
} from 'date-fns'
export default class ChaturbateAuth {
constructor(appContext, opts) {
this.appContext = appContext;
this.logger = appContext.logger;
this.tokenRequest;
this.cookieJar = new CookieJar(new FileCookieStore(path.join(appContext.dataDir, "cookie.json")));
}
async fetchCookiesIfNeeded(roomUrl) {
let cookies = await this.cookieJar.getCookies(roomUrl);
let tokenCookie = ChaturbateAuth.getTokenCookie(cookies)
this.logger.log({
level: 'debug',
message: `tokenCookie is as follows. ${tokenCookie}`
})
if (
tokenCookie === undefined ||
isBefore(tokenCookie.expires, new Date())
) {
this.logger.log({
level: 'debug',
message: 'tokenCookie is either undefined or expired. Getting new cookie.'
})
let res = await this.appContext.gotClient(roomUrl, {
"credentials": "omit",
"headers": {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1"
},
"method": "GET",
"mode": "cors"
})
const rawHeaders = res.headers.raw()
if (Array.isArray(rawHeaders["set-cookie"]))
cookies = rawHeaders["set-cookie"].map(Cookie.parse);
else cookies = [Cookie.parse(rawHeaders["set-cookie"])];
for (const cookie of cookies) {
this.logger.log({ level: 'debug', message: `adding cookie ${JSON.stringify(cookie)} to the cookieJar` })
await this.cookieJar.setCookie(cookie, roomUrl)
}
} else {
this.logger.log({ level: 'debug', message: `🍪✅ cookies are valid.` })
}
}
static getTokenCookie(cookies) {
return cookies.find((c) => c.key == 'csrftoken');
}
static getSbrCookie(cookies) {
return cookies.find((c) => c.key == 'sbr');
}
async getTokenRequest(url, form, cookie) {
this.pushServiceAuth = await this.getPushServiceAuth(url, form, cookie)
return this.pushServiceAuth.token_request
}
async getCsrfToken(roomUrl) {
let cookies = await this.cookieJar.getCookies(roomUrl);
let tokenCookie = ChaturbateAuth.getTokenCookie(cookies)
return tokenCookie.value
}
/**
* make a request against CB's server which
* - verifies custom credentials
* - issues signed Ably TokenRequest
*
* The response contains the following
* - token
* - channels
* - failures
* - token_request <-- signed Ably TokenRequest
* - client_id
* - settings
*
* GET https://chaturbate.com/push_service/auth/
*
* more info-- https://ably.com/docs/core-features/authentication#token-authentication
*/
async getPushServiceAuth(roomUrl, form, cookieString) {
// https://faqs.ably.com/40104-timestamp-not-current
// this was caused by signed token request re-use after it had expired.
// we must not cache the signed token request
// https://help.ably.io/error/40102
// caused by mismatching client_id, when using a new client_id issued by CB pushServiceAuth with an old ably connection
// (possibly) caused by not sending Cookie header during CB pushServiceAuth
// thus leading to CB thinking we're a new anon and issuing a new client_id
// @todo please verify if this is the cause
// 09:44:42.182 Ably: Transport.onIdleTimerExpire(): No activity seen from realtime in 25113ms; assuming connection has dropped
// saw this when my internet died. unsure of whether or not the connection comes back online.
// if it doesn't come back online, this could bite me in the future.
// this is a thing we can test by airgapping.
// if there's a way to implement an onIdleTimerExpire callback, that would be the solution.
// (solved by https://github.com/insanity54/futureporn/issues/208)
this.logger.log({
level: 'debug',
message: `getting pushServiceAuth.`
})
this.logger.log({
level: 'debug',
message: `using cookie string ${cookieString}`
})
this.pushServiceAuth = await this.appContext.gotClient("https://chaturbate.com/push_service/auth/", {
"headers": {
"Cookie": cookieString,
"User-Agent": this.ua,
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"X-Requested-With": "XMLHttpRequest",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"Sec-GPC": "1",
"Referrer": roomUrl,
},
"body": form,
"method": "POST"
}).json()
this.logger.log({
level: 'debug',
message: JSON.stringify(this.pushServiceAuth)
})
return this.pushServiceAuth
}
}

400
src/Room.js
View File

@ -1,400 +0,0 @@
import fetch from "node-fetch";
import cheerio from "cheerio";
import {
Cookie,
CookieJar
} from "tough-cookie";
import {
FileCookieStore
} from "tough-cookie-file-store";
import os from 'os';
import path from 'path';
import {
loggerFactory
} from './logger.js'
import {
isBefore
} from 'date-fns'
import Ably from 'ably';
import {
FormData
} from 'formdata-polyfill/esm.min.js'
const logger = loggerFactory({
defaultMeta: {
service: "futureporn/scout"
}
})
export default class Room {
constructor(opts) {
this.roomName = opts.roomName
this.roomId = opts.roomId
this.roomUrl = 'https://chaturbate.com/' + this.roomName
this.csrfToken = null
this.tokenRequest = null
this.ablyTokenRequest = null
this.dossier = null
this.realtimeHost = null
this.fallbackHosts = null
this.pushServiceAuth = null
this.datadir = path.join(os.homedir(), '.local/share/futureporn-scout')
this.cookieJar = new CookieJar(new FileCookieStore(path.join(this.datadir, "cookie.json")));
this.ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36';
this.onStart = opts.onStart || null;
this.onStop = opts.onStop || null;
this.onMessage = opts.onMessage || null;
this.onSilence = opts.onSilence || null;
this.onTitleChange = opts.onTitleChange || null;
this.onTip = opts.onTip || null;
this.realtime = null;
}
// deprecated. fragile because it depends on existing room dossier
// async getRoomId () {
// if (this.roomId !== '') {
// return this.roomId
// } else {
// const dossier = await getInitialRoomDossier(room)
// return dossier.room_uid
// }
// }
// deprecated. fragile because it depends on present room dossier
// async getInitialRoomDossier() {
// try {
// const res = await fetch(this.roomUrl);
// const body = await res.text();
// const $ = cheerio.load(body);
// let rawScript = $('script:contains(window.initialRoomDossier)').html();
// if (!rawScript) {
// throw new Error('window.initialRoomDossier is null. This could mean the channel is in password mode');
// }
// let rawDossier = rawScript.slice(rawScript.indexOf('"'), rawScript.lastIndexOf('"') + 1);
// let dossier = JSON.parse(JSON.parse(rawDossier));
// return dossier;
// } catch (error) {
// // Handle the error gracefully
// logger.log({ level: 'error', message: `Error fetching initial room dossier: ${error.message}` });
// return null; // Or any other appropriate action you want to take
// }
// }
static getPermissionsForm(roomId, csrfToken) {
logger.log({ level: 'debug', message: `getting permission form with room id ${roomId}` })
if (!roomId) throw new Error('roomId is required but it was undefined')
let form = new FormData();
const topics = `{
"RoomTipAlertTopic#RoomTipAlertTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},
"RoomPurchaseTopic#RoomPurchaseTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},
"RoomFanClubJoinedTopic#RoomFanClubJoinedTopic:${roomId}": {
"broadcaster_uid":"${roomId}"
},
"RoomMessageTopic#RoomMessageTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},
"GlobalPushServiceBackendChangeTopic#GlobalPushServiceBackendChangeTopic":{
},
"RoomAnonPresenceTopic#RoomAnonPresenceTopic:${roomId}":{\
"broadcaster_uid":"${roomId}"
},
"QualityUpdateTopic#QualityUpdateTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomNoticeTopic#RoomNoticeTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomEnterLeaveTopic#RoomEnterLeaveTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomModeratorPromotedTopic#RoomModeratorPromotedTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomModeratorRevokedTopic#RoomModeratorRevokedTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomStatusTopic#RoomStatusTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomTitleChangeTopic#RoomTitleChangeTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomSilenceTopic#RoomSilenceTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomKickTopic#RoomKickTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomUpdateTopic#RoomUpdateTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
},"RoomSettingsTopic#RoomSettingsTopic:${roomId}":{
"broadcaster_uid":"${roomId}"
}
}`
form.append('topics', topics)
form.append('csrfmiddlewaretoken', csrfToken)
return form
}
async monitorRealtime () {
if (!this.pushServiceAuth) this.pushServiceAuth = await this.getPushServiceAuth();
if (!this.roomId) throw new Error('roomId was missing')
const roomMessageTopicString = this.pushServiceAuth.channels[`RoomMessageTopic#RoomMessageTopic:${this.roomId}`]
const roomSilenceTopicString = this.pushServiceAuth.channels[`RoomSilenceTopic#RoomSilenceTopic:${this.roomId}`]
const roomStatusTopicString = this.pushServiceAuth.channels[`RoomStatusTopic#RoomStatusTopic:${this.roomId}`]
const roomTitleChangeTopicString = this.pushServiceAuth.channels[`RoomTitleChangeTopic#RoomTitleChangeTopic:${this.roomId}`]
const roomTipAlertTopicString = this.pushServiceAuth.channels[`RoomTipAlertTopic#RoomTipAlertTopic:${this.roomId}`]
this.realtime = await this.getRealtime()
this.realtime.connection.once('connected', (idk) => {
logger.log({ level: 'info', message: 'CB Realtime Connected!' })
})
const messageChannel = this.realtime.channels.get(roomMessageTopicString);
messageChannel.subscribe((message) => {
this.onMessage(message)
})
const silenceChannel = this.realtime.channels.get(roomSilenceTopicString);
silenceChannel.subscribe((message) => {
this.onSilence(message)
})
const statusChannel = this.realtime.channels.get(roomStatusTopicString);
statusChannel.subscribe((message) => {
logger.log({ level: 'debug', message: `got statusChannel message! ${JSON.stringify(message)}`})
if (message.data.status === 'public') {
this.onStart(message)
} else if (message.data.status === 'offline') {
this.onStop(message)
}
logger.log({ level: 'debug', message: `Received room:status:<roomId>:0` })
logger.log({ level: 'debug', message: JSON.stringify(message, 0, 2) })
});
const titleChannel = this.realtime.channels.get(roomTitleChangeTopicString);
titleChannel.subscribe((message) => {
this.onTitleChange(message)
})
const tipAlertChannel = this.realtime.channels.get(roomTipAlertTopicString);
tipAlertChannel.subscribe((message) => {
logger.log({ level: 'debug', message: `got tipAlertChannel message! ${JSON.stringify(message)}`})
this.onTip(message)
})
await this.realtime.connect()
}
async refreshToken () {
const authToken = await getNewToken()
}
/**
* make a request against CB's server which
* - verifies custom credentials
* - issues signed Ably TokenRequest
*
* The response contains the following
* - token
* - channels
* - failures
* - token_request <-- Ably TokenRequest
* - client_id
* - settings
*
* GET https://chaturbate.com/push_service/auth/
*
* more info-- https://ably.com/docs/core-features/authentication#token-authentication
*/
async getPushServiceAuth () {
// https://faqs.ably.com/40104-timestamp-not-current
// this was caused by signed token request re-use after it had expired.
// we must not cache the signed token request
// https://help.ably.io/error/40102
// caused by mismatching client_id, when using a new client_id issued by CB pushServiceAuth with an old ably connection
// (possibly) caused by not sending Cookie header during CB pushServiceAuth
// thus leading to CB thinking we're a new anon and issuing a new client_id
// @todo please verify if this is the cause
// 09:44:42.182 Ably: Transport.onIdleTimerExpire(): No activity seen from realtime in 25113ms; assuming connection has dropped
// saw this when my internet died. unsure of whether or not the connection comes back online.
// if it doesn't come back online, this could bite me in the future.
// this is a thing we can test by airgapping.
// if there's a way to implement an onIdleTimerExpire callback, that would be the solution.
// @todo
logger.log({
level: 'debug',
message: `getting pushServiceAuth.`
})
await this.fetchCookiesIfNeeded()
this.csrfToken = await this.getCsrfToken()
const cookieString = await this.cookieJar.getCookieString(this.roomUrl)
logger.log({
level: 'debug',
message: `using cookie string ${cookieString}`
})
const form = Room.getPermissionsForm(this.roomId, this.csrfToken)
const res = await fetch("https://chaturbate.com/push_service/auth/", {
"credentials": "include",
"headers": {
"Cookie": cookieString,
"User-Agent": this.ua,
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"X-Requested-With": "XMLHttpRequest",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"Sec-GPC": "1"
},
"referrer": this.roomUrl,
"body": form,
"method": "POST",
"mode": "cors"
});
this.pushServiceAuth = await res.json()
logger.log({
level: 'debug',
message: JSON.stringify(this.pushServiceAuth)
})
return this.pushServiceAuth
}
async getTokenRequest() {
this.pushServiceAuth = await this.getPushServiceAuth()
return this.pushServiceAuth.token_request
}
async getRealtime() {
// deps
// csrfToken, tokenRequest, realtimeHost, fallbackHosts
if (!this.realtimeHost || !this.fallbackHosts) {
this.pushServiceAuth = await this.getPushServiceAuth()
this.realtimeHost = this.pushServiceAuth.settings.realtime_host
this.fallbackHosts = this.pushServiceAuth.settings.fallback_hosts
}
const realtime = new Ably.Realtime.Promise({
autoConnect: false,
closeOnUnload: true,
transportParams: {
remainPresentFor: '0'
},
realtimeHost: this.realtimeHost,
restHost: this.realtimeHost,
fallback_hosts: this.fallbackHosts,
authCallback: (async(tokenParams, cb) => {
// @see https://github.com/insanity54/futureporn/issues/203
// @todo get a new signed token from CB's servers
logger.log({
level: 'debug',
message: `Ably authCallback. Getting a fresh new Ably TokenRequest.`
})
// This is where we get a signed TokenRequest from CB!
this.tokenRequest = await this.getTokenRequest()
logger.log({
level: 'debug',
message: `Got a new TokenRequest`
})
logger.log({
level: 'debug',
message: JSON.stringify(this.tokenRequest)
})
cb(null, this.tokenRequest)
})
})
return realtime
}
static getTokenCookie(cookies) {
return cookies.find((c) => c.key == 'csrftoken');
}
static getSbrCookie(cookies) {
return cookies.find((c) => c.key == 'sbr');
}
async fetchCookiesIfNeeded () {
let cookies = await this.cookieJar.getCookies(this.roomUrl);
let tokenCookie = Room.getTokenCookie(cookies)
logger.log({
level: 'debug',
message: `tokenCookie is as follows. ${tokenCookie}`
})
if (
tokenCookie === undefined ||
isBefore(tokenCookie.expires, new Date())
) {
logger.log({
level: 'debug',
message: 'tokenCookie is either undefined or expired. Getting new cookie.'
})
let res = await fetch(this.roomUrl, {
"credentials": "omit",
"headers": {
"User-Agent": this.ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1"
},
"method": "GET",
"mode": "cors"
})
const rawHeaders = res.headers.raw()
if (Array.isArray(rawHeaders["set-cookie"]))
cookies = rawHeaders["set-cookie"].map(Cookie.parse);
else cookies = [Cookie.parse(rawHeaders["set-cookie"])];
for (const cookie of cookies) {
logger.log({ level: 'debug', message: `adding cookie ${JSON.stringify(cookie)} to the cookieJar` })
await this.cookieJar.setCookie(cookie, this.roomUrl)
}
} else {
logger.log({ level: 'debug', message: `🍪✅ cookies are valid.`})
}
}
async getCsrfToken() {
let cookies = await this.cookieJar.getCookies(this.roomUrl);
let tokenCookie = Room.getTokenCookie(cookies)
return tokenCookie.value
}
}

251
src/Room.ts Normal file
View File

@ -0,0 +1,251 @@
import {
FormData
} from 'formdata-polyfill/esm.min.js'
import cheerio from 'cheerio'
import ChaturbateAuth from './ChaturbateAuth'
import { Logger } from 'winston'
import { IAppContext } from './appContext'
import AblyRealtime from './AblyRealtime'
import { Realtime } from 'ably'
export interface IRoomOptions {
chaturbateAuth?: any;
ably?: any;
name: string;
onStart?: any;
onStop?: any;
onMessage?: any;
onSilence?: any;
onTitleChange?: any;
onTip?: any;
onPassword?: any;
id?: string;
}
export default class Room {
private id: string | Promise<string>;
private logger: Logger;
private appContext: IAppContext;
private chaturbateAuth: ChaturbateAuth;
private ably: AblyRealtime;
private name: string;
private url: string;
private csrfToken: string | null;
private tokenRequest: any;
private ablyTokenRequest: any;
private realtimeHost: string | null;
private fallbackHosts: string[] | null;
private pushServiceAuth: any;
private onStart: ((message: any) => void) | null;
private onStop: ((message: any) => void) | null;
private onMessage: ((message: any) => void) | null;
private onSilence: ((message: any) => void) | null;
private onTitleChange: ((message: any) => void) | null;
private onTip: ((message: any) => void) | null;
private onPassword: ((message: any) => void) | null;
private realtime: Realtime | null;
private dossier: any;
constructor(appContext: IAppContext, opts: IRoomOptions) {
if (opts?.name === undefined) throw new Error('Room constructor requires a room name.')
this.logger = appContext.logger // this needs to be defined before this.id
this.appContext = appContext
this.chaturbateAuth = opts.chaturbateAuth;
this.ably = opts.ably;
this.name = opts.name
this.url = 'https://chaturbate.com/' + this.name + '/'
this.csrfToken = null
this.tokenRequest = null
this.ablyTokenRequest = null
this.realtimeHost = null
this.fallbackHosts = null
this.pushServiceAuth = null
this.onStart = opts.onStart || null;
this.onStop = opts.onStop || null;
this.onMessage = opts.onMessage || null;
this.onSilence = opts.onSilence || null;
this.onTitleChange = opts.onTitleChange || null;
this.onTip = opts.onTip || null;
this.onPassword = opts.onPassword || null;
this.realtime = null;
this.dossier = null
// We put this last because they depend on other opts
// before making a network request
this.id = opts?.id || this.getRoomId()
}
// depends on initialRoomDossier existing, which is missing when room is passworded
// will return null if dossier is missing.
async getRoomId (): Promise<string | null> {
try {
const dossier = await this.getInitialRoomDossier()
return this.id = dossier.room_uid
} catch (e) {
this.logger.log({ level: 'warning', message: `error while getting room ID. ${e}` })
return this.id = null
}
}
async getInitialRoomDossier(): Promise<any | null> {
try {
const res = await this.appContext.gotClient(this.url);
const $ = cheerio.load(res.body);
let rawScript = $('script:contains(window.initialRoomDossier)').html();
if (!rawScript) {
throw new Error('window.initialRoomDossier is null. This could mean the channel is in password mode');
}
let rawDossier = rawScript.slice(rawScript.indexOf('"'), rawScript.lastIndexOf('"') + 1);
let dossier = JSON.parse(JSON.parse(rawDossier));
return dossier;
} catch (error) {
// Handle the error gracefully
this.logger.log({ level: 'warning', message: `Error fetching initial room dossier: ${error.message}` });
return null; // Or any other appropriate action you want to take
}
}
/**
*
* The list of default topics which are always subscribed to
*
*/
public async getDefaultTopics(): Promise<string[]> {
await this.id
if (!this.id) throw new Error(`cannot get Room's default topics because a room.id could not be fetched`);
return [`RoomStatusTopic#RoomStatusTopic:${this.id}`]
}
public async getAvailableTopics(): Promise<string[]> {
await this.id
if (!this.id) throw new Error(`cannot get Room's available topics because room.id could not be fetched`)
const defaultTopics = await this.getDefaultTopics()
return defaultTopics
}
// the number suffixes are assumed to be session ids.
// opening multiple tabs of the same room will not incr the :n suffix.
// however, opening different rooms in multiple tags will incr the :n suffix
// maybe Ably does request deduplication for same sesionIds???
// we don't need to compute the sessionId, as pushServiceAuth response contains it.
static getPermissionsForm(roomIds: string[], topics: string[], csrfToken: string): FormData {
if (!roomIds || roomIds.length === 0) {
throw new Error('roomIds is required but it was undefined or empty');
}
const generateTopic = (topicKey: string, topicValue: string) => {
return JSON.stringify({ [topicKey]: topicValue });
};
let form = new FormData();
let formTopics = {};
roomIds.forEach((roomId) => {
if (topics.includes('status')) {
formTopics[`RoomStatusTopic#RoomStatusTopic:${roomId}`] = {"broadcaster_uid":roomId}
}
if (topics.includes('silence')) {
formTopics[`RoomSilenceTopic#RoomSilenceTopic:${roomId}`] = {"broadcaster_uid":roomId}
}
});
console.log(formTopics)
// process.exit(3)
form.append('topics', JSON.stringify(formTopics))
form.append('csrfmiddlewaretoken', csrfToken);
return form;
}
// async monitorRealtime(): Promise<void> {
// if (!this.pushServiceAuth) this.pushServiceAuth = await this.getPushServiceAuth();
// if (!this.id) throw new Error('roomId was missing')
// const roomMessageTopicString = this.pushServiceAuth.channels[`RoomMessageTopic#RoomMessageTopic:${this.id}`]
// const roomSilenceTopicString = this.pushServiceAuth.channels[`RoomSilenceTopic#RoomSilenceTopic:${this.id}`]
// const roomStatusTopicString = this.pushServiceAuth.channels[`RoomStatusTopic#RoomStatusTopic:${this.id}`]
// const roomTitleChangeTopicString = this.pushServiceAuth.channels[`RoomTitleChangeTopic#RoomTitleChangeTopic:${this.id}`]
// const roomTipAlertTopicString = this.pushServiceAuth.channels[`RoomTipAlertTopic#RoomTipAlertTopic:${this.id}`]
// const roomPasswordProtectedString = this.pushServiceAuth.channels[`RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:${this.id}`]
// // We probably don't need to call this
// // once per room. Once per scout is probably enough.
// // we need to refactor for this to work.
// this.realtime = await this.ably.getRealtimeClient()
// this.realtime.connection.once('connected', (idk) => {
// this.logger.log({ level: 'info', message: 'CB Realtime Connected!' })
// })
// const messageChannel = this.realtime.channels.get(roomMessageTopicString);
// messageChannel.subscribe((message) => {
// this.onMessage(message)
// })
// const silenceChannel = this.realtime.channels.get(roomSilenceTopicString);
// silenceChannel.subscribe((message) => {
// this.onSilence(message)
// })
// const statusChannel = this.realtime.channels.get(roomStatusTopicString);
// statusChannel.subscribe((message) => {
// this.logger.log({ level: 'debug', message: `got statusChannel message! ${JSON.stringify(message)}`})
// if (message.data.status === 'public') {
// this.onStart(message)
// } else if (message.data.status === 'offline') {
// this.onStop(message)
// }
// this.logger.log({ level: 'debug', message: `Received room:status:<roomId>:0` })
// this.logger.log({ level: 'debug', message: JSON.stringify(message, null, 2) })
// });
// const titleChannel = this.realtime.channels.get(roomTitleChangeTopicString);
// titleChannel.subscribe((message) => {
// this.onTitleChange(message)
// })
// const tipAlertChannel = this.realtime.channels.get(roomTipAlertTopicString);
// tipAlertChannel.subscribe((message) => {
// this.logger.log({ level: 'debug', message: `got tipAlertChannel message! ${JSON.stringify(message)}`})
// this.onTip(message)
// })
// const passwordProtectedChannel = this.realtime.channels.get(roomPasswordProtectedString);
// passwordProtectedChannel.subscribe((message) => {
// this.logger.log({ level: 'debug', message: `got passwordProtectedChannel message! ${JSON.stringify(message)}` })
// this.onPassword(message)
// })
// await this.realtime.connect()
// }
// async refreshToken (): Promise<() => void> {
// const authToken = await this.getNewToken()
// }
}

88
src/TuiTable.js Normal file
View File

@ -0,0 +1,88 @@
// terminal UI
import Table from 'tty-table'
export default class TuiTable {
constructor() {
this.header = [
{
value: 'room',
headerColor: 'cyan',
color: 'white',
align: 'center'
},
{
value: 'pubOrPvt',
headerColor: 'cyan',
color: 'white',
align: 'center'
},
{
value: 'status',
headerColor: 'cyan',
color: 'white',
align: 'center'
},
{
value: 'message',
headerColor: 'cyan',
color: 'white',
align: 'center'
},
{
value: 'tip',
headerColor: 'cyan',
color: 'white',
align: 'center'
},
{
value: 'title',
headerColor: 'cyan',
color: 'white',
align: 'center'
},
{
value: 'silence',
headerColor: 'cyan',
color: 'white',
align: 'center'
}
]
// Example with objects as rows
this.rows = [
]
this.footer = [ ]
this.options = {
borderStyle: "solid",
borderColor: "green",
paddingBottom: 0,
headerAlign: "center",
headerColor: "green",
align: "center",
color: "white",
width: "80%"
}
const t1 = Table(this.header, this.rows, this.footer, this.options).render()
console.log(t1)
}
addRow(data) {
this.rows.push(data)
const t1 = Table(this.header, this.rows, this.footer, this.options).render()
console.log(t1)
}
removeRow() {
}
}

26
src/WorkQueue.js Normal file
View File

@ -0,0 +1,26 @@
const fastq = require('fastq')
export default class WorkQueue() {
constructor(appContext, opts) {
this.concurrency = opts?.concurrency || 12
this.queue = fastq(this.worker, this.concurrency)
return this.queue
}
worker(arg, cb) {
cb(null, arg*2)
}
}
// queue.push(42, function (err, result) {
// if (err) { throw err }
// console.log('the result is', result)
// })
// function worker (arg, cb) {
// cb(null, arg * 2)
// }

18
src/Worker.js Normal file
View File

@ -0,0 +1,18 @@
export default class Worker {
constructor(appContext, opts) {
}
// called when a room goes live
async work () {
}
// called when a room is no longer live
async retire () {
}
}

22
src/appContext.js
View File

@ -1,22 +0,0 @@
import 'dotenv/config'
export const appEnv = new Array('PUBSUB_SERVER_URL')
export function getAppContext(appEnv, logger, db, sound, cb) {
return {
env: appEnv.reduce((acc, ev) => {
if (typeof process.env[ev] === 'undefined') throw new Error(`${ev} is undefined in env`);
acc[ev] = process.env[ev];
return acc;
}, {}),
logger,
db,
roomTimers: {},
sound,
cb
};
}

42
src/appContext.ts Normal file
View File

@ -0,0 +1,42 @@
import 'dotenv/config';
export const appEnv: string[] = ['PUBSUB_SERVER_URL'];
export function getAppContext(
appEnv: string[],
logger: any,
db: any,
sound: any,
cb: any,
dataDir: any,
gotClient: any
) {
return {
env: appEnv.reduce((acc: any, ev: string) => {
if (typeof process.env[ev] === 'undefined') throw new Error(`${ev} is undefined in env`);
acc[ev] = process.env[ev];
return acc;
}, {}),
logger,
db,
roomTimers: {},
sound,
cb,
dataDir,
gotClient,
faye: null,
};
}
export interface IAppContext {
env: Record<string, string>;
logger: any;
db: any;
roomTimers: {};
sound: any;
cb: any;
dataDir: any;
gotClient: any;
faye: any | null;
}

5
src/cb.js
View File

@ -1,10 +1,9 @@
import cheerio from 'cheerio'
import fetch from 'node-fetch'
import { execa } from 'execa'
import pRetry from 'p-retry'
export async function getRandomRoom () {
const res = await fetch('https://chaturbate.com/')
export async function getRandomRoom (appContext) {
const res = await appContext.gotClient.get('https://chaturbate.com/')
const body = await res.text()
const $ = cheerio.load(body)
let roomsRaw = $('a[data-room]')

5
src/cbCallbacks.js
View File

@ -3,6 +3,11 @@ import Stream from './Stream.js'
import { signalStart, sendSignal } from './faye.js'
// import { execa } from 'execa'
export const onCbPassword = async (appContext, roomName, message) => {
appContext.logger.log({ level: 'debug', message: `[ ] room is passworded`})
// const stmt = appContext.db.prepare(`INSERT INTO passwords VALUES `)
// @todo log passworded room events
}
export const onCbTip = async (appContext, roomName, message) => {
appContext.logger.log({ level: 'debug', message: `$$$ [TIP] ${JSON.stringify(message)}` })

10
src/chaturbate.js.old
View File

@ -80,7 +80,7 @@ export async function getRoomId (room) {
}
export async function getRandomRoom () {
const res = await fetch('https://chaturbate.com/')
const res = await gotClient('https://chaturbate.com/')
const body = await res.text()
const $ = cheerio.load(body)
let roomsRaw = $('a[data-room]')
@ -95,7 +95,7 @@ export async function getRandomRoom () {
}
// export async function getInitialRoomDossier(roomName = defaultRoomName) {
// const res = await fetch(`https://chaturbate.com/${roomName}`);
// const res = await gotClient(`https://chaturbate.com/${roomName}`);
// const body = await res.text();
// const $ = cheerio.load(body);
// let rawScript = $('script:contains(window.initialRoomDossier)').html();
@ -107,7 +107,7 @@ export async function getRandomRoom () {
// }
// export async function getViewerCount(roomName = defaultRoomName, presenceId = generateRandomString(millisecondsToHours(Date.now()))) {
// const res = await fetch(`https://chaturbate.com/push_service/room_user_count/${roomName}/?presence_id=${presenceId}`, {
// const res = await gotClient(`https://chaturbate.com/push_service/room_user_count/${roomName}/?presence_id=${presenceId}`, {
// "credentials": "include",
// "headers": {
// "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0",
@ -191,7 +191,7 @@ export async function getRandomRoom () {
// form.append('topics', topics)
// form.append('csrfmiddlewaretoken', csrfToken)
// const res = await fetch("https://chaturbate.com/push_service/auth/", {
// const res = await gotClient("https://chaturbate.com/push_service/auth/", {
// "credentials": "include",
// "headers": {
// "User-Agent": ua,
@ -257,7 +257,7 @@ export async function getRandomRoom () {
// if (typeof tokenCookie === 'undefined') {
// let res = await fetch(cbUrl, {
// let res = await gotClient(cbUrl, {
// "credentials": "omit",
// "headers": {
// "User-Agent": ua,

12
src/gotClient.js Normal file
View File

@ -0,0 +1,12 @@
import plugin from 'got-plugin-debounce';
import got from 'got';
// 1-2 seconds between requests, for courtesy
export default got
.extend(plugin)
.extend({ debounce: [1350, 2000] })
.extend({
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36',
}
})

23
src/logger.js → src/logger.ts
View File

@ -1,25 +1,26 @@
import winston from 'winston'
import winston, { Logger, LoggerOptions } from 'winston';
export const loggerFactory = (options) => {
const mergedOptions = Object.assign({}, {
export const loggerFactory = (options: LoggerOptions): Logger => {
const mergedOptions: LoggerOptions = {
level: 'info',
defaultMeta: { service: 'futureporn' },
format: winston.format.timestamp()
}, options)
const logger = winston.createLogger(mergedOptions);
format: winston.format.timestamp(),
...options
};
const logger: Logger = winston.createLogger(mergedOptions);
if (process.env.NODE_ENV !== 'production') {
logger.add(new winston.transports.Console({
level: 'debug',
format: winston.format.simple()
}))
}));
} else {
logger.add(new winston.transports.Console({
level: 'info',
format: winston.format.json()
}))
}));
}
return logger
}
return logger;
};

1
src/scrap2.json Normal file
View File

@ -0,0 +1 @@
{ "RoomUserPresenceTopic#RoomUserPresenceTopic:7XHKH7V:M79SV5L": { "broadcaster_uid": "7XHKH7V", "user_uid": "M79SV5L" }, "UserMessageTopic#UserMessageTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserChatMediaOpenedTopic#UserChatMediaOpenedTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserChatMediaRemovedTopic#UserChatMediaRemovedTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserPmReadTopic#UserPmReadTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserIgnoreTopic#UserIgnoreTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserTokenUpdateTopic#UserTokenUpdateTopic:M79SV5L": { "user_uid": "M79SV5L" }, "RoomTipAlertTopic#RoomTipAlertTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomPurchaseTopic#RoomPurchaseTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomFanClubJoinedTopic#RoomFanClubJoinedTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomMessageTopic#RoomMessageTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "GlobalPushServiceBackendChangeTopic#GlobalPushServiceBackendChangeTopic": { }, "QualityUpdateTopic#QualityUpdateTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "UserColorUpdateTopic#UserColorUpdateTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserAlertTopic#UserAlertTopic:M79SV5L": { "user_uid": "M79SV5L" }, "RoomNoticeTopic#RoomNoticeTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomEnterLeaveTopic#RoomEnterLeaveTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomModeratorPromotedTopic#RoomModeratorPromotedTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomModeratorRevokedTopic#RoomModeratorRevokedTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomStatusTopic#RoomStatusTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomTitleChangeTopic#RoomTitleChangeTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomSilenceTopic#RoomSilenceTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomKickTopic#RoomKickTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomUpdateTopic#RoomUpdateTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomSettingsTopic#RoomSettingsTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "RoomUserNoticeTopic#RoomUserNoticeTopic:7XHKH7V:M79SV5L": { "broadcaster_uid": "7XHKH7V", "user_uid": "M79SV5L" }, "RoomUserPrivateStatusTopic#RoomUserPrivateStatusTopic:7XHKH7V:M79SV5L": { "broadcaster_uid": "7XHKH7V", "user_uid": "M79SV5L" }, "RoomUserHiddenCamStatusTopic#RoomUserHiddenCamStatusTopic:7XHKH7V:M79SV5L": { "broadcaster_uid": "7XHKH7V", "user_uid": "M79SV5L" }, "RoomLightBlueTopic#RoomLightBlueTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "GameUpdateTopic#GameUpdateTopic:7XHKH7V": { "broadcaster_uid": "7XHKH7V" }, "UserSMCWatchingTopic#UserSMCWatchingTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UserNewsSeenTopic#UserNewsSeenTopic:M79SV5L": { "user_uid": "M79SV5L" }, "OfflineTipNotificationTopic#OfflineTipNotificationTopic:M79SV5L": { "user_uid": "M79SV5L" }, "UpdateOfflineTipNotificationTopic#UpdateOfflineTipNotificationTopic:M79SV5L": { "user_uid": "M79SV5L" } }

41
src/scrap3.json Normal file
View File

@ -0,0 +1,41 @@
// chaturbate channel names are on left hand side
// Ably channel names are on right hand side
{
"channels": {
"RoomUserPresenceTopic#RoomUserPresenceTopic:JQ2YJS5:M79SV5L": "room_user:presence:JQ2YJS5:M79SV5L:0",
"UserMessageTopic#UserMessageTopic:M79SV5L": "user:message:M79SV5L",
"UserChatMediaOpenedTopic#UserChatMediaOpenedTopic:M79SV5L": "user:chatmedia_open:M79SV5L",
"UserChatMediaRemovedTopic#UserChatMediaRemovedTopic:M79SV5L": "user:chatmedia_remove:M79SV5L",
"UserPmReadTopic#UserPmReadTopic:M79SV5L": "user:pm_read:M79SV5L",
"UserIgnoreTopic#UserIgnoreTopic:M79SV5L": "user:ignore:M79SV5L",
"UserTokenUpdateTopic#UserTokenUpdateTopic:M79SV5L": "user:token_update:M79SV5L",
"RoomTipAlertTopic#RoomTipAlertTopic:JQ2YJS5": "room:tip_alert:JQ2YJS5",
"RoomPurchaseTopic#RoomPurchaseTopic:JQ2YJS5": "room:purchase:JQ2YJS5",
"RoomFanClubJoinedTopic#RoomFanClubJoinedTopic:JQ2YJS5": "room:fanclub:JQ2YJS5",
"RoomMessageTopic#RoomMessageTopic:JQ2YJS5": "room:message:JQ2YJS5:0",
"GlobalPushServiceBackendChangeTopic#GlobalPushServiceBackendChangeTopic": "global:push_service",
"QualityUpdateTopic#QualityUpdateTopic:JQ2YJS5": "room:quality_update:JQ2YJS5",
"UserColorUpdateTopic#UserColorUpdateTopic:M79SV5L": "user:color_update:M79SV5L",
"UserAlertTopic#UserAlertTopic:M79SV5L": "user:alert:M79SV5L",
"RoomNoticeTopic#RoomNoticeTopic:JQ2YJS5": "room:notice:JQ2YJS5",
"RoomEnterLeaveTopic#RoomEnterLeaveTopic:JQ2YJS5": "room:enter_leave:JQ2YJS5",
"RoomPasswordProtectedTopic#RoomPasswordProtectedTopic:JQ2YJS5": "room:password_protected:JQ2YJS5:0",
"RoomModeratorPromotedTopic#RoomModeratorPromotedTopic:JQ2YJS5": "room:mod_promoted:JQ2YJS5",
"RoomModeratorRevokedTopic#RoomModeratorRevokedTopic:JQ2YJS5": "room:mod_revoked:JQ2YJS5",
"RoomStatusTopic#RoomStatusTopic:JQ2YJS5": "room:status:JQ2YJS5:0",
"RoomTitleChangeTopic#RoomTitleChangeTopic:JQ2YJS5": "room:title_change:JQ2YJS5",
"RoomSilenceTopic#RoomSilenceTopic:JQ2YJS5": "room:silence:JQ2YJS5",
"RoomKickTopic#RoomKickTopic:JQ2YJS5": "room:kick:JQ2YJS5",
"RoomUpdateTopic#RoomUpdateTopic:JQ2YJS5": "room:update:JQ2YJS5",
"RoomSettingsTopic#RoomSettingsTopic:JQ2YJS5": "room:settings:JQ2YJS5",
"RoomUserNoticeTopic#RoomUserNoticeTopic:JQ2YJS5:M79SV5L": "room_user:notice:JQ2YJS5:M79SV5L",
"RoomUserPrivateStatusTopic#RoomUserPrivateStatusTopic:JQ2YJS5:M79SV5L": "room_user:private_status:JQ2YJS5:M79SV5L",
"RoomUserHiddenCamStatusTopic#RoomUserHiddenCamStatusTopic:JQ2YJS5:M79SV5L": "room_user:status:JQ2YJS5:M79SV5L",
"RoomLightBlueTopic#RoomLightBlueTopic:JQ2YJS5": "lightblue:notice:JQ2YJS5",
"GameUpdateTopic#GameUpdateTopic:JQ2YJS5": "room:game_update:JQ2YJS5",
"UserSMCWatchingTopic#UserSMCWatchingTopic:M79SV5L": "user:smc_watching:M79SV5L"
}
}

1
src/scrap6.json Normal file
View File

File diff suppressed because one or more lines are too long

45
taco.html Normal file
View File

@ -0,0 +1,45 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width">
<title>JS Bin</title>
<script>
(function() {
var tsQueue = [];
var tsWatcher = setInterval(function() {
var tsInstance = window['tsInstance'];
if (tsInstance !== undefined) {
clearInterval(tsWatcher);
while (tsQueue.length > 0) {
tsQueue.shift()(tsInstance);
}
}
}, 50);
window['tsExec'] = function(func) {
if (window['tsInstance'] !== undefined && tsQueue.length <= 0) {
func(window['tsInstance']);
} else {
tsQueue.push(func);
}
}
})();;
if (window.self !== window.top && window.top["onFrameLoad"] !== undefined) {
window.top["onFrameLoad"](window.self)
};
</script>
<script>
window.tsInstance = {
sure: 'ok',
maybe: 'idk'
}
</script>
</head>
<body>
</body>
</html>

2
test/integration/Room.test.js
View File

@ -2,7 +2,7 @@
import chai, { expect } from "chai";
import { EventEmitter } from "node:events";
import chaiEvents from 'chai-events'
import Room from '../../src/Room.js'
import Room from '../../src/Room.ts'
import { projektMelodyCbRoomId } from '../../src/constants.js'

24
test/integration/gotBounce.js Normal file
View File

@ -0,0 +1,24 @@
import got from 'got';
import plugin from 'got-plugin-debounce';
const client = got
.extend(plugin) // load plugin
.extend({ debounce: 2000 }) // random value from 350ms to 450ms between 2 requests
const startedAt = Date.now();
const report = () => new Promise((resolve) => {console.log(`elapsed:${Date.now()-startedAt}`); resolve()})
await Promise.all([
client('https://www.google.com'),
report(),
client('https://www.google.com'),
report(),
client('https://www.google.com'),
report(),
client('https://www.google.com'),
report(),
]);
const elapsed = Date.now() - startedAt;
console.log(elapsed > 1000); // true

83
test/integration/realtime.test.js Normal file
View File

@ -0,0 +1,83 @@
import Room from '../../src/Room.ts';
import ChaturbateAuth from '../../src/ChaturbateAuth.js';
import Realtime from '../../src/Realtime.js';
import chai, { expect } from "chai";
import chaiEvents from 'chai-events';
import gotClient from '../../src/gotClient.js';
import path from 'node:path';
import os from 'node:os';
chai.use(chaiEvents);
describe('realtime', function () {
describe('Can I use my own permission form?', function () {
this.timeout(60*1000)
it('should subscribe to Ably realtime, error free', async function () {
const fakeAppContext = {
logger: {
log: (msg) => console.log(JSON.stringify(msg)),
},
dataDir: path.join(os.homedir(), '.local/share/futureporn-scout'),
gotClient
}
expect(fakeAppContext.logger.log).to.be.a('function')
const cb = new ChaturbateAuth(fakeAppContext)
const ably = new Realtime(fakeAppContext, { chaturbate: cb })
const rooms = [
'projektmelody',
'el_xox',
'skyeanette',
// 'athena_airis'
].map((r) => new Room(fakeAppContext, { name: r, cb, ably }))
const elRoom = rooms[1]
expect(elRoom.url).to.equal('https://chaturbate.com/el_xox/')
expect(elRoom.id).to.be.a('promise')
// get cookie from cb. We only need one for `chaturbate.com/` (not one per room.)
await cb.fetchCookiesIfNeeded(elRoom.url)
const csrfToken = await cb.getCsrfToken(elRoom.url) // doesn't really matter which roomUrl for this token, as long as it is on chaturbate.com domain
const cookieString = await cb.cookieJar.getCookieString(elRoom.url)
console.log('here is the cookiestring')
console.log(cookieString)
await Promise.allSettled((rooms.map((r) => r.id)))
expect(elRoom.id).to.equal('JQ2YJS5')
// @todo in index.js, flag any rooms which don't have an ID. This would suggest a passworded room.
const openRooms = rooms.filter((r) => (!!r.id))
// get permission form for all the rooms we are interested in
const permissionForm = Room.getPermissionsForm(openRooms.map((r) => r.id), ['status', 'silence'], csrfToken)
for (let pair of permissionForm.entries()) {
console.log(pair[0] + ': ' + pair[1]);
}
// get signed ably tokenrequest from cb
const token = await cb.getTokenRequest(elRoom.url, permissionForm, cookieString)
expect(token).to.have.property('keyName')
console.log(token)
// get realtime connection with merged permission form(?)
})
})
})

45
test/unit/tui.test.js Normal file
View File

@ -0,0 +1,45 @@
import TuiTable from '../../src/TuiTable.js'
describe('TuiTable', function() {
it('should display a table', function() {
const tui = new TuiTable()
tui.addRow({
room: 'foo',
pubOrPvt: 'sure',
status: 'idk',
message: '???',
tip: '$',
title: '#',
silence: 'shhh'
})
// Spawn a worker
const worker1 = { room: 'Room 1', pubOrPass: 'Pub', status: 'Running', message: 'Working...', tip: 'Tip 1', title: 'Worker 1', silence: 'No' };
tui.addRow({
room: 'foo',
pubOrPvt: 'sure',
status: 'idk',
message: '???',
tip: '$',
title: '#',
silence: 'shhh'
});
// Simulate worker retirement after some time
setTimeout(() => {
// tui.removeRow(worker1);
tui.addRow({
room: 'foo',
pubOrPvt: 'sure',
status: 'idk',
message: '???',
tip: '$',
title: '#',
silence: 'shhh'
});
}, 1000);
})
})

78
testdex.js Normal file
View File

@ -0,0 +1,78 @@
// import twitter from './src/twitter.js'
import 'dotenv/config'
// import { chat, getViewerCount, monitorRealtimeStatus } from './src/chaturbate.js'
import Room from './src/Room.ts'
import { containsCBInviteLink } from "./src/tweetProcess.js"
import { loggerFactory } from "./logger"
import postgres from 'postgres'
if (typeof process.env.POSTGRES_HOST === 'undefined') throw new Error('POSTGRES_HOST undef');
if (typeof process.env.POSTGRES_USERNAME === 'undefined') throw new Error('POSTGRES_USERNAME undef');
if (typeof process.env.POSTGRES_PASSWORD === 'undefined') throw new Error('POSTGRES_PASSWORD undef');
const logger = loggerFactory({
defaultMeta: { service: 'futureporn/scout' }
})
const sql = postgres({
user: process.env.POSTGRES_USERNAME,
password: process.env.POSTGRES_PASSWORD,
host: process.env.POSTGRES_HOST
})
/**
* tweetConsumer
*
* this is the function that is called when a tweet is detected
*/
const tweetConsumer = (tweet) => {
logger.log({ level: 'debug', message: ` [*] Tweet: ${JSON.stringify(tweet, 0, 2)}` })
if (containsCBInviteLink(tweet)) {
logger.log({ level: 'debug', message: ` [*] The tweet contains a CB invite link.` })
aedes.publish('futureporn/scout/tweet', tweet)
}
}
const onCbStart = () => {
sql.notify('scout/stream/start', { date: new Date().valueOf() })
}
const onCbStop = () => {
sql.notify('scout/stream/stop', { date: new Date().valueOf() })
}
/**
* main
*
* - connect to twitter and listen for new tweets
* - connect to chaturbate chat and watch for spikes in messages per minute
*/
async function main () {
// twitter(tweetConsumer)
// monitorRealtimeStatus('projektmelody', onCbStart, onCbStop)
const room = new Room({
roomName: 'kronniekray',
onStart: onCbStart,
onStop: onCbStop
})
room.monitorRealtime()
}
logger.log({ level: 'info', message: 'hello' })
logger.log({ level: 'info', message: `process.env.NODE_ENV:${process.env.NODE_ENV}` })
main()

15
tsconfig.json Normal file
View File

@ -0,0 +1,15 @@
{
"compilerOptions": {
"allowJs": true,
"module": "es6",
"esModuleInterop": true,
"target": "es6",
"moduleResolution": "node",
"sourceMap": true,
"allowImportingTsExtensions": true,
"outDir": "dist"
},
"lib": [
"es2015"
]
}