refactor
This commit is contained in:
Chris Grimmett
parent
f4934eebd3
commit
2fc77d6963
|
|
@ -1,3 +1,5 @@
|
|||
accounts.db
|
||||
|
||||
.env*
|
||||
venv*
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"extension": [
|
||||
"ts"
|
||||
],
|
||||
"loader": "ts-node/esm",
|
||||
"spec": "test/**/*.spec.ts",
|
||||
"require": "ts-node/register"
|
||||
}
|
||||
257
lib/Cluster.js
257
lib/Cluster.js
|
|
@ -1,257 +0,0 @@
|
|||
|
||||
|
||||
import dotenv from 'dotenv'
|
||||
dotenv.config()
|
||||
import got from 'got'
|
||||
import https from 'https'
|
||||
import path from 'node:path'
|
||||
import { FormData } from 'formdata-node'
|
||||
import fs, { createWriteStream } from 'node:fs'
|
||||
import { pipeline } from 'node:stream'
|
||||
import { promisify } from 'node:util'
|
||||
import { fileFromPath } from "formdata-node/file-from-path"
|
||||
import { loggerFactory } from './logger.js'
|
||||
|
||||
|
||||
// const ipfsClusterExecutable = '/usr/local/bin/ipfs-cluster-ctl'
|
||||
// const ipfsClusterUri = 'https://cluster.sbtp.xyz:9094'
|
||||
|
||||
// const IPFS_CLUSTER_HTTP_API_USERNAME = process.env.IPFS_CLUSTER_HTTP_API_USERNAME;
|
||||
// const IPFS_CLUSTER_HTTP_API_PASSWORD = process.env.IPFS_CLUSTER_HTTP_API_PASSWORD;
|
||||
// const IPFS_CLUSTER_HTTP_API_MULTIADDR = process.env.IPFS_CLUSTER_HTTP_API_MULTIADDR;
|
||||
|
||||
|
||||
// if (typeof IPFS_CLUSTER_HTTP_API_USERNAME === 'undefined') throw new Error('IPFS_CLUSTER_HTTP_API_USERNAME in env is undefined');
|
||||
// if (typeof IPFS_CLUSTER_HTTP_API_PASSWORD === 'undefined') throw new Error('IPFS_CLUSTER_HTTP_API_PASSWORD in env is undefined');
|
||||
// if (typeof IPFS_CLUSTER_HTTP_API_MULTIADDR === 'undefined') throw new Error('IPFS_CLUSTER_HTTP_API_MULTIADDR in env is undefined');
|
||||
|
||||
|
||||
const logger = loggerFactory({
|
||||
defaultMeta: {
|
||||
service: 'futureporn/common'
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
const getArgs = function () {
|
||||
let args = [
|
||||
'--no-check-certificate',
|
||||
'--host', IPFS_CLUSTER_HTTP_API_MULTIADDR,
|
||||
'--basic-auth', `${IPFS_CLUSTER_HTTP_API_USERNAME}:${IPFS_CLUSTER_HTTP_API_PASSWORD}`
|
||||
]
|
||||
return args
|
||||
}
|
||||
|
||||
|
||||
const getHttpsAgent = () => {
|
||||
const httpsAgent = new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
});
|
||||
return httpsAgent
|
||||
}
|
||||
|
||||
|
||||
const fixInvalidJson = (invalidJson) => {
|
||||
return invalidJson
|
||||
.split('\n')
|
||||
.filter((i) => i !== '')
|
||||
.map((datum) => JSON.parse(datum))
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* query the cluster for a list of all the pins
|
||||
*
|
||||
* @resolves {String}
|
||||
*/
|
||||
const ipfsClusterPinsQuery = async () => {
|
||||
const httpsAgent = getHttpsAgent()
|
||||
const res = await fetch(`${ipfsClusterUri}/pins?stream-channels=false`, {
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(IPFS_CLUSTER_HTTP_API_USERNAME+':'+IPFS_CLUSTER_HTTP_API_PASSWORD, "utf-8").toString("base64")}`
|
||||
},
|
||||
agent: httpsAgent
|
||||
})
|
||||
const b = await res.text()
|
||||
const c = b.split('\n')
|
||||
const d = c.filter((i) => i !== '')
|
||||
const e = d.map((datum) => JSON.parse(datum))
|
||||
return e
|
||||
}
|
||||
|
||||
|
||||
const ipfsClusterStatus = async (pin) => {
|
||||
const httpsAgent = getHttpsAgent()
|
||||
const res = await fetch(`${ipfsClusterUri}/pins/${pin}`, {
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(IPFS_CLUSTER_HTTP_API_USERNAME+':'+IPFS_CLUSTER_HTTP_API_PASSWORD, "utf-8").toString("base64")}`
|
||||
},
|
||||
agent: httpsAgent
|
||||
})
|
||||
const b = await res.text()
|
||||
return fixInvalidJson(b)
|
||||
}
|
||||
|
||||
|
||||
const ipfsClusterStatusAll = async (pin) => {
|
||||
const httpsAgent = getHttpsAgent()
|
||||
const res = await fetch(`${ipfsClusterUri}/pins`, {
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(IPFS_CLUSTER_HTTP_API_USERNAME+':'+IPFS_CLUSTER_HTTP_API_PASSWORD, "utf-8").toString("base64")}`
|
||||
},
|
||||
agent: httpsAgent
|
||||
})
|
||||
const b = await res.text()
|
||||
return fixInvalidJson(b)
|
||||
}
|
||||
|
||||
function countPinnedStatus(obj) {
|
||||
let count = 0;
|
||||
console.log(obj.peer_map)
|
||||
for (let key in obj.peer_map) {
|
||||
console.log(`comparing ${obj.peer_map[key].status}`)
|
||||
if (obj.peer_map[key].status === "pinned") {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
export default class Cluster {
|
||||
constructor(opts) {
|
||||
this.username = opts.username
|
||||
this.password = opts.password
|
||||
this.uri = opts.uri || 'https://cluster.sbtp.xyz:9094'
|
||||
if (typeof this.username === 'undefined') throw new Error('username not defined');
|
||||
if (typeof this.password === 'undefined') throw new Error('password not defined');
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* adds pin(s) to the cluster.
|
||||
*/
|
||||
async pinAdd(cid) {
|
||||
if (Array.isArray(cid)) {
|
||||
const results = await Promise.all(cid.map((cid) => this.pinAdd(cid)));
|
||||
return results;
|
||||
}
|
||||
|
||||
if (!cid) return;
|
||||
const opts = {
|
||||
https: { rejectUnauthorized: false },
|
||||
headers: {
|
||||
'Accept': '*/*',
|
||||
'Authorization': `Basic ${Buffer.from(this.username+':'+this.password).toString('base64')}`
|
||||
},
|
||||
isStream: false
|
||||
}
|
||||
|
||||
const res = await got.post(
|
||||
`${this.uri}/pins/${cid}?stream-channels=false`,
|
||||
opts
|
||||
);
|
||||
if (res.ok) {
|
||||
return cid
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async getPinCount(cid) {
|
||||
if (Array.isArray(cid)) throw new Error('getPinCount only supports a string CID as argument')
|
||||
const res = await this.pinStatus(cid);
|
||||
let count = 0;
|
||||
for (const peer of Object.values(res.peer_map)) {
|
||||
if (peer.status === 'pinned') {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
async pinStatus(cid) {
|
||||
if (!cid) throw new Error('required arg cid was not defined');
|
||||
|
||||
const opts = {
|
||||
timeout: {
|
||||
request: 60000,
|
||||
},
|
||||
https: { rejectUnauthorized: false },
|
||||
headers: {
|
||||
'Accept': '*/*',
|
||||
'Authorization': `Basic ${Buffer.from(this.username+':'+this.password).toString('base64')}`
|
||||
},
|
||||
isStream: false
|
||||
};
|
||||
|
||||
try {
|
||||
const json = await got.get(`${this.uri}/pins/${cid}?stream-channels=false`, opts).json();
|
||||
return json;
|
||||
} catch (error) {
|
||||
console.error('THERE WAS AN ERROR');
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async add (filename, fileSize) {
|
||||
const streamPipeline = promisify(pipeline);
|
||||
|
||||
const form = new FormData()
|
||||
form.set('file', await fileFromPath(filename))
|
||||
|
||||
const opts = {
|
||||
https: { rejectUnauthorized: false },
|
||||
body: form,
|
||||
headers: {
|
||||
'Accept': '*/*',
|
||||
'Authorization': `Basic ${Buffer.from(this.username+':'+this.password).toString('base64')}`
|
||||
},
|
||||
isStream: true
|
||||
}
|
||||
|
||||
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
let bytesReport = 0
|
||||
let timer
|
||||
let output
|
||||
try {
|
||||
|
||||
timer = setInterval(() => {
|
||||
if (typeof fileSize !== 'undefined') {
|
||||
logger.log({ level: 'info', message: `adding to IPFS. Progress: ${(bytesReport/fileSize*100).toFixed(2)}%`})
|
||||
} else {
|
||||
logger.log({ level: 'info', message: `adding to IPFS. Bytes transferred: ${bytesReport}` })
|
||||
}
|
||||
}, 60000*5)
|
||||
|
||||
logger.log({ level: 'info', message: `Adding ${filename} to IPFS cluster. Attempt ${i+1}` });
|
||||
const res = await got.post(`${this.uri}/add?cid-version=1&progress=1`, opts);
|
||||
|
||||
// progress updates are streamed from the cluster
|
||||
// for each update, just display it
|
||||
// when a cid exists in the output, it's done.
|
||||
for await (const chunk of res) {
|
||||
const data = JSON.parse(chunk.toString());
|
||||
bytesReport = data?.bytes
|
||||
|
||||
if (data?.cid) {
|
||||
clearInterval(timer)
|
||||
return data.cid;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logger.log({ level: 'error', message: `error while uploading! ${e}` });
|
||||
if (i < 4) {
|
||||
logger.log({ level: 'info', message: `Retrying the upload...` });
|
||||
}
|
||||
clearInterval(timer)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
export interface IB2File {
|
||||
id: number;
|
||||
url: string;
|
||||
key: string;
|
||||
uploadId: string;
|
||||
cdnUrl: string;
|
||||
}
|
||||
|
||||
export function unmarshallB2File(d: any): IB2File | null {
|
||||
if (!d) return null;
|
||||
return {
|
||||
id: d.id,
|
||||
url: d.attributes.url,
|
||||
key: d.attributes.key,
|
||||
uploadId: d.attributes.uploadId,
|
||||
cdnUrl: d.attributes?.cdnUrl
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
10
lib/boss.ts
10
lib/boss.ts
|
|
@ -1,10 +0,0 @@
|
|||
import PgBoss from 'pg-boss';
|
||||
|
||||
|
||||
|
||||
const boss = function(DSN) {
|
||||
const b = new PgBoss(DSN);
|
||||
return b;
|
||||
}
|
||||
|
||||
export default boss;
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
export const ipfsHashRegex = /Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}/;
|
||||
export const strapiUrl = (process.env.NODE_ENV === 'production') ? 'https://portal.futureporn.net' : 'http://localhost:1337'
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
import { execa } from 'execa';
|
||||
import path, { join } from 'node:path';
|
||||
|
||||
|
||||
|
||||
const reportInterval = 60000
|
||||
|
||||
|
||||
async function getTotalFrameCount (filename) {
|
||||
const { exitCode, killed, stdout, stderr } = await execa('ffprobe', [
|
||||
'-v', 'error',
|
||||
'-select_streams', 'v:0',
|
||||
'-show_entries', 'stream=nb_frames',
|
||||
'-of', 'default=nokey=1:noprint_wrappers=1',
|
||||
filename
|
||||
])
|
||||
if (exitCode !== 0 || killed !== false) {
|
||||
throw new Error(`problem while getting frame count. exitCode:${exitCode}, killed:${killed}, stdout:${stdout}, stderr:${stderr}`);
|
||||
}
|
||||
return parseInt(stdout)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @param {string} input
|
||||
* @resolves {string} output
|
||||
*/
|
||||
export async function get240Transcode (appContext, filename) {
|
||||
if (typeof filename === 'undefined') throw new Error('filename is undefined');
|
||||
const progressFilePath = path.join(appContext.env.TMPDIR, 'ffmpeg-progress.log')
|
||||
const outputFilePath = path.join(appContext.env.TMPDIR, path.basename(filename, '.mp4')+'_240p.mp4')
|
||||
const totalFrames = await getTotalFrameCount(filename)
|
||||
|
||||
appContext.logger.log({ level: 'debug', message: `transcoding ${filename} to ${outputFilePath} and saving progress log to ${progressFilePath}` })
|
||||
let progressReportTimer = setInterval(async () => {
|
||||
try {
|
||||
const frame = await getLastFrameNumber(progressFilePath)
|
||||
appContext.logger.log({ level: 'info', message: `transcoder progress-- ${(frame/totalFrames*100).toFixed(2)}%` })
|
||||
} catch (e) {
|
||||
appContext.logger.log({ level: 'info', message: 'we got an error thingy while reading the ffmpeg-progress log but its ok we can just ignore and try again later.' })
|
||||
}
|
||||
}, reportInterval)
|
||||
const { exitCode, killed, stdout, stderr } = await execa('ffmpeg', [
|
||||
'-y',
|
||||
'-i', filename,
|
||||
'-vf', 'scale=w=-2:h=240',
|
||||
'-b:v', '386k',
|
||||
'-b:a', '45k',
|
||||
'-progress', progressFilePath,
|
||||
outputFilePath
|
||||
]);
|
||||
if (exitCode !== 0 || killed !== false) {
|
||||
throw new RemuxError(`exitCode:${exitCode}, killed:${killed}, stdout:${stdout}, stderr:${stderr}`);
|
||||
}
|
||||
appContext.logger.log({ level: 'info', message: 'transcode COMPLETE!' })
|
||||
clearInterval(progressReportTimer)
|
||||
return outputFilePath
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
export const getFilename = (appContext, roomName) => {
|
||||
const name = `${roomName}_${new Date().toISOString()}.ts`
|
||||
return join(appContext.env.FUTUREPORN_WORKDIR, 'recordings', name);
|
||||
}
|
||||
|
||||
64
lib/fleek.js
64
lib/fleek.js
|
|
@ -1,64 +0,0 @@
|
|||
import { request, gql } from 'graphql-request';
|
||||
import { getFleek } from './strapi.js';
|
||||
import { debounce } from 'lodash-es';
|
||||
|
||||
|
||||
export async function triggerWebsiteBuild(appContext) {
|
||||
appContext.logger.log({ level: 'info', message: 'fleek.triggerWebsiteBuild() was executed. Will it actually run? that\'s up to debounce.' })
|
||||
appContext.build = (appContext.build) ? appContext.build : debounce(() => {
|
||||
__triggerWebsiteBuild(appContext)
|
||||
}, 1000*60*30, { leading: true })
|
||||
appContext.build()
|
||||
}
|
||||
|
||||
export async function __triggerWebsiteBuild(appContext) {
|
||||
const fleek = await getFleek(appContext)
|
||||
const { jwt, siteId, endpoint, lastBuild } = fleek.attributes
|
||||
console.log(`jwt:${jwt}, siteId:${siteId}, endpoint:${endpoint}`)
|
||||
const document = gql`
|
||||
mutation triggerDeploy($siteId: ID!) {
|
||||
triggerDeploy(siteId: $siteId) {
|
||||
...DeployDetail
|
||||
__typename
|
||||
}
|
||||
}
|
||||
|
||||
fragment DeployDetail on Deploy {
|
||||
id
|
||||
startedAt
|
||||
completedAt
|
||||
status
|
||||
ipfsHash
|
||||
log
|
||||
published
|
||||
previewImage
|
||||
repository {
|
||||
url
|
||||
name
|
||||
owner
|
||||
branch
|
||||
commit
|
||||
message
|
||||
__typename
|
||||
}
|
||||
gitEvent
|
||||
pullRequestUrl
|
||||
taskId
|
||||
__typename
|
||||
}
|
||||
`
|
||||
const headers = {
|
||||
'authorization': `Bearer ${jwt}`
|
||||
}
|
||||
|
||||
const variables = {
|
||||
siteId,
|
||||
}
|
||||
|
||||
await request(
|
||||
endpoint,
|
||||
document,
|
||||
variables,
|
||||
headers
|
||||
)
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import { IVod } from './vods.js'
|
||||
|
||||
export function getVideoSrcB2LocalFilePath (env: NodeJS.ProcessEnv, vod: IVod) {
|
||||
if (!vod?.videoSrcB2?.key) throw new Error(`vod is missing videoSrcB2.key which is required to download`)
|
||||
const key = vod.videoSrcB2.key
|
||||
const localFilePath = path.join(env.TMPDIR, key)
|
||||
return localFilePath
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
|
||||
export function getRandomInt(min, max) {
|
||||
min = Math.ceil(min);
|
||||
max = Math.floor(max);
|
||||
return Math.floor(Math.random() * (max - min + 1)) + min;
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
import { got } from 'got'
|
||||
|
||||
export async function getVod (appContext, vodId) {
|
||||
const { data } = await got.get(`${appContext.env.STRAPI_URL}/api/vods/${vodId}?populate=*`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
}
|
||||
}).json()
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
export async function updateVod (appContext, vodId, data) {
|
||||
const { data: output } = await got.put(`${appContext.env.STRAPI_URL}/api/vods/${vodId}`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
},
|
||||
json: {
|
||||
data
|
||||
}
|
||||
}).json()
|
||||
return output
|
||||
}
|
||||
|
||||
|
||||
export async function getFleek (appContext) {
|
||||
const { data } = await got.get(`${appContext.env.STRAPI_URL}/api/fleek`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
}
|
||||
}).json()
|
||||
return data
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
import { getVod, updateVod } from './strapi.js'
|
||||
import { getVideoSrcB2LocalFilePath } from './fsCommon.js'
|
||||
import { get240Transcode } from './ffmpeg.js'
|
||||
|
||||
export default async function taskAddVideo240Hash(appContext, body) {
|
||||
appContext.logger.log({ level: 'info', message: `[TASK] AddVideo240Hash begin` })
|
||||
if (body.model === 'vod') {
|
||||
if (body?.entry?.publishedAt) {
|
||||
const vod = await getVod(appContext, body.entry.id)
|
||||
const video240Hash = vod?.attributes?.video240Hash
|
||||
if (!video240Hash) {
|
||||
const videoSrcB2LocalFilePath = getVideoSrcB2LocalFilePath(appContext, vod)
|
||||
const video240LocalFilePath = await get240Transcode(appContext, videoSrcB2LocalFilePath)
|
||||
const cid = await appContext.cluster.add(video240LocalFilePath)
|
||||
const data = await updateVod(appContext, vod.id, { video240Hash: cid })
|
||||
appContext.changed = true
|
||||
appContext.logger.log({ level: 'info', message: `Added ${cid} as video240Hash` })
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- video240Hash already exists'})
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- vod is not published.'})
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- entry is not a vod.'})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
import { getVod, updateVod } from './strapi.js'
|
||||
import { getVideoSrcB2LocalFilePath } from './fsCommon.js'
|
||||
|
||||
|
||||
export default async function taskAddVideoSrcHash(appContext, body) {
|
||||
// if vod
|
||||
// if published
|
||||
// if missing videoSrcHash
|
||||
// ipfs add vod.videoSrcB2 => videoSrcHash
|
||||
// logger.info added
|
||||
// else
|
||||
// logger.debug doing nothing-- videoSrcHash already exists
|
||||
// //if missing video240Hash // too slow, adding in a different task
|
||||
// // transcode
|
||||
// // ipfs add /tmp/vod-1-240.mp4 => video240Hash
|
||||
// //else
|
||||
// // logger.debug doing nothing-- video240Hash already exists
|
||||
// else
|
||||
// logger.debug "doing nothing-- not published"
|
||||
// else
|
||||
// logger.debug "doing nothing-- not a vod"
|
||||
appContext.logger.log({ level: 'info', message: `[TASK] AddVideoSrcHash begin` })
|
||||
if (body.model === 'vod') {
|
||||
if (body?.entry?.publishedAt) {
|
||||
const vod = await getVod(appContext, body.entry.id)
|
||||
const videoSrcHash = vod?.attributes?.videoSrcHash
|
||||
appContext.logger.log({ level: 'debug', message: `>>>>>>here is the videoSrcHash:${videoSrcHash}`})
|
||||
if (!videoSrcHash) {
|
||||
const cid = await appContext.cluster.add(getVideoSrcB2LocalFilePath(appContext, vod))
|
||||
const data = await updateVod(appContext, vod.id, { videoSrcHash: cid })
|
||||
appContext.changed = true
|
||||
appContext.logger.log({ level: 'info', message: `Added ${cid} as videoSrcHash` })
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- videoSrcHash already exists'})
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- entry is not published.'})
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- entry is not a vod.'})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,171 +0,0 @@
|
|||
import { got } from 'got';
|
||||
import EleventyFetch from "@11ty/eleventy-fetch";
|
||||
|
||||
|
||||
export async function getPatreonCampaign() {
|
||||
return EleventyFetch('https://www.patreon.com/api/campaigns/8012692', {
|
||||
duration: "1d",
|
||||
type: "json",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
export async function getPatreonCampaignPledgeSum() {
|
||||
const campaign = await getPatreonCampaign()
|
||||
return campaign.data.attributes.pledge_sum
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Calculate how many mux allocations the site should have, based on the dollar amount of pledges from patreon
|
||||
*
|
||||
* @param {Number} pledgeSum - USD cents
|
||||
*/
|
||||
export function getMuxAllocationCount(pledgeSum) {
|
||||
const dollarAmount = pledgeSum / 100; // convert USD cents to USD dollars
|
||||
const muxAllocationCount = Math.floor(dollarAmount / 50); // calculate the number of mux allocations required
|
||||
return muxAllocationCount;
|
||||
}
|
||||
|
||||
export async function getAllPublishedVodsSortedByDate(appContext) {
|
||||
const { data } = await got.get(`${appContext.env.STRAPI_URL}/api/vods?sort[0]=date%3Adesc&populate[0]=muxAsset&populate[1]=videoSrcB2`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
}
|
||||
}).json()
|
||||
return data
|
||||
}
|
||||
|
||||
export async function idempotentlyAddMuxToVod(appContext, vod) {
|
||||
if (!vod?.attributes?.videoSrcB2?.data?.attributes?.url) throw new Error(`vod is missing videoSrcB2 url which is required to add to Mux`);
|
||||
const isActNeeded = (!vod?.attributes?.muxAsset?.data?.attributes?.playbackId) ? true : false
|
||||
|
||||
|
||||
if (isActNeeded) {
|
||||
// const { data: muxData } = await got.post(`${appContext.env.STRAPI_API_KEY}/mux-video-uploader/submitRemoteUpload`, {
|
||||
// headers: {
|
||||
// 'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
// },
|
||||
// form: {
|
||||
// title: vod.attributes.date,
|
||||
// url: vod.attributes.videoSrcB2.url
|
||||
// }
|
||||
// }).json()
|
||||
|
||||
|
||||
appContext.logger.log({ level: 'debug', message: `Creating Mux asset for vod ${vod.id} (${vod.attributes.videoSrcB2.data.attributes.cdnUrl})` })
|
||||
const { data: muxData } = await got.post('https://api.mux.com/video/v1/assets', {
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(`${appContext.env.MUX_TOKEN_ID}:${appContext.env.MUX_TOKEN_SECRET}`).toString('base64')}`
|
||||
},
|
||||
json: {
|
||||
"input": vod.attributes.videoSrcB2.data.attributes.cdnUrl,
|
||||
"playback_policy": [
|
||||
"signed"
|
||||
]
|
||||
}
|
||||
}).json()
|
||||
|
||||
|
||||
appContext.logger.log({level: 'debug', message: `Adding Mux Asset to strapi`})
|
||||
const { data: muxAssetData } = await got.post(`${appContext.env.STRAPI_URL}/api/mux-assets`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
},
|
||||
json: {
|
||||
data: {
|
||||
playbackId: muxData.playback_ids.find((p) => p.policy === 'signed').id,
|
||||
assetId: muxData.id
|
||||
}
|
||||
}
|
||||
}).json()
|
||||
|
||||
|
||||
appContext.logger.log({ level: 'debug', message: `Relating Mux Asset to Vod ${vod.id}` })
|
||||
const { data: strapiData } = await got.put(`${appContext.env.STRAPI_URL}/api/vods/${vod.id}`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
},
|
||||
json: {
|
||||
data: {
|
||||
muxAsset: muxAssetData.id
|
||||
}
|
||||
}
|
||||
}).json()
|
||||
appContext.changed = true
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing. Mux Asset is already present.' })
|
||||
}
|
||||
}
|
||||
|
||||
export async function idempotentlyRemoveMuxFromVod(appContext, vod) {
|
||||
// first see if a Mux playback ID is already absent
|
||||
// second, optionally act to ensure that the Mux playback ID is absent.
|
||||
// if we acted, also delete the Mux asset
|
||||
|
||||
// if (actNeeded)
|
||||
// remove Mux playback ID from Vod
|
||||
// delete Mux asset
|
||||
|
||||
const isActNeeded = (vod?.attributes?.muxAsset?.data?.attributes?.assetId) ? true : false
|
||||
if (isActNeeded) {
|
||||
appContext.logger.log({ level: 'debug', message: `Deleting Mux Asset for vod ${vod.id}` })
|
||||
const { data: muxData } = await got.delete(`https://api.mux.com/video/v1/assets/${vod.attributes.muxAsset.data.attributes.assetId}`, {
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(`${appContext.env.MUX_TOKEN_ID}:${appContext.env.MUX_TOKEN_SECRET}`).toString('base64')}`
|
||||
}
|
||||
}).json()
|
||||
|
||||
appContext.logger.log({ level: 'debug', message: `Removing Mux Asset relation from Vod ${vod.id}` })
|
||||
const { data: strapiData } = await got.put(`${appContext.env.STRAPI_URL}/api/vods/${vod.id}`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${appContext.env.STRAPI_API_KEY}`
|
||||
},
|
||||
json: {
|
||||
data: {
|
||||
muxAsset: null
|
||||
}
|
||||
}
|
||||
}).json()
|
||||
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing. Mux Asset is already absent.' })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export function getMuxTargetVods(appContext, muxAllocationCount, vods) {
|
||||
// get last N published vods
|
||||
// where N is muxAllocationCount
|
||||
return {
|
||||
vodsForMux: vods.slice(0, muxAllocationCount),
|
||||
vodsNotForMux: vods.slice(muxAllocationCount)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export async function createMuxAsset(appContext, videoUrl) {
|
||||
const { data } = await got.post('https://api.mux.com/video/v1/assets', {
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(`${appContext.env.MUX_TOKEN_ID}:${appContext.env.MUX_TOKEN_SECRET}`).toString('base64')}`
|
||||
},
|
||||
json: {
|
||||
input: videoUrl,
|
||||
playback_policy: ['signed']
|
||||
}
|
||||
}).json()
|
||||
}
|
||||
|
||||
export async function taskAllocateMux(appContext) {
|
||||
appContext.logger.log({ level: 'info', message: 'taskAllocateMux begin' })
|
||||
const pledgeSum = await getPatreonCampaignPledgeSum()
|
||||
const muxAllocationCount = getMuxAllocationCount(pledgeSum)
|
||||
appContext.logger.log({ level: 'debug', message: `pledgeSum:${pledgeSum}, muxAllocationCount:${muxAllocationCount}` })
|
||||
|
||||
const vods = await getAllPublishedVodsSortedByDate(appContext)
|
||||
const { vodsForMux, vodsNotForMux } = getMuxTargetVods(appContext, muxAllocationCount, vods)
|
||||
appContext.logger.log({ level: 'debug', message: `vodsForMux:${vodsForMux.map((v)=>v.id)}, vodsNotForMux:${vodsNotForMux.map((v)=>v.id)}`})
|
||||
for (const vod of vodsForMux) { await idempotentlyAddMuxToVod(appContext, vod) };
|
||||
for (const vod of vodsNotForMux) { await idempotentlyRemoveMuxFromVod(appContext, vod) }
|
||||
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
|
||||
// const dest = (process.env.NODE_ENV === 'production') ? '/bin' : join(homedir(), '.local', 'bin')
|
||||
// console.log(`ffmpeg dest:${dest}`)
|
||||
// await ffbinaries.downloadFiles({
|
||||
// destination: dest,
|
||||
// platform: 'linux'
|
||||
// }, function (err, data) {
|
||||
// if (err) {
|
||||
// console.log('error while downloading ffmpeg binaries ')
|
||||
// throw err
|
||||
// } else {
|
||||
// console.log(data)
|
||||
// }
|
||||
// });
|
||||
|
||||
|
||||
|
||||
import { join } from 'path';
|
||||
import { spawn } from 'child_process';
|
||||
import ffbinaries from 'ffbinaries';
|
||||
import fs from 'node:fs';
|
||||
|
||||
export const getFilename = (appContext, roomName) => {
|
||||
const name = `${roomName}_${new Date().toISOString()}.ts`
|
||||
return join(appContext.env.FUTUREPORN_WORKDIR, 'recordings', name);
|
||||
}
|
||||
|
||||
|
||||
export const assertDirectory = (appContext, directoryPath) => {
|
||||
appContext.logger.log({ level: 'info', message: `asserting existence of ${directoryPath}` })
|
||||
if (fs.statSync(directoryPath, { throwIfNoEntry: false }) === undefined) fs.mkdirSync(directoryPath, { recursive: true });
|
||||
}
|
||||
|
||||
export default async function assertFFmpeg(appContext) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const childProcess = spawn('ffmpeg', ['-version']);
|
||||
|
||||
childProcess.on('error', (err) => {
|
||||
appContext.logger.log({
|
||||
level: 'warn',
|
||||
message: `ffmpeg -version failed, which likely means ffmpeg is not installed or not on $PATH`,
|
||||
});
|
||||
appContext.logger.log({
|
||||
level: 'info',
|
||||
message: 'downloading ffmpeg binary'
|
||||
})
|
||||
|
||||
const dest = join(appContext.env.FUTUREPORN_WORKDIR, 'bin');
|
||||
assertDirectory(appContext, dest);
|
||||
|
||||
appContext.logger.log({
|
||||
level: 'info',
|
||||
message: 'downloading ffmpeg'
|
||||
})
|
||||
ffbinaries.downloadFiles({ destination: dest, platform: 'linux' }, function (err, data) {
|
||||
if (err) reject(err)
|
||||
else resolve()
|
||||
});
|
||||
});
|
||||
|
||||
childProcess.on('exit', (code) => {
|
||||
if (code !== 0) reject(`'ffmpeg -version' exited with code ${code}`)
|
||||
if (code === 0) {
|
||||
appContext.logger.log({ level: 'info', message: `ffmpeg PRESENT.` });
|
||||
resolve()
|
||||
}
|
||||
});
|
||||
})
|
||||
};
|
||||
|
||||
export const assertDependencyDirectory = (appContext) => {
|
||||
// Extract the directory path from the filename
|
||||
const directoryPath = join(appContext.env.FUTUREPORN_WORKDIR, 'recordings');
|
||||
console.log(`asserting ${directoryPath} exists`)
|
||||
|
||||
// Check if the directory exists, and create it if it doesn't
|
||||
if (!fs.existsSync(directoryPath)) {
|
||||
fs.mkdirSync(directoryPath, { recursive: true });
|
||||
console.log(`Created directory: ${directoryPath}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
export default async function taskContinueOnlyIfPublished (appContext, body) {
|
||||
appContext.logger.log({ level: 'info', message: 'taskContinueOnlyIfPublished started.' })
|
||||
appContext.logger.log({ level: 'info', message: JSON.stringify(body, 0, 2) })
|
||||
if (body.model !== 'vod' || (body.entry && body.entry.publishedAt === null) ) {
|
||||
appContext.logger.log({ level: 'info', message: `WEEE WOO WEE WOOO this is not a vod!... or it's not published. either way we stop doing tasks here.` })
|
||||
throw new Error('Aborting tasks because this is not a vod or not published.')
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: `taskContinueOnlyIfPublished is complete. This is a published vod so we are continuing with tasks.` })
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
import { downloadVideoSrcB2 } from './b2.js';
|
||||
import { getVod } from './strapi.js';
|
||||
import fs from 'node:fs';
|
||||
import { getVideoSrcB2LocalFilePath } from './fsCommon.js';
|
||||
|
||||
|
||||
|
||||
export default async function taskDownloadVideoSrcB2 (appContext, body) {
|
||||
appContext.logger.log({ level: 'info', message: 'taskDownloadVideoSrcB2 started.' })
|
||||
if (body.model === 'vod') {
|
||||
const vod = await getVod(appContext, body.entry.id)
|
||||
// download is only necessary when thumbnail or videoSrcHash is missing
|
||||
const hasThumbnail = (vod?.attributes?.thumbnailB2?.data?.attributes?.cdnUrl) ? true : false
|
||||
const hasVideoSrcHash = (vod?.attributes?.videoSrcHash) ? true : false
|
||||
if (!hasThumbnail || !hasVideoSrcHash) {
|
||||
await downloadVideoSrcB2(appContext, vod)
|
||||
} else {
|
||||
appContext.logger.log({ level: 'info', message: 'Doing nothing-- No need for downloading videoSrcB2.' })
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'info', message: 'Doing nothing-- entry is not a vod.'})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
|
||||
|
||||
|
||||
export async function idempotentlyPinIpfsContent(appContext, body) {
|
||||
let results = []
|
||||
const cids = [
|
||||
body?.entry?.videoSrcHash,
|
||||
body?.entry?.video240Hash,
|
||||
body?.entry?.thiccHash
|
||||
]
|
||||
appContext.logger.log({ level: 'info', message: `Here are the CIDs yoinked fresh from the webhook:${JSON.stringify(cids)}` })
|
||||
|
||||
const validCids = cids.filter((c) => c !== '' && c !== null && c !== undefined)
|
||||
appContext.logger.log({ level: 'info', message: `Here are the valid CIDs:${JSON.stringify(validCids)}` })
|
||||
if (validCids.length === 0) return results
|
||||
for (const vc of validCids) {
|
||||
appContext.logger.log({ level: 'info', message: `checking to see if ${vc} is pinned` })
|
||||
const pinCount = await appContext.cluster.getPinCount(vc)
|
||||
if (pinCount < 1) {
|
||||
appContext.logger.log({ level: 'info', message: `${vc} is pinned on ${pinCount} appContext.cluster peers.` })
|
||||
const pinnedCid = await appContext.cluster.pinAdd(vc)
|
||||
results.push(pinnedCid)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
export async function taskPinIpfsContent(appContext, body) {
|
||||
appContext.logger.log({ level: 'info', message: `idempotentlyPinIpfsContent` })
|
||||
appContext.logger.log({ level: 'info', message: JSON.stringify(body) })
|
||||
const pins = await idempotentlyPinIpfsContent(appContext, body, appContext.cluster)
|
||||
appContext.logger.log({ level: 'info', message: `${JSON.stringify(pins)}` })
|
||||
|
||||
if (pins.length > 0) {
|
||||
appContext.logger.log({ level: 'info', message: `Pinned ${pins}` })
|
||||
return {
|
||||
message: `Pinned ${pins}`
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'info', message: `Nothing to pin!` })
|
||||
return {
|
||||
message: `Nothing to pin`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
|
||||
export default async function taskTriggerWebsiteBuild (appContext) {
|
||||
|
||||
if (appContext.changed) {
|
||||
appContext.logger.log({
|
||||
level: 'info',
|
||||
message: `@TODO -- trigger website build. This is not automated at the moment because we build on local dev machine and we don't have a good way of triggering a build there. Maybe some zerotier network with faye would be a solution here.`
|
||||
})
|
||||
}
|
||||
}
|
||||
113
lib/vtubers.ts
113
lib/vtubers.ts
|
|
@ -1,113 +0,0 @@
|
|||
|
||||
import { IVod } from './vods.js'
|
||||
import { strapiUrl } from './constants.js';
|
||||
import { getSafeDate } from './dates.js';
|
||||
import qs from 'qs';
|
||||
|
||||
export interface IVtuber {
|
||||
id: number;
|
||||
slug: string;
|
||||
displayName: string;
|
||||
chaturbate?: string;
|
||||
twitter?: string;
|
||||
patreon?: string;
|
||||
twitch?: string;
|
||||
tiktok?: string;
|
||||
onlyfans?: string;
|
||||
youtube?: string;
|
||||
linktree?: string;
|
||||
carrd?: string;
|
||||
fansly?: string;
|
||||
pornhub?: string;
|
||||
discord?: string;
|
||||
reddit?: string;
|
||||
throne?: string;
|
||||
instagram?: string;
|
||||
facebook?: string;
|
||||
merch?: string;
|
||||
vods: IVod[];
|
||||
description1: string;
|
||||
description2?: string;
|
||||
image: string;
|
||||
imageBlur?: string;
|
||||
themeColor: string;
|
||||
}
|
||||
|
||||
|
||||
export function unmarshallVtuber(d: any): IVtuber {
|
||||
if (!d) {
|
||||
console.error('panick! unmarshallVTuber was called with undefined data')
|
||||
console.trace()
|
||||
}
|
||||
return {
|
||||
id: d.id,
|
||||
slug: d.attributes?.slug,
|
||||
displayName: d.attributes.displayName,
|
||||
chaturbate: d.attributes?.chaturbate,
|
||||
twitter: d.attributes?.twitter,
|
||||
patreon: d.attributes?.patreon,
|
||||
twitch: d.attributes?.twitch,
|
||||
tiktok: d.attributes?.tiktok,
|
||||
onlyfans: d.attributes?.onlyfans,
|
||||
youtube: d.attributes?.youtube,
|
||||
linktree: d.attributes?.linktree,
|
||||
carrd: d.attributes?.carrd,
|
||||
fansly: d.attributes?.fansly,
|
||||
pornhub: d.attributes?.pornhub,
|
||||
discord: d.attributes?.discord,
|
||||
reddit: d.attributes?.reddit,
|
||||
throne: d.attributes?.throne,
|
||||
instagram: d.attributes?.instagram,
|
||||
facebook: d.attributes?.facebook,
|
||||
merch: d.attributes?.merch,
|
||||
description1: d.attributes.description1,
|
||||
description2: d.attributes?.description2,
|
||||
image: d.attributes.image,
|
||||
imageBlur: d.attributes?.imageBlur,
|
||||
themeColor: d.attributes.themeColor,
|
||||
vods: d.attributes.vods
|
||||
}
|
||||
}
|
||||
|
||||
export async function getVtuberBySlug(slug: string): Promise<IVtuber> {
|
||||
const query = qs.stringify(
|
||||
{
|
||||
filters: {
|
||||
slug: {
|
||||
$eq: slug
|
||||
}
|
||||
},
|
||||
// populate: {
|
||||
// vods: {
|
||||
// fields: ['id', 'videoSrcHash'],
|
||||
// populate: ['vtuber']
|
||||
// }
|
||||
// }
|
||||
}
|
||||
)
|
||||
|
||||
return fetch(`${strapiUrl}/api/vtubers?${query}`)
|
||||
.then((res) => res.json())
|
||||
.then((d) => {
|
||||
const vtuberData = d.data[0]
|
||||
|
||||
return unmarshallVtuber(vtuberData)
|
||||
})
|
||||
}
|
||||
|
||||
export async function getVtuberById(id: number): Promise<IVtuber> {
|
||||
return fetch(`${strapiUrl}/api/vtubers?filters[id][$eq]=${id}`)
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
return unmarshallVtuber(data.data[0])
|
||||
})
|
||||
}
|
||||
|
||||
export async function getVtubers(): Promise<IVtuber[]> {
|
||||
return fetch(`${strapiUrl}/api/vtubers`)
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
return data.data.map((d: IVtuber) => unmarshallVtuber(d))
|
||||
})
|
||||
}
|
||||
|
||||
115
manager.ts
115
manager.ts
|
|
@ -1,39 +1,92 @@
|
|||
import dotenv from "dotenv"
|
||||
dotenv.config()
|
||||
import PgBoss from 'pg-boss';
|
||||
import dotenv from "dotenv";
|
||||
dotenv.config();
|
||||
import pkg from './package.json' assert {type: 'json'};
|
||||
|
||||
|
||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env')
|
||||
const DATABASE_URL = process.env.DATABASE_URL
|
||||
import { Queue } from 'bullmq';
|
||||
import { REDIS_HOST, REDIS_PORT, PORT, TASK_LIST } from './src/env.js';
|
||||
import express, { Request, Response } from 'express';
|
||||
import { createBullBoard } from "@bull-board/api";
|
||||
import { ExpressAdapter } from '@bull-board/express';
|
||||
import { BullMQAdapter } from '@bull-board/api/bullMQAdapter.js';
|
||||
import { default as Redis } from 'ioredis'
|
||||
import { registerRoutes } from "./src/register-routes.js";
|
||||
import * as swaggerJson from './dist/futureporn-qa/swagger/swagger.json';
|
||||
import * as swaggerUI from 'swagger-ui-express';
|
||||
|
||||
console.log(`futureporn-qa manager version ${pkg.version}`)
|
||||
|
||||
async function main () {
|
||||
const boss = new PgBoss({
|
||||
connectionString: DATABASE_URL,
|
||||
monitorStateIntervalSeconds: 5
|
||||
});
|
||||
const s = await boss.start();
|
||||
s.on('monitor-states', (states) => {
|
||||
// console.log(states)
|
||||
const { queues, active, created, completed, failed } = states;
|
||||
for (const q in queues) {
|
||||
if (queues.hasOwnProperty(q)) {
|
||||
const queueState = queues[q];
|
||||
// Now you can work with queueName and queueState
|
||||
console.log(`${q}-- active:${active}, created:${created}, completed:${completed}, failed:${failed}`);
|
||||
// console.log(q, queueState);
|
||||
}
|
||||
const connection = new Redis.default({
|
||||
port: parseInt(REDIS_PORT),
|
||||
host: REDIS_HOST,
|
||||
maxRetriesPerRequest: null,
|
||||
});
|
||||
|
||||
|
||||
const taskList = TASK_LIST.split(',');
|
||||
let adapters: BullMQAdapter[] = [];
|
||||
|
||||
async function main() {
|
||||
|
||||
|
||||
for (const task of taskList) {
|
||||
console.log(`> setting up ${task}`)
|
||||
const queue: Queue = new Queue(task, {
|
||||
connection
|
||||
});
|
||||
adapters.push(new BullMQAdapter(queue));
|
||||
if (task === 'identifyVodIssues') {
|
||||
await queue.add(
|
||||
'identifyVodIssues',
|
||||
{
|
||||
foo: 'blah',
|
||||
},
|
||||
{
|
||||
repeat: {
|
||||
every: 10000
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
// const printQueueStats = (queueName, queue) => {
|
||||
// console.log(` ${queueName} created:${queue.created}, retry:${queue.retry}, active:${queue.active}, completed:${queue.completed}, failed:${queue.failed}`);
|
||||
// };
|
||||
// printQueueStats('generateThumbnail', queues.generateThumbnail);
|
||||
// printQueueStats('identifyVodIssues', queues.identifyVodIssues);
|
||||
}
|
||||
|
||||
|
||||
|
||||
const serverAdapter = new ExpressAdapter();
|
||||
serverAdapter.setBasePath('/admin/queues');
|
||||
|
||||
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
|
||||
queues: adapters,
|
||||
serverAdapter: serverAdapter,
|
||||
});
|
||||
await boss.send('identifyVodIssues', { priority: 10 });
|
||||
await boss.schedule('identifyVodIssues', '* * * * *', { priority: 10 }); // every minute
|
||||
|
||||
|
||||
const app = express();
|
||||
app.use('/admin/queues', serverAdapter.getRouter());
|
||||
registerRoutes(app);
|
||||
|
||||
app.use(["/openapi", "/docs", "/swagger"], swaggerUI.serve, swaggerUI.setup(swaggerJson));
|
||||
app.get('/', (req: Request, res: Response) => {
|
||||
res.send(`
|
||||
<h1>futureporn-qa</h1>
|
||||
<h2>Meta</h2>
|
||||
<ul>
|
||||
<li>version ${pkg.version}</li>
|
||||
<li><a href="https://gitea.futureporn.net/futureporn/futureporn-qa">git repo</a></li>
|
||||
</ul>
|
||||
<h2>Pages</h2>
|
||||
<ul>
|
||||
<li><a href="/admin/queues">Bull Dashboard (queues)</a></li>
|
||||
<li><a href="/openapi">Swagger (API reference)</a></li>
|
||||
</ul>
|
||||
`)
|
||||
})
|
||||
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Listening on ${PORT}`);
|
||||
console.log(`For the UI, open http://localhost:${PORT}/admin/queues`);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
main();
|
||||
|
|
|
|||
61
package.json
61
package.json
|
|
@ -1,46 +1,61 @@
|
|||
{
|
||||
"name": "qa",
|
||||
"version": "3.0.3",
|
||||
"name": "futureporn-qa",
|
||||
"version": "4.0.0",
|
||||
"main": "index.js",
|
||||
"license": "Unlicense",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@11ty/eleventy-fetch": "^4.0.0",
|
||||
"@aws-sdk/client-s3": "^3.328.0",
|
||||
"@aws-sdk/client-s3": "^3.438.0",
|
||||
"@bull-board/api": "5.8.4",
|
||||
"@bull-board/express": "5.8.4",
|
||||
"@bull-board/ui": "5.8.4",
|
||||
"@multiformats/multiaddr-to-uri": "^9.0.7",
|
||||
"@paralleldrive/cuid2": "^2.2.2",
|
||||
"@types/node": "^20.6.5",
|
||||
"@swc/core": "^1.3.95",
|
||||
"@tsoa/runtime": "^5.0.0",
|
||||
"@types/node": "^20.8.10",
|
||||
"@types/qs": "^6.9.9",
|
||||
"bullmq": "^4.12.7",
|
||||
"date-fns": "^2.30.0",
|
||||
"date-fns-tz": "^2.0.0",
|
||||
"discord.js": "^14.9.0",
|
||||
"dotenv": "^16.0.3",
|
||||
"execa": "^7.1.1",
|
||||
"fastify": "^4.11.0",
|
||||
"fastq": "^1.15.0",
|
||||
"dotenv": "^16.3.1",
|
||||
"execa": "^7.2.0",
|
||||
"express": "^4.18.2",
|
||||
"ffbinaries": "^1.1.5",
|
||||
"fluent-json-schema": "^4.1.0",
|
||||
"formdata-node": "^5.0.0",
|
||||
"got": "^12.6.0",
|
||||
"graphql-request": "^6.0.0",
|
||||
"fluent-json-schema": "^4.1.2",
|
||||
"formdata-node": "^5.0.1",
|
||||
"got": "^12.6.1",
|
||||
"ioredis": "^5.3.2",
|
||||
"lodash-es": "^4.17.21",
|
||||
"method-override": "^3.0.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"pg-boss": "^9.0.3",
|
||||
"prevvy": "^6.0.0",
|
||||
"only-allow": "^1.2.1",
|
||||
"prevvy": "^7.0.1",
|
||||
"python-shell": "^5.0.0",
|
||||
"qs": "^6.11.2",
|
||||
"swagger-ui-express": "^5.0.0",
|
||||
"ts-node": "^10.9.1",
|
||||
"ts-node-esm": "^0.0.6",
|
||||
"tsoa": "^5.1.1",
|
||||
"tsx": "^3.14.0",
|
||||
"typescript": "^5.2.2",
|
||||
"winston": "^3.8.2",
|
||||
"winston": "^3.11.0",
|
||||
"yargs": "^17.7.2"
|
||||
},
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"preinstall": "npx only-allow pnpm",
|
||||
"start:manager": "node ./dist/manager.js",
|
||||
"start:worker": "node ./dist/worker.js",
|
||||
"test": "mocha",
|
||||
"build": "tsc --project ./tsconfig.json",
|
||||
"tunnel": "wg-quick up ./tunnel.conf; echo 'press enter to close tunnel'; read _; wg-quick down ./tunnel.conf",
|
||||
"dev:manager": "nodemon --watch manager.ts --watch .env --watch package.json --watch tasks --ext ts,json,js --exec \"node --inspect --loader ts-node/esm ./manager.ts\"",
|
||||
"dev:worker": "nodemon --watch worker.ts --watch .env --watch package.json --watch tasks --ext ts,json,js --exec \"node --inspect --loader ts-node/esm ./worker.ts\"",
|
||||
"tsoa": "tsoa spec-and-routes",
|
||||
"dev:manager.old": "nodemon --watch manager.ts --watch .env --watch package.json --watch tasks --ext ts,json,js --exec \"node --inspect --loader ts-node/esm ./manager.ts\"",
|
||||
"dev:manager": "tsx watch ./manager.ts",
|
||||
"dev:worker.old": "nodemon --watch worker.ts --watch .env --watch package.json --watch tasks --ext ts,json,js --exec \"node --inspect --loader ts-node/esm ./worker.ts\"",
|
||||
"dev:worker": "tsx watch ./worker.ts",
|
||||
"dev:old": "nodemon --watch lib --watch .env --watch package.json --watch index.js index.js"
|
||||
},
|
||||
"exports": [
|
||||
|
|
@ -48,9 +63,15 @@
|
|||
"./worker.js"
|
||||
],
|
||||
"devDependencies": {
|
||||
"chai": "^4.3.7",
|
||||
"concurrently": "^8.0.1",
|
||||
"@types/chai": "^4.3.9",
|
||||
"@types/chai-as-promised": "^7.1.7",
|
||||
"@types/mocha": "^10.0.3",
|
||||
"chai": "^4.3.10",
|
||||
"chai-as-promised": "^7.1.1",
|
||||
"client": "link:fetch-mock/esm/client",
|
||||
"concurrently": "^8.2.2",
|
||||
"mocha": "^10.2.0",
|
||||
"nock": "^13.3.7",
|
||||
"nodemon": "^2.0.22"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
4499
pnpm-lock.yaml
4499
pnpm-lock.yaml
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,179 @@
|
|||
accounts.db
|
||||
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/python
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=python
|
||||
|
||||
### Python ###
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
### Python Patch ###
|
||||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
|
||||
poetry.toml
|
||||
|
||||
# ruff
|
||||
.ruff_cache/
|
||||
|
||||
# LSP config files
|
||||
pyrightconfig.json
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/python
|
||||
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
aiosqlite==0.19.0
|
||||
anyio==4.0.0
|
||||
asyncio==3.4.3
|
||||
certifi==2023.7.22
|
||||
fake-useragent==1.3.0
|
||||
h11==0.14.0
|
||||
httpcore==0.18.0
|
||||
httpx==0.25.0
|
||||
idna==3.4
|
||||
loguru==0.7.2
|
||||
sniffio==1.3.0
|
||||
twscrape==0.9.0
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import argparse
|
||||
import asyncio
|
||||
from twscrape import API, gather
|
||||
from twscrape.logger import set_log_level
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Twitter Scraper")
|
||||
parser.add_argument("--accounts", nargs='+', required=True, help="List of account credentials in the format 'username:password:email:mail_password'")
|
||||
parser.add_argument("--search_query", required=True, help="Search query")
|
||||
return parser.parse_args()
|
||||
|
||||
async def main():
|
||||
args = parse_args()
|
||||
|
||||
api = API() # or API("path-to.db") - default is `accounts.db`
|
||||
|
||||
for account in args.accounts:
|
||||
username, password, email, mail_password = account.split(':')
|
||||
await api.pool.add_account(username, password, email, mail_password)
|
||||
|
||||
await api.pool.login_all()
|
||||
|
||||
# search
|
||||
async for tweet in api.search(args.search_query, limit=10):
|
||||
print(tweet.json())
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
|
||||
import { type Logger } from 'winston';
|
||||
import { ConnectionOptions, type Job } from 'bullmq';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import { IVod } from "./vods.js";
|
||||
|
||||
export interface ITaskOptions {
|
||||
env: NodeJS.ProcessEnv;
|
||||
logger: Logger;
|
||||
job: Job;
|
||||
connection: ConnectionOptions;
|
||||
}
|
||||
|
||||
|
||||
export interface ITask {
|
||||
name: string;
|
||||
runTask: IRunTaskFunction;
|
||||
}
|
||||
|
||||
export interface IRunTaskFunction {
|
||||
(options: ITaskOptions): Promise<any>;
|
||||
}
|
||||
|
||||
|
||||
export default class Task {
|
||||
private logger: Logger; // Assuming you have a Logger type
|
||||
|
||||
constructor(opts: { logger: Logger }) {
|
||||
this.logger = opts.logger;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export interface IIssueDefinition {
|
||||
name: string;
|
||||
check: ((env: NodeJS.ProcessEnv, logger: Logger, vod: IVod) => Promise<boolean>);
|
||||
solution: string;
|
||||
}
|
||||
|
||||
|
||||
export async function loadTaskDefinitions(logger: Logger, tasksDirectory: string, taskList: string[]): Promise<ITask[]> {
|
||||
const activeTasks = await Promise.all(
|
||||
taskList.map(async (taskName) => {
|
||||
const taskModulePath = path.join(tasksDirectory, `${taskName}.ts`); // Assuming your task files have .js extension
|
||||
logger.log({ level: 'info', message: `📁 ${taskModulePath}` });
|
||||
if (fs.existsSync(taskModulePath)) {
|
||||
const taskModule = await import(taskModulePath);
|
||||
if (taskModule.runTask) {
|
||||
return taskModule;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
})
|
||||
);
|
||||
return activeTasks.filter((task) => !!task)
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
import { createId } from '@paralleldrive/cuid2';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import { getVideoSrcB2LocalFilePath } from './fsCommon.js'
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
|
||||
const urlPrefix = 'https://f000.backblazeb2.com/b2api/v1/b2_download_file_by_id?fileId='
|
||||
|
||||
export async function downloadVideoSrcB2 (appContext, vod) {
|
||||
const localFilePath = getVideoSrcB2LocalFilePath(appContext, vod);
|
||||
const key = vod.attributes.videoSrcB2.data.attributes.key
|
||||
|
||||
const s3 = new S3Client({
|
||||
endpoint: appContext.env.B2_ENDPOINT,
|
||||
region: appContext.env.B2_REGION,
|
||||
credentials: {
|
||||
accessKeyId: process.env.B2_KEY,
|
||||
secretAccessKey: process.env.B2_SECRET,
|
||||
},
|
||||
});
|
||||
var params = {Bucket: appContext.env.B2_BUCKET, Key: key};
|
||||
const s3Result = await s3.send(new GetObjectCommand(params));
|
||||
if (!s3Result.Body) {
|
||||
throw new Error('received empty body from S3');
|
||||
}
|
||||
await pipeline(s3Result.Body, fs.createWriteStream(localFilePath));
|
||||
|
||||
return localFilePath
|
||||
}
|
||||
|
||||
|
||||
export async function uploadToB2 (appContext, filePath) {
|
||||
const keyName = `${createId()}-${path.basename(filePath)}`
|
||||
const bucketName = appContext.env.B2_BUCKET
|
||||
const s3 = new S3Client({
|
||||
endpoint: appContext.env.B2_ENDPOINT,
|
||||
region: appContext.env.B2_REGION,
|
||||
credentials: {
|
||||
accessKeyId: process.env.B2_KEY,
|
||||
secretAccessKey: process.env.B2_SECRET,
|
||||
}
|
||||
});
|
||||
var params = {Bucket: bucketName, Key: keyName, Body: fs.createReadStream(filePath)};
|
||||
const res = await s3.send(new PutObjectCommand(params));
|
||||
return {
|
||||
uploadId: res.VersionId,
|
||||
key: keyName,
|
||||
url: `${urlPrefix}${res.VersionId}`
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
export interface IB2File {
|
||||
data: {
|
||||
id: number;
|
||||
attributes: {
|
||||
url: string;
|
||||
key: string;
|
||||
uploadId: string;
|
||||
cdnUrl: string;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1 +1,3 @@
|
|||
export const ipfsHashRegex = /Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}/;
|
||||
export const strapiUrl = (process.env.NODE_ENV === 'production') ? 'https://portal.futureporn.net' : 'http://localhost:1337';
|
||||
export const projektMelodyEpoch = new Date('2020-02-07T23:21:48.000Z');
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
This is for API routes, facilitated by tsoa
|
||||
|
||||
https://tsoa-community.github.io/docs/generating.html
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import { Body, Post, Route, Tags, Example } from 'tsoa';
|
||||
import { ICrawl, ICrawlMonth } from '../models/crawl.js';
|
||||
import {
|
||||
ICreateCrawlRequest,
|
||||
ICreateCrawlMonthRequest,
|
||||
} from "../services/crawlService.js";
|
||||
import { queueCrawl, queueCrawlMonth } from '../services/crawlService.js';
|
||||
|
||||
@Route("crawl")
|
||||
export class CrawlController {
|
||||
/**
|
||||
* Creates a crawl job which will find the dates of past livestreams via the specified platform
|
||||
*/
|
||||
@Post()
|
||||
@Tags("Crawl")
|
||||
public async CreateCrawl(@Body() request: ICreateCrawlRequest): Promise<ICrawl> {
|
||||
const cr = await queueCrawl(request);
|
||||
return cr;
|
||||
}
|
||||
}
|
||||
|
||||
@Route("crawlMonth")
|
||||
export class CrawlMonthController {
|
||||
@Post()
|
||||
@Tags("Crawl")
|
||||
public async CreateCrawlMonth(@Body() request: ICreateCrawlMonthRequest): Promise<ICrawlMonth> {
|
||||
const cr = await queueCrawlMonth(request);
|
||||
return cr;
|
||||
}
|
||||
}
|
||||
|
|
@ -16,3 +16,21 @@ export function getDateFromSafeDate(safeDate: string): Date {
|
|||
return utcDate;
|
||||
}
|
||||
|
||||
|
||||
export function getMonthsBetweenDates(startDate: Date, endDate: Date): [Date, Date][] {
|
||||
const months: [Date, Date][] = [];
|
||||
|
||||
while (startDate <= endDate) {
|
||||
const year = startDate.getUTCFullYear();
|
||||
const month = startDate.getUTCMonth();
|
||||
const firstDayOfMonth = new Date(Date.UTC(year, month, 1));
|
||||
const lastDayOfMonth = new Date(Date.UTC(year, month + 1, 0));
|
||||
|
||||
months.push([firstDayOfMonth, lastDayOfMonth]);
|
||||
|
||||
// Move to the next month
|
||||
startDate.setUTCMonth(startDate.getUTCMonth() + 1);
|
||||
}
|
||||
|
||||
return months;
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
import dotenv from "dotenv";
|
||||
dotenv.config();
|
||||
|
||||
function checkEnvVariable(name: string) {
|
||||
if (!process.env[name]) {
|
||||
throw new Error(`${name} is missing from env`);
|
||||
}
|
||||
}
|
||||
|
||||
const requiredEnvVariables = [
|
||||
"TASK_LIST",
|
||||
"REDIS_HOST",
|
||||
"REDIS_PORT",
|
||||
"PORT",
|
||||
"IPFS_CLUSTER_HTTP_API_MULTIADDR",
|
||||
"IPFS_CLUSTER_HTTP_API_USERNAME",
|
||||
"IPFS_CLUSTER_HTTP_API_PASSWORD",
|
||||
"TWITTER_ACCOUNTS",
|
||||
"STRAPI_API_KEY",
|
||||
"STRAPI_URL",
|
||||
];
|
||||
|
||||
requiredEnvVariables.forEach(checkEnvVariable);
|
||||
|
||||
export const REDIS_PORT = process.env.REDIS_PORT as string;
|
||||
export const TASK_LIST = process.env.TASK_LIST as string;
|
||||
export const REDIS_HOST = process.env.REDIS_HOST as string;
|
||||
export const PORT = process.env.PORT as string;
|
||||
export const IPFS_CLUSTER_HTTP_API_MULTIADDR = process.env.IPFS_CLUSTER_HTTP_API_MULTIADDR as string;
|
||||
export const IPFS_CLUSTER_HTTP_API_USERNAME = process.env.IPFS_CLUSTER_HTTP_API_USERNAME as string;
|
||||
export const IPFS_CLUSTER_HTTP_API_PASSWORD = process.env.IPFS_CLUSTER_HTTP_API_PASSWORD as string;
|
||||
export const TWITTER_ACCOUNTS = process.env.TWITTER_ACCOUNTS as string;
|
||||
export const STRAPI_API_KEY = process.env.STRAPI_API_KEY as string;
|
||||
export const STRAPI_URL = process.env.STRAPI_URL as string;
|
||||
|
|
@ -0,0 +1,378 @@
|
|||
/**
|
||||
* Hypertext Transfer Protocol (HTTP) response status codes.
|
||||
* @see {@link https://en.wikipedia.org/wiki/List_of_HTTP_status_codes}
|
||||
*/
|
||||
export enum HttpStatusCode {
|
||||
/**
|
||||
* The server has received the request headers and the client should proceed to send the request body
|
||||
* (in the case of a request for which a body needs to be sent; for example, a POST request).
|
||||
* Sending a large request body to a server after a request has been rejected for inappropriate headers would be inefficient.
|
||||
* To have a server check the request's headers, a client must send Expect: 100-continue as a header in its initial request
|
||||
* and receive a 100 Continue status code in response before sending the body. The response 417 Expectation Failed indicates the request should not be continued.
|
||||
*/
|
||||
CONTINUE = 100,
|
||||
|
||||
/**
|
||||
* The requester has asked the server to switch protocols and the server has agreed to do so.
|
||||
*/
|
||||
SWITCHING_PROTOCOLS = 101,
|
||||
|
||||
/**
|
||||
* A WebDAV request may contain many sub-requests involving file operations, requiring a long time to complete the request.
|
||||
* This code indicates that the server has received and is processing the request, but no response is available yet.
|
||||
* This prevents the client from timing out and assuming the request was lost.
|
||||
*/
|
||||
PROCESSING = 102,
|
||||
|
||||
/**
|
||||
* Standard response for successful HTTP requests.
|
||||
* The actual response will depend on the request method used.
|
||||
* In a GET request, the response will contain an entity corresponding to the requested resource.
|
||||
* In a POST request, the response will contain an entity describing or containing the result of the action.
|
||||
*/
|
||||
OK = 200,
|
||||
|
||||
/**
|
||||
* The request has been fulfilled, resulting in the creation of a new resource.
|
||||
*/
|
||||
CREATED = 201,
|
||||
|
||||
/**
|
||||
* The request has been accepted for processing, but the processing has not been completed.
|
||||
* The request might or might not be eventually acted upon, and may be disallowed when processing occurs.
|
||||
*/
|
||||
ACCEPTED = 202,
|
||||
|
||||
/**
|
||||
* SINCE HTTP/1.1
|
||||
* The server is a transforming proxy that received a 200 OK from its origin,
|
||||
* but is returning a modified version of the origin's response.
|
||||
*/
|
||||
NON_AUTHORITATIVE_INFORMATION = 203,
|
||||
|
||||
/**
|
||||
* The server successfully processed the request and is not returning any content.
|
||||
*/
|
||||
NO_CONTENT = 204,
|
||||
|
||||
/**
|
||||
* The server successfully processed the request, but is not returning any content.
|
||||
* Unlike a 204 response, this response requires that the requester reset the document view.
|
||||
*/
|
||||
RESET_CONTENT = 205,
|
||||
|
||||
/**
|
||||
* The server is delivering only part of the resource (byte serving) due to a range header sent by the client.
|
||||
* The range header is used by HTTP clients to enable resuming of interrupted downloads,
|
||||
* or split a download into multiple simultaneous streams.
|
||||
*/
|
||||
PARTIAL_CONTENT = 206,
|
||||
|
||||
/**
|
||||
* The message body that follows is an XML message and can contain a number of separate response codes,
|
||||
* depending on how many sub-requests were made.
|
||||
*/
|
||||
MULTI_STATUS = 207,
|
||||
|
||||
/**
|
||||
* The members of a DAV binding have already been enumerated in a preceding part of the (multistatus) response,
|
||||
* and are not being included again.
|
||||
*/
|
||||
ALREADY_REPORTED = 208,
|
||||
|
||||
/**
|
||||
* The server has fulfilled a request for the resource,
|
||||
* and the response is a representation of the result of one or more instance-manipulations applied to the current instance.
|
||||
*/
|
||||
IM_USED = 226,
|
||||
|
||||
/**
|
||||
* Indicates multiple options for the resource from which the client may choose (via agent-driven content negotiation).
|
||||
* For example, this code could be used to present multiple video format options,
|
||||
* to list files with different filename extensions, or to suggest word-sense disambiguation.
|
||||
*/
|
||||
MULTIPLE_CHOICES = 300,
|
||||
|
||||
/**
|
||||
* This and all future requests should be directed to the given URI.
|
||||
*/
|
||||
MOVED_PERMANENTLY = 301,
|
||||
|
||||
/**
|
||||
* This is an example of industry practice contradicting the standard.
|
||||
* The HTTP/1.0 specification (RFC 1945) required the client to perform a temporary redirect
|
||||
* (the original describing phrase was "Moved Temporarily"), but popular browsers implemented 302
|
||||
* with the functionality of a 303 See Other. Therefore, HTTP/1.1 added status codes 303 and 307
|
||||
* to distinguish between the two behaviours. However, some Web applications and frameworks
|
||||
* use the 302 status code as if it were the 303.
|
||||
*/
|
||||
FOUND = 302,
|
||||
|
||||
/**
|
||||
* SINCE HTTP/1.1
|
||||
* The response to the request can be found under another URI using a GET method.
|
||||
* When received in response to a POST (or PUT/DELETE), the client should presume that
|
||||
* the server has received the data and should issue a redirect with a separate GET message.
|
||||
*/
|
||||
SEE_OTHER = 303,
|
||||
|
||||
/**
|
||||
* Indicates that the resource has not been modified since the version specified by the request headers If-Modified-Since or If-None-Match.
|
||||
* In such case, there is no need to retransmit the resource since the client still has a previously-downloaded copy.
|
||||
*/
|
||||
NOT_MODIFIED = 304,
|
||||
|
||||
/**
|
||||
* SINCE HTTP/1.1
|
||||
* The requested resource is available only through a proxy, the address for which is provided in the response.
|
||||
* Many HTTP clients (such as Mozilla and Internet Explorer) do not correctly handle responses with this status code, primarily for security reasons.
|
||||
*/
|
||||
USE_PROXY = 305,
|
||||
|
||||
/**
|
||||
* No longer used. Originally meant "Subsequent requests should use the specified proxy."
|
||||
*/
|
||||
SWITCH_PROXY = 306,
|
||||
|
||||
/**
|
||||
* SINCE HTTP/1.1
|
||||
* In this case, the request should be repeated with another URI; however, future requests should still use the original URI.
|
||||
* In contrast to how 302 was historically implemented, the request method is not allowed to be changed when reissuing the original request.
|
||||
* For example, a POST request should be repeated using another POST request.
|
||||
*/
|
||||
TEMPORARY_REDIRECT = 307,
|
||||
|
||||
/**
|
||||
* The request and all future requests should be repeated using another URI.
|
||||
* 307 and 308 parallel the behaviors of 302 and 301, but do not allow the HTTP method to change.
|
||||
* So, for example, submitting a form to a permanently redirected resource may continue smoothly.
|
||||
*/
|
||||
PERMANENT_REDIRECT = 308,
|
||||
|
||||
/**
|
||||
* The server cannot or will not process the request due to an apparent client error
|
||||
* (e.g., malformed request syntax, too large size, invalid request message framing, or deceptive request routing).
|
||||
*/
|
||||
BAD_REQUEST = 400,
|
||||
|
||||
/**
|
||||
* Similar to 403 Forbidden, but specifically for use when authentication is required and has failed or has not yet
|
||||
* been provided. The response must include a WWW-Authenticate header field containing a challenge applicable to the
|
||||
* requested resource. See Basic access authentication and Digest access authentication. 401 semantically means
|
||||
* "unauthenticated",i.e. the user does not have the necessary credentials.
|
||||
*/
|
||||
UNAUTHORIZED = 401,
|
||||
|
||||
/**
|
||||
* Reserved for future use. The original intention was that this code might be used as part of some form of digital
|
||||
* cash or micro payment scheme, but that has not happened, and this code is not usually used.
|
||||
* Google Developers API uses this status if a particular developer has exceeded the daily limit on requests.
|
||||
*/
|
||||
PAYMENT_REQUIRED = 402,
|
||||
|
||||
/**
|
||||
* The request was valid, but the server is refusing action.
|
||||
* The user might not have the necessary permissions for a resource.
|
||||
*/
|
||||
FORBIDDEN = 403,
|
||||
|
||||
/**
|
||||
* The requested resource could not be found but may be available in the future.
|
||||
* Subsequent requests by the client are permissible.
|
||||
*/
|
||||
NOT_FOUND = 404,
|
||||
|
||||
/**
|
||||
* A request method is not supported for the requested resource;
|
||||
* for example, a GET request on a form that requires data to be presented via POST, or a PUT request on a read-only resource.
|
||||
*/
|
||||
METHOD_NOT_ALLOWED = 405,
|
||||
|
||||
/**
|
||||
* The requested resource is capable of generating only content not acceptable according to the Accept headers sent in the request.
|
||||
*/
|
||||
NOT_ACCEPTABLE = 406,
|
||||
|
||||
/**
|
||||
* The client must first authenticate itself with the proxy.
|
||||
*/
|
||||
PROXY_AUTHENTICATION_REQUIRED = 407,
|
||||
|
||||
/**
|
||||
* The server timed out waiting for the request.
|
||||
* According to HTTP specifications:
|
||||
* "The client did not produce a request within the time that the server was prepared to wait. The client MAY repeat the request without modifications at any later time."
|
||||
*/
|
||||
REQUEST_TIMEOUT = 408,
|
||||
|
||||
/**
|
||||
* Indicates that the request could not be processed because of conflict in the request,
|
||||
* such as an edit conflict between multiple simultaneous updates.
|
||||
*/
|
||||
CONFLICT = 409,
|
||||
|
||||
/**
|
||||
* Indicates that the resource requested is no longer available and will not be available again.
|
||||
* This should be used when a resource has been intentionally removed and the resource should be purged.
|
||||
* Upon receiving a 410 status code, the client should not request the resource in the future.
|
||||
* Clients such as search engines should remove the resource from their indices.
|
||||
* Most use cases do not require clients and search engines to purge the resource, and a "404 Not Found" may be used instead.
|
||||
*/
|
||||
GONE = 410,
|
||||
|
||||
/**
|
||||
* The request did not specify the length of its content, which is required by the requested resource.
|
||||
*/
|
||||
LENGTH_REQUIRED = 411,
|
||||
|
||||
/**
|
||||
* The server does not meet one of the preconditions that the requester put on the request.
|
||||
*/
|
||||
PRECONDITION_FAILED = 412,
|
||||
|
||||
/**
|
||||
* The request is larger than the server is willing or able to process. Previously called "Request Entity Too Large".
|
||||
*/
|
||||
PAYLOAD_TOO_LARGE = 413,
|
||||
|
||||
/**
|
||||
* The URI provided was too long for the server to process. Often the result of too much data being encoded as a query-string of a GET request,
|
||||
* in which case it should be converted to a POST request.
|
||||
* Called "Request-URI Too Long" previously.
|
||||
*/
|
||||
URI_TOO_LONG = 414,
|
||||
|
||||
/**
|
||||
* The request entity has a media type which the server or resource does not support.
|
||||
* For example, the client uploads an image as image/svg+xml, but the server requires that images use a different format.
|
||||
*/
|
||||
UNSUPPORTED_MEDIA_TYPE = 415,
|
||||
|
||||
/**
|
||||
* The client has asked for a portion of the file (byte serving), but the server cannot supply that portion.
|
||||
* For example, if the client asked for a part of the file that lies beyond the end of the file.
|
||||
* Called "Requested Range Not Satisfiable" previously.
|
||||
*/
|
||||
RANGE_NOT_SATISFIABLE = 416,
|
||||
|
||||
/**
|
||||
* The server cannot meet the requirements of the Expect request-header field.
|
||||
*/
|
||||
EXPECTATION_FAILED = 417,
|
||||
|
||||
/**
|
||||
* This code was defined in 1998 as one of the traditional IETF April Fools' jokes, in RFC 2324, Hyper Text Coffee Pot Control Protocol,
|
||||
* and is not expected to be implemented by actual HTTP servers. The RFC specifies this code should be returned by
|
||||
* teapots requested to brew coffee. This HTTP status is used as an Easter egg in some websites, including Google.com.
|
||||
*/
|
||||
I_AM_A_TEAPOT = 418,
|
||||
|
||||
/**
|
||||
* The request was directed at a server that is not able to produce a response (for example because a connection reuse).
|
||||
*/
|
||||
MISDIRECTED_REQUEST = 421,
|
||||
|
||||
/**
|
||||
* The request was well-formed but was unable to be followed due to semantic errors.
|
||||
*/
|
||||
UNPROCESSABLE_ENTITY = 422,
|
||||
|
||||
/**
|
||||
* The resource that is being accessed is locked.
|
||||
*/
|
||||
LOCKED = 423,
|
||||
|
||||
/**
|
||||
* The request failed due to failure of a previous request (e.g., a PROPPATCH).
|
||||
*/
|
||||
FAILED_DEPENDENCY = 424,
|
||||
|
||||
/**
|
||||
* The client should switch to a different protocol such as TLS/1.0, given in the Upgrade header field.
|
||||
*/
|
||||
UPGRADE_REQUIRED = 426,
|
||||
|
||||
/**
|
||||
* The origin server requires the request to be conditional.
|
||||
* Intended to prevent "the 'lost update' problem, where a client
|
||||
* GETs a resource's state, modifies it, and PUTs it back to the server,
|
||||
* when meanwhile a third party has modified the state on the server, leading to a conflict."
|
||||
*/
|
||||
PRECONDITION_REQUIRED = 428,
|
||||
|
||||
/**
|
||||
* The user has sent too many requests in a given amount of time. Intended for use with rate-limiting schemes.
|
||||
*/
|
||||
TOO_MANY_REQUESTS = 429,
|
||||
|
||||
/**
|
||||
* The server is unwilling to process the request because either an individual header field,
|
||||
* or all the header fields collectively, are too large.
|
||||
*/
|
||||
REQUEST_HEADER_FIELDS_TOO_LARGE = 431,
|
||||
|
||||
/**
|
||||
* A server operator has received a legal demand to deny access to a resource or to a set of resources
|
||||
* that includes the requested resource. The code 451 was chosen as a reference to the novel Fahrenheit 451.
|
||||
*/
|
||||
UNAVAILABLE_FOR_LEGAL_REASONS = 451,
|
||||
|
||||
/**
|
||||
* A generic error message, given when an unexpected condition was encountered and no more specific message is suitable.
|
||||
*/
|
||||
INTERNAL_SERVER_ERROR = 500,
|
||||
|
||||
/**
|
||||
* The server either does not recognize the request method, or it lacks the ability to fulfill the request.
|
||||
* Usually this implies future availability (e.g., a new feature of a web-service API).
|
||||
*/
|
||||
NOT_IMPLEMENTED = 501,
|
||||
|
||||
/**
|
||||
* The server was acting as a gateway or proxy and received an invalid response from the upstream server.
|
||||
*/
|
||||
BAD_GATEWAY = 502,
|
||||
|
||||
/**
|
||||
* The server is currently unavailable (because it is overloaded or down for maintenance).
|
||||
* Generally, this is a temporary state.
|
||||
*/
|
||||
SERVICE_UNAVAILABLE = 503,
|
||||
|
||||
/**
|
||||
* The server was acting as a gateway or proxy and did not receive a timely response from the upstream server.
|
||||
*/
|
||||
GATEWAY_TIMEOUT = 504,
|
||||
|
||||
/**
|
||||
* The server does not support the HTTP protocol version used in the request
|
||||
*/
|
||||
HTTP_VERSION_NOT_SUPPORTED = 505,
|
||||
|
||||
/**
|
||||
* Transparent content negotiation for the request results in a circular reference.
|
||||
*/
|
||||
VARIANT_ALSO_NEGOTIATES = 506,
|
||||
|
||||
/**
|
||||
* The server is unable to store the representation needed to complete the request.
|
||||
*/
|
||||
INSUFFICIENT_STORAGE = 507,
|
||||
|
||||
/**
|
||||
* The server detected an infinite loop while processing the request.
|
||||
*/
|
||||
LOOP_DETECTED = 508,
|
||||
|
||||
/**
|
||||
* Further extensions to the request are required for the server to fulfill it.
|
||||
*/
|
||||
NOT_EXTENDED = 510,
|
||||
|
||||
/**
|
||||
* The client needs to authenticate to gain network access.
|
||||
* Intended for use by intercepting proxies used to control access to the network (e.g., "captive portals" used
|
||||
* to require agreement to Terms of Service before granting full Internet access via a Wi-Fi hotspot).
|
||||
*/
|
||||
NETWORK_AUTHENTICATION_REQUIRED = 511,
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
|
||||
import { type Logger } from 'winston';
|
||||
import { multiaddrToUri } from '@multiformats/multiaddr-to-uri';
|
||||
import https from 'https';
|
||||
import fetch from 'node-fetch';
|
||||
|
||||
export async function pinCid (env: NodeJS.ProcessEnv, logger: Logger, cid: string): Promise<string> {
|
||||
const controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
const httpsAgent = new https.Agent({
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
const uri = multiaddrToUri(env.IPFS_CLUSTER_HTTP_API_MULTIADDR);
|
||||
|
||||
logger.log({ level: 'debug', message: `uri=${uri}` });
|
||||
|
||||
const duration: number = 60*1000*3;
|
||||
const timer = setTimeout(() => {
|
||||
logger.log({ level: 'debug', message: `aborting after ${duration}ms` })
|
||||
controller.abort();
|
||||
}, duration);
|
||||
|
||||
const res = await fetch(`${uri}/pins/${cid}`, {
|
||||
signal,
|
||||
agent: httpsAgent,
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(`${env.IPFS_CLUSTER_HTTP_API_USERNAME}:${env.IPFS_CLUSTER_HTTP_API_PASSWORD}`, "utf-8").toString("base64")}`
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
if (res.ok) {
|
||||
clearTimeout(timer); // Cancel the timeout
|
||||
|
||||
const data = await res.json() as { cid: string; };
|
||||
logger.log({ level: 'debug', message: `data=${JSON.stringify(data)}` })
|
||||
|
||||
if (data.cid !== cid) throw new Error(`CID did not match`);
|
||||
return data.cid
|
||||
|
||||
} else {
|
||||
throw new Error(`fetch failed with status ${res.status} ${res.statusText}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function isCidPinned (env: NodeJS.ProcessEnv, logger: Logger, cid: string): Promise<boolean> {
|
||||
const controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
const uri = multiaddrToUri(env.IPFS_CLUSTER_HTTP_API_MULTIADDR);
|
||||
|
||||
logger.log({ level: 'debug', message: `uri=${uri}` });
|
||||
const httpsAgent = new https.Agent({
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
|
||||
const duration: number = 60*1000;
|
||||
const timer = setTimeout(() => {
|
||||
logger.log({ level: 'debug', message: `aborting after ${duration}ms` })
|
||||
controller.abort();
|
||||
}, duration);
|
||||
|
||||
const res = await fetch(`${uri}/pins/${cid}`, {
|
||||
signal,
|
||||
agent: httpsAgent,
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(`${env.IPFS_CLUSTER_HTTP_API_USERNAME}:${env.IPFS_CLUSTER_HTTP_API_PASSWORD}`, "utf-8").toString("base64")}`
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
clearTimeout(timer); // Cancel the timeout
|
||||
|
||||
const data = await res.json();
|
||||
logger.log({ level: 'debug', message: `data=${JSON.stringify(data)}` })
|
||||
return (isCidPinnedOnPeers(data, cid))
|
||||
|
||||
} else {
|
||||
return false; // Fetch failed
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
function isCidPinnedOnPeers(clusterData: any, targetCid: string): boolean {
|
||||
const peerMap = clusterData.peer_map;
|
||||
|
||||
// Extract the CID from the top-level clusterData
|
||||
const topLevelCid = clusterData.cid;
|
||||
|
||||
for (const peerId in peerMap) {
|
||||
if (peerMap.hasOwnProperty(peerId)) {
|
||||
const peerInfo = peerMap[peerId];
|
||||
|
||||
// Check if the peer has the target CID pinned
|
||||
if (peerInfo.status === 'pinned' && topLevelCid === targetCid) {
|
||||
return true; // CID is pinned on at least 1 peer
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false; // CID is not pinned on any peer
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
|
||||
/**
|
||||
* Crawls specifications for finding the dates of past hentai livestreams using social media platforms
|
||||
*/
|
||||
export interface ICrawl {
|
||||
username: string;
|
||||
jobId: string;
|
||||
vtuberId: number;
|
||||
}
|
||||
|
||||
|
||||
|
||||
export interface ICrawlMonth {
|
||||
/**
|
||||
* @example "el_xox"
|
||||
*/
|
||||
username: string;
|
||||
since: Date;
|
||||
until: Date;
|
||||
vtuberId: number;
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import express, { Request, Response, Next } from "express";
|
||||
import methodOverride from "method-override";
|
||||
import { RegisterRoutes } from "../dist/futureporn-qa/routes/routes.js";
|
||||
import { ValidateError } from "tsoa";
|
||||
import { HttpStatusCode } from "./http-status-code.js";
|
||||
|
||||
|
||||
interface IError {
|
||||
status?: number;
|
||||
fields?: string[];
|
||||
message?: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
|
||||
|
||||
export const registerRoutes = (app: express.Express) => {
|
||||
app
|
||||
.use(express.urlencoded({ extended: true }))
|
||||
.use(express.json())
|
||||
.use(methodOverride())
|
||||
.use((_req: Request, res: Response, next: Next) => {
|
||||
res.header("Access-Control-Allow-Origin", "*");
|
||||
res.header(
|
||||
"Access-Control-Allow-Headers",
|
||||
`Origin, X-Requested-With, Content-Type, Accept, Authorization`
|
||||
);
|
||||
next();
|
||||
});
|
||||
|
||||
RegisterRoutes(app);
|
||||
|
||||
const getErrorBody = (err: unknown) => {
|
||||
if (err instanceof ValidateError) {
|
||||
return {
|
||||
message: err.message,
|
||||
status: HttpStatusCode.BAD_REQUEST,
|
||||
fields: err.fields,
|
||||
name: err.name,
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
message: "UNKNOWN_ERROR",
|
||||
status: HttpStatusCode.INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
app.use(
|
||||
(
|
||||
err: IError,
|
||||
_req: express.Request,
|
||||
res: express.Response,
|
||||
next: express.NextFunction
|
||||
) => {
|
||||
console.error(err);
|
||||
|
||||
|
||||
const body = getErrorBody(err);
|
||||
res.status(body.status).json(body);
|
||||
next();
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import { ICrawl } from '../models/crawl.js';
|
||||
import { Queue } from 'bullmq';
|
||||
import { REDIS_HOST, REDIS_PORT } from '../env.js';
|
||||
import { default as Redis } from 'ioredis'
|
||||
|
||||
/**
|
||||
* @example {
|
||||
* "username": "projektmelody",
|
||||
* "vtuberId": 1
|
||||
* }
|
||||
*/
|
||||
export type ICreateCrawlRequest = {
|
||||
username: string;
|
||||
vtuberId: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* @example {
|
||||
* "username": "projektmelody",
|
||||
* "since": "2023-10-02T00:00:00.000Z",
|
||||
* "until": "2023-10-31T00:00:00.000Z",
|
||||
* "vtuberId": 1
|
||||
* }
|
||||
*/
|
||||
export type ICreateCrawlMonthRequest = {
|
||||
username: string;
|
||||
since: Date;
|
||||
until: Date;
|
||||
vtuberId: number;
|
||||
}
|
||||
|
||||
|
||||
export async function queueCrawl(req: ICreateCrawlRequest): Promise<ICrawl> {
|
||||
|
||||
const connection = new Redis.default({
|
||||
port: parseInt(REDIS_PORT),
|
||||
host: REDIS_HOST,
|
||||
maxRetriesPerRequest: null,
|
||||
});
|
||||
|
||||
const { username, vtuberId } = req;
|
||||
|
||||
// add crawl job to queue
|
||||
const queue: Queue = new Queue('crawlTwitter', {
|
||||
connection
|
||||
});
|
||||
|
||||
const job = await queue.add(
|
||||
'crawlTwitter',
|
||||
{
|
||||
username: username,
|
||||
vtuberId: vtuberId,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
username: username,
|
||||
jobId: job?.id || '',
|
||||
vtuberId: vtuberId,
|
||||
}
|
||||
}
|
||||
|
||||
export async function queueCrawlMonth(req: ICreateCrawlMonthRequest): Promise<any> {
|
||||
|
||||
const connection = new Redis.default({
|
||||
port: parseInt(REDIS_PORT),
|
||||
host: REDIS_HOST,
|
||||
maxRetriesPerRequest: null,
|
||||
});
|
||||
|
||||
const { username, since, until, vtuberId } = req;
|
||||
|
||||
// add crawl job to queue
|
||||
const queue: Queue = new Queue('crawlTwitterMonth', {
|
||||
connection
|
||||
});
|
||||
|
||||
const job = await queue.add(
|
||||
'crawlTwitterMonth',
|
||||
{
|
||||
username: username,
|
||||
since: since,
|
||||
until: until,
|
||||
vtuberId: vtuberId,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
username: username,
|
||||
since: since,
|
||||
until: until,
|
||||
vtuberId: vtuberId,
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
import { IVtuber } from "./vtubers.ts";
|
||||
import { Logger } from "winston";
|
||||
import { IStrapiTweet } from "./tweets.ts";
|
||||
import qs from 'qs';
|
||||
import { STRAPI_API_KEY, STRAPI_URL } from "./env.ts";
|
||||
|
||||
|
||||
export interface IStream {
|
||||
id: number;
|
||||
attributes: {
|
||||
date: string;
|
||||
vtuber: IVtuber;
|
||||
}
|
||||
}
|
||||
|
||||
export interface IStreamResponse {
|
||||
data: IStream[];
|
||||
meta: {
|
||||
pagination: {
|
||||
total: number;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function createStream (logger: Logger, tweet: IStrapiTweet, vtuberId: number): Promise<IStream> {
|
||||
logger.log({ level: 'debug', message: `Searching for stream with related tweet id_str=${tweet.attributes.id_str}` });
|
||||
const existingStreamQuery = qs.stringify({
|
||||
populate: {
|
||||
tweet: {
|
||||
fields: ['id_str'],
|
||||
},
|
||||
},
|
||||
filters: {
|
||||
tweet: {
|
||||
id_str: {
|
||||
$eq: tweet.attributes.id_str
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
const existingStreamRes = await fetch(`${STRAPI_URL}/api/streams?${existingStreamQuery}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'authorization': `Bearer ${STRAPI_API_KEY}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
});
|
||||
const existingStreamJson = await existingStreamRes.json() as IStreamResponse;
|
||||
logger.log({ level: 'debug', message: `Here is the existingStreamJson=${JSON.stringify(existingStreamJson)}` });
|
||||
if (!!existingStreamJson?.data && !!existingStreamJson?.data[0]) return existingStreamJson.data[0];
|
||||
|
||||
logger.log({ level: 'debug', message: `There is not an existing stream in Strapi with tweet matching id_str=${tweet.attributes.id_str} so we create it.` });
|
||||
const streamRes = await fetch(`${STRAPI_URL}/api/streams`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'authorization': `Bearer ${STRAPI_API_KEY}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
date: new Date(tweet.attributes.date).toISOString(),
|
||||
date_str: tweet.attributes.date,
|
||||
tweet: tweet.id
|
||||
}
|
||||
})
|
||||
})
|
||||
const bod = await streamRes.json();
|
||||
if (!streamRes.ok) {
|
||||
throw new Error(`failed to create stream entry in Strapi. status=${streamRes.status}, statusText=${streamRes.statusText} body=${bod}`);
|
||||
}
|
||||
|
||||
logger.log({ level: 'info', message: `created stream in Strapi. bod=${JSON.stringify(bod)}` });
|
||||
return bod.data
|
||||
|
||||
}
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
import Prevvy from 'prevvy';
|
||||
import path from 'node:path';
|
||||
import { got } from 'got';
|
||||
import { getVod } from './strapi.js';
|
||||
import { getVideoSrcB2LocalFilePath } from './fsCommon.js';
|
||||
import { uploadToB2 } from './b2.js'
|
||||
import { getVod } from '../lib/strapi.js';
|
||||
import { getVideoSrcB2LocalFilePath } from '../lib/fsCommon.js';
|
||||
import { uploadToB2 } from '../lib/b2.js'
|
||||
|
||||
|
||||
export async function generateThumbnail () {
|
||||
export async function generateThumbnail (vod) {
|
||||
const fileName = `vod-${vod?.id}-thumb.png`
|
||||
const thumbnailFilePath = path.join(appContext.env.TMPDIR, fileName)
|
||||
appContext.logger.log({ level: 'info', message: `Creating thumbnail from ${thumbnailFilePath}`})
|
||||
|
|
@ -83,7 +83,7 @@ export default async function taskAssertThumbnail (appContext, body) {
|
|||
if (generateANewThumbnail) {
|
||||
appContext.logger.log({ level: 'debug', message: `Generating a new thumbnail for vod ${vod.attributes.id}`})
|
||||
const uploadData = await generateThumbnail(appContext, body)
|
||||
await associateB2WithVod(uploadData)
|
||||
await associateB2WithVod(appContext, uploadData)
|
||||
}
|
||||
} else {
|
||||
appContext.logger.log({ level: 'debug', message: 'Doing nothing-- entry is not a vod.'})
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import { IRunTaskFunction, ITaskOptions } from "../Task.js";
|
||||
import { projektMelodyEpoch } from "../constants.js";
|
||||
import { getMonthsBetweenDates } from "../dates.js";
|
||||
import { Queue } from "bullmq";
|
||||
|
||||
export const name = 'crawlTwitter';
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job, connection }: ITaskOptions) => {
|
||||
// @todo this is the initial crawl function
|
||||
// this function does not do any searching.
|
||||
// instead, this function queues many twitter searches ("crawlTwitterMonth" task)
|
||||
// one search per month since projektmelody epoch until present date.
|
||||
const { username, vtuberId } = job.data;
|
||||
if (!vtuberId) throw new Error('vtuberId was not present in job data');
|
||||
if (!username) throw new Error('username was not present in job data');
|
||||
|
||||
|
||||
const queue: Queue = new Queue('crawlTwitterMonth', {
|
||||
connection
|
||||
});
|
||||
|
||||
|
||||
|
||||
const months = getMonthsBetweenDates(projektMelodyEpoch, new Date());
|
||||
|
||||
|
||||
// for each month, queue a search
|
||||
for (const month of months) {
|
||||
await queue.add(
|
||||
'crawlTwitterMonth',
|
||||
{
|
||||
since: month[0],
|
||||
until: month[1],
|
||||
username: username,
|
||||
vtuberId: vtuberId
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
return months
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
import { IRunTaskFunction, ITaskOptions } from "../Task.js";
|
||||
import { PythonShell, Options } from 'python-shell';
|
||||
import path, { dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { TWITTER_ACCOUNTS, STRAPI_URL, STRAPI_API_KEY } from '../env.js';
|
||||
import { ITweetScrape, IStrapiTweet, isChaturbateInviteLinkPresent, createTweet } from '../tweets.js';
|
||||
import { createStream } from "../stream.ts";
|
||||
import { Logger } from "winston";
|
||||
import qs from 'qs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
|
||||
|
||||
export const name = 'crawlTwitterMonth';
|
||||
/**
|
||||
* search on twitter
|
||||
*
|
||||
* Get tweets for a given month.
|
||||
* For each tweet, look for a chaturbate invite link
|
||||
* If such a link is present, create a livestream entry in the db
|
||||
*
|
||||
*
|
||||
*/
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptions) => {
|
||||
logger.log({ level: 'info', message: `🀄 crawlTwitterMonth begins. since=${job.data?.since}, until=${job.data?.until}, username=${job.data?.username}` });
|
||||
|
||||
if (!TWITTER_ACCOUNTS) throw new Error('TWITTER_ACCOUNTS was undefined in env, but it is required.');
|
||||
if (!job.data?.since) throw new Error('job.data.since was undefined');
|
||||
if (!job.data?.until) throw new Error('job.data.until was undefined');
|
||||
if (!job.data?.username) throw new Error('job.data.username was undefined');
|
||||
if (!job.data?.vtuberId) throw new Error('job.data.vtuberId was undefined, but it is required.');
|
||||
|
||||
const twitterAccounts = TWITTER_ACCOUNTS.split(',');
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'python_scripts');
|
||||
const pythonPath = path.join(__dirname, '..', '..', 'python_scripts', 'venv', 'bin', 'python3');
|
||||
logger.log({ level: 'debug', message: `__dirname=${__dirname} import.meta.url=${import.meta.url}, scriptPath=${scriptPath}` })
|
||||
|
||||
const opts: Options = {
|
||||
mode: "json",
|
||||
pythonPath: pythonPath,
|
||||
scriptPath: scriptPath,
|
||||
args: [
|
||||
'--accounts', ...twitterAccounts,
|
||||
'--search_query', `(from:${job.data.username}) since:${job.data.since} until:${job.data.until}`
|
||||
]
|
||||
}
|
||||
|
||||
const tweets: ITweetScrape[] = await PythonShell.run('twitterScraper.py', opts);
|
||||
|
||||
await Promise.all(tweets.map(async (tweet) => {
|
||||
const tweetInStrapi = await createTweet(logger, tweet, job.data.vtuberId); // assert tweet's existence
|
||||
const isInviteTweet = await isChaturbateInviteLinkPresent(tweet);
|
||||
if (isInviteTweet) {
|
||||
logger.log({ level: 'debug', message: `Tweet contains CB invite link. One sec while I create a stream entry in Strapi.` });
|
||||
const streamInStrapi = await createStream(logger, tweetInStrapi, job.data.vtuberId);
|
||||
logger.log({ level: 'debug', message: `streamInStrapi id=${streamInStrapi.id}` });
|
||||
} else {
|
||||
logger.log({ level: 'debug', message: `tweet ${tweet.id_str} is NOT an invite tweet`})
|
||||
}
|
||||
}))
|
||||
|
||||
|
||||
|
||||
|
||||
logger.log({ level: 'info', message: `🀄 crawlTwitterMonth ends. Processed ${tweets.length} tweets.` });
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
import { IMuxAsset, IVod } from "../vods.js";
|
||||
import fetch from 'node-fetch';
|
||||
import { IJobData } from "../types.js";
|
||||
import { getVod } from "../vods.js";
|
||||
import { Logger } from "winston";
|
||||
|
||||
export interface ICreateMuxAssetResponse {
|
||||
id: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* BIG WARNING
|
||||
*
|
||||
* I don't want to implement this at the moment
|
||||
*
|
||||
* because...
|
||||
*
|
||||
* I made a huge mistake with Mux API that cost $500
|
||||
* I need to be very cautious with this code
|
||||
* to avoid a repeat of that mistake.
|
||||
*/
|
||||
|
||||
|
||||
// export async function createMuxAsset(job: PgBoss.Job, env: NodeJS.ProcessEnv, logger: Logger) {
|
||||
// const data = job.data as IJobData
|
||||
// const id = data.id;
|
||||
// const vod = await getVod(id, env);
|
||||
// if (!vod) {
|
||||
// const msg = `panick! vod was not fetched`
|
||||
// console.error(msg)
|
||||
// throw new Error(msg)
|
||||
// }
|
||||
// if (!vod?.videoSrcB2?.cdnUrl) {
|
||||
// const msg = `panick! videoSrcB2 missing on vod ${vod.id}`
|
||||
// console.error(msg)
|
||||
// throw new Error(msg)
|
||||
// }
|
||||
// logger.log({ level: 'info', message: `Creating Mux asset for vod ${vod.id} (${vod.videoSrcB2.cdnUrl})` });
|
||||
|
||||
// if (!vod.videoSrcB2?.cdnUrl) {
|
||||
// const msg = 'panic! videoSrcB2.cdnUrl is missing!';
|
||||
// console.error(msg);
|
||||
// throw new Error(msg);
|
||||
// }
|
||||
|
||||
// // Create Mux Asset
|
||||
// const muxResponse = await fetch('https://api.mux.com/video/v1/assets', {
|
||||
// method: 'POST',
|
||||
// headers: {
|
||||
// 'Authorization': `Basic ${Buffer.from(`${env.MUX_TOKEN_ID}:${env.MUX_TOKEN_SECRET}`).toString('base64')}`,
|
||||
// 'Content-Type': 'application/json',
|
||||
// },
|
||||
// body: JSON.stringify({
|
||||
// "input": vod.videoSrcB2.cdnUrl,
|
||||
// "playback_policy": [
|
||||
// "signed"
|
||||
// ]
|
||||
// }),
|
||||
// });
|
||||
|
||||
// const muxData = await muxResponse.json() as { playback_ids: Array<string>; id: string };
|
||||
|
||||
// logger.log({ level: 'info', message: JSON.stringify(muxData, null, 2) });
|
||||
|
||||
// if (!muxData?.playback_ids) {
|
||||
// const msg = `panick! muxData was missing playback_ids`
|
||||
// console.error(msg)
|
||||
// throw new Error(msg)
|
||||
// }
|
||||
|
||||
// logger.log({ level: 'info', message: `Adding Mux Asset to strapi` });
|
||||
|
||||
// const playbackId = muxData.playback_ids.find((p: any) => p.policy === 'signed')
|
||||
// if (!playbackId) {
|
||||
// const msg = `panick: playbackId was not found in the muxData`
|
||||
// console.error(msg)
|
||||
// throw new Error(msg)
|
||||
// }
|
||||
|
||||
// // Add Mux Asset to Strapi
|
||||
// const muxAssetResponse = await fetch(`${env.STRAPI_URL}/api/mux-assets`, {
|
||||
// method: 'POST',
|
||||
// headers: {
|
||||
// 'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
// 'Content-Type': 'application/json',
|
||||
// },
|
||||
// body: JSON.stringify({
|
||||
// data: {
|
||||
// playbackId: playbackId,
|
||||
// assetId: muxData.id
|
||||
// }
|
||||
// }),
|
||||
// });
|
||||
|
||||
// const muxAssetData = await muxAssetResponse.json() as ICreateMuxAssetResponse;
|
||||
|
||||
// logger.log({ level: 'debug', message: `Relating Mux Asset to Vod ${vod.id}` });
|
||||
|
||||
// // Relate Mux Asset to Vod
|
||||
// const strapiResponse = await fetch(`${env.STRAPI_URL}/api/vods/${vod.id}`, {
|
||||
// method: 'PUT',
|
||||
// headers: {
|
||||
// 'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
// 'Content-Type': 'application/json',
|
||||
// },
|
||||
// body: JSON.stringify({
|
||||
// data: {
|
||||
// muxAsset: muxAssetData.id
|
||||
// }
|
||||
// }),
|
||||
// });
|
||||
|
||||
// const strapiData = await strapiResponse.json();
|
||||
// }
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
|
||||
|
||||
import { strapiUrl } from "../constants.js";
|
||||
import fetch from 'node-fetch';
|
||||
import { getVod } from '../vods.js';
|
||||
import { ITaskOptions, IRunTaskFunction } from "src/Task.js";
|
||||
import { IJobData } from "../types.js";
|
||||
|
||||
|
||||
|
||||
export const name = 'deleteThumbnail';
|
||||
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptions) => {
|
||||
const jobData = job.data as IJobData;
|
||||
const vod = await getVod(jobData.id, env);
|
||||
|
||||
if (!vod?.thumbnail?.data?.id) throw new Error('vod.thumbnail was missing');
|
||||
const res = await fetch(`${strapiUrl}/api/b2-files/${vod?.thumbnail?.data?.id}`, {
|
||||
method: 'DELETE',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`
|
||||
}
|
||||
});
|
||||
if (!res.ok) {
|
||||
logger.log({ level: 'info', message: `Response code: ${res.status} (ok:${res.ok})` })
|
||||
const msg = `could not delete thumbnail due to fetch response error ${res.status} ${res.body}`
|
||||
logger.log({ level: 'error', message: msg });
|
||||
throw new Error(msg);
|
||||
} else {
|
||||
logger.log({ level: 'info', message: ` thumbnail ${vod.thumbnail.data.id} deleted.` })
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
|
||||
// import Prevvy from 'prevvy';
|
||||
import Prevvy from '/home/chris/Documents/prevvy/index.js';
|
||||
import path from 'node:path';
|
||||
import { getRawVod } from '../vods.js';
|
||||
import { getVideoSrcB2LocalFilePath } from '../fsCommon.js';
|
||||
import { uploadToB2 } from '../b2.js'
|
||||
import { IRawVod } from '../vods.js';
|
||||
import { IJobData } from '../types.js';
|
||||
import { IB2File } from '../b2File.js';
|
||||
import { Logger } from 'winston';
|
||||
import { IRunTaskFunction, ITaskOptions } from 'src/Task.js';
|
||||
import { Job } from 'bullmq';
|
||||
|
||||
|
||||
export interface IUploadData {
|
||||
key: string;
|
||||
uploadId: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
|
||||
export const name = 'generateThumbnail';
|
||||
|
||||
export async function __generateThumbnail(vod: IRawVod, env: NodeJS.ProcessEnv, logger: Logger, job: Job): Promise<string> {
|
||||
const fileName = `vod-${vod?.id}-thumb.png`;
|
||||
const thumbnailFilePath = path.join(env.TMPDIR, fileName);
|
||||
const videoInputUrl = vod.attributes.videoSrcB2?.data?.attributes?.cdnUrl;
|
||||
if (!videoInputUrl) {
|
||||
console.error(vod?.attributes?.videoSrcB2);
|
||||
throw new Error(`videoInputUrl in __generateThumbnail was undefined`);
|
||||
}
|
||||
logger.log({ level: 'info', message: `🫰 Creating thumbnail from ${videoInputUrl} ---> ${thumbnailFilePath}` });
|
||||
|
||||
const thumb = new Prevvy({
|
||||
input: videoInputUrl,
|
||||
output: thumbnailFilePath,
|
||||
throttleTimeout: 2000,
|
||||
width: 128,
|
||||
cols: 5,
|
||||
rows: 5,
|
||||
});
|
||||
|
||||
thumb.on('progress', async (data: { percentage: number }) => {
|
||||
await job.updateProgress(data.percentage);
|
||||
});
|
||||
|
||||
await thumb.generate();
|
||||
return thumbnailFilePath;
|
||||
|
||||
}
|
||||
|
||||
export async function associateB2WithVod(vod: IRawVod, uploadData: IUploadData, env: NodeJS.ProcessEnv, logger: Logger) {
|
||||
logger.log({ level: 'info', message: `🥤 lets create b2-file in Strapi` });
|
||||
|
||||
// Create the B2 file
|
||||
const thumbResponse = await fetch(`${env.STRAPI_URL}/api/b2-files`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
key: uploadData.key,
|
||||
uploadId: uploadData.uploadId,
|
||||
url: uploadData.url,
|
||||
cdnUrl: `https://futureporn-b2.b-cdn.net/${uploadData.key}`
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!thumbResponse.ok) {
|
||||
const msg = `🟠 Failed to create B2 file: ${thumbResponse.statusText}`
|
||||
console.error(msg)
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const thumbData = await thumbResponse.json() as IB2File;
|
||||
|
||||
logger.log({ level: 'info', message: `📀 B2 file creation complete for B2 file id: ${thumbData.data.id}` });
|
||||
|
||||
logger.log({ level: 'info', message: `🪇 lets associate B2-file with VOD ${vod.id} in Strapi` });
|
||||
|
||||
// Associate B2 file with VOD
|
||||
const associateResponse = await fetch(`${env.STRAPI_URL}/api/vods/${vod.id}`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
thumbnail: thumbData.data.id,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!associateResponse.ok) {
|
||||
const msg = `💀 Failed to associate B2 file with VOD: ${associateResponse.statusText}`;
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
|
||||
logger.log({ level: 'info', message: `🫚 Association complete` });
|
||||
}
|
||||
|
||||
|
||||
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptions) => {
|
||||
const data = job.data as IJobData;
|
||||
if (!data.id) {
|
||||
const msg = 'panic! no vod id was passed to generateThumbnail.'
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
const vod = await getRawVod(data.id, env);
|
||||
if (!vod) {
|
||||
const msg = `panic! vod ${data.id} missing`
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
|
||||
logger.log({ level: 'info', message: '🖼️ __generateThumbnail begin' })
|
||||
const thumbnailFilePath = await __generateThumbnail(vod, env, logger, job);
|
||||
|
||||
logger.log({ level: 'info', message: `🆙 uploading thumbnail ${thumbnailFilePath} for vod ${data.id} to B2` });
|
||||
const uploadData = await uploadToB2(env, thumbnailFilePath);
|
||||
if (!uploadData) {
|
||||
const msg = 'panic! uploadData missing'
|
||||
console.error(msg)
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
logger.log({ level: 'info', message: `👍 associating thumbnail for vod ${data.id} with strapi` })
|
||||
await associateB2WithVod(vod, uploadData, env, logger)
|
||||
logger.log({ level: 'info', message: `👍👍 thumbnail associated with vod ${data.id}` });
|
||||
|
||||
return 'DONE'
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
import fetch, { AbortError } from 'node-fetch';
|
||||
import { ITaskOptions, IRunTaskFunction, IIssueDefinition } from "src/Task.js";
|
||||
import { multiaddrToUri } from '@multiformats/multiaddr-to-uri';
|
||||
|
||||
// /api/v0/pin/add
|
||||
|
||||
// https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-pin-add
|
||||
|
||||
// processOrder.ts
|
||||
// import { Boss, Job } from 'pg-boss';
|
||||
|
||||
// export class ProcessOrderTask {
|
||||
// private boss: Boss;
|
||||
|
||||
// constructor(boss: Boss) {
|
||||
// this.boss = boss;
|
||||
// }
|
||||
|
||||
// async executeJob(job: Job<any>) {
|
||||
// // Your task logic here
|
||||
// const data = job.data;
|
||||
// // Perform the actual work here
|
||||
// // ...
|
||||
// job.done(); // Mark the job as completed
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
/**
|
||||
* identifyVideoSrcHashIssues
|
||||
*
|
||||
* This does an in-depth check of the CID
|
||||
*
|
||||
* Takes about 3 minutes to determine if a CID is reachable or not
|
||||
*
|
||||
*/
|
||||
|
||||
export const name = 'identifyVideoSrcHashIssues';
|
||||
|
||||
// const issueDefinitions: IIssueDefinition[] = [
|
||||
// {
|
||||
// name: 'videoSrcHashUnreachable',
|
||||
// check: async (vod, env) => {
|
||||
// // if (!vod?.thumbnail?.data?.attributes?.cdnUrl) return true;
|
||||
// // else return false;
|
||||
|
||||
// },
|
||||
// solution: 'pinVideoSrcHash'
|
||||
// },
|
||||
// ];
|
||||
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job, connection }: ITaskOptions) => {
|
||||
logger.log({ level: 'info', message: `🐓 identifyVideoSrcHashIssues runTask begin. 🌱` });
|
||||
|
||||
// for (const iDef of issueDefinitions) {
|
||||
// const isProblem = await iDef.check(vod, env);
|
||||
// const status = isProblem ? '🔴 FAIL' : '🟢 PASS';
|
||||
// logger.log({ level: 'info', message: `${status} ${iDef.name} ${isProblem ? `(queueing ${iDef.solution} to solve.)` : ''}` });
|
||||
// if (isProblem) {
|
||||
// logger.log({ level: 'info', message: `🚩 VOD Problem detected. Adding ${iDef.solution} task to the queue.` })
|
||||
// const queue = new Queue(iDef.solution, {
|
||||
// connection
|
||||
// })
|
||||
// queue.add(iDef.solution, { id: vod.id });
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
const uri = multiaddrToUri(env.IPFS_CLUSTER_HTTP_API_MULTIADDR)
|
||||
const controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
controller.abort();
|
||||
}, 60*3000);
|
||||
|
||||
try {
|
||||
logger.log({ level: 'info', message: `>> IDK IDK uri=${uri}` })
|
||||
const res = await fetch(`${uri}/api/v0/pins/${job.data.CID}`, { signal });
|
||||
if (res.ok) {
|
||||
clearTimeout(timeout); // Cancel the timeout
|
||||
logger.log({ level: 'info', message: `🌚 identifyVideoSrcHashIssues runTask end. 🌊` });
|
||||
return false;
|
||||
} else {
|
||||
logger.log({ level: 'warn', message: `>> fetch failed !! res.status=${res.status} controller.signal=${controller.signal}` })
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.log({ level: 'error', message: `>> fetch failed due to error ${JSON.stringify(error)}` })
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
import { getVod, IVod, getRandomVod } from "../vods.js";
|
||||
import fetch, { AbortError } from 'node-fetch';
|
||||
import { isBefore } from 'date-fns';
|
||||
import { IJobData } from "../types.js";
|
||||
import { ITaskOptions, IRunTaskFunction, IIssueDefinition } from "src/Task.js";
|
||||
import { Queue } from 'bullmq';
|
||||
import { isCidPinned } from '../ipfsCluster.js';
|
||||
|
||||
|
||||
|
||||
export const name = 'identifyVodIssues';
|
||||
|
||||
/*
|
||||
The check functions return true if there is an issue
|
||||
This is an exhaustive list of all possible problems that can exist on a vod.
|
||||
*/
|
||||
const issueDefinitions: IIssueDefinition[] = [
|
||||
{
|
||||
name: 'thumbnailMissing',
|
||||
check: async (env, logger, vod) => {
|
||||
if (!vod?.thumbnail?.data?.attributes?.cdnUrl) return true;
|
||||
else return false;
|
||||
},
|
||||
solution: 'generateThumbnail'
|
||||
},
|
||||
{
|
||||
name: 'thumbnailUnreachable',
|
||||
check: async (env, logger, vod) => {
|
||||
if (!vod?.thumbnail?.data?.attributes?.cdnUrl) return false; // false because the problem isn't explicitly that the thumb is unreachable
|
||||
const response = await fetch(vod.thumbnail.data.attributes.cdnUrl);
|
||||
if (!response.ok) return true;
|
||||
else return false;
|
||||
},
|
||||
solution: 'deleteThumbnail'
|
||||
},
|
||||
// {
|
||||
// // WARNING: Mux quickly becomes unaffordable.
|
||||
// // @todo implement human-in-the-loop encoding approval
|
||||
// name: 'muxAssetMissing',
|
||||
// check: async (vod) => {
|
||||
// // we only want to allocate new videos
|
||||
// // so we only consider vods published after
|
||||
// // a certain date
|
||||
// const allocationCutoffDate = new Date('2023-06-01T00:00:00.000Z');
|
||||
// const vodDate = new Date(vod.date2);
|
||||
// const isVodOld = isBefore(vodDate, allocationCutoffDate)
|
||||
// logger.log({ level: 'info', message: `muxAsset:${vod?.muxAsset?.assetId}, vodDate:${vod.date2}, allocationCutoffDate:${allocationCutoffDate.toISOString()}, isVodOld:${isVodOld}` })
|
||||
// if (isVodOld) return false;
|
||||
// if (!!vod?.muxAsset?.assetId) return false;
|
||||
// logger.log({ level: 'error', message: `vod ${vod.id} is missing a muxAsset!` })
|
||||
// return true;
|
||||
// },
|
||||
// solution: 'createMuxAsset'
|
||||
// },
|
||||
{
|
||||
name: 'videoSrcHashUnpinned',
|
||||
check: async (env, logger, vod) => {
|
||||
const isPinned = await isCidPinned(env, logger, vod.videoSrcHash);
|
||||
return (!isPinned);
|
||||
},
|
||||
solution: 'pinVideoSrcHash'
|
||||
},
|
||||
// {
|
||||
// name: 'videoSrcHashUnreachable',
|
||||
// check: async (vod, env) => {
|
||||
// // /api/v0/pin/add the CID to see if it takes long or not.
|
||||
// // if the pin add is taking > 5 minutes, consider the CID not present on the provider
|
||||
// // if the pin is not present, get the file from backup
|
||||
// // and then /api/v0/add the file to IPFS.
|
||||
// // @todo
|
||||
// const res = fetch(`${uri}/api/v0/pin/add?arg=${vod.videoSrcHash}&progress=1`, {
|
||||
// method: 'POST'
|
||||
// })
|
||||
// return false;
|
||||
// },
|
||||
// solution: 'pinVideoSrcHashFromBackup'
|
||||
// },
|
||||
|
||||
|
||||
// {
|
||||
// name: 'video240HashUnreachable',
|
||||
// check: async (vod) => {
|
||||
|
||||
// },
|
||||
// solution: 'pinFromBackup'
|
||||
// }
|
||||
]
|
||||
|
||||
|
||||
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job, connection }: ITaskOptions) => {
|
||||
|
||||
logger.log({ level: 'debug', message: `🆕 identifyVodIssues begin` });
|
||||
|
||||
const data = job.data as IJobData;
|
||||
let vod: IVod | null;
|
||||
|
||||
// determine if we received a vod id or if we need to choose a random vod
|
||||
if (!data?.id) {
|
||||
// get a random vod
|
||||
vod = await getRandomVod(env);
|
||||
} else {
|
||||
// get a vod by id
|
||||
vod = await getVod(data.id, env);
|
||||
}
|
||||
|
||||
if (!vod) {
|
||||
const msg = '☠️ Panic! Could not get a vod';
|
||||
logger.log({ level: 'error', message: msg });
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const id = vod?.id;
|
||||
if (!vod.id) throw new Error(`vod was missing ID`);
|
||||
|
||||
logger.log({ level: 'info', message: `📹 VOD ${id}` });
|
||||
for (const iDef of issueDefinitions) {
|
||||
const isProblem = await iDef.check(env, logger, vod);
|
||||
const status = isProblem ? '🔴 FAIL' : '🟢 PASS';
|
||||
logger.log({ level: 'info', message: `${status} ${iDef.name} ${isProblem ? `(queueing ${iDef.solution} to solve.)` : ''}` });
|
||||
if (isProblem) {
|
||||
logger.log({ level: 'info', message: `🚩 VOD Problem detected. Adding ${iDef.solution} task to the queue.` })
|
||||
const queue = new Queue(iDef.solution, {
|
||||
connection
|
||||
})
|
||||
queue.add(iDef.solution, { id: vod.id });
|
||||
}
|
||||
}
|
||||
return 'OK'
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
|
||||
|
||||
|
||||
import { getVod } from '../vods.js';
|
||||
import { ITaskOptions, IRunTaskFunction } from "src/Task.js";
|
||||
import { IJobData } from "../types.js";
|
||||
import { pinCid } from "../ipfsCluster.js";
|
||||
|
||||
|
||||
export const name = 'pinVideoSrcHash';
|
||||
|
||||
export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptions) => {
|
||||
const jobData = job.data as IJobData;
|
||||
const vod = await getVod(jobData.id, env);
|
||||
const cid = await pinCid(env, logger, vod.videoSrcHash);
|
||||
return cid;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
import fetch from 'node-fetch';
|
||||
import { Logger } from 'winston';
|
||||
import qs from 'qs';
|
||||
import { STRAPI_API_KEY, STRAPI_URL } from './env.ts';
|
||||
|
||||
const chaturbateLinkRegex = /chaturbate.com/;
|
||||
|
||||
/**
|
||||
* This link map exists because some of
|
||||
* projektmelody's repeatedly-used shorturls
|
||||
* are expired as of Nov 2023.
|
||||
* Instead of redirecting to her chaturbate,
|
||||
* The visitor is redirected to centralreach.com
|
||||
*
|
||||
* To handle these broken links, we're taking note of
|
||||
* the links which are embedded in old tweets.
|
||||
* any instance of these specific links equates to a CB invite link.
|
||||
*/
|
||||
export const brokenLinkMap = new Map([
|
||||
['http://shorturl.at/tNUVY', 'https://chaturbate.com/projektmelody']
|
||||
]);
|
||||
|
||||
export interface IStrapiTweet {
|
||||
id: number;
|
||||
attributes: {
|
||||
id_str: string;
|
||||
url: string;
|
||||
date: string;
|
||||
json: string;
|
||||
}
|
||||
}
|
||||
|
||||
export interface IMeta {
|
||||
pagination: {
|
||||
total: number;
|
||||
}
|
||||
}
|
||||
|
||||
export interface IStrapiTweetResponse {
|
||||
data: IStrapiTweet[];
|
||||
meta: IMeta;
|
||||
}
|
||||
|
||||
export interface IStrapiTweetSingleResponse {
|
||||
data: IStrapiTweet;
|
||||
meta: IMeta;
|
||||
}
|
||||
|
||||
export interface ITweetScrape {
|
||||
id: number;
|
||||
id_str: string;
|
||||
url: string;
|
||||
date: string;
|
||||
user: {
|
||||
id: number;
|
||||
id_str: string;
|
||||
url: string;
|
||||
username: string;
|
||||
displayname: string;
|
||||
rawDescription: string;
|
||||
created: string;
|
||||
followersCount: number;
|
||||
friendsCount: number;
|
||||
statusesCount: number;
|
||||
favouritesCount: number;
|
||||
listedCount: number;
|
||||
mediaCount: number;
|
||||
location: string;
|
||||
profileImageUrl: string;
|
||||
profileBannerUrl: string;
|
||||
protected: boolean | null;
|
||||
verified: boolean;
|
||||
blue: boolean;
|
||||
blueType: string | null;
|
||||
descriptionLinks: {
|
||||
url: string;
|
||||
text: string;
|
||||
tcourl: string;
|
||||
}[];
|
||||
_type: string;
|
||||
};
|
||||
lang: string;
|
||||
rawContent: string;
|
||||
replyCount: number;
|
||||
retweetCount: number;
|
||||
likeCount: number;
|
||||
quoteCount: number;
|
||||
conversationId: number;
|
||||
hashtags: string[];
|
||||
cashtags: string[];
|
||||
mentionedUsers: any[];
|
||||
links: {
|
||||
url: string;
|
||||
text: string;
|
||||
tcourl: string;
|
||||
}[];
|
||||
viewCount: number;
|
||||
retweetedTweet: any;
|
||||
quotedTweet: any;
|
||||
place: any;
|
||||
coordinates: any;
|
||||
inReplyToTweetId: any;
|
||||
inReplyToUser: any;
|
||||
source: string;
|
||||
sourceUrl: string;
|
||||
sourceLabel: string;
|
||||
media: {
|
||||
photos: {
|
||||
url: string;
|
||||
}[];
|
||||
videos: any[];
|
||||
animated: any[];
|
||||
};
|
||||
_type: string;
|
||||
}
|
||||
|
||||
|
||||
export async function createTweet (logger: Logger, tweet: ITweetScrape, vtuberId: number): Promise<IStrapiTweet> {
|
||||
logger.log({ level: 'debug', message: `Searching for tweet with same id. id_str=${tweet.id_str}` });
|
||||
const existingTweetQuery = qs.stringify({
|
||||
filters: {
|
||||
id_str: {
|
||||
$eq: tweet.id_str
|
||||
}
|
||||
}
|
||||
})
|
||||
const existingTweetRes = await fetch(`${STRAPI_URL}/api/tweets?${existingTweetQuery}`, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'authorization': `Bearer ${STRAPI_API_KEY}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
});
|
||||
const existingTweetJson = await existingTweetRes.json() as IStrapiTweetResponse;
|
||||
if (!!existingTweetJson.data[0]?.attributes?.id_str) return existingTweetJson.data[0];
|
||||
|
||||
// Create tweet if it isnt already saved in Strapi
|
||||
const tweetRes = await fetch(`${STRAPI_URL}/api/tweets`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'authorization': `Bearer ${STRAPI_API_KEY}`,
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
date: new Date(tweet.date).toISOString(),
|
||||
id_str: tweet.id_str,
|
||||
url: tweet.url,
|
||||
json: JSON.stringify(tweet),
|
||||
vtuber: vtuberId
|
||||
}
|
||||
})
|
||||
})
|
||||
const bod = await tweetRes.json() as IStrapiTweetSingleResponse;
|
||||
if (!tweetRes.ok) {
|
||||
throw new Error(`failed to create tweet entry in Strapi. status=${tweetRes.status}, statusText=${tweetRes.statusText} body=${bod}`);
|
||||
}
|
||||
|
||||
logger.log({ level: 'info', message: `Created tweet in Strapi. bod=${JSON.stringify(bod)}` });
|
||||
|
||||
return bod.data;
|
||||
}
|
||||
|
||||
/*
|
||||
* expands a shortened url such as those from tinyUrl or other URL shortening service.
|
||||
*
|
||||
*/
|
||||
export async function expandUrl(url: string, redirects: number = 0): Promise<string> {
|
||||
if (!url) throw new Error('expandUrl expects URL as an argument, but it was empty.');
|
||||
if (!URL.canParse(url)) return url;
|
||||
if (brokenLinkMap.has(url)) return brokenLinkMap.get(url)!;
|
||||
if (chaturbateLinkRegex.test(url)) return url;
|
||||
|
||||
if (redirects > 19) {
|
||||
throw new Error(`Too many redirects while expanding url=${url}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
method: 'HEAD',
|
||||
redirect: 'manual'
|
||||
});
|
||||
|
||||
if ([301, 302, 303, 308].includes(res.status)) {
|
||||
const location = res.headers.get('location');
|
||||
if (!location) throw new Error('Failed to get location header');
|
||||
else return expandUrl(location, redirects + 1);
|
||||
} else {
|
||||
// We return the input url in case of 404s or any other non-redirect scenario.
|
||||
return url;
|
||||
}
|
||||
} catch (e) {
|
||||
// It's better to return the original input URL than it is to crash.
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* Looks at all the URLs in the tweet, expanding them (from link shortener) if necessary
|
||||
* Resolves true if there are any chaturbate invite links in the tweet.
|
||||
*
|
||||
*/
|
||||
export async function isChaturbateInviteLinkPresent(tweet: ITweetScrape): Promise<boolean> {
|
||||
const expandedLinks = await Promise.all(tweet.links.map(async (link) => {
|
||||
const expandedLink = await expandUrl(link.url);
|
||||
return expandedLink;
|
||||
}));
|
||||
return expandedLinks.some(isChaturbateLink);
|
||||
}
|
||||
|
||||
|
||||
export function isChaturbateLink(link: string): boolean {
|
||||
return (chaturbateLinkRegex.test(link));
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import { type Logger } from 'winston';
|
||||
|
||||
|
||||
export interface IVod {
|
||||
id: number;
|
||||
attributes: {
|
||||
thumbnail: {
|
||||
cdnUrl: string;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface IJobData {
|
||||
id?: number;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ import { getDateFromSafeDate, getSafeDate } from './dates.js'
|
|||
import { unmarshallVtuber, IVtuber } from './vtubers.js'
|
||||
import qs from 'qs'
|
||||
import { ITagVodRelation, unmarshallTagVodRelation } from './tag-vod-relations.js'
|
||||
import { IB2File, unmarshallB2File } from './b2File.js'
|
||||
import { IB2File } from './b2File.js'
|
||||
import fetch from 'node-fetch'
|
||||
|
||||
export interface IMuxAsset {
|
||||
|
|
@ -37,6 +37,25 @@ export interface IMarshalledVod {
|
|||
}
|
||||
}
|
||||
|
||||
export interface IRawVod {
|
||||
id: number;
|
||||
attributes: {
|
||||
title?: string;
|
||||
date: string;
|
||||
date2: string;
|
||||
muxAsset: IMuxAsset;
|
||||
thumbnail: IB2File | null;
|
||||
vtuber: IVtuber;
|
||||
tagVodRelations: ITagVodRelation[];
|
||||
video240Hash: string;
|
||||
videoSrcHash: string;
|
||||
videoSrcB2: IB2File | null;
|
||||
announceTitle: string;
|
||||
announceUrl: string;
|
||||
note: string;
|
||||
}
|
||||
}
|
||||
|
||||
export interface IVod {
|
||||
id: number;
|
||||
title?: string;
|
||||
|
|
@ -73,12 +92,12 @@ export function unmarshallVod(d: any): IVod {
|
|||
playbackId: d.attributes?.muxAsset?.data?.attributes?.playbackId,
|
||||
assetId: d.attributes?.muxAsset?.data?.attributes?.assetId,
|
||||
},
|
||||
thumbnail: unmarshallB2File(d.attributes?.thumbnail?.data),
|
||||
thumbnail: d.attributes?.thumbnail,
|
||||
vtuber: unmarshallVtuber(d.attributes?.vtuber?.data),
|
||||
tagVodRelations: d.attributes?.tagVodRelations?.data.map(unmarshallTagVodRelation),
|
||||
video240Hash: d.attributes?.video240Hash,
|
||||
videoSrcHash: d.attributes?.videoSrcHash,
|
||||
videoSrcB2: unmarshallB2File(d.attributes?.videoSrcB2?.data),
|
||||
videoSrcB2: d.attributes?.videoSrcB2,
|
||||
announceTitle: d.attributes.announceTitle,
|
||||
announceUrl: d.attributes.announceUrl,
|
||||
note: d.attributes.note,
|
||||
|
|
@ -125,11 +144,7 @@ export async function getVodForDate(date: Date): Promise<IVod> {
|
|||
|
||||
export async function getRandomVod(env: NodeJS.ProcessEnv): Promise<IVod | null> {
|
||||
try {
|
||||
const d = await fetch(`${strapiUrl}/api/vod/random`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`
|
||||
}
|
||||
})
|
||||
const d = await fetch(`${strapiUrl}/api/vod/random`)
|
||||
.then((res) => res.json()) as IVod;
|
||||
return d;
|
||||
} catch (e) {
|
||||
|
|
@ -139,7 +154,8 @@ export async function getRandomVod(env: NodeJS.ProcessEnv): Promise<IVod | null>
|
|||
}
|
||||
}
|
||||
|
||||
export async function getVod(id: number, env: NodeJS.ProcessEnv): Promise<IVod | null> {
|
||||
|
||||
export async function getRawVod(id: number, env: NodeJS.ProcessEnv): Promise<IRawVod | null> {
|
||||
const query = qs.stringify(
|
||||
{
|
||||
filters: {
|
||||
|
|
@ -168,22 +184,22 @@ export async function getVod(id: number, env: NodeJS.ProcessEnv): Promise<IVod |
|
|||
}
|
||||
)
|
||||
try {
|
||||
const d = await fetch(`${strapiUrl}/api/vods?${query}`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`
|
||||
}
|
||||
})
|
||||
.then((res) => res.json())
|
||||
.then((data: any) => unmarshallVod(data.data[0]))
|
||||
|
||||
return d
|
||||
const res: any = await fetch(`${strapiUrl}/api/vods?${query}`)
|
||||
const data: IMarshalledVod = await res.json();
|
||||
return data.data[0];
|
||||
} catch (e) {
|
||||
console.error(`there was an error while fetching vod ${id}`)
|
||||
console.error(e)
|
||||
return null
|
||||
console.error(`there was an error while fetching vod ${id}`);
|
||||
console.error(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export async function getVod(id: number, env: NodeJS.ProcessEnv): Promise<IVod | null> {
|
||||
const rawVod = await getRawVod(id, env);
|
||||
return unmarshallVod(rawVod);
|
||||
}
|
||||
|
||||
export async function getVods(page: number = 1, pageSize: number = 25, sortDesc = true): Promise<IVods> {
|
||||
const query = qs.stringify(
|
||||
{
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
|
||||
import { IVod } from './vods.js'
|
||||
import { strapiUrl } from './constants.js';
|
||||
import { getSafeDate } from './dates.js';
|
||||
import qs from 'qs';
|
||||
|
||||
export interface IVtuber {
|
||||
id: number;
|
||||
attributes: {
|
||||
slug: string;
|
||||
displayName: string;
|
||||
chaturbate?: string;
|
||||
twitter?: string;
|
||||
patreon?: string;
|
||||
twitch?: string;
|
||||
tiktok?: string;
|
||||
onlyfans?: string;
|
||||
youtube?: string;
|
||||
linktree?: string;
|
||||
carrd?: string;
|
||||
fansly?: string;
|
||||
pornhub?: string;
|
||||
discord?: string;
|
||||
reddit?: string;
|
||||
throne?: string;
|
||||
instagram?: string;
|
||||
facebook?: string;
|
||||
merch?: string;
|
||||
vods: IVod[];
|
||||
description1: string;
|
||||
description2?: string;
|
||||
image: string;
|
||||
imageBlur?: string;
|
||||
themeColor: string;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// export function unmarshallVtuber(d: any): IVtuber {
|
||||
// if (!d) {
|
||||
// console.error('panick! unmarshallVTuber was called with undefined data')
|
||||
// console.trace()
|
||||
// }
|
||||
// return {
|
||||
// id: d.id,
|
||||
// slug: d.attributes?.slug,
|
||||
// displayName: d.attributes.displayName,
|
||||
// chaturbate: d.attributes?.chaturbate,
|
||||
// twitter: d.attributes?.twitter,
|
||||
// patreon: d.attributes?.patreon,
|
||||
// twitch: d.attributes?.twitch,
|
||||
// tiktok: d.attributes?.tiktok,
|
||||
// onlyfans: d.attributes?.onlyfans,
|
||||
// youtube: d.attributes?.youtube,
|
||||
// linktree: d.attributes?.linktree,
|
||||
// carrd: d.attributes?.carrd,
|
||||
// fansly: d.attributes?.fansly,
|
||||
// pornhub: d.attributes?.pornhub,
|
||||
// discord: d.attributes?.discord,
|
||||
// reddit: d.attributes?.reddit,
|
||||
// throne: d.attributes?.throne,
|
||||
// instagram: d.attributes?.instagram,
|
||||
// facebook: d.attributes?.facebook,
|
||||
// merch: d.attributes?.merch,
|
||||
// description1: d.attributes.description1,
|
||||
// description2: d.attributes?.description2,
|
||||
// image: d.attributes.image,
|
||||
// imageBlur: d.attributes?.imageBlur,
|
||||
// themeColor: d.attributes.themeColor,
|
||||
// vods: d.attributes.vods
|
||||
// }
|
||||
// }
|
||||
|
||||
export async function getVtuberBySlug(slug: string): Promise<IVtuber> {
|
||||
const query = qs.stringify(
|
||||
{
|
||||
filters: {
|
||||
slug: {
|
||||
$eq: slug
|
||||
}
|
||||
},
|
||||
// populate: {
|
||||
// vods: {
|
||||
// fields: ['id', 'videoSrcHash'],
|
||||
// populate: ['vtuber']
|
||||
// }
|
||||
// }
|
||||
}
|
||||
)
|
||||
|
||||
return fetch(`${strapiUrl}/api/vtubers?${query}`)
|
||||
.then((res) => res.json())
|
||||
.then((d) => {
|
||||
return d.data[0]
|
||||
})
|
||||
}
|
||||
|
||||
export async function getVtuberById(id: number): Promise<IVtuber> {
|
||||
return fetch(`${strapiUrl}/api/vtubers?filters[id][$eq]=${id}`)
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
return data.data[0]
|
||||
})
|
||||
}
|
||||
|
||||
export async function getVtubers(): Promise<IVtuber[]> {
|
||||
return fetch(`${strapiUrl}/api/vtubers`)
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
return data.data
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1,101 +0,0 @@
|
|||
import { IMuxAsset, IVod } from "../lib/vods.js";
|
||||
import fetch from 'node-fetch';
|
||||
import { IJobData } from "../worker.js";
|
||||
import PgBoss from "pg-boss";
|
||||
import { getVod } from "../lib/vods.js";
|
||||
|
||||
export interface ICreateMuxAssetResponse {
|
||||
id: string;
|
||||
}
|
||||
|
||||
export async function createMuxAsset(job: PgBoss.Job, env: NodeJS.ProcessEnv) {
|
||||
const data = job.data as IJobData
|
||||
const id = data.id;
|
||||
const vod = await getVod(id, env);
|
||||
if (!vod) {
|
||||
const msg = `panick! vod was not fetched`
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
if (!vod?.videoSrcB2?.cdnUrl) {
|
||||
const msg = `panick! videoSrcB2 missing on vod ${vod.id}`
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
console.log(`Creating Mux asset for vod ${vod.id} (${vod.videoSrcB2.cdnUrl})`);
|
||||
|
||||
if (!vod.videoSrcB2?.cdnUrl) {
|
||||
const msg = 'panic! videoSrcB2.cdnUrl is missing!';
|
||||
console.error(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
// Create Mux Asset
|
||||
const muxResponse = await fetch('https://api.mux.com/video/v1/assets', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Basic ${Buffer.from(`${env.MUX_TOKEN_ID}:${env.MUX_TOKEN_SECRET}`).toString('base64')}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
"input": vod.videoSrcB2.cdnUrl,
|
||||
"playback_policy": [
|
||||
"signed"
|
||||
]
|
||||
}),
|
||||
});
|
||||
|
||||
const muxData = await muxResponse.json() as { playback_ids: Array<string>; id: string };
|
||||
|
||||
console.log(muxData)
|
||||
|
||||
if (!muxData?.playback_ids) {
|
||||
const msg = `panick! muxData was missing playback_ids`
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
|
||||
console.log(`Adding Mux Asset to strapi`);
|
||||
|
||||
const playbackId = muxData.playback_ids.find((p: any) => p.policy === 'signed')
|
||||
if (!playbackId) {
|
||||
const msg = `panick: playbackId was not found in the muxData`
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
|
||||
// Add Mux Asset to Strapi
|
||||
const muxAssetResponse = await fetch(`${env.STRAPI_URL}/api/mux-assets`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
playbackId: playbackId,
|
||||
assetId: muxData.id
|
||||
}
|
||||
}),
|
||||
});
|
||||
|
||||
const muxAssetData = await muxAssetResponse.json() as ICreateMuxAssetResponse;
|
||||
|
||||
console.log({ level: 'debug', message: `Relating Mux Asset to Vod ${vod.id}` });
|
||||
|
||||
// Relate Mux Asset to Vod
|
||||
const strapiResponse = await fetch(`${env.STRAPI_URL}/api/vods/${vod.id}`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
muxAsset: muxAssetData.id
|
||||
}
|
||||
}),
|
||||
});
|
||||
|
||||
const strapiData = await strapiResponse.json();
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
|
||||
|
||||
import PgBoss from "pg-boss";
|
||||
import { strapiUrl } from "../lib/constants.js";
|
||||
import fetch from 'node-fetch';
|
||||
import { getVod } from '../lib/vods.js';
|
||||
|
||||
interface IJobData {
|
||||
id: number;
|
||||
}
|
||||
|
||||
export async function deleteThumbnail(data: PgBoss.Job, boss: PgBoss, env: NodeJS.ProcessEnv) {
|
||||
const jobData = data.data as IJobData;
|
||||
const vod = await getVod(jobData.id, env)
|
||||
|
||||
if (!vod?.thumbnail?.id) throw new Error('vod.thumbnail was missing')
|
||||
const res = await fetch(`${strapiUrl}/api/b2-files/${vod?.thumbnail.id}`, {
|
||||
method: 'DELETE',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`
|
||||
}
|
||||
})
|
||||
if (!res.ok) {
|
||||
console.log(`Response code: ${res.status} (ok:${res.ok})`)
|
||||
const msg = `could not delete thumbnail due to fetch response error ${res.body}`
|
||||
console.error(msg);
|
||||
throw new Error(msg);
|
||||
} else {
|
||||
console.log(` thumbnail ${vod.thumbnail.id} deleted.`)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -1,163 +0,0 @@
|
|||
|
||||
import Prevvy from 'prevvy';
|
||||
import path from 'node:path';
|
||||
import { got } from 'got';
|
||||
import { getVod } from '../lib/vods.js';
|
||||
import { getVideoSrcB2LocalFilePath } from '../lib/fsCommon.js';
|
||||
import { uploadToB2 } from '../lib/b2.js'
|
||||
import { IVod } from '../lib/vods.js';
|
||||
import { IJobData } from '../worker.js';
|
||||
import PgBoss from 'pg-boss';
|
||||
import { IB2File } from '../lib/b2File.js';
|
||||
|
||||
|
||||
export interface IUploadData {
|
||||
key: string;
|
||||
uploadId: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export async function __generateThumbnail (vod: IVod, env: NodeJS.ProcessEnv): Promise<string> {
|
||||
const fileName = `vod-${vod?.id}-thumb.png`
|
||||
const thumbnailFilePath = path.join(env.TMPDIR, fileName)
|
||||
const videoInputUrl = vod.videoSrcB2?.cdnUrl;
|
||||
console.log(`Creating thumbnail from ${videoInputUrl} ---> ${thumbnailFilePath}`)
|
||||
const thumb = new Prevvy({
|
||||
input: videoInputUrl,
|
||||
output: thumbnailFilePath,
|
||||
throttleTimeout: 2000,
|
||||
width: 128,
|
||||
cols: 5,
|
||||
rows: 5,
|
||||
})
|
||||
await thumb.generate();
|
||||
return thumbnailFilePath
|
||||
}
|
||||
|
||||
export async function associateB2WithVod(vod: IVod, uploadData: IUploadData, env: NodeJS.ProcessEnv) {
|
||||
console.log(`lets create b2-file in Strapi`);
|
||||
|
||||
// Create the B2 file
|
||||
const thumbResponse = await fetch(`${env.STRAPI_URL}/api/b2-files`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
key: uploadData.key,
|
||||
uploadId: uploadData.uploadId,
|
||||
url: uploadData.url,
|
||||
cdnUrl: `https://futureporn-b2.b-cdn.net/${uploadData.key}`
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!thumbResponse.ok) {
|
||||
const msg = `Failed to create B2 file: ${thumbResponse.statusText}`
|
||||
console.error(msg)
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const thumbData = await thumbResponse.json() as { data: IB2File };
|
||||
|
||||
console.log(` B2 file creation complete for B2 file id: ${thumbData.data.id}`);
|
||||
console.log(` ^v^v^v^v^v^v`);
|
||||
console.log(thumbData);
|
||||
|
||||
console.log(`lets associate B2-file with VOD ${vod.id} in Strapi`);
|
||||
|
||||
// Associate B2 file with VOD
|
||||
const associateResponse = await fetch(`${env.STRAPI_URL}/api/vods/${vod.id}`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${env.STRAPI_API_KEY}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
data: {
|
||||
thumbnail: thumbData.data.id,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!associateResponse.ok) {
|
||||
const msg = `Failed to associate B2 file with VOD: ${associateResponse.statusText}`;
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
|
||||
console.log(`Association complete`);
|
||||
}
|
||||
|
||||
|
||||
// export default async function taskAssertThumbnail (appContext, body) {
|
||||
// if (body.model === 'vod') {
|
||||
// const vod = await getVod(appContext, body.entry.id)
|
||||
// appContext.logger.log({ level: 'info', message: 'taskAssertThumbnail begin' })
|
||||
|
||||
// const cdnUrl = vod?.attributes?.thumbnail?.data?.attributes?.cdnUrl;
|
||||
// const thumbnailKey = vod?.attributes?.thumbnail?.data?.attributes?.key
|
||||
|
||||
// let generateANewThumbnail = false; // default
|
||||
|
||||
// // If thumbnail is absent, create it
|
||||
// if (!thumbnailKey && !cdnUrl) {
|
||||
// generateANewThumbnail = true;
|
||||
|
||||
// // if thumbnail is present, verify HTTP 200
|
||||
// } else {
|
||||
// const response = await got(cdnUrl, { method: 'HEAD', throwHttpErrors: false });
|
||||
// if (!response.ok) {
|
||||
// generateANewThumbnail = true
|
||||
// } else {
|
||||
// // response was OK, so thumb must be good
|
||||
// appContext.logger.log({ level: 'debug', message: 'Doing nothing-- thumbnail already exists.'})
|
||||
// }
|
||||
// }
|
||||
|
||||
// if (generateANewThumbnail) {
|
||||
// appContext.logger.log({ level: 'debug', message: `Generating a new thumbnail for vod ${vod.attributes.id}`})
|
||||
// const uploadData = await generateThumbnail(appContext, body)
|
||||
// await associateB2WithVod(appContext, uploadData)
|
||||
// }
|
||||
// } else {
|
||||
// appContext.logger.log({ level: 'debug', message: 'Doing nothing-- entry is not a vod.'})
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
export async function generateThumbnail(job: PgBoss.Job, env: NodeJS.ProcessEnv) {
|
||||
const data = job.data as IJobData;
|
||||
if (!data.id) {
|
||||
const msg = 'panic! no vod id was passed to generateThumbnail.'
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
const vod = await getVod(data.id, env);
|
||||
if (!vod) {
|
||||
const msg = 'panic! vod missing'
|
||||
console.error(msg)
|
||||
throw new Error(msg)
|
||||
}
|
||||
|
||||
console.log('__generateThumbnail begin')
|
||||
const thumbnailFilePath = await __generateThumbnail(vod, env);
|
||||
|
||||
console.log(` uploading thumbnail ${thumbnailFilePath} for vod ${data.id} to B2`);
|
||||
const uploadData = await uploadToB2(env, thumbnailFilePath);
|
||||
if (!uploadData) {
|
||||
const msg = 'panic! uploadData missing'
|
||||
console.error(msg)
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
console.log(` associating thumbnail for vod ${data.id} with strapi`)
|
||||
await associateB2WithVod(vod, uploadData, env)
|
||||
console.log(` 👍👍👍 thumbnail associated with vod ${data.id}`);
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
import PgBoss from "pg-boss";
|
||||
import { getVods, getVod, IVod, getRandomVod } from "../lib/vods.js";
|
||||
import fetch from 'node-fetch';
|
||||
import { isBefore } from 'date-fns';
|
||||
import { IJobData } from "../worker.js";
|
||||
import { env } from "node:process";
|
||||
|
||||
|
||||
interface IIssueDefinition {
|
||||
name: string;
|
||||
check: ((vod: IVod) => Promise<boolean>);
|
||||
solution: string;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
The check functions return true if there is an issue
|
||||
|
||||
This is an exhaustive list of all possible problems that can exist on a vod.
|
||||
*/
|
||||
const issueDefinitions: IIssueDefinition[] = [
|
||||
{
|
||||
name: 'thumbnailMissing',
|
||||
check: async (vod) => {
|
||||
if (!vod?.thumbnail?.cdnUrl) return true;
|
||||
else return false;
|
||||
},
|
||||
solution: 'generateThumbnail'
|
||||
},
|
||||
{
|
||||
name: 'thumbnailUnreachable',
|
||||
check: async (vod) => {
|
||||
if (!vod?.thumbnail?.cdnUrl) return false; // false because the problem isn't explicitly that the thumb is unreachable
|
||||
const response = await fetch(vod.thumbnail.cdnUrl);
|
||||
if (!response.ok) return true;
|
||||
else return false;
|
||||
},
|
||||
solution: 'deleteThumbnail'
|
||||
},
|
||||
// {
|
||||
// Disabled because Mux has no price cap.
|
||||
// With use, Mux quickly becomes unaffordable.
|
||||
// name: 'muxAssetMissing',
|
||||
// check: async (vod) => {
|
||||
// // we only want to allocate new videos
|
||||
// // so we only consider vods published after
|
||||
// // a certain date
|
||||
// const allocationCutoffDate = new Date('2019-09-24T00:00:00.000Z');
|
||||
// const vodDate = new Date(vod.date2);
|
||||
// const isVodOld = isBefore(vodDate, allocationCutoffDate)
|
||||
// console.log(`muxAsset:${vod?.muxAsset?.assetId}, vodDate:${vod.date2}, allocationCutoffDate:${allocationCutoffDate.toISOString()}, isVodOld:${isVodOld}`)
|
||||
// if (isVodOld) return false;
|
||||
// if (!!vod?.muxAsset?.assetId) return false;
|
||||
// console.info(`vod ${vod.id} is missing a muxAsset!`)
|
||||
// return true;
|
||||
// },
|
||||
// solution: 'createMuxAsset'
|
||||
// }
|
||||
]
|
||||
|
||||
|
||||
|
||||
export async function identifyVodIssues(job: PgBoss.Job, boss: PgBoss, env: NodeJS.ProcessEnv) {
|
||||
const data = job.data as IJobData;
|
||||
let vod: IVod | null;
|
||||
|
||||
// determine if we received a vod id or if we need to choose a random vod
|
||||
if (!data?.id) {
|
||||
// get a random vod
|
||||
vod = await getRandomVod(env);
|
||||
} else {
|
||||
// get a vod by id
|
||||
vod = await getVod(data.id, env);
|
||||
}
|
||||
|
||||
if (!vod) {
|
||||
const msg = 'Panic! Could not get a vod';
|
||||
console.error(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
const id = vod?.id;
|
||||
console.log(`## VOD ${id} ##`);
|
||||
for (const iDef of issueDefinitions) {
|
||||
const isProblem = await iDef.check(vod);
|
||||
const status = isProblem ? '🔴 FAIL' : '🟢 PASS';
|
||||
console.log(` ${status} ${iDef.name} ${isProblem ? `(queueing ${iDef.solution} to solve.)` : ''}`);
|
||||
if (isProblem) {
|
||||
boss.send(iDef.solution, { id: vod.id }, { priority: 20 });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// import { PrismaClient } from '@prisma/client'
|
||||
|
||||
// const prisma = new PrismaClient()
|
||||
|
||||
// async function main() {
|
||||
// const users = await prisma.user.findMany()
|
||||
// console.log(users)
|
||||
// }
|
||||
|
||||
// main()
|
||||
// .then(async () => {
|
||||
// await prisma.$disconnect()
|
||||
// })
|
||||
// .catch(async (e) => {
|
||||
// console.error(e)
|
||||
// await prisma.$disconnect()
|
||||
// process.exit(1)
|
||||
// })
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
{
|
||||
"id": 1695546540419428484,
|
||||
"id_str": "1695546540419428484",
|
||||
"url": "https://twitter.com/ProjektMelody/status/1695546540419428484",
|
||||
"date": "2023-08-26 21:19:31+00:00",
|
||||
"user": {
|
||||
"id": 1148121315943075841,
|
||||
"id_str": "1148121315943075841",
|
||||
"url": "https://twitter.com/ProjektMelody",
|
||||
"username": "ProjektMelody",
|
||||
"displayname": "ProjektMelody \ud83e\udd6f VSHOJO",
|
||||
"rawDescription": "\ud83e\udd471st Hentai A.I. Cam Girl\n\ud83e\uddea Science Team\n\ud83d\udcbb 3D/2D Streamer\n\u270f\ufe0f Fan Art: #Mel34\n\ud83d\udd27 Live2D: @henxtie + Rig: @iron_vertex\n\n\u2728\ufe0f\ud83d\udd1e https://t.co/qGruPkRGcn",
|
||||
"created": "2019-07-08 06:47:07+00:00",
|
||||
"followersCount": 642022,
|
||||
"friendsCount": 1715,
|
||||
"statusesCount": 11860,
|
||||
"favouritesCount": 18805,
|
||||
"listedCount": 1617,
|
||||
"mediaCount": 2225,
|
||||
"location": "Isekai'd to Casting Couch",
|
||||
"profileImageUrl": "https://pbs.twimg.com/profile_images/1686520956120977408/VXr38IaP_normal.jpg",
|
||||
"profileBannerUrl": "https://pbs.twimg.com/profile_banners/1148121315943075841/1694201047",
|
||||
"protected": null,
|
||||
"verified": false,
|
||||
"blue": true,
|
||||
"blueType": null,
|
||||
"descriptionLinks": [
|
||||
{
|
||||
"url": "http://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/qGruPkRGcn"
|
||||
},
|
||||
{
|
||||
"url": "https://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/dgQ8JdfvJs"
|
||||
}
|
||||
],
|
||||
"_type": "snscrape.modules.twitter.User"
|
||||
},
|
||||
"lang": "en",
|
||||
"rawContent": "WE'RE LIVE! Sorry for the lateness, my ass too fat. \n\nWe're LIVE!!!!! --> https://t.co/BdKdvTbn7q https://t.co/vnA4MrAfXU",
|
||||
"replyCount": 19,
|
||||
"retweetCount": 65,
|
||||
"likeCount": 1205,
|
||||
"quoteCount": 0,
|
||||
"conversationId": 1695546540419428484,
|
||||
"hashtags": [],
|
||||
"cashtags": [],
|
||||
"mentionedUsers": [],
|
||||
"links": [
|
||||
{
|
||||
"url": "http://shorturl.at/tNUVY",
|
||||
"text": "shorturl.at/tNUVY",
|
||||
"tcourl": "https://t.co/BdKdvTbn7q"
|
||||
}
|
||||
],
|
||||
"viewCount": 63118,
|
||||
"retweetedTweet": null,
|
||||
"quotedTweet": null,
|
||||
"place": null,
|
||||
"coordinates": null,
|
||||
"inReplyToTweetId": null,
|
||||
"inReplyToUser": null,
|
||||
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
|
||||
"sourceUrl": "https://mobile.twitter.com",
|
||||
"sourceLabel": "Twitter Web App",
|
||||
"media": {
|
||||
"photos": [
|
||||
{
|
||||
"url": "https://pbs.twimg.com/media/F4fJuPyWsAAS4Zh.png"
|
||||
}
|
||||
],
|
||||
"videos": [],
|
||||
"animated": []
|
||||
},
|
||||
"_type": "snscrape.modules.twitter.Tweet"
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
{
|
||||
"id": 1702461599452193113,
|
||||
"id_str": "1702461599452193113",
|
||||
"url": "https://twitter.com/ProjektMelody/status/1702461599452193113",
|
||||
"date": "2023-09-14 23:17:30+00:00",
|
||||
"user": {
|
||||
"id": 1148121315943075841,
|
||||
"id_str": "1148121315943075841",
|
||||
"url": "https://twitter.com/ProjektMelody",
|
||||
"username": "ProjektMelody",
|
||||
"displayname": "ProjektMelody \ud83e\udd6f VSHOJO",
|
||||
"rawDescription": "\ud83e\udd471st Hentai A.I. Cam Girl\n\ud83e\uddea Science Team\n\ud83d\udcbb 3D/2D Streamer\n\u270f\ufe0f Fan Art: #Mel34\n\ud83d\udd27 Live2D: @henxtie + Rig: @iron_vertex\n\n\u2728\ufe0f\ud83d\udd1e https://t.co/qGruPkRGcn",
|
||||
"created": "2019-07-08 06:47:07+00:00",
|
||||
"followersCount": 642064,
|
||||
"friendsCount": 1715,
|
||||
"statusesCount": 11860,
|
||||
"favouritesCount": 18805,
|
||||
"listedCount": 1618,
|
||||
"mediaCount": 2225,
|
||||
"location": "Isekai'd to Casting Couch",
|
||||
"profileImageUrl": "https://pbs.twimg.com/profile_images/1686520956120977408/VXr38IaP_normal.jpg",
|
||||
"profileBannerUrl": "https://pbs.twimg.com/profile_banners/1148121315943075841/1694201047",
|
||||
"protected": null,
|
||||
"verified": false,
|
||||
"blue": true,
|
||||
"blueType": null,
|
||||
"descriptionLinks": [
|
||||
{
|
||||
"url": "http://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/qGruPkRGcn"
|
||||
},
|
||||
{
|
||||
"url": "https://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/dgQ8JdfvJs"
|
||||
}
|
||||
],
|
||||
"_type": "snscrape.modules.twitter.User"
|
||||
},
|
||||
"lang": "in",
|
||||
"rawContent": "seymour butz --------->https://t.co/BdKdvTbn7q",
|
||||
"replyCount": 1,
|
||||
"retweetCount": 26,
|
||||
"likeCount": 702,
|
||||
"quoteCount": 0,
|
||||
"conversationId": 1702461424352506331,
|
||||
"hashtags": [],
|
||||
"cashtags": [],
|
||||
"mentionedUsers": [],
|
||||
"links": [
|
||||
{
|
||||
"url": "http://shorturl.at/tNUVY",
|
||||
"text": "shorturl.at/tNUVY",
|
||||
"tcourl": "https://t.co/BdKdvTbn7q"
|
||||
}
|
||||
],
|
||||
"viewCount": 53875,
|
||||
"retweetedTweet": null,
|
||||
"quotedTweet": null,
|
||||
"place": null,
|
||||
"coordinates": null,
|
||||
"inReplyToTweetId": 1702461424352506331,
|
||||
"inReplyToUser": {
|
||||
"id": 1148121315943075841,
|
||||
"username": "ProjektMelody",
|
||||
"displayname": "ProjektMelody \ud83e\udd6f VSHOJO",
|
||||
"_type": "snscrape.modules.twitter.UserRef"
|
||||
},
|
||||
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
|
||||
"sourceUrl": "https://mobile.twitter.com",
|
||||
"sourceLabel": "Twitter Web App",
|
||||
"media": {
|
||||
"photos": [],
|
||||
"videos": [],
|
||||
"animated": []
|
||||
},
|
||||
"_type": "snscrape.modules.twitter.Tweet"
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
{
|
||||
"id": 1719169779766866207,
|
||||
"id_str": "1719169779766866207",
|
||||
"url": "https://twitter.com/el_XoX34/status/1719169779766866207",
|
||||
"date": "2023-10-31 01:49:50+00:00",
|
||||
"user": {
|
||||
"id": 1328005180131061760,
|
||||
"id_str": "1328005180131061760",
|
||||
"url": "https://twitter.com/el_XoX34",
|
||||
"username": "el_XoX34",
|
||||
"displayname": "el_XoX\ud83d\udc30\ud83d\udd1e",
|
||||
"rawDescription": "\ud83d\udc30 Dorky Lewd Bun | \ud83d\udc99FA@NSLY IN WEBSITE \u2b07\ufe0f | \ud83d\udc8b Hentai Star | \u2728 Game Show Host | \ud83d\udd1eAUDIOS: https://t.co/9gHcDH2lEz | \ud83d\udc8c: elxoxbusiness@gmail.com",
|
||||
"created": "2020-11-15 16:02:21+00:00",
|
||||
"followersCount": 90357,
|
||||
"friendsCount": 1086,
|
||||
"statusesCount": 12322,
|
||||
"favouritesCount": 44788,
|
||||
"listedCount": 329,
|
||||
"mediaCount": 2204,
|
||||
"location": "pansexual | nonbinary \ud83c\udff3\ufe0f\u200d\ud83c\udf08 \u2661",
|
||||
"profileImageUrl": "https://pbs.twimg.com/profile_images/1626401510971084808/_Yq1TSKA_normal.jpg",
|
||||
"profileBannerUrl": "https://pbs.twimg.com/profile_banners/1328005180131061760/1696191257",
|
||||
"protected": null,
|
||||
"verified": false,
|
||||
"blue": true,
|
||||
"blueType": null,
|
||||
"descriptionLinks": [
|
||||
{
|
||||
"url": "http://patreon.com/el_XoX",
|
||||
"text": "patreon.com/el_XoX",
|
||||
"tcourl": "https://t.co/9gHcDH2lEz"
|
||||
},
|
||||
{
|
||||
"url": "https://elxox.carrd.co/",
|
||||
"text": "elxox.carrd.co",
|
||||
"tcourl": "https://t.co/cq4UlP6i2P"
|
||||
}
|
||||
],
|
||||
"_type": "snscrape.modules.twitter.User"
|
||||
},
|
||||
"lang": "en",
|
||||
"rawContent": "The rest would be available for Tier 2 or for $15!! \ud83d\udc95\u2935\ufe0f\u2935\ufe0f\nhttps://t.co/769MZLgLPT",
|
||||
"replyCount": 0,
|
||||
"retweetCount": 1,
|
||||
"likeCount": 17,
|
||||
"quoteCount": 0,
|
||||
"conversationId": 1719169756807274889,
|
||||
"hashtags": [],
|
||||
"cashtags": [],
|
||||
"mentionedUsers": [],
|
||||
"links": [
|
||||
{
|
||||
"url": "https://fansly.com/post/575088286796623872",
|
||||
"text": "fansly.com/post/575088286\u2026",
|
||||
"tcourl": "https://t.co/769MZLgLPT"
|
||||
}
|
||||
],
|
||||
"viewCount": 5771,
|
||||
"retweetedTweet": null,
|
||||
"quotedTweet": null,
|
||||
"place": null,
|
||||
"coordinates": null,
|
||||
"inReplyToTweetId": 1719169756807274889,
|
||||
"inReplyToUser": {
|
||||
"id": 1328005180131061760,
|
||||
"username": "el_XoX34",
|
||||
"displayname": "el_XoX\ud83d\udc30\ud83d\udd1e",
|
||||
"_type": "snscrape.modules.twitter.UserRef"
|
||||
},
|
||||
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
|
||||
"sourceUrl": "https://mobile.twitter.com",
|
||||
"sourceLabel": "Twitter Web App",
|
||||
"media": {
|
||||
"photos": [],
|
||||
"videos": [],
|
||||
"animated": []
|
||||
},
|
||||
"_type": "snscrape.modules.twitter.Tweet"
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
{
|
||||
"id": 1717957394909860027,
|
||||
"id_str": "1717957394909860027",
|
||||
"url": "https://twitter.com/ProjektMelody/status/1717957394909860027",
|
||||
"date": "2023-10-27 17:32:15+00:00",
|
||||
"user": {
|
||||
"id": 1148121315943075841,
|
||||
"id_str": "1148121315943075841",
|
||||
"url": "https://twitter.com/ProjektMelody",
|
||||
"username": "ProjektMelody",
|
||||
"displayname": "ProjektMelody \ud83e\udd6f VSHOJO",
|
||||
"rawDescription": "\ud83e\udd471st Hentai A.I. Cam Girl\n\ud83e\uddea Science Team\n\ud83d\udcbb 3D/2D Streamer\n\u270f\ufe0f Fan Art: #Mel34\n\ud83d\udd27 Live2D: @henxtie + Rig: @iron_vertex\n\n\u2728\ufe0f\ud83d\udd1e https://t.co/qGruPkRGcn",
|
||||
"created": "2019-07-08 06:47:07+00:00",
|
||||
"followersCount": 641554,
|
||||
"friendsCount": 1715,
|
||||
"statusesCount": 11855,
|
||||
"favouritesCount": 18796,
|
||||
"listedCount": 1618,
|
||||
"mediaCount": 2224,
|
||||
"location": "Isekai'd to Casting Couch",
|
||||
"profileImageUrl": "https://pbs.twimg.com/profile_images/1686520956120977408/VXr38IaP_normal.jpg",
|
||||
"profileBannerUrl": "https://pbs.twimg.com/profile_banners/1148121315943075841/1694201047",
|
||||
"protected": null,
|
||||
"verified": false,
|
||||
"blue": true,
|
||||
"blueType": null,
|
||||
"descriptionLinks": [
|
||||
{
|
||||
"url": "http://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/qGruPkRGcn"
|
||||
},
|
||||
{
|
||||
"url": "https://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/dgQ8JdfvJs"
|
||||
}
|
||||
],
|
||||
"_type": "snscrape.modules.twitter.User"
|
||||
},
|
||||
"lang": "en",
|
||||
"rawContent": "Look at these smug baddies",
|
||||
"replyCount": 16,
|
||||
"retweetCount": 102,
|
||||
"likeCount": 2204,
|
||||
"quoteCount": 0,
|
||||
"conversationId": 1717957394909860027,
|
||||
"hashtags": [],
|
||||
"cashtags": [],
|
||||
"mentionedUsers": [],
|
||||
"links": [],
|
||||
"viewCount": 63584,
|
||||
"retweetedTweet": null,
|
||||
"quotedTweet": {
|
||||
"id": 1714537964171121073,
|
||||
"id_str": "1714537964171121073",
|
||||
"url": "https://twitter.com/mayoartworks/status/1714537964171121073",
|
||||
"date": "2023-10-18 07:04:39+00:00",
|
||||
"user": {
|
||||
"id": 1333288287792979968,
|
||||
"id_str": "1333288287792979968",
|
||||
"url": "https://twitter.com/mayoartworks",
|
||||
"username": "mayoartworks",
|
||||
"displayname": "mayo.artworks",
|
||||
"rawDescription": "manga artist! \nVRChat dweller. \n\n\u2615 buy me a coffee! \nhttps://t.co/nw0cLgKXaV\n\nhttps://t.co/4DmjzSw2f4",
|
||||
"created": "2020-11-30 05:54:42+00:00",
|
||||
"followersCount": 11919,
|
||||
"friendsCount": 111,
|
||||
"statusesCount": 1424,
|
||||
"favouritesCount": 2072,
|
||||
"listedCount": 47,
|
||||
"mediaCount": 556,
|
||||
"location": "Guayaquil, EC",
|
||||
"profileImageUrl": "https://pbs.twimg.com/profile_images/1701740733881880576/mDwpOE75_normal.jpg",
|
||||
"profileBannerUrl": "https://pbs.twimg.com/profile_banners/1333288287792979968/1697242200",
|
||||
"protected": null,
|
||||
"verified": false,
|
||||
"blue": true,
|
||||
"blueType": null,
|
||||
"descriptionLinks": [
|
||||
{
|
||||
"url": "http://ko-fi.com/mayoartworks",
|
||||
"text": "ko-fi.com/mayoartworks",
|
||||
"tcourl": "https://t.co/nw0cLgKXaV"
|
||||
},
|
||||
{
|
||||
"url": "http://instagram.com/mayo.artworks/",
|
||||
"text": "instagram.com/mayo.artworks/",
|
||||
"tcourl": "https://t.co/4DmjzSw2f4"
|
||||
},
|
||||
{
|
||||
"url": "https://www.patreon.com/mayoartworks",
|
||||
"text": "patreon.com/mayoartworks",
|
||||
"tcourl": "https://t.co/UYpvLLKtxC"
|
||||
}
|
||||
],
|
||||
"_type": "snscrape.modules.twitter.User"
|
||||
},
|
||||
"lang": "en",
|
||||
"rawContent": "#Mel34 n @CosDestiny . commission. thank you for your support! https://t.co/LexOTzgcV5",
|
||||
"replyCount": 2,
|
||||
"retweetCount": 63,
|
||||
"likeCount": 659,
|
||||
"quoteCount": 1,
|
||||
"conversationId": 1714537964171121073,
|
||||
"hashtags": [
|
||||
"Mel34"
|
||||
],
|
||||
"cashtags": [],
|
||||
"mentionedUsers": [
|
||||
{
|
||||
"id": 1260107292,
|
||||
"username": "CosDestiny",
|
||||
"displayname": "\u25e4Destiny(\u904b\u547d) \u25e2 \u25e4 LEWD\u25e2 Nally.Vtube \u25e4",
|
||||
"_type": "snscrape.modules.twitter.UserRef"
|
||||
}
|
||||
],
|
||||
"links": [],
|
||||
"viewCount": 91564,
|
||||
"retweetedTweet": null,
|
||||
"quotedTweet": null,
|
||||
"place": null,
|
||||
"coordinates": null,
|
||||
"inReplyToTweetId": null,
|
||||
"inReplyToUser": null,
|
||||
"source": "<a href=\"http://twitter.com/#!/download/ipad\" rel=\"nofollow\">Twitter for iPad</a>",
|
||||
"sourceUrl": "http://twitter.com/#!/download/ipad",
|
||||
"sourceLabel": "Twitter for iPad",
|
||||
"media": {
|
||||
"photos": [
|
||||
{
|
||||
"url": "https://pbs.twimg.com/media/F8tDL2qXUAA00AK.jpg"
|
||||
}
|
||||
],
|
||||
"videos": [],
|
||||
"animated": []
|
||||
},
|
||||
"_type": "snscrape.modules.twitter.Tweet"
|
||||
},
|
||||
"place": null,
|
||||
"coordinates": null,
|
||||
"inReplyToTweetId": null,
|
||||
"inReplyToUser": null,
|
||||
"source": "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
|
||||
"sourceUrl": "http://twitter.com/download/android",
|
||||
"sourceLabel": "Twitter for Android",
|
||||
"media": {
|
||||
"photos": [],
|
||||
"videos": [],
|
||||
"animated": []
|
||||
},
|
||||
"_type": "snscrape.modules.twitter.Tweet"
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
{
|
||||
"id": 1718708096208425378,
|
||||
"id_str": "1718708096208425378",
|
||||
"url": "https://twitter.com/ProjektMelody/status/1718708096208425378",
|
||||
"date": "2023-10-29 19:15:16+00:00",
|
||||
"user": {
|
||||
"id": 1148121315943075841,
|
||||
"id_str": "1148121315943075841",
|
||||
"url": "https://twitter.com/ProjektMelody",
|
||||
"username": "ProjektMelody",
|
||||
"displayname": "ProjektMelody \ud83e\udd6f VSHOJO",
|
||||
"rawDescription": "\ud83e\udd471st Hentai A.I. Cam Girl\n\ud83e\uddea Science Team\n\ud83d\udcbb 3D/2D Streamer\n\u270f\ufe0f Fan Art: #Mel34\n\ud83d\udd27 Live2D: @henxtie + Rig: @iron_vertex\n\n\u2728\ufe0f\ud83d\udd1e https://t.co/qGruPkRGcn",
|
||||
"created": "2019-07-08 06:47:07+00:00",
|
||||
"followersCount": 641554,
|
||||
"friendsCount": 1715,
|
||||
"statusesCount": 11855,
|
||||
"favouritesCount": 18796,
|
||||
"listedCount": 1618,
|
||||
"mediaCount": 2224,
|
||||
"location": "Isekai'd to Casting Couch",
|
||||
"profileImageUrl": "https://pbs.twimg.com/profile_images/1686520956120977408/VXr38IaP_normal.jpg",
|
||||
"profileBannerUrl": "https://pbs.twimg.com/profile_banners/1148121315943075841/1694201047",
|
||||
"protected": null,
|
||||
"verified": false,
|
||||
"blue": true,
|
||||
"blueType": null,
|
||||
"descriptionLinks": [
|
||||
{
|
||||
"url": "http://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/qGruPkRGcn"
|
||||
},
|
||||
{
|
||||
"url": "https://linktr.ee/projektmelody",
|
||||
"text": "linktr.ee/projektmelody",
|
||||
"tcourl": "https://t.co/dgQ8JdfvJs"
|
||||
}
|
||||
],
|
||||
"_type": "snscrape.modules.twitter.User"
|
||||
},
|
||||
"lang": "en",
|
||||
"rawContent": "LAST MINUTE COLLLLAB! Zen orchastrated a Ghostbuster thingy with a bunch of cool peeps. Somehow I was invited. Let's jam!\n\nhttps://t.co/ZXn20dReuo https://t.co/NdbgfR2766",
|
||||
"replyCount": 15,
|
||||
"retweetCount": 80,
|
||||
"likeCount": 2042,
|
||||
"quoteCount": 0,
|
||||
"conversationId": 1718708096208425378,
|
||||
"hashtags": [],
|
||||
"cashtags": [],
|
||||
"mentionedUsers": [],
|
||||
"links": [
|
||||
{
|
||||
"url": "http://twitch.tv/projektmelody",
|
||||
"text": "twitch.tv/projektmelody",
|
||||
"tcourl": "https://t.co/ZXn20dReuo"
|
||||
}
|
||||
],
|
||||
"viewCount": 53145,
|
||||
"retweetedTweet": null,
|
||||
"quotedTweet": null,
|
||||
"place": null,
|
||||
"coordinates": null,
|
||||
"inReplyToTweetId": null,
|
||||
"inReplyToUser": null,
|
||||
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
|
||||
"sourceUrl": "https://mobile.twitter.com",
|
||||
"sourceLabel": "Twitter Web App",
|
||||
"media": {
|
||||
"photos": [
|
||||
{
|
||||
"url": "https://pbs.twimg.com/media/F9oTe1gWsAABvoX.jpg"
|
||||
}
|
||||
],
|
||||
"videos": [],
|
||||
"animated": []
|
||||
},
|
||||
"_type": "snscrape.modules.twitter.Tweet"
|
||||
}
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
import dotenv from 'dotenv'
|
||||
import Cluster from '../lib/Cluster.js'
|
||||
|
||||
// if (process.env.IPFS_CLUSTER_HTTP_API_MULTIADDR === undefined) throw new Error('IPFS_CLUSTER_HTTP_API_MULTIADDR undef in env');
|
||||
if (process.env.IPFS_CLUSTER_HTTP_API_USERNAME === undefined) throw new Error('IPFS_CLUSTER_HTTP_API_USERNAME undef in env');
|
||||
if (process.env.IPFS_CLUSTER_HTTP_API_PASSWORD === undefined) throw new Error('IPFS_CLUSTER_HTTP_API_PASSWORD undef in env');
|
||||
|
||||
|
||||
async function main() {
|
||||
const cluster = new Cluster({
|
||||
username: process.env.IPFS_CLUSTER_HTTP_API_USERNAME,
|
||||
password: process.env.IPFS_CLUSTER_HTTP_API_PASSWORD
|
||||
})
|
||||
// const statuses = await cluster.getPinCount('bafybeiclkyiomuru53rapmaekxyzyuiicc2ddtqx3el5pxaq2apqwdpnr4')
|
||||
// console.log(statuses)
|
||||
const eeeeee = await idempotentlyPinIpfsContent(cluster, {
|
||||
entry: {
|
||||
videoSrcHash: 'bafkreibsuow7tcfweysasilsslt2h3rlxa4deud43p7kx2fc25tw6urfcu'
|
||||
}
|
||||
})
|
||||
console.log(eeeeee)
|
||||
}
|
||||
|
||||
|
||||
async function idempotentlyPinIpfsContent(cluster, data) {
|
||||
let results = []
|
||||
const cids = [
|
||||
data?.entry?.videoSrcHash,
|
||||
data?.entry?.video240Hash,
|
||||
data?.entry?.thiccHash
|
||||
]
|
||||
const validCids = cids.filter((c) => c !== '' && c !== undefined)
|
||||
console.log(`Here are the valid CIDs:${JSON.stringify(validCids)}`)
|
||||
if (validCids.length === 0) return results
|
||||
for (const vc of validCids) {
|
||||
const pinCount = await cluster.getPinCount(vc)
|
||||
if (pinCount < 1) {
|
||||
const pinnedCid = await cluster.pinAdd(vc)
|
||||
results.push(pinnedCid)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
main()
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import { describe } from "node:test"
|
||||
import chai, { expect } from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import { expandUrl, isChaturbateInviteLinkPresent, isChaturbateLink } from "../src/tweets.js"
|
||||
import nock from 'nock';
|
||||
import tweetExpiredCb from './fixtures/tweetExpiredCb.json' assert { type: 'json' };
|
||||
import tweetCb from './fixtures/tweetCb.json' assert { type: 'json' };
|
||||
import tweetFansly from './fixtures/tweetFansly.json' assert { type: 'json' };
|
||||
import tweetLinkless from './fixtures/tweetLinkless.json' assert { type: 'json' };
|
||||
import tweetTwitch from './fixtures/tweetTwitch.json' assert { type: 'json' };
|
||||
|
||||
chai.use(chaiAsPromised);
|
||||
|
||||
describe('tweets', function () {
|
||||
describe('isChaturbateInviteLinkPresent', function () {
|
||||
it('should load json fixtures properly', function () {
|
||||
expect(tweetExpiredCb).to.have.property('_type', 'snscrape.modules.twitter.Tweet');
|
||||
expect(tweetCb).to.have.property('_type', 'snscrape.modules.twitter.Tweet');
|
||||
expect(tweetFansly).to.have.property('_type', 'snscrape.modules.twitter.Tweet');
|
||||
expect(tweetLinkless).to.have.property('_type', 'snscrape.modules.twitter.Tweet');
|
||||
expect(tweetTwitch).to.have.property('_type', 'snscrape.modules.twitter.Tweet');
|
||||
})
|
||||
it('should resolve true to a tweet with link to expired url shortener redirecting to cb', function () {
|
||||
return expect(isChaturbateInviteLinkPresent(tweetExpiredCb)).to.eventually.be.true;
|
||||
})
|
||||
it('should resolve true to a tweet with link to a cb room', function () {
|
||||
return expect(isChaturbateInviteLinkPresent(tweetCb)).to.eventually.be.true;
|
||||
})
|
||||
it('should resolve false to a tweet with link to a fansly room', function () {
|
||||
return expect(isChaturbateInviteLinkPresent(tweetFansly)).to.eventually.be.false;
|
||||
})
|
||||
it('should resolve false for a tweet with no links', function () {
|
||||
return expect(isChaturbateInviteLinkPresent(tweetLinkless)).to.eventually.be.false;
|
||||
})
|
||||
it('should resolve false for a tweet with a twitch link', function() {
|
||||
return expect(isChaturbateInviteLinkPresent(tweetTwitch)).to.eventually.be.false;
|
||||
})
|
||||
})
|
||||
|
||||
describe('isChaturbateLink', function () {
|
||||
it('should detect chaturbate.com/b/:room/', function () {
|
||||
const link = 'https://chaturbate.com/b/projektmelody/'
|
||||
expect(isChaturbateLink(link)).to.be.true;
|
||||
})
|
||||
it('should detect chaturbate.com/:room/', function () {
|
||||
const link = 'https://chaturbate.com/projektmelody/'
|
||||
expect(isChaturbateLink(link)).to.be.true;
|
||||
})
|
||||
})
|
||||
|
||||
describe('expandUrl', function () {
|
||||
it('should handle an invalid URL', function () {
|
||||
const invalid = '/users/198128/artworks';
|
||||
return expect(expandUrl(invalid)).to.eventually.equal(invalid);
|
||||
})
|
||||
it('should handle a 404', function () {
|
||||
const jast = 'https://jast.us/melody';
|
||||
return expect(expandUrl(jast)).to.eventually.equal(jast);
|
||||
})
|
||||
it('should expand tinyurl link', async function () {
|
||||
const shortUrl = new URL('https://tinyurl.com/58p5k9ru');
|
||||
const expectedUrl = 'https://www.tacobell.com/';
|
||||
nock(shortUrl.origin)
|
||||
.head(shortUrl.pathname)
|
||||
.reply(308, 'mocked by nock', {
|
||||
'location': expectedUrl
|
||||
})
|
||||
const expandedUrl = await expandUrl(shortUrl.toString());
|
||||
expect(expandedUrl).to.equal(expectedUrl);
|
||||
})
|
||||
it('should expand shorturl link', async function () {
|
||||
const shortUrl = new URL('https://www.shorturl.at/ejlLT');
|
||||
const expectedUrl = 'https://www.tacobell.com/';
|
||||
nock(shortUrl.origin)
|
||||
.head(shortUrl.pathname)
|
||||
.reply(302, 'mocked by nock', {
|
||||
'location': expectedUrl
|
||||
})
|
||||
const expandedUrl = await expandUrl(shortUrl.toString());
|
||||
expect(expandedUrl).to.equal(expectedUrl);
|
||||
})
|
||||
it('should return an already expanded link', async function () {
|
||||
const expandedUrl = 'https://chaturbate.com/b/projektmelody/';
|
||||
const doubleExpandedUrl = await expandUrl(expandedUrl);
|
||||
expect(doubleExpandedUrl).to.equal(expandedUrl);
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
import { got } from 'got';
|
||||
import dotenv from 'dotenv';
|
||||
dotenv.config()
|
||||
|
||||
if (process.env.PORT === 'undefined') throw new Error('PORT must be defined in process.env')
|
||||
|
||||
async function main() {
|
||||
await got.post(`http://localhost:${process.env.PORT}/webhook`, {
|
||||
json: {
|
||||
"event": "entry.create",
|
||||
"createdAt": "2020-01-10T08:47:36.649Z",
|
||||
"model": "vod",
|
||||
"entry": {
|
||||
"id": 1,
|
||||
"backup": {
|
||||
"url": "https://f000.backblazeb2.com/b2api/v1/b2_download_file_by_id?fileId=4_zfd1367af11f33a3973a30b18_f226a2d1743f4e230_d20220907_m223453_c000_v0001077_t0027_u01662590093450"
|
||||
},
|
||||
"videoSrcHash": '',
|
||||
"video240Hash": '',
|
||||
"date": null,
|
||||
"thiccHash": '',
|
||||
"createdAt": "2020-01-10T08:47:36.264Z",
|
||||
"updatedAt": "2020-01-10T08:47:36.264Z",
|
||||
}
|
||||
}
|
||||
}).json()
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,21 +1,45 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"outDir": "dist",
|
||||
"allowJs": true,
|
||||
"target": "es6",
|
||||
"esModuleInterop": true,
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"strict": false,
|
||||
},
|
||||
"include": [
|
||||
"**/*.ts",
|
||||
"dist/types/**/*.ts", "cli.ts.noexec",
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "NodeNext",
|
||||
"experimentalDecorators": true,
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"noImplicitAny": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"skipLibCheck": true,
|
||||
"declaration": true,
|
||||
"emitDeclarationOnly": true,
|
||||
"typeRoots": [
|
||||
"node_modules/@types",
|
||||
"src/typings/**/*.d.ts",
|
||||
"reflect-metadata"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules"
|
||||
"resolveJsonModule": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
},
|
||||
"paths": {
|
||||
"@/lib/*": [
|
||||
"lib/*"
|
||||
],
|
||||
"@/src/*": [
|
||||
"src/*"
|
||||
],
|
||||
"@/tasks/*": [
|
||||
"tasks/*"
|
||||
],
|
||||
"@/issues/*": [
|
||||
"issues/*"
|
||||
]
|
||||
}
|
||||
},
|
||||
"include": [
|
||||
"**/*.ts",
|
||||
"dist/types/**/*.ts",
|
||||
"cli.ts.noexec", "test/tweets.test.js",
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"tests"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"entryFile": "manager.ts",
|
||||
"noImplicitAdditionalProperties": "throw-on-extras",
|
||||
"controllerPathGlobs": ["src/**/*Controller.ts"],
|
||||
"spec": {
|
||||
"outputDirectory": "dist/futureporn-qa/swagger",
|
||||
"specVersion": 3
|
||||
},
|
||||
"routes": {
|
||||
"routesDir": "dist/futureporn-qa/routes"
|
||||
},
|
||||
"esm": true
|
||||
}
|
||||
99
worker.ts
99
worker.ts
|
|
@ -1,29 +1,80 @@
|
|||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as dotenv from 'dotenv'
|
||||
import PgBoss from 'pg-boss';
|
||||
import {identifyVodIssues} from './tasks/identifyVodIssues.js';
|
||||
import {generateThumbnail} from './tasks/generateThumbnail.js';
|
||||
import {deleteThumbnail} from './tasks/deleteThumbnail.js';
|
||||
import pkg from './package.json' assert {type: 'json'};
|
||||
import { loggerFactory } from './src/logger.js';
|
||||
import { type Logger } from 'winston';
|
||||
import { dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { type ITask, loadTaskDefinitions } from './src/Task.js';
|
||||
import { Worker, Job, Queue } from 'bullmq'
|
||||
import { default as Redis } from 'ioredis'
|
||||
import { REDIS_HOST, REDIS_PORT, TASK_LIST, IPFS_CLUSTER_HTTP_API_MULTIADDR } from './src/env.js';
|
||||
|
||||
export interface IJobData {
|
||||
id?: number;
|
||||
env: NodeJS.ProcessEnv;
|
||||
const connection = new Redis.default({
|
||||
port: parseInt(REDIS_PORT),
|
||||
host: REDIS_HOST,
|
||||
maxRetriesPerRequest: null,
|
||||
});
|
||||
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
dotenv.config();
|
||||
|
||||
|
||||
const taskList = TASK_LIST.split(',');
|
||||
|
||||
|
||||
const logger = loggerFactory({ defaultMeta: { service: 'futureporn-qa' }});
|
||||
|
||||
logger.log({ level: 'info', message: `🧃 futureporn-qa worker version ${pkg.version}` });
|
||||
logger.log({ level: 'info', message: `📃 worker for the following tasks:${taskList}` });
|
||||
|
||||
|
||||
|
||||
async function main(logger: Logger, taskList: string[]) {
|
||||
|
||||
const tasksDirectory = path.join(__dirname, 'src', 'tasks'); // Update the path as needed
|
||||
const activeTasks = await loadTaskDefinitions(logger, tasksDirectory, taskList);
|
||||
|
||||
// Print the list of active tasks
|
||||
logger.log({ level: 'info', message: `👷 Active Tasks: ${activeTasks.map((task) => task.name)}` });
|
||||
|
||||
|
||||
for (const task of activeTasks) {
|
||||
logger.log({ level: 'info', message: `🈺 Initializing queue for ${task.name}`});
|
||||
|
||||
logger.log({ level: 'info', message: `💪 Initializing worker for ${task.name}` });
|
||||
const worker = new Worker(
|
||||
task.name,
|
||||
async (job: Job) => {
|
||||
logger.log({ level: 'info', message: `🏃 Worker is doing the thing ${JSON.stringify(job.data)} ${task.name}` })
|
||||
const res = await task.runTask({
|
||||
env: process.env,
|
||||
logger: logger,
|
||||
job: job,
|
||||
connection: connection,
|
||||
})
|
||||
logger.log({ level: 'info', message: `🍻 Worker completed task ${task.name} (job ID ${job.id})`})
|
||||
return res;
|
||||
}, {
|
||||
concurrency: 1,
|
||||
autorun: false,
|
||||
connection,
|
||||
}
|
||||
);
|
||||
worker.on('error', (err) => {
|
||||
logger.log({ level: 'error', message: `🔥 Worker ${worker.id} encountered an error` })
|
||||
if (err instanceof Error) {
|
||||
logger.log({ level: 'error', message: `🔥 ${err.message}` })
|
||||
}
|
||||
})
|
||||
worker.on('failed', (job: Job, error: Error) => {
|
||||
logger.log({ level: 'warn', message: `💔 worker failed on job ${job.id}`});
|
||||
})
|
||||
worker.run();
|
||||
}
|
||||
}
|
||||
|
||||
dotenv.config()
|
||||
|
||||
if (!process.env.DATABASE_URL) throw new Error('DATABASE_URL is missing in env')
|
||||
const DATABASE_URL = process.env.DATABASE_URL
|
||||
|
||||
console.log(`futureporn-qa worker version ${pkg.version}`)
|
||||
|
||||
|
||||
async function main () {
|
||||
const boss = new PgBoss(DATABASE_URL);
|
||||
await boss.start()
|
||||
await boss.work('identifyVodIssues', (job: PgBoss.Job) => identifyVodIssues(job, boss, process.env));
|
||||
await boss.work('generateThumbnail', { teamConcurrency: 1, teamSize: 1 }, (job: PgBoss.Job) => generateThumbnail(job, process.env));
|
||||
await boss.work('deleteThumbnail', (job: PgBoss.Job) => deleteThumbnail(job, boss, process.env));
|
||||
}
|
||||
|
||||
main()
|
||||
main(logger, taskList);
|
||||
|
|
|
|||
Reference in New Issue