Makefile

@@ -11,9 +11,6 @@ crds:
 cert-manager:
 	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.yaml
 	
-secrets:
-	./scripts/k8s-secrets.sh
-
 flux:
 	flux bootstrap git --url="ssh://git@gitea.futureporn.net:2222/futureporn/fp" --branch=main --path="clusters/production" --private-key-file=/home/cj/.ssh/fp-flux
 

charts/fp/templates/scout.yaml

@@ -34,16 +34,6 @@ spec:
           value: "{{ .Values.scout.cdnBucketUrl }}"
         - name: STRAPI_URL
           value: https://strapi.piko.sbtp.xyz
-        - name: S3_BUCKET_APPLICATION_KEY
-          valueFrom:
-            secretKeyRef:
-              name: scout
-              key: s3BucketApplicationKey
-        - name: S3_BUCKET_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: scout
-              key: s3BucketKeyId
         - name: SCOUT_NITTER_ACCESS_KEY
           valueFrom:
             secretKeyRef:

packages/next/app/components/archive-progress.tsx

@@ -1,4 +1,4 @@
-import { getAllStreamsForVtuber, getStreamCountForVtuber } from "@/lib/streams";
+import { getAllStreamsForVtuber } from "@/lib/streams";
 import { getVodsForVtuber } from "@/lib/vods";
 import { IVtuber } from "@/lib/vtubers";
 
@@ -7,18 +7,21 @@ export interface IArchiveProgressProps {
 }
 
 export default async function ArchiveProgress ({ vtuber }: IArchiveProgressProps) {
+    // const vods = await getVodsForVtuber(vtuber.id)
+    // const streams = await getAllStreamsForVtuber(vtuber.id);
+    // const goodStreams = await getAllStreamsForVtuber(vtuber.id, ['good']);
+    // const issueStreams = await getAllStreamsForVtuber(vtuber.id, ['issue']);
+    // const totalStreams = streams.length;
+    // const eligibleStreams = issueStreams.length+goodStreams.length;
 
     // // Check if totalStreams is not zero before calculating completedPercentage
     // const completedPercentage = (totalStreams !== 0) ? Math.round(eligibleStreams / totalStreams * 100) : 0;
-    const totalStreams = await getStreamCountForVtuber(vtuber.id);
+    const completedPercentage = 50
-    const eligibleStreams = await getStreamCountForVtuber(vtuber.id, ['good', 'issue']);
+    const totalStreams = 500
-    const completedPercentage = (eligibleStreams / totalStreams) * 100
+    const eligibleStreams = 50
     return (
         <div>
-            {/* <p>
+            <p>@todo</p>
-                {totalStreams} known streams<br />
-                {eligibleStreams} archived<br />
-            </p>  */}
             <p className="heading">{eligibleStreams}/{totalStreams} Streams Archived ({completedPercentage}%)</p>
             <progress className="progress is-success" value={eligibleStreams} max={totalStreams}>{completedPercentage}%</progress>
         </div>

packages/next/app/components/funding-goal.tsx

@@ -1,7 +1,7 @@
 
 import { getCampaign } from "@/lib/patreon";
 import { getGoals, IGoals } from '@/lib/pm'
-import Image from "next/legacy/image";
+import Image from 'next/image';
 import React from 'react';
 import Link from 'next/link'
 

packages/next/app/components/stream-page.tsx

@@ -4,7 +4,7 @@ import { IStream } from "@/lib/streams";
 // import NotFound from "app/streams/[cuid]/not-found";
 import { IVod } from "@/lib/vods";
 import Link from "next/link";
-import Image from "next/legacy/image";
+import Image from "next/image";
 import { LocalizedDate } from "./localized-date";
 import { FontAwesomeIcon, FontAwesomeIconProps } from "@fortawesome/react-fontawesome";
 import { faTriangleExclamation, faCircleInfo, faThumbsUp, IconDefinition, faO, faX, faExternalLinkAlt } from "@fortawesome/free-solid-svg-icons";
@@ -169,8 +169,7 @@ export default function StreamPage({ stream }: IStreamProps) {
                                 <span className="title is-1"><FontAwesomeIcon icon={icon}></FontAwesomeIcon></span>
                                 <p className="mt-3">{desc1}</p>
                                 <p className="mt-5">{desc2}<br />
-                                {/* <Link href={`/upload?cuid=${stream.attributes.cuid}`}>Upload it here.</Link></p> */}
+                                <Link href={`/upload?cuid=${stream.attributes.cuid}`}>Upload it here.</Link></p>
-                                <Link style={{ cursor: 'not-allowed' }} href={`/upload?cuid=${stream.attributes.cuid}`}><i>Uploads coming soon.</i></Link></p>
                             </div>
                         </article>
                     </div>

packages/next/app/components/stream.tsx

@@ -4,7 +4,7 @@ import { LocalizedDate } from "./localized-date";
 import Link from "next/link";
 import ChaturbateIcon from "@/components/icons/chaturbate";
 import FanslyIcon from "@/components/icons/fansly";
-import Image from "next/legacy/image";
+import Image from "next/image";
 
 export interface IStreamProps {
     stream: IStream;

packages/next/app/components/streams-table.tsx

@@ -8,7 +8,7 @@ import {
   useQuery,
 } from '@tanstack/react-query'
 import { format } from 'date-fns'
-import Image from "next/legacy/image"
+import Image from 'next/image'
 import {
   PaginationState,
   useReactTable,
@@ -65,7 +65,6 @@ export default function StreamsTable() {
                           src={image}
                           alt={displayName}
                           placeholder="blur"
-                          objectFit='cover'
                           blurDataURL={imageBlur}
                           width={32}
                           height={32}
@@ -81,11 +80,8 @@ export default function StreamsTable() {
             }
         },
         {
-            header: 'Date2',
+            header: 'Date',
             accessorFn: d => format(new Date(d.attributes.date2), 'yyyy-MM-dd HH:mm'),
-            // accessorFn: d => new Date(d.attributes.date2),
-            sortingFn: 'datetime',
-            sortDescFirst: true,
             cell: info => (<Link href={`/archive/${info.row.original.attributes.cuid}`}>{info.getValue() as string}</Link>)
         },
         {

packages/next/app/components/thumbnail.tsx

@@ -1,6 +1,6 @@
 'use client'
 import LinkableHeading from "./linkable-heading"
-import Image from "next/legacy/image"
+import Image from "next/image"
 import { useState } from "react"
 import styles from '@/assets/styles/fp.module.css'
 

packages/next/app/components/toys.tsx

@@ -2,7 +2,7 @@ import React from 'react';
 import { IToy, IToysResponse } from '@/lib/toys';
 import { IVtuber } from '@/lib/vtubers';
 import Link from 'next/link';
-import Image from "next/legacy/image";
+import Image from 'next/image';
 
 export interface IToyProps {
   toy: IToy;
@@ -56,7 +56,8 @@ export function ToyItem({ toy }: IToyProps) {
             src={toy.attributes.image2}
             alt={displayName}
             objectFit='contain'
-            layout='fill'
+            
+            fill
           />
         </figure>
         <p className="heading">{toy.attributes.model}</p>

packages/next/app/components/vod-card.tsx

@@ -4,7 +4,7 @@ import { faPatreon } from "@fortawesome/free-brands-svg-icons";
 import { faVideo } from "@fortawesome/free-solid-svg-icons";
 import { getSafeDate, getDateFromSafeDate } from '@/lib/dates';
 import { IVtuber } from '@/lib/vtubers';
-import Image from "next/legacy/image"
+import Image from 'next/image'
 import { LocalizedDate } from '@/components/localized-date'
 import { IMuxAsset, IMuxAssetResponse } from "@/lib/types";
 import { IB2File } from "@/lib/b2File";

packages/next/app/components/vod-nav.tsx

@@ -3,7 +3,7 @@
 import { faVideo, faExternalLinkAlt, faShareAlt } from "@fortawesome/free-solid-svg-icons";
 import { faXTwitter } from '@fortawesome/free-brands-svg-icons';
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
-import Image from "next/legacy/image";
+import Image from 'next/image';
 import Link from 'next/link';
 import { IVod } from '@/lib/vods';
 import { buildIpfsUrl } from '@/lib/ipfs';

packages/next/app/components/vod-page.tsx

@@ -7,7 +7,7 @@ import { faChevronLeft, faChevronRight, faGlobe, faImage, faLink } from "@fortaw
 import { notFound } from 'next/navigation';
 import { IpfsCid } from './ipfs-cid';
 import LinkableHeading from './linkable-heading';
-import Image from "next/legacy/image";
+import Image from 'next/image';
 import Thumbnail from './thumbnail';
 
 export function getVodTitle(vod: IVod): string {

packages/next/app/components/vtuber-button.tsx

@@ -1,4 +1,4 @@
-import Image from "next/legacy/image"
+import Image from "next/image"
 
 interface VtuberButtonProps {
     image: string;

packages/next/app/components/vtuber-card.tsx

@@ -1,7 +1,7 @@
 import Link from "next/link";
 import type { IVtuber } from '@/lib/vtubers';
 import { getVodsForVtuber } from "@/lib/vods";
-import Image from "next/legacy/image"
+import Image from 'next/image'
 import NotFound from "app/vt/[slug]/not-found";
 import ArchiveProgress from "./archive-progress";
 
@@ -24,7 +24,6 @@ export default async function VTuberCard(vtuber: IVtuber) {
                                     className="is-rounded"
                                     src={image}
                                     alt={displayName}
-                                    objectFit="cover"
                                     placeholder="blur"
                                     blurDataURL={imageBlur}
                                     width={48}

packages/next/app/lib/streams.ts

@@ -335,8 +335,7 @@ export async function fetchStreamData({ pageIndex, pageSize }: { pageIndex: numb
             start: offset,
             limit: pageSize,
             withCount: true
-        },
+        }
-        sort: ['date:desc']
     })
     const response = await fetch(
         `${strapiUrl}/api/streams?${query}`
@@ -352,29 +351,7 @@ export async function fetchStreamData({ pageIndex, pageSize }: { pageIndex: numb
     return d;
 }
 
-export async function getStreamCountForVtuber(vtuberId: number, archiveStatuses = ['missing', 'issue', 'good']): Promise<number> {
+  
-    if (!vtuberId) throw new Error(`getStreamCountForVtuber requires a vtuberId, but it was undefined.`);
-    // @todo possible performance improvement is to only request the meta field, since we don't use any of the data.attributes
-    const query = qs.stringify(
-        {
-            filters: {
-                vtuber: {
-                    id: {
-                        $eq: vtuberId
-                    }
-                },
-                archiveStatus: {
-                    '$in': archiveStatuses
-                }
-            }
-        }
-    )
-    const res = await fetch(`${strapiUrl}/api/streams?${query}`, fetchStreamsOptions)
-    const data = await res.json()
-    console.log(`getStreamCountForVtuber with archiveStatuses=${archiveStatuses}`)
-    console.log(JSON.stringify(data, null, 2))
-    return data.meta.pagination.total
-}
 
 export async function getStreamsForVtuber(vtuberId: number, page: number = 1, pageSize: number = 25, sortDesc = true): Promise<IStreamsResponse> {
     console.log(`getStreamsForVtuber() with strapiUrl=${strapiUrl}`)

packages/next/app/lib/types.ts

@@ -26,21 +26,3 @@ export interface IMuxAssetResponse {
 export interface IMeta {
 	pagination: IPagination;
 }
-
-
-
-export interface IPlatformNotification {
-    id: number;
-    attributes: {
-        source: string;
-        platform: string;
-        date: string;
-		date2: string;
-		vtuber: number;
-    }
-}
-
-export interface IPlatformNotificationResponse {
-    data: IPlatformNotification;
-    meta: IMeta;
-}

packages/next/app/lib/vtubers.ts

@@ -2,7 +2,9 @@
 
 import { IVod } from './vods'
 import { strapiUrl, siteUrl } from './constants';
+import { getSafeDate } from './dates';
 import qs from 'qs';
+import { resourceLimits } from 'worker_threads';
 import { IMeta } from './types';
 
 
@@ -41,9 +43,6 @@ export interface IVtuber {
         image: string;
         imageBlur?: string;
         themeColor: string;
-        fanslyId?: string;
-        chaturbateId?: string;
-        twitterId?: string;
     }
 }
 

packages/next/app/vt/[slug]/page.tsx

@@ -4,7 +4,7 @@ import { getVtuberBySlug } from '@/lib/vtubers'
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faExternalLinkAlt, faBagShopping } from "@fortawesome/free-solid-svg-icons";
 import { faFacebook, faInstagram, faPatreon, faYoutube, faTwitch, faTiktok, faXTwitter, faReddit, faDiscord } from "@fortawesome/free-brands-svg-icons";
-import Image from "next/legacy/image";
+import Image from 'next/image';
 import OnlyfansIcon from "@/components/icons/onlyfans";
 import PornhubIcon from '@/components/icons/pornhub';
 import ThroneIcon from '@/components/icons/throne';
@@ -63,8 +63,7 @@ export default async function Page({ params }: { params: { slug: string } }) {
                     className="is-rounded"
                     alt={vtuber.attributes.displayName}
                     src={vtuber.attributes.image}
-                    layout='fill'
+                    fill={true}
-                    objectFit='cover'
                     placeholder='blur'
                     blurDataURL={vtuber.attributes.imageBlur}
                   />

packages/next/next.config.js

@@ -14,12 +14,6 @@ const nextConfig = {
         port: '',
         pathname: '/**',
       },
-      {
-        protocol: 'https',
-        hostname: 'fp-dev.b-cdn.net',
-        port: '',
-        pathname: '/**',
-      },
     ],
   }
 };

packages/scout/src/cb.js

@@ -2,10 +2,12 @@ import cheerio from 'cheerio'
 
 /**
  * 
+ * @param {Object} limiter     An instance of node-rate-limiter, see https://github.com/jhurliman/node-rate-limiter
  * @param {String} roomUrl     example: https://chaturbate.com/projektmelody
  * @returns {Object} initialRoomDossier
  */
-export async function getInitialRoomDossier(roomUrl) {
+export async function getInitialRoomDossier(limiter, roomUrl) {
+    await limiter.removeTokens(1);
     try {
       const res = await fetch(roomUrl, {
         headers: {

packages/scout/src/cb.spec.js

@@ -1,11 +1,13 @@
 import { describe } from 'mocha'
 import { expect } from 'chai';
 import { getInitialRoomDossier } from './cb.js'
+import { RateLimiter } from "limiter";
 
 describe('cb', function () {
+    let limiter = new RateLimiter({ tokensPerInterval: 10, interval: "minute" })
     describe('getInitialRoomDossier', function () {
         it('should return json', async function () {
-            const dossier = await getInitialRoomDossier('https://chaturbate.com/projektmelody')
+            const dossier = await getInitialRoomDossier(limiter, 'https://chaturbate.com/projektmelody')
             expect(dossier).to.have.property('wschat_host')
         })
     })

packages/scout/src/fansly.js

@@ -12,11 +12,12 @@ const normalize = (url) => {
 
 const fromUsername = (username) => `https://fansly.com/${username}`
 
-const image = async function image (fanslyUserId) {
+const image = async function image (limiter, fanslyUserId) {
-    if (!fanslyUserId) throw new Error(`first arg passed to fansly.data.image must be a {string} fanslyUserId`);
+    if (!limiter) throw new Error(`first arg passed to fansly.data.image must be a node-rate-limiter instance`);
+    if (!fanslyUserId) throw new Error(`second arg passed to fansly.data.image must be a {string} fanslyUserId`);
     const url = `https://api.fansly.com/api/v1/account/${fanslyUserId}/avatar`
     const filePath = getTmpFile('avatar.jpg')
-    return download({ filePath, url })
+    return download({ filePath, limiter, url })
 }
 
 const url = {

packages/scout/src/imap.js

@@ -4,6 +4,10 @@ import EventEmitter from 'node:events';
 import 'dotenv/config';
 import { simpleParser } from 'mailparser';
 
+// pinned to v2.0.1 due to https://github.com/jhurliman/node-rate-limiter/issues/80
+import * as $limiter from 'limiter';
+const { RateLimiter } = $limiter
+
 
 
 if (!process.env.SCOUT_IMAP_SERVER) throw new Error('SCOUT_IMAP_SERVER is missing from env');
@@ -11,6 +15,8 @@ if (!process.env.SCOUT_IMAP_PORT) throw new Error('SCOUT_IMAP_PORT is missing fr
 if (!process.env.SCOUT_IMAP_USERNAME) throw new Error('SCOUT_IMAP_USERNAME is missing from env');
 if (!process.env.SCOUT_IMAP_PASSWORD) throw new Error('SCOUT_IMAP_PASSWORD is missing from env');
 
+const limiter = new RateLimiter({ tokensPerInterval: 1, interval: 3000 });
+
 // https://stackoverflow.com/a/49428486/1004931
 function streamToString(stream) {
     const chunks = [];
@@ -31,6 +37,7 @@ export class Email extends EventEmitter {
     }
 
     async archiveMessage(uid) {
+        await limiter.removeTokens(1);
         await this.client.messageDelete(uid, { uid: true })
     }
 

packages/scout/src/index.ts

@@ -25,29 +25,32 @@ async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageOb
         // console.log('    ✏️ checking e-mail')
         const { isMatch, url, platform, channel, displayName, date, userId, avatar }: NotificationData = (await checkEmail(body) ) 
 
+
         if (isMatch) {
             const wfId = `process-email-${createId()}`
             // console.log(`    ✏️ [DRY] starting Temporal workflow ${wfId} @todo actually start temporal workflow`)
             // await signalRealtime({ url, platform, channel, displayName, date, userId, avatar })
             // @todo invoke a Temporal workflow here
-            console.log('    ✏️✏️ starting Temporal workflow')
             const handle = await client.workflow.start(processEmail, {
                 workflowId: wfId,
                 taskQueue: 'scout',
                 args: [{ url, platform, channel, displayName, date, userId, avatar }]
             });
             // const handle = client.getHandle(workflowId);
-            const result = await handle.result()
+            const result = await handle.result();
             console.log(`result of the workflow is as follows`)
             console.log(result)
+            throw new Error('!todo we are stopping after just one (for now) @todo')
+
+            // console.log('    ✏️✏️ creating stream entry in db')
+            // await createStreamInDb({ source: 'email', platform, channel, date, url, userId, avatar })
         }
 
-        console.log('    ✏️ archiving e-mail')
+        // console.log('    ✏️ archiving e-mail')
-        await email.archiveMessage(msg.uid)
+        // await email.archiveMessage(msg.uid)
-
     } catch (e) {
         // console.error('error encoutered')
-        console.error(`An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`)
+        console.error(`  An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`)
         console.error(e)
     }
 }
@@ -55,6 +58,6 @@ async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageOb
 
 (async () => {
     const email = new Email()
-    email.on('message', (msg: FetchMessageObject) => handleMessage({ email, msg }))
+    email.once('message', (msg: FetchMessageObject) => handleMessage({ email, msg }))
     await email.connect()
 })()

packages/scout/src/s3.js

@@ -11,8 +11,6 @@ import fs from 'node:fs'
 
 if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
 if (!process.env.SCOUT_NITTER_URL) throw new Error('SCOUT_NITTER_URL was undefined in env');
-if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID was undefined in env');
-if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY was undefined in env');
 
 
 

packages/scout/src/signals.js

@@ -4,6 +4,7 @@ import qs from 'qs'
 import { subMinutes, addMinutes } from 'date-fns'
 import { fpSlugify, download } from './utils.js'
 import { getProminentColor } from './image.js'
+import { RateLimiter } from "limiter"
 import { getImage } from './vtuber.js'
 import fansly from './fansly.js'
 
@@ -35,6 +36,7 @@ if (!process.env.CDN_BUCKET_URL) throw new Error('CDN_BUCKET_URL is undefined in
  */
 export async function createStreamInDb ({ source, platform, channel, date, url, userId }) {
     
+    // const limiter = new RateLimiter({ tokensPerInterval: 0.3, interval: "second" });
     let vtuberId, streamId
 
     console.log('>> # Step 1')
@@ -115,7 +117,7 @@ export async function createStreamInDb ({ source, platform, channel, date, url,
         const b2FileData = await s3.uploadFile(imageFile)
 
         // get b2 cdn link to image
-        const imageCdnLink = `${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
+        const imageCdnLink = `https://${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
 
 
         const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {

packages/scout/src/temporal/activities.ts

@@ -6,12 +6,12 @@
 import fetch from "node-fetch"
 import { NotificationData, processEmail } from "./workflows.js"
 import qs from 'qs'
-import { IPlatformNotificationResponse, IVtuberResponse, IStreamResponse } from 'next'
+import { IVtuberResponse } from 'next'
 import { getImage } from '../vtuber.js'
-import { fpSlugify } from '../utils.js'
+import { fpSlugify, download } from '../utils.js'
+import fansly from '../fansly.js'
 import { getProminentColor } from '../image.js'
 import { uploadFile } from '../s3.js'
-import { addMinutes, subMinutes } from 'date-fns'
 
 export type ChargeResult = {
   status: string;
@@ -42,286 +42,113 @@ export async function upsertVtuber({ platform, userId, url, channel }: Notificat
 
   // GET /api/:pluralApiId?filters[field][operator]=value
   const findVtubersFilters = (() => {
-    if (platform === 'chaturbate') {
+      if (platform === 'chaturbate') {
-      return { chaturbate: { $eq: url } }
+          return { chaturbate: { $eq: url } }
-    } else if (platform === 'fansly') {
+      } else if (platform === 'fansly') {
-      if (!userId) throw new Error('Fansly userId was undefined, but it is required.')
+          if (!userId) throw new Error('Fansly userId was undefined, but it is required.')
-      return { fanslyId: { $eq: userId } }
+          return { fanslyId: { $eq: userId } }
-    }
+      }
   })()
   console.log('>>>>> the following is findVtubersFilters.')
   console.log(findVtubersFilters)
   
   const findVtubersQueryString = qs.stringify({
-    filters: findVtubersFilters
+      filters: findVtubersFilters
   }, { encode: false })
   console.log(`>>>>> platform=${platform}, url=${url}, userId=${userId}`)
   
   console.log('>> findVtuber')
   const findVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers?${findVtubersQueryString}`, {
-    method: 'GET',
+      method: 'GET',
-    headers: {
+      headers: {
-      'content-type': 'application/json'
+          'content-type': 'application/json'
-    }
+      }
   })
   const findVtuberJson = await findVtuberRes.json() as IVtuberResponse
   console.log('>> here is the vtuber json')
   console.log(findVtuberJson)
   if (findVtuberJson?.data && findVtuberJson.data.length > 0) {
-    console.log('>> a vtuber was FOUND')
+      console.log('>>a vtuber was FOUND')
-    if (findVtuberJson.data.length > 1) throw new Error('There was more than one vtuber match. There must only be one.')
+      if (findVtuberJson.data.length > 1) throw new Error('There was more than one vtuber match. There must only be one.')
-    vtuberId = findVtuberJson.data[0].id
+      vtuberId = findVtuberJson.data[0].id
-    console.log('here is the findVtuberJson (as follows)')
+      console.log('here is the findVtuberJson (as follows)')
-    console.log(findVtuberJson)
+      console.log(findVtuberJson)
-    console.log(`the matching vtuber has ID=${vtuberId} (${findVtuberJson.data[0].attributes.displayName})`)
+      console.log(`the matching vtuber has ID=${vtuberId} (${findVtuberJson.data[0].attributes.displayName})`)
   }
 
   if (!vtuberId) {
-    console.log('>> vtuberId was not found so we create')
+      console.log('>> vtuberId was not found so we create')
 
-    /**
+      /**
-     * We are creating a vtuber record.
+       * We are creating a vtuber record.
-     * We need a few things.
+       * We need a few things.
-     *   * image URL
+       *   * image URL
-     *   * themeColor
+       *   * themeColor
-     * 
+       * 
-     * To get an image, we have to do a few things.
+       * To get an image, we have to do a few things.
-     *   * [x] download image from platform
+       *   * [x] download image from platform
-     *   * [x] get themeColor from image
+       *   * [x] get themeColor from image
-     *   * [x] upload image to b2
+       *   * [x] upload image to b2
-     *   * [x] get B2 cdn link to image
+       *   * [x] get B2 cdn link to image
-     * 
+       * 
-     * To get themeColor, we need the image locally where we can then run 
+       * To get themeColor, we need the image locally where we can then run 
-     */
+       */
 
-    // download image from platform
+      // download image from platform
-    // vtuber.getImage expects a vtuber object, which we don't have yet, so we create a dummy one
+      // vtuber.getImage expects a vtuber object, which we don't have yet, so we create a dummy one
-    const dummyVtuber = {
+      const dummyVtuber = {
-      attributes: {
+          attributes: {
-        slug: fpSlugify(channel),
+              slug: fpSlugify(channel),
-        fanslyId: (platform === 'fansly') ? userId : null
+              fansly: fansly.url.fromUsername(channel)
+          }
       }
-    }
+      const platformImageUrl = await getImage(dummyVtuber)
-    const imageFile = await getImage(dummyVtuber)
+      const imageFile = await download({ url: platformImageUrl })
 
-    // get themeColor from image
+      // get themeColor from image
-    const themeColor = await getProminentColor(imageFile)
+      const themeColor = await getProminentColor(imageFile)
 
-    // upload image to b2
+      // upload image to b2
-    const b2FileData = await uploadFile(imageFile)
+      const b2FileData = await uploadFile(imageFile)
 
-    // get b2 cdn link to image
+      // get b2 cdn link to image
-    const imageCdnLink = `${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
+      const imageCdnLink = `https://${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
 
 
-    const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {
+      const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {
-      method: 'POST',
+          method: 'POST',
-      headers: {
+          headers: {
-        'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
+              'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
-        'content-type': 'application/json'
+              'content-type': 'application/json'
-      },
+          },
-      body: JSON.stringify({
+          body: JSON.stringify({
-        data: {
+              data: {
-          displayName: channel,
+                  displayName: channel,
-          fansly: (platform === 'fansly') ? url : null,
+                  fansly: (platform === 'fansly') ? url : null,
-          fanslyId: (platform === 'fansly') ? userId : null,
+                  fanslyId: (platform === 'fansly') ? userId : null,
-          chaturbate: (platform === 'chaturbate') ? url : null,
+                  chaturbate: (platform === 'chaturbate') ? url : null,
-          slug: fpSlugify(channel),
+                  slug: fpSlugify(channel),
-          description1: ' ',
+                  description1: ' ',
-          image: imageCdnLink,
+                  image: imageCdnLink,
-          themeColor: themeColor || '#dde1ec'
+                  themeColor: themeColor || '#dde1ec'
-        }
+              }
+          })
       })
-    })
+      const createVtuberJson = await createVtuberRes.json() as IVtuberResponse
-    const createVtuberJson = await createVtuberRes.json() as IVtuberResponse
+      console.log('>> createVtuberJson as follows')
-    console.log('>> createVtuberJson as follows')
+      console.log(JSON.stringify(createVtuberJson, null, 2))
-    console.log(JSON.stringify(createVtuberJson, null, 2))
+      if (createVtuberJson.data) {
-    if (createVtuberJson.data) {
+          vtuberId = createVtuberJson.data.id
-      vtuberId = createVtuberJson.data.id
+          console.log(`>>> vtuber created with id=${vtuberId}`)
-      console.log(`>>> vtuber created with id=${vtuberId}`)
+      }
-    }
   }
-  return vtuberId
+  return 777
 }
-export async function upsertPlatformNotification({ source, date, platform, vtuberId }: { source: string, date: string, platform: string, vtuberId: number }): Promise<number> {
+export async function upsertPlatformNotification(): Promise<number> {
-  if (!source) throw new Error(`upsertPlatformNotification requires source arg, but it was undefined`);
+  return 777
-  if (!date) throw new Error(`upsertPlatformNotification requires date arg, but it was undefined`);
-  if (!platform) throw new Error(`upsertPlatformNotification requires platform arg, but it was undefined`);
-  if (!vtuberId) throw new Error(`upsertPlatformNotification requires vtuberId arg, but it was undefined`);
-
-  let pNotifId
-  // # Step 2.
-  // Next we create the platform-notification record.
-  // This probably doesn't already exist, so we don't check for a pre-existing platform-notification.
-  const pNotifPayload = {
-    data: {
-      source: source,
-      date: date,
-      date2: date,
-      platform: platform,
-      vtuber: vtuberId,
-    }
-  }
-  console.log('pNotifPayload as follows')
-  console.log(pNotifPayload)
-
-  const pNotifCreateRes = await fetch(`${process.env.STRAPI_URL}/api/platform-notifications`, {
-    method: 'POST',
-    headers: {
-      'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
-      'Content-Type': 'application/json'
-    },
-    body: JSON.stringify(pNotifPayload)
-  })
-  const pNotifData = await pNotifCreateRes.json() as IPlatformNotificationResponse
-  if (pNotifData.error) {
-    console.error('>> we failed to create platform-notification, there was an error in the response')
-    console.error(JSON.stringify(pNotifData.error, null, 2))
-    throw new Error(pNotifData.error)
-  }
-  console.log(`>> pNotifData (json response) is as follows`)
-  console.log(pNotifData)
-  if (!pNotifData.data?.id) throw new Error('failed to created pNotifData! The response was missing an id');
-
-  pNotifId = pNotifData.data.id
-  if (!pNotifId) throw new Error('failed to get Platform Notification ID');
-  return pNotifId
-
 }
-export async function upsertStream({
+export async function upsertStream(): Promise<number> {
-  date,
+  return 777
-  vtuberId,
-  platform,
-  pNotifId
-}: {
-  date: string,
-  vtuberId: number,
-  platform: string,
-  pNotifId: number
-}): Promise<number> {
-
-  if (!date) throw new Error(`upsertStream requires date in the arg object, but it was undefined`);
-  if (!vtuberId) throw new Error(`upsertStream requires vtuberId in the arg object, but it was undefined`);
-  if (!platform) throw new Error(`upsertStream requires platform in the arg object, but it was undefined`);
-  if (!pNotifId) throw new Error(`upsertStream requires pNotifId in the arg object, but it was undefined`);
-
-  let streamId
-  // # Step 3.
-  // Finally we find or create the stream record
-  // The stream may already be in the db (the streamer is multi-platform streaming), so we look for that record.
-  // This gets a bit tricky. How do we determine one stream from another?
-  // For now, the rule is 30 minutes of separation. 
-  // Anything <=30m is interpreted as the same stream. Anything >30m is interpreted as a different stream.
-  // If the stream is not in the db, we create the stream record
-  const dateSinceRange = subMinutes(new Date(date), 30)
-  const dateUntilRange = addMinutes(new Date(date), 30)
-  console.log(`Find a stream within + or - 30 mins of the notif date=${new Date(date).toISOString()}. dateSinceRange=${dateSinceRange.toISOString()}, dateUntilRange=${dateUntilRange.toISOString()}`)
-  const findStreamQueryString = qs.stringify({
-    populate: 'platform-notifications',
-    filters: {
-      date: {
-        $gte: dateSinceRange,
-        $lte: dateUntilRange
-      },
-      vtuber: {
-        id: {
-          '$eq': vtuberId
-        }
-      }
-    }
-  }, { encode: false })
-
-  console.log('>> findStream')
-  const findStreamRes = await fetch(`${process.env.STRAPI_URL}/api/streams?${findStreamQueryString}`, {
-    method: 'GET',
-    headers: {
-      'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
-      'Content-Type': 'application/json'
-    }
-  })
-  const findStreamData = await findStreamRes.json() as IStreamResponse
-  if (findStreamData?.data && findStreamData.data.length > 0) {
-    console.log('>> we found a findStreamData json. (there is an existing stream for this e-mail/notification)')
-    console.log(JSON.stringify(findStreamData, null, 2))
-    streamId = findStreamData.data[0].id
-
-    // Before we're done here, we need to do something extra. We need to populate isChaturbateStream and/or isFanslyStream.
-    // We know which of these booleans to set based on the stream's related platformNotifications
-    // We go through each pNotif and look at it's platform
-    let isFanslyStream = false
-    let isChaturbateStream = false
-    if (findStreamData.data[0].attributes.platformNotifications) {
-      for (const pn of findStreamData.data[0].attributes.platformNotifications) {
-        if (pn.platform === 'fansly') {
-          isFanslyStream = true
-        } else if (pn.platform === 'chaturbate') {
-          isChaturbateStream = true
-        }
-      }
-    }
-
-
-    console.log(`>>> updating stream ${streamId}. isFanslyStream=${isFanslyStream}, isChaturbateStream=${isChaturbateStream}`)
-    const updateStreamRes = await fetch(`${process.env.STRAPI_URL}/api/streams/${streamId}`, {
-      method: 'PUT',
-      headers: {
-        'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
-        'content-type': 'application/json'
-      },
-      body: JSON.stringify({
-        data: {
-          isFanslyStream: isFanslyStream,
-          isChaturbateStream: isChaturbateStream,
-          platformNotifications: [
-            pNotifId
-          ]
-        }
-      })
-    })
-    const updateStreamJson = await updateStreamRes.json() as IStreamResponse
-    if (updateStreamJson?.error) throw new Error(JSON.stringify(updateStreamJson, null, 2));
-    console.log(`>> assuming a successful update to the stream record. response as follows.`)
-    console.log(JSON.stringify(updateStreamJson, null, 2))
-  }
-
-  if (!streamId) {
-    console.log('>> did not find a streamId, so we go ahead and create a stream record in the db.')
-    const createStreamPayload = {
-      data: {
-        isFanslyStream: (platform === 'fansly') ? true : false,
-        isChaturbateStream: (platform === 'chaturbate') ? true : false,
-        archiveStatus: 'missing',
-        date: date,
-        date2: date,
-        date_str: date,
-        vtuber: vtuberId,
-        platformNotifications: [
-          pNotifId
-        ]
-      }
-    }
-    console.log('>> createStreamPayload as follows')
-    console.log(createStreamPayload)
-    const createStreamRes = await fetch(`${process.env.STRAPI_URL}/api/streams`, {
-      method: 'POST',
-      headers: {
-        'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
-        "Content-Type": "application/json"
-      },
-      body: JSON.stringify(createStreamPayload)
-    })
-    const createStreamJson = await createStreamRes.json() as IStreamResponse
-    console.log('>> we got the createStreamJson')
-    console.log(createStreamJson)
-    if (createStreamJson.error) {
-      console.error(JSON.stringify(createStreamJson.error, null, 2))
-      throw new Error('Failed to create stream in DB due to an error. (see above)')
-    }
-    streamId = createStreamJson.id
-  }
-
-  if (!streamId) throw new Error('failed to get streamId')
-  return streamId
 }
 
 

packages/scout/src/temporal/worker.ts

@@ -1,7 +1,6 @@
 import path from "path"
 import { NativeConnection, Worker } from "@temporalio/worker"
 import * as activities from "./activities.js"
-import pRetry from 'p-retry'
 
 
 if (!process.env.TEMPORAL_SERVICE_ADDRESS) throw new Error(`TEMPORAL_SERVICE_ADDRESS is missing in env`);
@@ -50,11 +49,7 @@ async function run() {
   await worker.run();
 }
 
-await pRetry(run, { 
+run().catch((err) => {
-  forever: true,
+  console.error(err);
-  onFailedAttempt: (e) => {
+  process.exit(1);
-    console.error(e);
-    console.error(`there was an error during scout-worker run(). run() will now restart.`)
-    console.log(`P.S., check out these booba --> (.)(.)`)
-  }
 })

packages/scout/src/temporal/workflows.ts

@@ -15,6 +15,9 @@ export type NotificationData = {
 };
 
 const { 
+  chargeUser, 
+  checkAndDecrementInventory, 
+  incrementInventory,
   upsertPlatformNotification,
   upsertStream,
   upsertVtuber,
@@ -36,9 +39,10 @@ export async function processEmail({
   // Step 1
 
   const vtuberId = await upsertVtuber({ url, platform, channel, displayName, date, userId, avatar })
-  const pNotifId = await upsertPlatformNotification({ vtuberId, source: 'email', date, platform })
+  console.log('we have finished upsertVtuber and the vtuberId is '+vtuberId)
-  const streamId = await upsertStream({ date, vtuberId, platform, pNotifId })
+  throw new Error('Error: Error: error: erorreorororr; @todo');
-
+  const pNotifId = await upsertPlatformNotification()
+  const streamId = await upsertStream()
   return { vtuberId, pNotifId, streamId }
 }
 

packages/scout/src/twitter.js

@@ -19,8 +19,9 @@ const normalize = (url) => {
 }
 
 
-const image = async function image (twitterUsername) {
+const image = async function image (limiter, twitterUsername) {
-    if (!twitterUsername) throw new Error('first arg to twitter.data.image must be a twitterUsername. It was undefined.');
+    if (!limiter) throw new Error('first arg to twitter.data.image must be an instance of node-rate-limiter');
+    if (!twitterUsername) throw new Error('second arg to twitter.data.image must be a twitterUsername. It was undefined.');
     const requestDataFromNitter = async () => {
         const url = `${process.env.SCOUT_NITTER_URL}/${twitterUsername}/rss?key=${process.env.SCOUT_NITTER_ACCESS_KEY}`
         // console.log(`fetching from url=${url}`)
@@ -36,7 +37,7 @@ const image = async function image (twitterUsername) {
         const dom = htmlparser2.parseDocument(body);
         const $ = load(dom, { _useHtmlParser2: true })
         const urls = $('url:contains("profile_images")').first()
-        const downloadedImageFile = await download({ url: urls.text() })
+        const downloadedImageFile = await download({ limiter, url: urls.text() })
         return downloadedImageFile
     } catch (e) {
         console.error(`while fetching rss from nitter, the following error was encountered.`)

packages/scout/src/twitter.spec.js

@@ -2,6 +2,7 @@ import { expect } from 'chai'
 import twitter from './twitter.js'
 import { describe } from 'mocha'
 import { tmpFileRegex } from './utils.js'
+import { RateLimiter } from 'limiter'
 
 describe('twitter', function () {
     describe('regex', function () {
@@ -18,9 +19,10 @@ describe('twitter', function () {
     })
     describe('data', function () {
         this.timeout(1000*30)
+        const limiter = new RateLimiter({ tokensPerInterval: 10, interval: "second" })
         describe('image', function () {
             it("should download the twitter users's avatar and save it to disk", async function () {
-                const imgFile = await twitter.data.image('projektmelody')
+                const imgFile = await twitter.data.image(limiter, 'projektmelody')
                 expect(imgFile).to.match(tmpFileRegex)
             })
         })

packages/scout/src/utils.js

@@ -24,16 +24,19 @@ export function getTmpFile(str) {
 
 /**
  * 
+ * @param {Object} limiter [https://github.com/jhurliman/node-rate-limiter](node-rate-limiter) instance
  * @param {String} url 
  * @returns {String} filePath
  * 
  * greetz https://stackoverflow.com/a/74722818/1004931
  */
-export async function download({ url, filePath }) {
+export async function download({ limiter, url, filePath }) {
+    if (!limiter) throw new Error(`first arg passed to download() must be a node-rate-limiter instance.`);
     if (!url) throw new Error(`second arg passed to download() must be a {string} url`);
     const fileBaseName = basename(url)
     filePath = filePath || path.join(os.tmpdir(), `${createId()}_${fileBaseName}`)
     const stream = fs.createWriteStream(filePath)
+    await limiter.removeTokens(1);
 
     const requestData = async () => {
         const response = await fetch(url, {

packages/scout/src/utils.spec.js

@@ -1,6 +1,7 @@
 import { fpSlugify, getTmpFile, download } from './utils.js'
 import { expect } from 'chai'
 import { describe } from 'mocha'
+import { RateLimiter } from "limiter"
 
 
 describe('utils', function () {
@@ -17,8 +18,9 @@ describe('utils', function () {
         })
     }),
     describe('download', function () {
+        const limiter = new RateLimiter({ tokensPerInterval: 100, interval: "second" })
         it('should get the file', async function () {
-            const file = await download({ url: 'https://futureporn-b2.b-cdn.net/sample.webp' })
+            const file = await download({ limiter, url: 'https://futureporn-b2.b-cdn.net/sample.webp' })
             expect(file).to.match(/\/tmp\/.*sample\.webp$/)
         })
     })

packages/scout/src/vtuber.js

@@ -20,22 +20,25 @@ import fansly from './fansly.js'
  * 
  * We depend on one of these social media URLs. If there is neither Twitter or fansly listed, we throw an error. 
  * 
+ * @param {Object} limiter      -- instance of node-rate-limiter
  * @param {Object} vtuber       -- vtuber instance from strapi
  * @returns {String} filePath   -- path on disk where the image was saved
  */
-export async function getImage(vtuber) {
+export async function getImage(limiter, vtuber) {
-    if (!vtuber) throw new Error('first arg must be vtuber instance');
+    if (!limiter) throw new Error('first arg must be node-rate-limiter instace');
+    if (!vtuber) throw new Error('second arg must be vtuber instance');
+    await limiter.removeTokens(1);
     
     const { twitter: twitterUrl, fanslyId: fanslyId } = vtuber.attributes
     const twitterUsername = twitterUrl && twitter.regex.username.exec(twitterUrl).at(1)
 
     let img;
     if (twitterUrl) {
-        img = await twitter.data.image(twitterUsername)
+        img = await twitter.data.image(limiter, twitterUsername)
     } else if (fanslyId) {
-        img = await fansly.data.image(fanslyId)
+        img = await fansly.data.image(limiter, fanslyId)
     } else {
-        const msg = ` while attempting to get vtuber image, there was neither twitterUrl nor fanslyId listed. One of these must exist for us to download an image. \nvtuber=${JSON.stringify(vtuber, null, 2)}`
+        const msg = 'while attempting to get vtuber image, there was neither twitter nor fansly listed. One of these must exist for us to download an image.'
         console.error(msg)
         throw new Error(msg)
     }

packages/scout/src/vtuber.spec.js

@@ -1,6 +1,7 @@
 
 import { expect } from 'chai'
 import { describe } from 'mocha'
+import { RateLimiter } from 'limiter'
 import { getImage } from './vtuber.js'
 import { tmpFileRegex } from './utils.js'
 
@@ -24,12 +25,13 @@ const vtuberFixture1 = {
 describe('vtuber', function () {
     this.timeout(1000*60)
     describe('getImage', function () {
+        const limiter = new RateLimiter({ tokensPerInterval: 1, interval: "second" })
         it('should download an avatar image from twitter', async function () {
-            const file = await getImage(vtuberFixture0)
+            const file = await getImage(limiter, vtuberFixture0)
             expect(file).to.match(tmpFileRegex)
         })
         it('should download an avatar image from fansly', async function () {
-            const file = await getImage(vtuberFixture1)
+            const file = await getImage(limiter, vtuberFixture1)
             expect(file).to.match(tmpFileRegex)
         })
     })

packages/strapi/README.md

@@ -5,9 +5,3 @@
   * ironmouse "Thank you" (for testing): 4760169
   * cj_clippy "Full library access" (for production): 9380584
   * cj_clippy "Your URL displayed on Futureporn.net": 10663202
-
-### Content-Type Builder (Docker caveat)
-
-Don't use the web UI to create or update Content-Types! The changes will be lost. This is a side-effect of our hacked together solution for Strapi with pnpm in docker.
-
-Instead, content-type schemas must be hand-edited in ./src/api/(...). For the changes to take effect, trigger a strapi resource update in Tilt.

packages/strapi/src/api/platform-notification/content-types/platform-notification/schema.json

@@ -45,12 +45,6 @@
             "type": "relation",
             "relation": "oneToOne",
             "target": "api::vtuber.vtuber"
-        },
-        "stream": {
-            "type": "relation",
-            "relation": "manyToOne",
-            "target": "api::stream.stream",
-            "inversedBy": "platformNotifications"
         }
     }
 }

packages/strapi/src/api/stream/content-types/stream/lifecycles.js

@@ -22,6 +22,11 @@ module.exports = {
     async afterUpdate(event) {
 
 
+        /**
+         * NOTE
+         * 
+         * These hooks do not fire in response to API calls. They only fire in response to UI saves.
+         */
 
         console.log(`>>>>>>>>>>>>>> STREAM is afterUpdate !!!!!!!!!!!!`);
         const { data, where, select, populate } = event.params;
@@ -79,7 +84,7 @@ module.exports = {
          * If any platformNotification is from chaturbate, isChaturbateStream is set to true.
          */
         const existingData2 = await strapi.entityService.findOne("api::stream.stream", id, {
-            populate: {'platformNotifications': true}
+            populate: ['platform_notifications']
         })
 
         let isFanslyStream = false
@@ -88,14 +93,12 @@ module.exports = {
         console.log(`lets find the platformNotifications`)
         console.log(JSON.stringify(existingData2, null, 2))
 
-        // Iterate through all platformNotifications to determine platform
+        // Iterate through all vods to determine archiveStatus
-        if (existingData2?.platformNotifications) {
+        for (const pn of existingData2.platform_notifications) {
-            for (const pn of existingData2.platformNotifications) {
+            if (pn.platform === 'fansly') {
-                if (pn.platform === 'fansly') {
+                isFanslyStream = true
-                    isFanslyStream = true
+            } else if (pn.platform === 'chaturbate') {
-                } else if (pn.platform === 'chaturbate') {
+                isChaturbateStream = true
-                    isChaturbateStream = true
-                }
             }
         }
 

packages/strapi/src/api/stream/content-types/stream/schema.json

@@ -12,12 +12,6 @@
   },
   "pluginOptions": {},
   "attributes": {
-    "platformNotifications": {
-      "type": "relation",
-      "relation": "oneToMany",
-      "target": "api::platform-notification.platform-notification",
-      "mappedBy": "stream"
-    },
     "date_str": {
       "type": "string",
       "required": true,

packages/strapi/src/api/vtuber/content-types/vtuber/schema.json

@@ -79,7 +79,7 @@
     },
     "description1": {
       "type": "text",
-      "required": false
+      "required": true
     },
     "description2": {
       "type": "text"
@@ -113,15 +113,6 @@
       "relation": "oneToMany",
       "target": "api::stream.stream",
       "mappedBy": "vtuber"
-    },
-    "fanslyId": {
-      "type": "string"
-    },
-    "chaturbateId": {
-      "type": "string"
-    },
-    "twitterId": {
-      "type": "string"
     }
   }
 }

scripts/k8s-secrets.sh

@@ -39,9 +39,7 @@ kubectl --namespace futureporn create secret generic scout \
 --from-literal=imapUsername=${SCOUT_IMAP_USERNAME} \
 --from-literal=imapPassword=${SCOUT_IMAP_PASSWORD} \
 --from-literal=imapAccessToken=${SCOUT_IMAP_ACCESS_TOKEN} \
---from-literal=nitterAccessKey=${SCOUT_NITTER_ACCESS_KEY} \
+--from-literal=nitterAccessKey=${SCOUT_NITTER_ACCESS_KEY}
---from-literal=s3BucketKeyId=${S3_BUCKET_KEY_ID} \
---from-literal=s3BucketApplicationKey=${S3_BUCKET_APPLICATION_KEY}
 
 kubectl --namespace futureporn delete secret link2cid --ignore-not-found
 kubectl --namespace futureporn create secret generic link2cid \