diff --git a/Makefile b/Makefile
index dcaba5e..dc13ecc 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,10 @@ crds:
cert-manager:
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.yaml
-
+
+secrets:
+ ./scripts/k8s-secrets.sh
+
flux:
flux bootstrap git --url="ssh://git@gitea.futureporn.net:2222/futureporn/fp" --branch=main --path="clusters/production" --private-key-file=/home/cj/.ssh/fp-flux
diff --git a/charts/fp/templates/scout.yaml b/charts/fp/templates/scout.yaml
index a5ce104..d9cdbc6 100644
--- a/charts/fp/templates/scout.yaml
+++ b/charts/fp/templates/scout.yaml
@@ -34,6 +34,16 @@ spec:
value: "{{ .Values.scout.cdnBucketUrl }}"
- name: STRAPI_URL
value: https://strapi.piko.sbtp.xyz
+ - name: S3_BUCKET_APPLICATION_KEY
+ valueFrom:
+ secretKeyRef:
+ name: scout
+ key: s3BucketApplicationKey
+ - name: S3_BUCKET_KEY_ID
+ valueFrom:
+ secretKeyRef:
+ name: scout
+ key: s3BucketKeyId
- name: SCOUT_NITTER_ACCESS_KEY
valueFrom:
secretKeyRef:
diff --git a/packages/next/app/components/archive-progress.tsx b/packages/next/app/components/archive-progress.tsx
index 4b19d9a..fe12fd4 100644
--- a/packages/next/app/components/archive-progress.tsx
+++ b/packages/next/app/components/archive-progress.tsx
@@ -1,4 +1,4 @@
-import { getAllStreamsForVtuber } from "@/lib/streams";
+import { getAllStreamsForVtuber, getStreamCountForVtuber } from "@/lib/streams";
import { getVodsForVtuber } from "@/lib/vods";
import { IVtuber } from "@/lib/vtubers";
@@ -8,7 +8,7 @@ export interface IArchiveProgressProps {
export default async function ArchiveProgress ({ vtuber }: IArchiveProgressProps) {
// const vods = await getVodsForVtuber(vtuber.id)
- // const streams = await getAllStreamsForVtuber(vtuber.id);
+ const streams = await getStreamCountForVtuber(vtuber.id);
// const goodStreams = await getAllStreamsForVtuber(vtuber.id, ['good']);
// const issueStreams = await getAllStreamsForVtuber(vtuber.id, ['issue']);
// const totalStreams = streams.length;
@@ -21,7 +21,11 @@ export default async function ArchiveProgress ({ vtuber }: IArchiveProgressProps
const eligibleStreams = 50
return (
-
@todo
+
+
+ {JSON.stringify(streams, null, 2)}
+
+
{eligibleStreams}/{totalStreams} Streams Archived ({completedPercentage}%)
diff --git a/packages/next/app/components/stream-page.tsx b/packages/next/app/components/stream-page.tsx
index b3619cc..4392b5d 100644
--- a/packages/next/app/components/stream-page.tsx
+++ b/packages/next/app/components/stream-page.tsx
@@ -169,7 +169,8 @@ export default function StreamPage({ stream }: IStreamProps) {
{desc1}
{desc2}
- Upload it here.
+ {/* Upload it here. */}
+ Uploads coming soon.
diff --git a/packages/next/app/components/streams-table.tsx b/packages/next/app/components/streams-table.tsx
index e8ee028..0082cdd 100644
--- a/packages/next/app/components/streams-table.tsx
+++ b/packages/next/app/components/streams-table.tsx
@@ -65,6 +65,7 @@ export default function StreamsTable() {
src={image}
alt={displayName}
placeholder="blur"
+ objectFit='contain'
blurDataURL={imageBlur}
width={32}
height={32}
diff --git a/packages/next/app/components/vtuber-card.tsx b/packages/next/app/components/vtuber-card.tsx
index a324dc2..0c512fd 100644
--- a/packages/next/app/components/vtuber-card.tsx
+++ b/packages/next/app/components/vtuber-card.tsx
@@ -24,6 +24,7 @@ export default async function VTuberCard(vtuber: IVtuber) {
className="is-rounded"
src={image}
alt={displayName}
+ objectFit="cover"
placeholder="blur"
blurDataURL={imageBlur}
width={48}
diff --git a/packages/next/app/lib/streams.ts b/packages/next/app/lib/streams.ts
index b8d8a18..cdd4d15 100644
--- a/packages/next/app/lib/streams.ts
+++ b/packages/next/app/lib/streams.ts
@@ -351,7 +351,25 @@ export async function fetchStreamData({ pageIndex, pageSize }: { pageIndex: numb
return d;
}
-
+export async function getStreamCountForVtuber(vtuberId: number): Promise {
+ if (!vtuberId) throw new Error(`getStreamCountForVtuber requires a vtuberId, but it was undefined.`);
+ const query = qs.stringify(
+ {
+ filters: {
+ vtuber: {
+ id: {
+ $eq: vtuberId
+ }
+ }
+ }
+ }
+ )
+ const res = await fetch(`${strapiUrl}/api/streams?${query}`, fetchStreamsOptions)
+ const data = await res.json()
+ console.log('getStreamCountForVtuber')
+ console.log(JSON.stringify(data, null, 2))
+ return data.meta.pagination.total
+}
export async function getStreamsForVtuber(vtuberId: number, page: number = 1, pageSize: number = 25, sortDesc = true): Promise {
console.log(`getStreamsForVtuber() with strapiUrl=${strapiUrl}`)
diff --git a/packages/next/app/lib/types.ts b/packages/next/app/lib/types.ts
index 8a2cc29..f34d4b0 100644
--- a/packages/next/app/lib/types.ts
+++ b/packages/next/app/lib/types.ts
@@ -26,3 +26,21 @@ export interface IMuxAssetResponse {
export interface IMeta {
pagination: IPagination;
}
+
+
+
+export interface IPlatformNotification {
+ id: number;
+ attributes: {
+ source: string;
+ platform: string;
+ date: string;
+ date2: string;
+ vtuber: number;
+ }
+}
+
+export interface IPlatformNotificationResponse {
+ data: IPlatformNotification;
+ meta: IMeta;
+}
\ No newline at end of file
diff --git a/packages/next/app/lib/vtubers.ts b/packages/next/app/lib/vtubers.ts
index b2abc65..23795d7 100644
--- a/packages/next/app/lib/vtubers.ts
+++ b/packages/next/app/lib/vtubers.ts
@@ -2,9 +2,7 @@
import { IVod } from './vods'
import { strapiUrl, siteUrl } from './constants';
-import { getSafeDate } from './dates';
import qs from 'qs';
-import { resourceLimits } from 'worker_threads';
import { IMeta } from './types';
@@ -43,6 +41,9 @@ export interface IVtuber {
image: string;
imageBlur?: string;
themeColor: string;
+ fanslyId?: string;
+ chaturbateId?: string;
+ twitterId?: string;
}
}
diff --git a/packages/next/app/vt/[slug]/page.tsx b/packages/next/app/vt/[slug]/page.tsx
index a2773c7..c12eba5 100644
--- a/packages/next/app/vt/[slug]/page.tsx
+++ b/packages/next/app/vt/[slug]/page.tsx
@@ -64,6 +64,7 @@ export default async function Page({ params }: { params: { slug: string } }) {
alt={vtuber.attributes.displayName}
src={vtuber.attributes.image}
fill={true}
+ objectFit='cover'
placeholder='blur'
blurDataURL={vtuber.attributes.imageBlur}
/>
diff --git a/packages/next/next.config.js b/packages/next/next.config.js
index 1706724..ccc33e5 100644
--- a/packages/next/next.config.js
+++ b/packages/next/next.config.js
@@ -14,6 +14,12 @@ const nextConfig = {
port: '',
pathname: '/**',
},
+ {
+ protocol: 'https',
+ hostname: 'fp-dev.b-cdn.net',
+ port: '',
+ pathname: '/**',
+ },
],
}
};
diff --git a/packages/scout/src/cb.js b/packages/scout/src/cb.js
index 49e230b..eb75787 100644
--- a/packages/scout/src/cb.js
+++ b/packages/scout/src/cb.js
@@ -2,12 +2,10 @@ import cheerio from 'cheerio'
/**
*
- * @param {Object} limiter An instance of node-rate-limiter, see https://github.com/jhurliman/node-rate-limiter
* @param {String} roomUrl example: https://chaturbate.com/projektmelody
* @returns {Object} initialRoomDossier
*/
-export async function getInitialRoomDossier(limiter, roomUrl) {
- await limiter.removeTokens(1);
+export async function getInitialRoomDossier(roomUrl) {
try {
const res = await fetch(roomUrl, {
headers: {
diff --git a/packages/scout/src/cb.spec.js b/packages/scout/src/cb.spec.js
index a6c57da..66a5d75 100644
--- a/packages/scout/src/cb.spec.js
+++ b/packages/scout/src/cb.spec.js
@@ -1,13 +1,11 @@
import { describe } from 'mocha'
import { expect } from 'chai';
import { getInitialRoomDossier } from './cb.js'
-import { RateLimiter } from "limiter";
describe('cb', function () {
- let limiter = new RateLimiter({ tokensPerInterval: 10, interval: "minute" })
describe('getInitialRoomDossier', function () {
it('should return json', async function () {
- const dossier = await getInitialRoomDossier(limiter, 'https://chaturbate.com/projektmelody')
+ const dossier = await getInitialRoomDossier('https://chaturbate.com/projektmelody')
expect(dossier).to.have.property('wschat_host')
})
})
diff --git a/packages/scout/src/fansly.js b/packages/scout/src/fansly.js
index baca4cd..df3f77d 100644
--- a/packages/scout/src/fansly.js
+++ b/packages/scout/src/fansly.js
@@ -12,12 +12,11 @@ const normalize = (url) => {
const fromUsername = (username) => `https://fansly.com/${username}`
-const image = async function image (limiter, fanslyUserId) {
- if (!limiter) throw new Error(`first arg passed to fansly.data.image must be a node-rate-limiter instance`);
- if (!fanslyUserId) throw new Error(`second arg passed to fansly.data.image must be a {string} fanslyUserId`);
+const image = async function image (fanslyUserId) {
+ if (!fanslyUserId) throw new Error(`first arg passed to fansly.data.image must be a {string} fanslyUserId`);
const url = `https://api.fansly.com/api/v1/account/${fanslyUserId}/avatar`
const filePath = getTmpFile('avatar.jpg')
- return download({ filePath, limiter, url })
+ return download({ filePath, url })
}
const url = {
diff --git a/packages/scout/src/imap.js b/packages/scout/src/imap.js
index a65d2c9..1b69d13 100644
--- a/packages/scout/src/imap.js
+++ b/packages/scout/src/imap.js
@@ -4,10 +4,6 @@ import EventEmitter from 'node:events';
import 'dotenv/config';
import { simpleParser } from 'mailparser';
-// pinned to v2.0.1 due to https://github.com/jhurliman/node-rate-limiter/issues/80
-import * as $limiter from 'limiter';
-const { RateLimiter } = $limiter
-
if (!process.env.SCOUT_IMAP_SERVER) throw new Error('SCOUT_IMAP_SERVER is missing from env');
@@ -15,8 +11,6 @@ if (!process.env.SCOUT_IMAP_PORT) throw new Error('SCOUT_IMAP_PORT is missing fr
if (!process.env.SCOUT_IMAP_USERNAME) throw new Error('SCOUT_IMAP_USERNAME is missing from env');
if (!process.env.SCOUT_IMAP_PASSWORD) throw new Error('SCOUT_IMAP_PASSWORD is missing from env');
-const limiter = new RateLimiter({ tokensPerInterval: 1, interval: 3000 });
-
// https://stackoverflow.com/a/49428486/1004931
function streamToString(stream) {
const chunks = [];
@@ -37,7 +31,6 @@ export class Email extends EventEmitter {
}
async archiveMessage(uid) {
- await limiter.removeTokens(1);
await this.client.messageDelete(uid, { uid: true })
}
diff --git a/packages/scout/src/index.ts b/packages/scout/src/index.ts
index c693c77..15ef8ad 100644
--- a/packages/scout/src/index.ts
+++ b/packages/scout/src/index.ts
@@ -25,32 +25,29 @@ async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageOb
// console.log(' ✏️ checking e-mail')
const { isMatch, url, platform, channel, displayName, date, userId, avatar }: NotificationData = (await checkEmail(body) )
-
if (isMatch) {
const wfId = `process-email-${createId()}`
// console.log(` ✏️ [DRY] starting Temporal workflow ${wfId} @todo actually start temporal workflow`)
// await signalRealtime({ url, platform, channel, displayName, date, userId, avatar })
// @todo invoke a Temporal workflow here
+ console.log(' ✏️✏️ starting Temporal workflow')
const handle = await client.workflow.start(processEmail, {
workflowId: wfId,
taskQueue: 'scout',
args: [{ url, platform, channel, displayName, date, userId, avatar }]
});
// const handle = client.getHandle(workflowId);
- const result = await handle.result();
+ const result = await handle.result()
console.log(`result of the workflow is as follows`)
console.log(result)
- throw new Error('!todo we are stopping after just one (for now) @todo')
-
- // console.log(' ✏️✏️ creating stream entry in db')
- // await createStreamInDb({ source: 'email', platform, channel, date, url, userId, avatar })
}
- // console.log(' ✏️ archiving e-mail')
- // await email.archiveMessage(msg.uid)
+ console.log(' ✏️ archiving e-mail')
+ await email.archiveMessage(msg.uid)
+
} catch (e) {
// console.error('error encoutered')
- console.error(` An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`)
+ console.error(`An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`)
console.error(e)
}
}
@@ -58,6 +55,6 @@ async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageOb
(async () => {
const email = new Email()
- email.once('message', (msg: FetchMessageObject) => handleMessage({ email, msg }))
+ email.on('message', (msg: FetchMessageObject) => handleMessage({ email, msg }))
await email.connect()
})()
\ No newline at end of file
diff --git a/packages/scout/src/s3.js b/packages/scout/src/s3.js
index 752fe61..07b259a 100644
--- a/packages/scout/src/s3.js
+++ b/packages/scout/src/s3.js
@@ -11,6 +11,8 @@ import fs from 'node:fs'
if (!process.env.S3_BUCKET_NAME) throw new Error('S3_BUCKET_NAME was undefined in env');
if (!process.env.SCOUT_NITTER_URL) throw new Error('SCOUT_NITTER_URL was undefined in env');
+if (!process.env.S3_BUCKET_KEY_ID) throw new Error('S3_BUCKET_KEY_ID was undefined in env');
+if (!process.env.S3_BUCKET_APPLICATION_KEY) throw new Error('S3_BUCKET_APPLICATION_KEY was undefined in env');
diff --git a/packages/scout/src/signals.js b/packages/scout/src/signals.js
index 1b385ed..bf2e250 100644
--- a/packages/scout/src/signals.js
+++ b/packages/scout/src/signals.js
@@ -4,7 +4,6 @@ import qs from 'qs'
import { subMinutes, addMinutes } from 'date-fns'
import { fpSlugify, download } from './utils.js'
import { getProminentColor } from './image.js'
-import { RateLimiter } from "limiter"
import { getImage } from './vtuber.js'
import fansly from './fansly.js'
@@ -36,7 +35,6 @@ if (!process.env.CDN_BUCKET_URL) throw new Error('CDN_BUCKET_URL is undefined in
*/
export async function createStreamInDb ({ source, platform, channel, date, url, userId }) {
- // const limiter = new RateLimiter({ tokensPerInterval: 0.3, interval: "second" });
let vtuberId, streamId
console.log('>> # Step 1')
@@ -117,7 +115,7 @@ export async function createStreamInDb ({ source, platform, channel, date, url,
const b2FileData = await s3.uploadFile(imageFile)
// get b2 cdn link to image
- const imageCdnLink = `https://${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
+ const imageCdnLink = `${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {
diff --git a/packages/scout/src/temporal/activities.ts b/packages/scout/src/temporal/activities.ts
index ae76a0e..a6fc2ce 100644
--- a/packages/scout/src/temporal/activities.ts
+++ b/packages/scout/src/temporal/activities.ts
@@ -6,12 +6,12 @@
import fetch from "node-fetch"
import { NotificationData, processEmail } from "./workflows.js"
import qs from 'qs'
-import { IVtuberResponse } from 'next'
+import { IPlatformNotificationResponse, IVtuberResponse, IStreamResponse } from 'next'
import { getImage } from '../vtuber.js'
-import { fpSlugify, download } from '../utils.js'
-import fansly from '../fansly.js'
+import { fpSlugify } from '../utils.js'
import { getProminentColor } from '../image.js'
import { uploadFile } from '../s3.js'
+import { addMinutes, subMinutes } from 'date-fns'
export type ChargeResult = {
status: string;
@@ -42,113 +42,286 @@ export async function upsertVtuber({ platform, userId, url, channel }: Notificat
// GET /api/:pluralApiId?filters[field][operator]=value
const findVtubersFilters = (() => {
- if (platform === 'chaturbate') {
- return { chaturbate: { $eq: url } }
- } else if (platform === 'fansly') {
- if (!userId) throw new Error('Fansly userId was undefined, but it is required.')
- return { fanslyId: { $eq: userId } }
- }
+ if (platform === 'chaturbate') {
+ return { chaturbate: { $eq: url } }
+ } else if (platform === 'fansly') {
+ if (!userId) throw new Error('Fansly userId was undefined, but it is required.')
+ return { fanslyId: { $eq: userId } }
+ }
})()
console.log('>>>>> the following is findVtubersFilters.')
console.log(findVtubersFilters)
-
+
const findVtubersQueryString = qs.stringify({
- filters: findVtubersFilters
+ filters: findVtubersFilters
}, { encode: false })
console.log(`>>>>> platform=${platform}, url=${url}, userId=${userId}`)
-
+
console.log('>> findVtuber')
const findVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers?${findVtubersQueryString}`, {
- method: 'GET',
- headers: {
- 'content-type': 'application/json'
- }
+ method: 'GET',
+ headers: {
+ 'content-type': 'application/json'
+ }
})
const findVtuberJson = await findVtuberRes.json() as IVtuberResponse
console.log('>> here is the vtuber json')
console.log(findVtuberJson)
if (findVtuberJson?.data && findVtuberJson.data.length > 0) {
- console.log('>>a vtuber was FOUND')
- if (findVtuberJson.data.length > 1) throw new Error('There was more than one vtuber match. There must only be one.')
- vtuberId = findVtuberJson.data[0].id
- console.log('here is the findVtuberJson (as follows)')
- console.log(findVtuberJson)
- console.log(`the matching vtuber has ID=${vtuberId} (${findVtuberJson.data[0].attributes.displayName})`)
+ console.log('>> a vtuber was FOUND')
+ if (findVtuberJson.data.length > 1) throw new Error('There was more than one vtuber match. There must only be one.')
+ vtuberId = findVtuberJson.data[0].id
+ console.log('here is the findVtuberJson (as follows)')
+ console.log(findVtuberJson)
+ console.log(`the matching vtuber has ID=${vtuberId} (${findVtuberJson.data[0].attributes.displayName})`)
}
if (!vtuberId) {
- console.log('>> vtuberId was not found so we create')
+ console.log('>> vtuberId was not found so we create')
- /**
- * We are creating a vtuber record.
- * We need a few things.
- * * image URL
- * * themeColor
- *
- * To get an image, we have to do a few things.
- * * [x] download image from platform
- * * [x] get themeColor from image
- * * [x] upload image to b2
- * * [x] get B2 cdn link to image
- *
- * To get themeColor, we need the image locally where we can then run
- */
+ /**
+ * We are creating a vtuber record.
+ * We need a few things.
+ * * image URL
+ * * themeColor
+ *
+ * To get an image, we have to do a few things.
+ * * [x] download image from platform
+ * * [x] get themeColor from image
+ * * [x] upload image to b2
+ * * [x] get B2 cdn link to image
+ *
+ * To get themeColor, we need the image locally where we can then run
+ */
- // download image from platform
- // vtuber.getImage expects a vtuber object, which we don't have yet, so we create a dummy one
- const dummyVtuber = {
- attributes: {
- slug: fpSlugify(channel),
- fansly: fansly.url.fromUsername(channel)
- }
+ // download image from platform
+ // vtuber.getImage expects a vtuber object, which we don't have yet, so we create a dummy one
+ const dummyVtuber = {
+ attributes: {
+ slug: fpSlugify(channel),
+ fanslyId: (platform === 'fansly') ? userId : null
}
- const platformImageUrl = await getImage(dummyVtuber)
- const imageFile = await download({ url: platformImageUrl })
+ }
+ const imageFile = await getImage(dummyVtuber)
- // get themeColor from image
- const themeColor = await getProminentColor(imageFile)
+ // get themeColor from image
+ const themeColor = await getProminentColor(imageFile)
- // upload image to b2
- const b2FileData = await uploadFile(imageFile)
+ // upload image to b2
+ const b2FileData = await uploadFile(imageFile)
- // get b2 cdn link to image
- const imageCdnLink = `https://${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
+ // get b2 cdn link to image
+ const imageCdnLink = `${process.env.CDN_BUCKET_URL}/${b2FileData.Key}`
- const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {
- method: 'POST',
- headers: {
- 'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
- 'content-type': 'application/json'
- },
- body: JSON.stringify({
- data: {
- displayName: channel,
- fansly: (platform === 'fansly') ? url : null,
- fanslyId: (platform === 'fansly') ? userId : null,
- chaturbate: (platform === 'chaturbate') ? url : null,
- slug: fpSlugify(channel),
- description1: ' ',
- image: imageCdnLink,
- themeColor: themeColor || '#dde1ec'
- }
- })
+ const createVtuberRes = await fetch(`${process.env.STRAPI_URL}/api/vtubers`, {
+ method: 'POST',
+ headers: {
+ 'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
+ 'content-type': 'application/json'
+ },
+ body: JSON.stringify({
+ data: {
+ displayName: channel,
+ fansly: (platform === 'fansly') ? url : null,
+ fanslyId: (platform === 'fansly') ? userId : null,
+ chaturbate: (platform === 'chaturbate') ? url : null,
+ slug: fpSlugify(channel),
+ description1: ' ',
+ image: imageCdnLink,
+ themeColor: themeColor || '#dde1ec'
+ }
})
- const createVtuberJson = await createVtuberRes.json() as IVtuberResponse
- console.log('>> createVtuberJson as follows')
- console.log(JSON.stringify(createVtuberJson, null, 2))
- if (createVtuberJson.data) {
- vtuberId = createVtuberJson.data.id
- console.log(`>>> vtuber created with id=${vtuberId}`)
- }
+ })
+ const createVtuberJson = await createVtuberRes.json() as IVtuberResponse
+ console.log('>> createVtuberJson as follows')
+ console.log(JSON.stringify(createVtuberJson, null, 2))
+ if (createVtuberJson.data) {
+ vtuberId = createVtuberJson.data.id
+ console.log(`>>> vtuber created with id=${vtuberId}`)
+ }
}
- return 777
+ return vtuberId
}
-export async function upsertPlatformNotification(): Promise {
- return 777
+export async function upsertPlatformNotification({ source, date, platform, vtuberId }: { source: string, date: string, platform: string, vtuberId: number }): Promise {
+ if (!source) throw new Error(`upsertPlatformNotification requires source arg, but it was undefined`);
+ if (!date) throw new Error(`upsertPlatformNotification requires date arg, but it was undefined`);
+ if (!platform) throw new Error(`upsertPlatformNotification requires platform arg, but it was undefined`);
+ if (!vtuberId) throw new Error(`upsertPlatformNotification requires vtuberId arg, but it was undefined`);
+
+ let pNotifId
+ // # Step 2.
+ // Next we create the platform-notification record.
+ // This probably doesn't already exist, so we don't check for a pre-existing platform-notification.
+ const pNotifPayload = {
+ data: {
+ source: source,
+ date: date,
+ date2: date,
+ platform: platform,
+ vtuber: vtuberId,
+ }
+ }
+ console.log('pNotifPayload as follows')
+ console.log(pNotifPayload)
+
+ const pNotifCreateRes = await fetch(`${process.env.STRAPI_URL}/api/platform-notifications`, {
+ method: 'POST',
+ headers: {
+ 'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify(pNotifPayload)
+ })
+ const pNotifData = await pNotifCreateRes.json() as IPlatformNotificationResponse
+ if (pNotifData.error) {
+ console.error('>> we failed to create platform-notification, there was an error in the response')
+ console.error(JSON.stringify(pNotifData.error, null, 2))
+ throw new Error(pNotifData.error)
+ }
+ console.log(`>> pNotifData (json response) is as follows`)
+ console.log(pNotifData)
+ if (!pNotifData.data?.id) throw new Error('failed to created pNotifData! The response was missing an id');
+
+ pNotifId = pNotifData.data.id
+ if (!pNotifId) throw new Error('failed to get Platform Notification ID');
+ return pNotifId
+
}
-export async function upsertStream(): Promise {
- return 777
+export async function upsertStream({
+ date,
+ vtuberId,
+ platform,
+ pNotifId
+}: {
+ date: string,
+ vtuberId: number,
+ platform: string,
+ pNotifId: number
+}): Promise {
+
+ if (!date) throw new Error(`upsertStream requires date in the arg object, but it was undefined`);
+ if (!vtuberId) throw new Error(`upsertStream requires vtuberId in the arg object, but it was undefined`);
+ if (!platform) throw new Error(`upsertStream requires platform in the arg object, but it was undefined`);
+ if (!pNotifId) throw new Error(`upsertStream requires pNotifId in the arg object, but it was undefined`);
+
+ let streamId
+ // # Step 3.
+ // Finally we find or create the stream record
+ // The stream may already be in the db (the streamer is multi-platform streaming), so we look for that record.
+ // This gets a bit tricky. How do we determine one stream from another?
+ // For now, the rule is 30 minutes of separation.
+ // Anything <=30m is interpreted as the same stream. Anything >30m is interpreted as a different stream.
+ // If the stream is not in the db, we create the stream record
+ const dateSinceRange = subMinutes(new Date(date), 30)
+ const dateUntilRange = addMinutes(new Date(date), 30)
+ console.log(`Find a stream within + or - 30 mins of the notif date=${new Date(date).toISOString()}. dateSinceRange=${dateSinceRange.toISOString()}, dateUntilRange=${dateUntilRange.toISOString()}`)
+ const findStreamQueryString = qs.stringify({
+ populate: 'platform-notifications',
+ filters: {
+ date: {
+ $gte: dateSinceRange,
+ $lte: dateUntilRange
+ },
+ vtuber: {
+ id: {
+ '$eq': vtuberId
+ }
+ }
+ }
+ }, { encode: false })
+
+ console.log('>> findStream')
+ const findStreamRes = await fetch(`${process.env.STRAPI_URL}/api/streams?${findStreamQueryString}`, {
+ method: 'GET',
+ headers: {
+ 'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
+ 'Content-Type': 'application/json'
+ }
+ })
+ const findStreamData = await findStreamRes.json() as IStreamResponse
+ if (findStreamData?.data && findStreamData.data.length > 0) {
+ console.log('>> we found a findStreamData json. (there is an existing stream for this e-mail/notification)')
+ console.log(JSON.stringify(findStreamData, null, 2))
+ streamId = findStreamData.data[0].id
+
+ // Before we're done here, we need to do something extra. We need to populate isChaturbateStream and/or isFanslyStream.
+ // We know which of these booleans to set based on the stream's related platformNotifications
+ // We go through each pNotif and look at it's platform
+ let isFanslyStream = false
+ let isChaturbateStream = false
+ if (findStreamData.data[0].attributes.platformNotifications) {
+ for (const pn of findStreamData.data[0].attributes.platformNotifications) {
+ if (pn.platform === 'fansly') {
+ isFanslyStream = true
+ } else if (pn.platform === 'chaturbate') {
+ isChaturbateStream = true
+ }
+ }
+ }
+
+
+ console.log(`>>> updating stream ${streamId}. isFanslyStream=${isFanslyStream}, isChaturbateStream=${isChaturbateStream}`)
+ const updateStreamRes = await fetch(`${process.env.STRAPI_URL}/api/streams/${streamId}`, {
+ method: 'PUT',
+ headers: {
+ 'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
+ 'content-type': 'application/json'
+ },
+ body: JSON.stringify({
+ data: {
+ isFanslyStream: isFanslyStream,
+ isChaturbateStream: isChaturbateStream,
+ platformNotifications: [
+ pNotifId
+ ]
+ }
+ })
+ })
+ const updateStreamJson = await updateStreamRes.json() as IStreamResponse
+ if (updateStreamJson?.error) throw new Error(JSON.stringify(updateStreamJson, null, 2));
+ console.log(`>> assuming a successful update to the stream record. response as follows.`)
+ console.log(JSON.stringify(updateStreamJson, null, 2))
+ }
+
+ if (!streamId) {
+ console.log('>> did not find a streamId, so we go ahead and create a stream record in the db.')
+ const createStreamPayload = {
+ data: {
+ isFanslyStream: (platform === 'fansly') ? true : false,
+ isChaturbateStream: (platform === 'chaturbate') ? true : false,
+ archiveStatus: 'missing',
+ date: date,
+ date2: date,
+ date_str: date,
+ vtuber: vtuberId,
+ platformNotifications: [
+ pNotifId
+ ]
+ }
+ }
+ console.log('>> createStreamPayload as follows')
+ console.log(createStreamPayload)
+ const createStreamRes = await fetch(`${process.env.STRAPI_URL}/api/streams`, {
+ method: 'POST',
+ headers: {
+ 'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`,
+ "Content-Type": "application/json"
+ },
+ body: JSON.stringify(createStreamPayload)
+ })
+ const createStreamJson = await createStreamRes.json() as IStreamResponse
+ console.log('>> we got the createStreamJson')
+ console.log(createStreamJson)
+ if (createStreamJson.error) {
+ console.error(JSON.stringify(createStreamJson.error, null, 2))
+ throw new Error('Failed to create stream in DB due to an error. (see above)')
+ }
+ streamId = createStreamJson.id
+ }
+
+ if (!streamId) throw new Error('failed to get streamId')
+ return streamId
}
diff --git a/packages/scout/src/temporal/worker.ts b/packages/scout/src/temporal/worker.ts
index 3e0024e..70c0e70 100644
--- a/packages/scout/src/temporal/worker.ts
+++ b/packages/scout/src/temporal/worker.ts
@@ -1,6 +1,7 @@
import path from "path"
import { NativeConnection, Worker } from "@temporalio/worker"
import * as activities from "./activities.js"
+import pRetry from 'p-retry'
if (!process.env.TEMPORAL_SERVICE_ADDRESS) throw new Error(`TEMPORAL_SERVICE_ADDRESS is missing in env`);
@@ -49,7 +50,11 @@ async function run() {
await worker.run();
}
-run().catch((err) => {
- console.error(err);
- process.exit(1);
-})
\ No newline at end of file
+await pRetry(run, {
+ forever: true,
+ onFailedAttempt: (e) => {
+ console.error(e);
+ console.error(`there was an error during scout-worker run(). run() will now restart.`)
+ console.log(`P.S., check out these booba --> (.)(.)`)
+ }
+})
diff --git a/packages/scout/src/temporal/workflows.ts b/packages/scout/src/temporal/workflows.ts
index 8b6fab7..48ff899 100644
--- a/packages/scout/src/temporal/workflows.ts
+++ b/packages/scout/src/temporal/workflows.ts
@@ -14,10 +14,7 @@ export type NotificationData = {
avatar: string;
};
-const {
- chargeUser,
- checkAndDecrementInventory,
- incrementInventory,
+const {
upsertPlatformNotification,
upsertStream,
upsertVtuber,
@@ -39,10 +36,9 @@ export async function processEmail({
// Step 1
const vtuberId = await upsertVtuber({ url, platform, channel, displayName, date, userId, avatar })
- console.log('we have finished upsertVtuber and the vtuberId is '+vtuberId)
- throw new Error('Error: Error: error: erorreorororr; @todo');
- const pNotifId = await upsertPlatformNotification()
- const streamId = await upsertStream()
+ const pNotifId = await upsertPlatformNotification({ vtuberId, source: 'email', date, platform })
+ const streamId = await upsertStream({ date, vtuberId, platform, pNotifId })
+
return { vtuberId, pNotifId, streamId }
}
diff --git a/packages/scout/src/twitter.js b/packages/scout/src/twitter.js
index e6bb2d0..083f732 100644
--- a/packages/scout/src/twitter.js
+++ b/packages/scout/src/twitter.js
@@ -19,9 +19,8 @@ const normalize = (url) => {
}
-const image = async function image (limiter, twitterUsername) {
- if (!limiter) throw new Error('first arg to twitter.data.image must be an instance of node-rate-limiter');
- if (!twitterUsername) throw new Error('second arg to twitter.data.image must be a twitterUsername. It was undefined.');
+const image = async function image (twitterUsername) {
+ if (!twitterUsername) throw new Error('first arg to twitter.data.image must be a twitterUsername. It was undefined.');
const requestDataFromNitter = async () => {
const url = `${process.env.SCOUT_NITTER_URL}/${twitterUsername}/rss?key=${process.env.SCOUT_NITTER_ACCESS_KEY}`
// console.log(`fetching from url=${url}`)
@@ -37,7 +36,7 @@ const image = async function image (limiter, twitterUsername) {
const dom = htmlparser2.parseDocument(body);
const $ = load(dom, { _useHtmlParser2: true })
const urls = $('url:contains("profile_images")').first()
- const downloadedImageFile = await download({ limiter, url: urls.text() })
+ const downloadedImageFile = await download({ url: urls.text() })
return downloadedImageFile
} catch (e) {
console.error(`while fetching rss from nitter, the following error was encountered.`)
diff --git a/packages/scout/src/twitter.spec.js b/packages/scout/src/twitter.spec.js
index 62b28aa..e2f2403 100644
--- a/packages/scout/src/twitter.spec.js
+++ b/packages/scout/src/twitter.spec.js
@@ -2,7 +2,6 @@ import { expect } from 'chai'
import twitter from './twitter.js'
import { describe } from 'mocha'
import { tmpFileRegex } from './utils.js'
-import { RateLimiter } from 'limiter'
describe('twitter', function () {
describe('regex', function () {
@@ -19,10 +18,9 @@ describe('twitter', function () {
})
describe('data', function () {
this.timeout(1000*30)
- const limiter = new RateLimiter({ tokensPerInterval: 10, interval: "second" })
describe('image', function () {
it("should download the twitter users's avatar and save it to disk", async function () {
- const imgFile = await twitter.data.image(limiter, 'projektmelody')
+ const imgFile = await twitter.data.image('projektmelody')
expect(imgFile).to.match(tmpFileRegex)
})
})
diff --git a/packages/scout/src/utils.js b/packages/scout/src/utils.js
index 60fb388..9295c1c 100644
--- a/packages/scout/src/utils.js
+++ b/packages/scout/src/utils.js
@@ -24,19 +24,16 @@ export function getTmpFile(str) {
/**
*
- * @param {Object} limiter [https://github.com/jhurliman/node-rate-limiter](node-rate-limiter) instance
* @param {String} url
* @returns {String} filePath
*
* greetz https://stackoverflow.com/a/74722818/1004931
*/
-export async function download({ limiter, url, filePath }) {
- if (!limiter) throw new Error(`first arg passed to download() must be a node-rate-limiter instance.`);
+export async function download({ url, filePath }) {
if (!url) throw new Error(`second arg passed to download() must be a {string} url`);
const fileBaseName = basename(url)
filePath = filePath || path.join(os.tmpdir(), `${createId()}_${fileBaseName}`)
const stream = fs.createWriteStream(filePath)
- await limiter.removeTokens(1);
const requestData = async () => {
const response = await fetch(url, {
diff --git a/packages/scout/src/utils.spec.js b/packages/scout/src/utils.spec.js
index 3a5f179..97c7595 100644
--- a/packages/scout/src/utils.spec.js
+++ b/packages/scout/src/utils.spec.js
@@ -1,7 +1,6 @@
import { fpSlugify, getTmpFile, download } from './utils.js'
import { expect } from 'chai'
import { describe } from 'mocha'
-import { RateLimiter } from "limiter"
describe('utils', function () {
@@ -18,9 +17,8 @@ describe('utils', function () {
})
}),
describe('download', function () {
- const limiter = new RateLimiter({ tokensPerInterval: 100, interval: "second" })
it('should get the file', async function () {
- const file = await download({ limiter, url: 'https://futureporn-b2.b-cdn.net/sample.webp' })
+ const file = await download({ url: 'https://futureporn-b2.b-cdn.net/sample.webp' })
expect(file).to.match(/\/tmp\/.*sample\.webp$/)
})
})
diff --git a/packages/scout/src/vtuber.js b/packages/scout/src/vtuber.js
index d99c283..3771394 100644
--- a/packages/scout/src/vtuber.js
+++ b/packages/scout/src/vtuber.js
@@ -20,25 +20,22 @@ import fansly from './fansly.js'
*
* We depend on one of these social media URLs. If there is neither Twitter or fansly listed, we throw an error.
*
- * @param {Object} limiter -- instance of node-rate-limiter
* @param {Object} vtuber -- vtuber instance from strapi
* @returns {String} filePath -- path on disk where the image was saved
*/
-export async function getImage(limiter, vtuber) {
- if (!limiter) throw new Error('first arg must be node-rate-limiter instace');
- if (!vtuber) throw new Error('second arg must be vtuber instance');
- await limiter.removeTokens(1);
+export async function getImage(vtuber) {
+ if (!vtuber) throw new Error('first arg must be vtuber instance');
const { twitter: twitterUrl, fanslyId: fanslyId } = vtuber.attributes
const twitterUsername = twitterUrl && twitter.regex.username.exec(twitterUrl).at(1)
let img;
if (twitterUrl) {
- img = await twitter.data.image(limiter, twitterUsername)
+ img = await twitter.data.image(twitterUsername)
} else if (fanslyId) {
- img = await fansly.data.image(limiter, fanslyId)
+ img = await fansly.data.image(fanslyId)
} else {
- const msg = 'while attempting to get vtuber image, there was neither twitter nor fansly listed. One of these must exist for us to download an image.'
+ const msg = ` while attempting to get vtuber image, there was neither twitterUrl nor fanslyId listed. One of these must exist for us to download an image. \nvtuber=${JSON.stringify(vtuber, null, 2)}`
console.error(msg)
throw new Error(msg)
}
diff --git a/packages/scout/src/vtuber.spec.js b/packages/scout/src/vtuber.spec.js
index fe8812f..b628368 100644
--- a/packages/scout/src/vtuber.spec.js
+++ b/packages/scout/src/vtuber.spec.js
@@ -1,7 +1,6 @@
import { expect } from 'chai'
import { describe } from 'mocha'
-import { RateLimiter } from 'limiter'
import { getImage } from './vtuber.js'
import { tmpFileRegex } from './utils.js'
@@ -25,13 +24,12 @@ const vtuberFixture1 = {
describe('vtuber', function () {
this.timeout(1000*60)
describe('getImage', function () {
- const limiter = new RateLimiter({ tokensPerInterval: 1, interval: "second" })
it('should download an avatar image from twitter', async function () {
- const file = await getImage(limiter, vtuberFixture0)
+ const file = await getImage(vtuberFixture0)
expect(file).to.match(tmpFileRegex)
})
it('should download an avatar image from fansly', async function () {
- const file = await getImage(limiter, vtuberFixture1)
+ const file = await getImage(vtuberFixture1)
expect(file).to.match(tmpFileRegex)
})
})
diff --git a/packages/strapi/README.md b/packages/strapi/README.md
index 34338c4..5259892 100644
--- a/packages/strapi/README.md
+++ b/packages/strapi/README.md
@@ -5,3 +5,9 @@
* ironmouse "Thank you" (for testing): 4760169
* cj_clippy "Full library access" (for production): 9380584
* cj_clippy "Your URL displayed on Futureporn.net": 10663202
+
+### Content-Type Builder (Docker caveat)
+
+Don't use the web UI to create or update Content-Types! The changes will be lost. This is a side-effect of our hacked together solution for Strapi with pnpm in docker.
+
+Instead, content-type schemas must be hand-edited in ./src/api/(...). For the changes to take effect, trigger a strapi resource update in Tilt.
\ No newline at end of file
diff --git a/packages/strapi/src/api/platform-notification/content-types/platform-notification/schema.json b/packages/strapi/src/api/platform-notification/content-types/platform-notification/schema.json
index 7c8b86e..c5561e6 100644
--- a/packages/strapi/src/api/platform-notification/content-types/platform-notification/schema.json
+++ b/packages/strapi/src/api/platform-notification/content-types/platform-notification/schema.json
@@ -45,6 +45,12 @@
"type": "relation",
"relation": "oneToOne",
"target": "api::vtuber.vtuber"
+ },
+ "stream": {
+ "type": "relation",
+ "relation": "manyToOne",
+ "target": "api::stream.stream",
+ "inversedBy": "platformNotifications"
}
}
}
\ No newline at end of file
diff --git a/packages/strapi/src/api/stream/content-types/stream/lifecycles.js b/packages/strapi/src/api/stream/content-types/stream/lifecycles.js
index 464dc68..32ea546 100644
--- a/packages/strapi/src/api/stream/content-types/stream/lifecycles.js
+++ b/packages/strapi/src/api/stream/content-types/stream/lifecycles.js
@@ -22,11 +22,6 @@ module.exports = {
async afterUpdate(event) {
- /**
- * NOTE
- *
- * These hooks do not fire in response to API calls. They only fire in response to UI saves.
- */
console.log(`>>>>>>>>>>>>>> STREAM is afterUpdate !!!!!!!!!!!!`);
const { data, where, select, populate } = event.params;
@@ -93,12 +88,14 @@ module.exports = {
console.log(`lets find the platformNotifications`)
console.log(JSON.stringify(existingData2, null, 2))
- // Iterate through all vods to determine archiveStatus
- for (const pn of existingData2.platform_notifications) {
- if (pn.platform === 'fansly') {
- isFanslyStream = true
- } else if (pn.platform === 'chaturbate') {
- isChaturbateStream = true
+ // Iterate through all platformNotifications to determine platform
+ if (existingData2.platformNotifications) {
+ for (const pn of existingData2.platformNotifications) {
+ if (pn.platform === 'fansly') {
+ isFanslyStream = true
+ } else if (pn.platform === 'chaturbate') {
+ isChaturbateStream = true
+ }
}
}
diff --git a/packages/strapi/src/api/stream/content-types/stream/schema.json b/packages/strapi/src/api/stream/content-types/stream/schema.json
index 1506582..63fab1f 100644
--- a/packages/strapi/src/api/stream/content-types/stream/schema.json
+++ b/packages/strapi/src/api/stream/content-types/stream/schema.json
@@ -12,6 +12,12 @@
},
"pluginOptions": {},
"attributes": {
+ "platformNotifications": {
+ "type": "relation",
+ "relation": "oneToMany",
+ "target": "api::platform-notification.platform-notification",
+ "mappedBy": "stream"
+ },
"date_str": {
"type": "string",
"required": true,
diff --git a/packages/strapi/src/api/vtuber/content-types/vtuber/schema.json b/packages/strapi/src/api/vtuber/content-types/vtuber/schema.json
index 91568cc..4c64af0 100644
--- a/packages/strapi/src/api/vtuber/content-types/vtuber/schema.json
+++ b/packages/strapi/src/api/vtuber/content-types/vtuber/schema.json
@@ -79,7 +79,7 @@
},
"description1": {
"type": "text",
- "required": true
+ "required": false
},
"description2": {
"type": "text"
@@ -113,6 +113,15 @@
"relation": "oneToMany",
"target": "api::stream.stream",
"mappedBy": "vtuber"
+ },
+ "fanslyId": {
+ "type": "string"
+ },
+ "chaturbateId": {
+ "type": "string"
+ },
+ "twitterId": {
+ "type": "string"
}
}
}
diff --git a/scripts/k8s-secrets.sh b/scripts/k8s-secrets.sh
index c98ca26..aa6ae3d 100755
--- a/scripts/k8s-secrets.sh
+++ b/scripts/k8s-secrets.sh
@@ -39,7 +39,9 @@ kubectl --namespace futureporn create secret generic scout \
--from-literal=imapUsername=${SCOUT_IMAP_USERNAME} \
--from-literal=imapPassword=${SCOUT_IMAP_PASSWORD} \
--from-literal=imapAccessToken=${SCOUT_IMAP_ACCESS_TOKEN} \
---from-literal=nitterAccessKey=${SCOUT_NITTER_ACCESS_KEY}
+--from-literal=nitterAccessKey=${SCOUT_NITTER_ACCESS_KEY} \
+--from-literal=s3BucketKeyId=${S3_BUCKET_KEY_ID} \
+--from-literal=s3BucketApplicationKey=${S3_BUCKET_APPLICATION_KEY}
kubectl --namespace futureporn delete secret link2cid --ignore-not-found
kubectl --namespace futureporn create secret generic link2cid \