diff --git a/d.scout.dockerfile b/d.scout.dockerfile index e8673ac..4adb739 100644 --- a/d.scout.dockerfile +++ b/d.scout.dockerfile @@ -2,29 +2,43 @@ FROM node:20 as base ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" WORKDIR /app -RUN corepack enable +RUN corepack enable && corepack prepare pnpm@9.2.0 --activate FROM base as build -WORKDIR /usr/src/app -RUN mkdir -p /usr/src/app/packages/scout && mkdir /usr/src/app/packages/taco && mkdir -p /prod/scout -COPY package.json pnpm-lock.yaml . -COPY ./packages/scout/pnpm-lock.yaml ./packages/scout/ -COPY ./packages/taco/pnpm-lock.yaml ./packages/taco/ -COPY ./packages/types/pnpm-lock.yaml ./packages/types/ -COPY ./packages/temporal-workflows/pnpm-lock.yaml ./packages/temporal-workflows/ -COPY ./packages/temporal-worker/pnpm-lock.yaml ./packages/temporal-worker/ -RUN pnpm fetch +WORKDIR /app +RUN mkdir -p /app/packages/scout && mkdir /app/packages/taco && mkdir -p /prod/scout + +## Copy manfiests, lockfiles, and configs into docker context +COPY package.json pnpm-lock.yaml .npmrc . +COPY ./packages/scout/pnpm-lock.yaml ./packages/scout/package.json ./packages/scout/ +COPY ./packages/utils/pnpm-lock.yaml ./packages/utils/package.json ./packages/utils/ +COPY ./packages/image/pnpm-lock.yaml ./packages/image/package.json ./packages/image/ +COPY ./packages/storage/pnpm-lock.yaml ./packages/storage/package.json ./packages/storage/ +COPY ./packages/taco/pnpm-lock.yaml ./packages/taco/package.json ./packages/taco/ +COPY ./packages/types/pnpm-lock.yaml ./packages/types/package.json ./packages/types/ +COPY ./packages/temporal-workflows/pnpm-lock.yaml ./packages/temporal-workflows/package.json ./packages/temporal-workflows/ +COPY ./packages/temporal-worker/pnpm-lock.yaml ./packages/temporal-worker/package.json ./packages/temporal-worker/ + +## Install npm packages RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --recursive --frozen-lockfile + +## Copy package code into docker context +COPY ./packages/utils/ ./packages/utils/ +COPY ./packages/image/ ./packages/image/ +COPY ./packages/storage/ ./packages/storage/ COPY ./packages/scout/ ./packages/scout/ COPY ./packages/taco/ ./packages/taco/ COPY ./packages/types/ ./packages/types/ COPY ./packages/temporal-workflows/ ./packages/temporal-workflows/ COPY ./packages/temporal-worker/ ./packages/temporal-worker/ + +## Build (transpile TS into JS) RUN pnpm -r build + +## Deploy (copy all production code into one place) RUN pnpm deploy --filter=scout --prod /prod/scout FROM base as scout COPY --from=build /prod/scout . -RUN ls -la . ENTRYPOINT ["pnpm", "start"] diff --git a/packages/scout/package.json b/packages/scout/package.json index 9210f45..ed5b101 100644 --- a/packages/scout/package.json +++ b/packages/scout/package.json @@ -9,10 +9,9 @@ }, "scripts": { "test": "mocha --require ts-node/register src/**/*.spec.ts", - "build:worker": "tsc --build ./tsconfig.json", - "start": "echo please use either start:manager or start:worker", - "start:manager": "node --loader ts-node/esm ./src/index.ts", - "start:worker": "node --loader ts-node/esm ./src/temporal/worker.ts" + "build": "tsc --build", + "dev": "nodemon --ext js,ts,json,yaml --exec \"node --loader ts-node/esm --disable-warning=ExperimentalWarning ./src/index.ts\"", + "start": "node /app/dist/index.js" }, "keywords": [], "author": "@CJ_Clippy", @@ -21,6 +20,8 @@ "@aws-sdk/client-s3": "^3.583.0", "@aws-sdk/lib-storage": "^3.588.0", "@aws-sdk/s3-request-presigner": "^3.588.0", + "@futureporn/temporal-workflows": "workspace:^", + "@futureporn/types": "workspace:^", "@paralleldrive/cuid2": "^2.2.2", "@temporalio/client": "^1.9.0", "@temporalio/worker": "^1.9.0", @@ -38,23 +39,24 @@ "limiter": "2.0.1", "mailparser": "^3.7.1", "node-fetch": "^3.3.0", - "nodemon": "^3.1.3", "p-retry": "^5.1.2", "prevvy": "^7.0.1", "qs": "^6.12.1", "sharp": "^0.33.4", "slugify": "^1.6.6", "tsx": "^4.7.2", - "@futureporn/types": "workspace:^", "xpath": "^0.0.34" }, "packageManager": "pnpm@9.2.0", "devDependencies": { "@types/chai": "^4.3.16", + "@types/cheerio": "^0.22.35", + "@types/mailparser": "^3.4.4", "@types/mocha": "^10.0.7", "chai": "^5.1.0", "mocha": "^10.4.0", + "nodemon": "^3.1.4", "ts-node": "^10.9.2", - "typescript": "^5.4.5" + "typescript": "^5.5.3" } } diff --git a/packages/scout/pnpm-lock.yaml b/packages/scout/pnpm-lock.yaml index 69281e7..166fc24 100644 --- a/packages/scout/pnpm-lock.yaml +++ b/packages/scout/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@aws-sdk/s3-request-presigner': specifier: ^3.588.0 version: 3.614.0 + '@futureporn/temporal-workflows': + specifier: workspace:^ + version: link:../temporal-workflows '@futureporn/types': specifier: workspace:^ version: link:../types @@ -71,9 +74,6 @@ importers: node-fetch: specifier: ^3.3.0 version: 3.3.2 - nodemon: - specifier: ^3.1.3 - version: 3.1.4 p-retry: specifier: ^5.1.2 version: 5.1.2 @@ -99,6 +99,12 @@ importers: '@types/chai': specifier: ^4.3.16 version: 4.3.16 + '@types/cheerio': + specifier: ^0.22.35 + version: 0.22.35 + '@types/mailparser': + specifier: ^3.4.4 + version: 3.4.4 '@types/mocha': specifier: ^10.0.7 version: 10.0.7 @@ -108,11 +114,14 @@ importers: mocha: specifier: ^10.4.0 version: 10.6.0 + nodemon: + specifier: ^3.1.4 + version: 3.1.4 ts-node: specifier: ^10.9.2 version: 10.9.2(@swc/core@1.6.13)(@types/node@20.14.10)(typescript@5.5.3) typescript: - specifier: ^5.4.5 + specifier: ^5.5.3 version: 5.5.3 packages: @@ -972,6 +981,9 @@ packages: '@types/chai@4.3.16': resolution: {integrity: sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==} + '@types/cheerio@0.22.35': + resolution: {integrity: sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==} + '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} @@ -996,6 +1008,9 @@ packages: '@types/luxon@3.4.2': resolution: {integrity: sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==} + '@types/mailparser@3.4.4': + resolution: {integrity: sha512-C6Znp2QVS25JqtuPyxj38Qh+QoFcLycdxsvcc6IZCGekhaMBzbdTXzwGzhGoYb3TfKu8IRCNV0sV1o3Od97cEQ==} + '@types/mocha@10.0.7': resolution: {integrity: sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==} @@ -3536,6 +3551,10 @@ snapshots: '@types/chai@4.3.16': {} + '@types/cheerio@0.22.35': + dependencies: + '@types/node': 20.14.10 + '@types/debug@4.1.12': dependencies: '@types/ms': 0.7.34 @@ -3564,6 +3583,11 @@ snapshots: '@types/luxon@3.4.2': {} + '@types/mailparser@3.4.4': + dependencies: + '@types/node': 20.14.10 + iconv-lite: 0.6.3 + '@types/mocha@10.0.7': {} '@types/ms@0.7.34': {} diff --git a/packages/scout/src/index.ts b/packages/scout/src/index.ts index 21cc9f2..17f0a39 100644 --- a/packages/scout/src/index.ts +++ b/packages/scout/src/index.ts @@ -4,57 +4,74 @@ * watches an e-mail inbox for going live notifications */ -import { checkEmail } from './parsers.js' -// import { createStreamInDb } from './signals.js' -import { Email } from './imap.js' +// import { checkEmail } from './parsers.js' +// // import { createStreamInDb } from './signals.js' +// import { Email } from './imap.js' import { Client, Connection } from '@temporalio/client' -import { NotificationData, processEmailNotification } from 'temporal-workflows' -import { FetchMessageObject } from 'imapflow' +// import { type NotificationData } from '@futureporn/types' +// import { type FetchMessageObject } from 'imapflow' import { createId } from '@paralleldrive/cuid2' +import { WorkflowA } from '@futureporn/temporal-workflows/workflows.js' +console.log('connecting to temporal...') const connection = await Connection.connect({ address: 'temporal-frontend.futureporn.svc.cluster.local:7233' }); const client = new Client({ connection, namespace: 'futureporn' }); -async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageObject }) { - try { - console.log(' ✏️ loading message') - const body = await email.loadMessage(msg.uid) +// async function handleMessage({ email, msg }: { email: Email, msg: FetchMessageObject }) { +// try { +// console.log(' ✏️ loading message') +// const body = await email.loadMessage(msg.uid) - // console.log(' ✏️ checking e-mail') - const { isMatch, url, platform, channel, displayName, date, userId, avatar }: NotificationData = (await checkEmail(body) ) +// console.log(' ✏️ checking e-mail') +// const { isMatch, url, platform, channel, displayName, date, userId, avatar }: NotificationData = (await checkEmail(body) ) - if (isMatch) { - const wfId = `process-email-${createId()}` - // console.log(` ✏️ [DRY] starting Temporal workflow ${wfId} @todo actually start temporal workflow`) - // await signalRealtime({ url, platform, channel, displayName, date, userId, avatar }) - // @todo invoke a Temporal workflow here - console.log(' ✏️✏️ starting Temporal workflow') - const handle = await client.workflow.start(processEmail, { - workflowId: wfId, - taskQueue: 'scout', - args: [{ url, platform, channel, displayName, date, userId, avatar }] - }); - // const handle = client.getHandle(workflowId); - const result = await handle.result() - console.log(`result of the workflow is as follows`) - console.log(result) - } +// if (isMatch) { +// const wfId = `process-email-${createId()}` +// // console.log(` ✏️ [DRY] starting Temporal workflow ${wfId} @todo actually start temporal workflow`) +// // await signalRealtime({ url, platform, channel, displayName, date, userId, avatar }) +// // @todo invoke a Temporal workflow here +// console.log(' ✏️✏️ starting Temporal workflow') +// // const handle = await client.workflow.start(WorkflowA, { +// // workflowId: wfId, +// // taskQueue: 'futureporn' +// // }); +// // // const handle = await client.workflow.start(processNotifEmail, { +// // // workflowId: wfId, +// // // taskQueue: 'futureporn', +// // // args: [{ url, platform, channel, displayName, date, userId, avatar }] +// // // }); +// // // const handle = client.getHandle(workflowId); +// // const result = await handle.result() +// // console.log(`result of the workflow is as follows`) +// // console.log(result) +// } - console.log(' ✏️ archiving e-mail') - await email.archiveMessage(msg.uid) +// console.log(' ✏️ archiving e-mail') +// await email.archiveMessage(msg.uid) - } catch (e) { - // console.error('error encoutered') - console.error(`An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`) - console.error(e) - } -} +// } catch (e) { +// // console.error('error encoutered') +// console.error(`An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.`) +// console.error(e) +// } +// } (async () => { - const email = new Email() - email.once('message', (msg: FetchMessageObject) => handleMessage({ email, msg })) - await email.connect() -})() \ No newline at end of file + // const email = new Email() + // email.once('message', (msg: FetchMessageObject) => handleMessage({ email, msg })) + // await email.connect() + console.log('scout is starting') + const wfId = `process-email-${createId()}` + const handle = await client.workflow.start(WorkflowA, { + workflowId: wfId, + taskQueue: 'futureporn' + }); + const result = await handle.result() + console.log(result) +})() + + +// console.log('init') \ No newline at end of file diff --git a/packages/scout/src/parsers.spec.js b/packages/scout/src/parsers.spec.ts similarity index 100% rename from packages/scout/src/parsers.spec.js rename to packages/scout/src/parsers.spec.ts diff --git a/packages/scout/src/parsers.js b/packages/scout/src/parsers.ts similarity index 86% rename from packages/scout/src/parsers.js rename to packages/scout/src/parsers.ts index c9d8523..364512b 100644 --- a/packages/scout/src/parsers.js +++ b/packages/scout/src/parsers.ts @@ -1,6 +1,7 @@ import { simpleParser } from 'mailparser'; import { load } from 'cheerio' +import { type NotificationData } from '@futureporn/types' const definitions = [ { @@ -14,10 +15,10 @@ const definitions = [ { platform: 'fansly', selectors: { - channel: ($) => $("a[href*='/live/']").attr('href').toString().split('/').at(-1), + channel: ($: any) => $("a[href*='/live/']").attr('href').toString().split('/').at(-1), displayName: 'div[class*="message-col"] div:nth-child(5)', - userId: ($) => $("img[src*='/api/v1/account/']").attr('src').toString().split('/').at(-2), - avatar: ($) => $("img[src*='/api/v1/account/']").attr('src').toString() + userId: ($: any) => $("img[src*='/api/v1/account/']").attr('src').toString().split('/').at(-2), + avatar: ($: any) => $("img[src*='/api/v1/account/']").attr('src').toString() }, from: 'no-reply@fansly.com', template: 'https://fansly.com/:channel', @@ -25,7 +26,7 @@ const definitions = [ } ] -function render(template, values) { +function render(template: string, values: ) { // console.log(`values=${values}`) // console.log(values) return template.replace(/:([a-zA-Z0-9_]+)/g, (match, key) => { @@ -53,7 +54,7 @@ function render(template, values) { * @returns {String|null} result.userId Varies by platform. Some platforms don't have the userId in the e-mail, so it's null. * fansly example: '555722198917066752' */ -export async function checkEmail (body) { +export async function checkEmail (body: string): Promise { const mail = await simpleParser(body) if (!mail?.html) { diff --git a/packages/temporal-workflows/package.json b/packages/temporal-workflows/package.json index 0f047cb..e4dbc45 100644 --- a/packages/temporal-workflows/package.json +++ b/packages/temporal-workflows/package.json @@ -22,7 +22,8 @@ "@futureporn/scout": "workspace:^", "@futureporn/storage": "workspace:^", "@futureporn/types": "workspace:*", - "@futureporn/utils": "workspace:*" + "@futureporn/utils": "workspace:*", + "@futureporn/temporal-workflows": "workspace:*" }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^5.62.0", diff --git a/packages/temporal-workflows/pnpm-lock.yaml b/packages/temporal-workflows/pnpm-lock.yaml index c7c1d11..22aa904 100644 --- a/packages/temporal-workflows/pnpm-lock.yaml +++ b/packages/temporal-workflows/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@futureporn/storage': specifier: workspace:^ version: link:../storage + '@futureporn/temporal-workflows': + specifier: workspace:* + version: 'link:' '@futureporn/types': specifier: workspace:* version: link:../types diff --git a/packages/temporal-workflows/src/activities.ts b/packages/temporal-workflows/src/activities.ts index d6c3cad..4d03412 100644 --- a/packages/temporal-workflows/src/activities.ts +++ b/packages/temporal-workflows/src/activities.ts @@ -1,6 +1,6 @@ export * from './workflowA/activities/activitiesA.js' export * from './workflowA/activities/activitiesB.js' -export * from './workflowB/activities.js' -export * from './processNotifEmail/activities/upsertPlatformNotification.js' -export * from './processNotifEmail/activities/upsertStream.js' -export * from './processNotifEmail/activities/upsertVtuber.js' +// export * from './workflowB/activities.js' +// export * from './processNotifEmail/activities/upsertPlatformNotification.js' +// export * from './processNotifEmail/activities/upsertStream.js' +// export * from './processNotifEmail/activities/upsertVtuber.js' diff --git a/packages/temporal-workflows/src/processNotifEmail/workflow.ts b/packages/temporal-workflows/src/processNotifEmail/workflow.ts index 15c4a7c..4631830 100644 --- a/packages/temporal-workflows/src/processNotifEmail/workflow.ts +++ b/packages/temporal-workflows/src/processNotifEmail/workflow.ts @@ -15,10 +15,12 @@ const { upsertPlatformNotification } = proxyActivities { - log.info('Hello from processNotifEmail workflow'); - const vtuberId = await upsertVtuber({ platform, userId, url, channel }); - const pNotifId = await upsertPlatformNotification({ source: 'email', date, platform, vtuberId }); - const streamId = await upsertStream({ date, vtuberId, platform, pNotifId }); - return `vtuberId: ${vtuberId} | pNotifId: ${pNotifId} | streamId: ${streamId}`; +export async function processNotifEmail(args: NotificationData): Promise { + return '@todo @todo @todo' + // const { url, platform, channel, displayName, date, userId, avatar } = args + // log.info('Hello from processNotifEmail workflow'); + // const vtuberId = await upsertVtuber({ platform, userId, url, channel }); + // const pNotifId = await upsertPlatformNotification({ source: 'email', date, platform, vtuberId }); + // const streamId = await upsertStream({ date, vtuberId, platform, pNotifId }); + // return `vtuberId: ${vtuberId} | pNotifId: ${pNotifId} | streamId: ${streamId}`; } diff --git a/packages/temporal-workflows/src/workflows.ts b/packages/temporal-workflows/src/workflows.ts index eaa4706..6163a3c 100644 --- a/packages/temporal-workflows/src/workflows.ts +++ b/packages/temporal-workflows/src/workflows.ts @@ -1,3 +1,3 @@ export * from './workflowA/workflow.js' -export * from './workflowB/workflow.js' -export * from './processNotifEmail/workflow.js' \ No newline at end of file +// export * from './workflowB/workflow.js' +// export * from './processNotifEmail/workflow.js' \ No newline at end of file diff --git a/packages/types/index.d.ts b/packages/types/index.d.ts index 0aabd5a..c604cd5 100644 --- a/packages/types/index.d.ts +++ b/packages/types/index.d.ts @@ -122,12 +122,12 @@ declare namespace Futureporn { type NotificationData = { isMatch?: boolean; - url: string; - platform: string; - channel: string; - displayName: string; - date: string; - userId: string | null; - avatar: string; + url?: string; + platform?: string; + channel?: string | null; + displayName?: string; + date?: string; + userId?: string | null; + avatar?: string; }; } \ No newline at end of file diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 18ec407..5bdb872 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,11 @@ +## We differentiate packages and services in order to make Dockerfile builds less terse. +## When we are building a docker image for a service, we need to include all their package dependencies. +## Instead of having to manually pick and choose every dependent package into the build context, +## we can just copy all the packages/* into build context. +## For example, when we are building fp/scout, we don't want to include @futureporn/next or @futureporn/strapi in the docker build context. +## @futureporn/scout, @futureporn/strapi, @futureporn/next would all be considered services. +## In other words, services/* depend on packages, but packages/* do not depend on services/* + packages: - 'packages/*' + - 'services/*' \ No newline at end of file