fix strict ts errors

This commit is contained in:
Chris Grimmett 2023-11-08 18:02:05 -08:00
parent c7fbc40399
commit 41acf588ea
19 changed files with 173 additions and 235 deletions

View File

@ -4,6 +4,10 @@ Quality Assurance
A long-running tasks
## Dependencies
* Node >= 18.17.0 (hard requirement)
## Issue tracker
Please use https://gitea.futureporn.net/futureporn/futureporn-qa/issues

View File

@ -4,6 +4,9 @@
"main": "index.js",
"license": "Unlicense",
"private": true,
"engines": {
"node": ">=18.17.0"
},
"dependencies": {
"@11ty/eleventy-fetch": "^4.0.0",
"@aws-sdk/client-s3": "^3.441.0",
@ -14,6 +17,8 @@
"@paralleldrive/cuid2": "^2.2.2",
"@swc/core": "^1.3.96",
"@tsoa/runtime": "^5.0.0",
"@types/express": "^4.17.21",
"@types/method-override": "^0.0.35",
"@types/node": "^20.8.10",
"@types/qs": "^6.9.9",
"bullmq": "^4.13.0",
@ -31,7 +36,7 @@
"method-override": "^3.0.0",
"node-fetch": "^3.3.2",
"only-allow": "^1.2.1",
"prevvy": "^7.2.0",
"prevvy": "^7.4.0",
"python-shell": "^5.0.0",
"qs": "^6.11.2",
"swagger-ui-express": "^5.0.0",

View File

@ -32,6 +32,12 @@ dependencies:
'@tsoa/runtime':
specifier: ^5.0.0
version: 5.0.0
'@types/express':
specifier: ^4.17.21
version: 4.17.21
'@types/method-override':
specifier: ^0.0.35
version: 0.0.35
'@types/node':
specifier: ^20.8.10
version: 20.8.10
@ -84,8 +90,8 @@ dependencies:
specifier: ^1.2.1
version: 1.2.1
prevvy:
specifier: ^7.2.0
version: 7.2.0(@babel/core@7.23.2)
specifier: ^7.4.0
version: 7.4.0(@babel/core@7.23.2)
python-shell:
specifier: ^5.0.0
version: 5.0.0
@ -3035,6 +3041,12 @@ packages:
'@types/node': 20.8.10
dev: false
/@types/debug@4.1.11:
resolution: {integrity: sha512-R2qflTjHDs4CL6D/6TkqBeIHr54WzZfIxN729xvCNlYIVp2LknlnCro5Yo3frNaX2E5gO9pZ3/QAPVdGmu+q9w==}
dependencies:
'@types/ms': 0.7.34
dev: false
/@types/estree@0.0.39:
resolution: {integrity: sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==}
dev: false
@ -3064,7 +3076,7 @@ packages:
/@types/fluent-ffmpeg@2.1.24:
resolution: {integrity: sha512-g5oQO8Jgi2kFS3tTub7wLvfLztr1s8tdXmRd8PiL/hLMLzTIAyMR2sANkTggM/rdEDAg3d63nYRRVepwBiCw5A==}
dependencies:
'@types/node': 20.8.10
'@types/node': 20.9.0
dev: false
/@types/html-minifier@3.5.3:
@ -3083,6 +3095,16 @@ packages:
resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==}
dev: false
/@types/luxon@3.3.4:
resolution: {integrity: sha512-H9OXxv4EzJwE75aTPKpiGXJq+y4LFxjpsdgKwSmr503P5DkWc3AG7VAFYrFNVvqemT5DfgZJV9itYhqBHSGujA==}
dev: false
/@types/method-override@0.0.35:
resolution: {integrity: sha512-HdhM5xiIV8fwsZ3B8e9IKWJOqEgmXXBJ/qQzhs5Z8idjsszqEX4j/7/QAcso27ArZ1tSBXg2XMlI1cIHAsCTXA==}
dependencies:
'@types/express': 4.17.21
dev: false
/@types/mime@1.3.4:
resolution: {integrity: sha512-1Gjee59G25MrQGk8bsNvC6fxNiRgUlGn2wlhGf95a59DrprnnHk80FIMMFG9XHMdrfsuA119ht06QPDXA1Z7tw==}
dev: false
@ -3095,6 +3117,10 @@ packages:
resolution: {integrity: sha512-RsOPImTriV/OE4A9qKjMtk2MnXiuLLbcO3nCXK+kvq4nr0iMfFgpjaX3MPLb6f7+EL1FGSelYvuJMV6REH+ZPQ==}
dev: true
/@types/ms@0.7.34:
resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
dev: false
/@types/multer@1.4.9:
resolution: {integrity: sha512-9NSvPJ2E8bNTc8XtJq1Cimx2Wrn2Ah48F15B2Du/hM8a8CHLhVbJMlF3ZCqhvMdht7Sa+YdP0aKP7N4fxDcrrg==}
dependencies:
@ -3107,6 +3133,12 @@ packages:
undici-types: 5.26.5
dev: false
/@types/node@20.9.0:
resolution: {integrity: sha512-nekiGu2NDb1BcVofVcEKMIwzlx4NjHlcjhoxxKBNLtz15Y1z7MYf549DFvkHSId02Ax6kGwWntIBPC3l/JZcmw==}
dependencies:
undici-types: 5.26.5
dev: false
/@types/qs@6.9.9:
resolution: {integrity: sha512-wYLxw35euwqGvTDx6zfY1vokBFnsK0HNrzc6xNHchxfO2hpuRg74GbkEW7e3sSmPvj0TjCDT1VCa6OtHXnubsg==}
dev: false
@ -3399,10 +3431,6 @@ packages:
engines: {node: '>=8'}
dev: true
/bluebird@3.7.2:
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
dev: false
/body-parser@1.19.2:
resolution: {integrity: sha512-SAAwOxgoCKMGs9uUAUFHygfLAyaniaoun6I8mFY9pRAJL9+Kec34aU+oIjDhTycub1jozEfEwx1W1IuOYxVSFw==}
engines: {node: '>= 0.8'}
@ -3789,17 +3817,6 @@ packages:
luxon: 3.4.3
dev: false
/cross-spawn@6.0.5:
resolution: {integrity: sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==}
engines: {node: '>=4.8'}
dependencies:
nice-try: 1.0.5
path-key: 2.0.1
semver: 5.7.2
shebang-command: 1.2.0
which: 1.3.1
dev: false
/cross-spawn@7.0.3:
resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
engines: {node: '>= 8'}
@ -4192,19 +4209,6 @@ packages:
resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
dev: false
/execa@0.10.0:
resolution: {integrity: sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==}
engines: {node: '>=4'}
dependencies:
cross-spawn: 6.0.5
get-stream: 3.0.0
is-stream: 1.1.0
npm-run-path: 2.0.2
p-finally: 1.0.0
signal-exit: 3.0.7
strip-eof: 1.0.0
dev: false
/execa@7.2.0:
resolution: {integrity: sha512-UduyVP7TLB5IcAQl+OzLyLcS/l32W/GLg+AhHJ+ow40FOk2U3SAllPwR44v4vmdFwIWqpdwxxpQbF1n5ta9seA==}
engines: {node: ^14.18.0 || ^16.14.0 || >=18.0.0}
@ -4220,6 +4224,21 @@ packages:
strip-final-newline: 3.0.0
dev: false
/execa@8.0.1:
resolution: {integrity: sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==}
engines: {node: '>=16.17'}
dependencies:
cross-spawn: 7.0.3
get-stream: 8.0.1
human-signals: 5.0.0
is-stream: 3.0.0
merge-stream: 2.0.0
npm-run-path: 5.1.0
onetime: 6.0.0
signal-exit: 4.1.0
strip-final-newline: 3.0.0
dev: false
/expand-tilde@2.0.2:
resolution: {integrity: sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==}
engines: {node: '>=0.10.0'}
@ -4580,16 +4599,16 @@ packages:
resolution: {integrity: sha512-7HuY/hebu4gryTDT7O/XY/fvY9wRByEGdK6QOa4of8npTcv0+NS6frFKABcf6S9EBAsveTuKTsZQQBFMMNILIg==}
dev: false
/get-stream@3.0.0:
resolution: {integrity: sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ==}
engines: {node: '>=4'}
dev: false
/get-stream@6.0.1:
resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==}
engines: {node: '>=10'}
dev: false
/get-stream@8.0.1:
resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==}
engines: {node: '>=16'}
dev: false
/get-symbol-description@1.0.0:
resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==}
engines: {node: '>= 0.4'}
@ -4858,6 +4877,11 @@ packages:
engines: {node: '>=14.18.0'}
dev: false
/human-signals@5.0.0:
resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==}
engines: {node: '>=16.17.0'}
dev: false
/iconv-lite@0.4.24:
resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==}
engines: {node: '>=0.10.0'}
@ -5048,11 +5072,6 @@ packages:
call-bind: 1.0.5
dev: false
/is-stream@1.1.0:
resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==}
engines: {node: '>=0.10.0'}
dev: false
/is-stream@2.0.1:
resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}
engines: {node: '>=8'}
@ -5560,10 +5579,6 @@ packages:
resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==}
dev: false
/nice-try@1.0.5:
resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==}
dev: false
/no-case@2.3.2:
resolution: {integrity: sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==}
dependencies:
@ -5656,13 +5671,6 @@ packages:
engines: {node: '>=14.16'}
dev: false
/npm-run-path@2.0.2:
resolution: {integrity: sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==}
engines: {node: '>=4'}
dependencies:
path-key: 2.0.1
dev: false
/npm-run-path@5.1.0:
resolution: {integrity: sha512-sJOdmRGrY2sjNTRMbSvluQqg+8X7ZK61yvzBEIDhz4f8z1TZFYABsqjjCBd/0PUNE9M6QDgHJXQkGUEm7Q+l9Q==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@ -5807,11 +5815,6 @@ packages:
resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==}
engines: {node: '>=0.10.0'}
/path-key@2.0.1:
resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==}
engines: {node: '>=4'}
dev: false
/path-key@3.1.1:
resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
engines: {node: '>=8'}
@ -5850,16 +5853,17 @@ packages:
resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
engines: {node: '>=8.6'}
/prevvy@7.2.0(@babel/core@7.23.2):
resolution: {integrity: sha512-QO8qMuge/LU6vJLr/PRXqaqWW3mQkHeUgh/wXmxfb1QqhVIF8NFbLnrWVqP4XITJVSUwsIBROgP8sJjo2e1qkQ==}
/prevvy@7.4.0(@babel/core@7.23.2):
resolution: {integrity: sha512-31zI0t4/P2kqnOvGU2Z9Zf4ILqP4W9ZiMG4WdMLVtNXNd9Fry37dmr9T822na1rGUnWpEJD5/z5L9itUzuCRIg==}
requiresBuild: true
dependencies:
'@babel/preset-env': 7.23.2(@babel/core@7.23.2)
'@types/debug': 4.1.11
'@types/fluent-ffmpeg': 2.1.24
'@types/node': 20.8.10
bluebird: 3.7.2
'@types/luxon': 3.3.4
'@types/node': 20.9.0
debug: 4.3.4(supports-color@8.1.1)
execa: 0.10.0
execa: 8.0.1
fluent-ffmpeg: 2.1.2
luxon: 1.28.1
typescript: 5.2.2
@ -6257,6 +6261,7 @@ packages:
/semver@5.7.2:
resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==}
hasBin: true
dev: true
/semver@6.3.1:
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
@ -6377,13 +6382,6 @@ packages:
resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
dev: false
/shebang-command@1.2.0:
resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==}
engines: {node: '>=0.10.0'}
dependencies:
shebang-regex: 1.0.0
dev: false
/shebang-command@2.0.0:
resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
engines: {node: '>=8'}
@ -6391,11 +6389,6 @@ packages:
shebang-regex: 3.0.0
dev: false
/shebang-regex@1.0.0:
resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==}
engines: {node: '>=0.10.0'}
dev: false
/shebang-regex@3.0.0:
resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
engines: {node: '>=8'}
@ -6417,6 +6410,11 @@ packages:
resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
dev: false
/signal-exit@4.1.0:
resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
engines: {node: '>=14'}
dev: false
/simple-swizzle@0.2.2:
resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
dependencies:
@ -6547,11 +6545,6 @@ packages:
dependencies:
ansi-regex: 5.0.1
/strip-eof@1.0.0:
resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==}
engines: {node: '>=0.10.0'}
dev: false
/strip-final-newline@3.0.0:
resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==}
engines: {node: '>=12'}

View File

@ -1,46 +1,21 @@
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { createId } from '@paralleldrive/cuid2';
import path from 'node:path';
import fs from 'node:fs';
import { getVideoSrcB2LocalFilePath } from './fsCommon.js'
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { B2_ENDPOINT, B2_REGION, B2_KEY, B2_SECRET } from './env.ts';
const urlPrefix = 'https://f000.backblazeb2.com/b2api/v1/b2_download_file_by_id?fileId='
// export async function downloadVideoSrcB2 (appContext, vod) {
// const localFilePath = getVideoSrcB2LocalFilePath(appContext, vod);
// const key = vod.attributes.videoSrcB2.data.attributes.key
// const s3 = new S3Client({
// endpoint: appContext.env.B2_ENDPOINT,
// region: appContext.env.B2_REGION,
// credentials: {
// accessKeyId: process.env.B2_KEY,
// secretAccessKey: process.env.B2_SECRET,
// },
// });
// var params = {Bucket: appContext.env.B2_BUCKET, Key: key};
// const s3Result = await s3.send(new GetObjectCommand(params));
// if (!s3Result.Body) {
// throw new Error('received empty body from S3');
// }
// await pipeline(s3Result.Body, fs.createWriteStream(localFilePath));
// return localFilePath
// }
export async function uploadToB2 (env: NodeJS.ProcessEnv, filePath: string) {
const keyName = `${createId()}-${path.basename(filePath)}`
const bucketName = env.B2_BUCKET
console.log(`uploadToB2 begin. bucketName:${bucketName}`)
const s3 = new S3Client({
endpoint: env.B2_ENDPOINT,
region: env.B2_REGION,
endpoint: B2_ENDPOINT,
region: B2_REGION,
credentials: {
accessKeyId: env.B2_KEY,
secretAccessKey: env.B2_SECRET,
accessKeyId: B2_KEY,
secretAccessKey: B2_SECRET,
}
});
var params = {Bucket: bucketName, Key: keyName, Body: fs.createReadStream(filePath)};

View File

@ -18,6 +18,11 @@ const requiredEnvVariables = [
"TWITTER_ACCOUNTS",
"STRAPI_API_KEY",
"STRAPI_URL",
"B2_ENDPOINT",
"B2_REGION",
"B2_KEY",
"B2_SECRET",
"TMPDIR",
];
requiredEnvVariables.forEach(checkEnvVariable);
@ -32,3 +37,8 @@ export const IPFS_CLUSTER_HTTP_API_PASSWORD = process.env.IPFS_CLUSTER_HTTP_API_
export const TWITTER_ACCOUNTS = process.env.TWITTER_ACCOUNTS as string;
export const STRAPI_API_KEY = process.env.STRAPI_API_KEY as string;
export const STRAPI_URL = process.env.STRAPI_URL as string;
export const B2_ENDPOINT = process.env.B2_ENDPOINT as string;
export const B2_REGION = process.env.B2_REGION as string;
export const B2_KEY = process.env.B2_KEY as string;
export const B2_SECRET = process.env.B2_SECRET as string;
export const TMPDIR = process.env.TMPDIR as string;

View File

@ -1,9 +0,0 @@
import path from 'node:path';
import fs from 'node:fs';
export function getVideoSrcB2LocalFilePath (appContext, vod) {
if (!vod?.attributes?.videoSrcB2?.data?.attributes?.key) throw new Error(`vod is missing videoSrcB2.key which is required to download`)
const key = vod.attributes.videoSrcB2.data.attributes.key
const localFilePath = path.join(appContext.env.TMPDIR, key)
return localFilePath
}

View File

@ -3,6 +3,7 @@ import { type Logger } from 'winston';
import { multiaddrToUri } from '@multiformats/multiaddr-to-uri';
import https from 'https';
import fetch from 'node-fetch';
import { IPFS_CLUSTER_HTTP_API_MULTIADDR } from './env.js'
export async function pinCid (env: NodeJS.ProcessEnv, logger: Logger, cid: string): Promise<string> {
const controller = new AbortController();
@ -10,7 +11,7 @@ export async function pinCid (env: NodeJS.ProcessEnv, logger: Logger, cid: strin
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
});
const uri = multiaddrToUri(env.IPFS_CLUSTER_HTTP_API_MULTIADDR);
const uri = multiaddrToUri(IPFS_CLUSTER_HTTP_API_MULTIADDR);
logger.log({ level: 'debug', message: `uri=${uri}` });
@ -47,7 +48,7 @@ export async function pinCid (env: NodeJS.ProcessEnv, logger: Logger, cid: strin
export async function isCidPinned (env: NodeJS.ProcessEnv, logger: Logger, cid: string): Promise<boolean> {
const controller = new AbortController();
const signal = controller.signal;
const uri = multiaddrToUri(env.IPFS_CLUSTER_HTTP_API_MULTIADDR);
const uri = multiaddrToUri(IPFS_CLUSTER_HTTP_API_MULTIADDR);
logger.log({ level: 'debug', message: `uri=${uri}` });
const httpsAgent = new https.Agent({

View File

@ -1,6 +1,12 @@
import winston from 'winston'
export const loggerFactory = (options) => {
interface IOptions {
defaultMeta: {
service: string;
}
}
export const loggerFactory = (options: IOptions) => {
const mergedOptions = Object.assign({}, {
level: 'info',
defaultMeta: { service: 'futureporn' }

View File

@ -1,4 +1,4 @@
import express, { Request, Response, Next } from "express";
import express, { Request, Response, NextFunction } from "express";
import methodOverride from "method-override";
import { RegisterRoutes } from "../dist/futureporn-qa/routes/routes.js";
import { ValidateError } from "tsoa";
@ -19,7 +19,7 @@ export const registerRoutes = (app: express.Express) => {
.use(express.urlencoded({ extended: true }))
.use(express.json())
.use(methodOverride())
.use((_req: Request, res: Response, next: Next) => {
.use((_req: Request, res: Response, next: NextFunction) => {
res.header("Access-Control-Allow-Origin", "*");
res.header(
"Access-Control-Allow-Headers",

View File

@ -60,7 +60,8 @@ export async function createStream (logger: Logger, tweet: IStrapiTweet, vtuberI
data: {
date: new Date(tweet.attributes.date).toISOString(),
date_str: tweet.attributes.date,
tweet: tweet.id
tweet: tweet.id,
vtuber: vtuberId
}
})
})

View File

@ -3,7 +3,7 @@
import { strapiUrl } from "../constants.js";
import fetch from 'node-fetch';
import { getVod } from '../vods.js';
import { ITaskOptions, IRunTaskFunction } from "src/Task.js";
import { ITaskOptions, IRunTaskFunction } from "../Task.js";
import { IJobData } from "../types.js";
@ -14,8 +14,8 @@ export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptio
const jobData = job.data as IJobData;
const vod = await getVod(jobData.id, env);
if (!vod?.thumbnail?.data?.id) throw new Error('vod.thumbnail was missing');
const res = await fetch(`${strapiUrl}/api/b2-files/${vod?.thumbnail?.data?.id}`, {
if (!vod?.attributes?.thumbnail?.data?.id) throw new Error('vod.thumbnail was missing');
const res = await fetch(`${strapiUrl}/api/b2-files/${vod?.attributes?.thumbnail?.data?.id}`, {
method: 'DELETE',
headers: {
'Authorization': `Bearer ${env.STRAPI_API_KEY}`
@ -23,11 +23,11 @@ export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptio
});
if (!res.ok) {
logger.log({ level: 'info', message: `Response code: ${res.status} (ok:${res.ok})` })
const msg = `could not delete thumbnail due to fetch response error ${res.status} ${res.body}`
const msg = `Could not delete thumbnail due to fetch response error ${res.status} ${res.body}`
logger.log({ level: 'error', message: msg });
throw new Error(msg);
} else {
logger.log({ level: 'info', message: ` thumbnail ${vod.thumbnail.data.id} deleted.` })
logger.log({ level: 'info', message: ` thumbnail ${vod.attributes.thumbnail.data.id} deleted.` })
return
}
}

View File

@ -1,18 +1,16 @@
// import Prevvy from 'prevvy';
import Prevvy from '/home/chris/Documents/prevvy';
import Prevvy from '/home/chris/Documents/prevvy/index.js';
import path from 'node:path';
import { getRawVod } from '../vods.js';
import { getVideoSrcB2LocalFilePath } from '../fsCommon.js';
import { uploadToB2 } from '../b2.js'
import { IRawVod } from '../vods.js';
import { IVod, getVod } from '../vods.js';
import { IJobData } from '../types.js';
import { IB2File } from '../b2File.js';
import { Logger } from 'winston';
import { IRunTaskFunction, ITaskOptions } from '../Task.js';
import { Job } from 'bullmq';
import { TMPDIR } from '../env.js';
export interface IUploadData {
key: string;
@ -23,9 +21,9 @@ export interface IUploadData {
export const name = 'generateThumbnail';
export async function __generateThumbnail(vod: IRawVod, env: NodeJS.ProcessEnv, logger: Logger, job: Job): Promise<string> {
export async function __generateThumbnail(vod: IVod, env: NodeJS.ProcessEnv, logger: Logger, job: Job): Promise<string> {
const fileName = `vod-${vod?.id}-thumb.png`;
const thumbnailFilePath = path.join(env.TMPDIR, fileName);
const thumbnailFilePath = path.join(TMPDIR, fileName);
const videoInputUrl = vod.attributes.videoSrcB2?.data?.attributes?.cdnUrl;
if (!videoInputUrl) {
console.error(vod?.attributes?.videoSrcB2);
@ -51,7 +49,7 @@ export async function __generateThumbnail(vod: IRawVod, env: NodeJS.ProcessEnv,
}
export async function associateB2WithVod(vod: IRawVod, uploadData: IUploadData, env: NodeJS.ProcessEnv, logger: Logger) {
export async function associateB2WithVod(vod: IVod, uploadData: IUploadData, env: NodeJS.ProcessEnv, logger: Logger) {
logger.log({ level: 'info', message: `🥤 lets create b2-file in Strapi` });
// Create the B2 file
@ -115,7 +113,7 @@ export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptio
console.error(msg)
throw new Error(msg)
}
const vod = await getRawVod(data.id, env);
const vod = await getVod(data.id, env);
if (!vod) {
const msg = `panic! vod ${data.id} missing`
console.error(msg)

View File

@ -1,11 +1,10 @@
import { getVod, IVod, getRandomVod } from "../vods.js";
import fetch, { AbortError } from 'node-fetch';
import { isBefore } from 'date-fns';
import { IJobData } from "../types.js";
import { ITaskOptions, IRunTaskFunction, IIssueDefinition } from "src/Task.js";
import { ITaskOptions, IRunTaskFunction, IIssueDefinition } from "../Task.js";
import { Queue } from 'bullmq';
import { isCidPinned } from '../ipfsCluster.js';
import { Logger } from "winston";
export const name = 'identifyVodIssues';
@ -18,7 +17,7 @@ const issueDefinitions: IIssueDefinition[] = [
{
name: 'thumbnailMissing',
check: async (env, logger, vod) => {
if (!vod?.thumbnail?.data?.attributes?.cdnUrl) return true;
if (!vod?.attributes?.thumbnail?.data?.attributes?.cdnUrl) return true;
else return false;
},
solution: 'generateThumbnail'
@ -26,8 +25,8 @@ const issueDefinitions: IIssueDefinition[] = [
{
name: 'thumbnailUnreachable',
check: async (env, logger, vod) => {
if (!vod?.thumbnail?.data?.attributes?.cdnUrl) return false; // false because the problem isn't explicitly that the thumb is unreachable
const response = await fetch(vod.thumbnail.data.attributes.cdnUrl);
if (!vod?.attributes?.thumbnail?.data?.attributes?.cdnUrl) return false; // false because the problem isn't explicitly that the thumb is unreachable
const response = await fetch(vod.attributes.thumbnail.data.attributes.cdnUrl);
if (!response.ok) return true;
else return false;
},
@ -54,8 +53,8 @@ const issueDefinitions: IIssueDefinition[] = [
// },
{
name: 'videoSrcHashUnpinned',
check: async (env, logger, vod) => {
const isPinned = await isCidPinned(env, logger, vod.videoSrcHash);
check: async (env: NodeJS.ProcessEnv, logger: Logger, vod: IVod) => {
const isPinned = await isCidPinned(env, logger, vod.attributes.videoSrcHash);
return (!isPinned);
},
solution: 'pinVideoSrcHash'

View File

@ -2,7 +2,7 @@
import { getVod } from '../vods.js';
import { ITaskOptions, IRunTaskFunction } from "src/Task.js";
import { ITaskOptions, IRunTaskFunction } from "../Task.js";
import { IJobData } from "../types.js";
import { pinCid } from "../ipfsCluster.js";
@ -12,7 +12,8 @@ export const name = 'pinVideoSrcHash';
export const runTask: IRunTaskFunction = async ({ logger, env, job }: ITaskOptions) => {
const jobData = job.data as IJobData;
const vod = await getVod(jobData.id, env);
const cid = await pinCid(env, logger, vod.videoSrcHash);
if (!vod) throw new Error('pinVideoSrcHash received no vod, which is required.');
const cid = await pinCid(env, logger, vod.attributes.videoSrcHash);
return cid;
}

View File

@ -116,7 +116,8 @@ export interface ITweetScrape {
export async function createTweet (logger: Logger, tweet: ITweetScrape, vtuberId: number): Promise<IStrapiTweet> {
logger.log({ level: 'debug', message: `Searching for tweet with same id. id_str=${tweet.id_str}` });
logger.log({ level: 'debug', message: `Searching for tweet with same id. id_str=${tweet.id_str} vtuberId=${vtuberId}` });
if (!vtuberId) throw new Error(`createTweet was missing vtuberId argument, which is required.`);
const existingTweetQuery = qs.stringify({
filters: {
id_str: {

View File

@ -1,7 +1,7 @@
import { strapiUrl } from './constants.js'
import { getDateFromSafeDate, getSafeDate } from './dates.js'
import { unmarshallVtuber, IVtuber } from './vtubers.js'
import { IVtuber } from './vtubers.js'
import qs from 'qs'
import { ITagVodRelation, unmarshallTagVodRelation } from './tag-vod-relations.js'
import { IB2File } from './b2File.js'
@ -23,7 +23,7 @@ export interface IVods {
}
}
export interface IMarshalledVod {
export interface IVodsResponse {
data: {
attributes: {
pledge_sum: number;
@ -37,7 +37,7 @@ export interface IMarshalledVod {
}
}
export interface IRawVod {
export interface IVod {
id: number;
attributes: {
title?: string;
@ -56,55 +56,10 @@ export interface IRawVod {
}
}
export interface IVod {
id: number;
title?: string;
date: string;
date2: string;
muxAsset: IMuxAsset;
thumbnail: IB2File | null;
vtuber: IVtuber;
tagVodRelations: ITagVodRelation[];
video240Hash: string;
videoSrcHash: string;
videoSrcB2: IB2File | null;
announceTitle: string;
announceUrl: string;
note: string;
}
export function unmarshallVod(d: any): IVod {
if (!d?.attributes) {
throw new Error(`panick! vod doesnt have an attributes`)
}
if (!d.attributes?.vtuber?.data) {
throw new Error("panick! vod data doesn't contain vtuber. please populate.")
}
const vod = {
id: d.id,
title: d.attributes.title,
date: d.attributes.date,
date2: d.attributes.date2,
muxAsset: {
playbackId: d.attributes?.muxAsset?.data?.attributes?.playbackId,
assetId: d.attributes?.muxAsset?.data?.attributes?.assetId,
},
thumbnail: d.attributes?.thumbnail,
vtuber: unmarshallVtuber(d.attributes?.vtuber?.data),
tagVodRelations: d.attributes?.tagVodRelations?.data.map(unmarshallTagVodRelation),
video240Hash: d.attributes?.video240Hash,
videoSrcHash: d.attributes?.videoSrcHash,
videoSrcB2: d.attributes?.videoSrcB2,
announceTitle: d.attributes.announceTitle,
announceUrl: d.attributes.announceUrl,
note: d.attributes.note,
}
return vod
}
export async function getVodForDate(date: Date): Promise<IVod> {
const iso8601DateString = date.toISOString().split('T')[0];
@ -139,7 +94,6 @@ export async function getVodForDate(date: Date): Promise<IVod> {
return fetch(`${strapiUrl}/api/vods?${query}`)
.then((res) => res.json())
.then((d: any) => d.data[0])
.then(unmarshallVod)
}
export async function getRandomVod(env: NodeJS.ProcessEnv): Promise<IVod | null> {
@ -155,7 +109,7 @@ export async function getRandomVod(env: NodeJS.ProcessEnv): Promise<IVod | null>
}
export async function getRawVod(id: number, env: NodeJS.ProcessEnv): Promise<IRawVod | null> {
export async function getVod(id: number, env: NodeJS.ProcessEnv): Promise<IVod | null> {
const query = qs.stringify(
{
filters: {
@ -185,8 +139,9 @@ export async function getRawVod(id: number, env: NodeJS.ProcessEnv): Promise<IRa
)
try {
const res: any = await fetch(`${strapiUrl}/api/vods?${query}`)
const data: IMarshalledVod = await res.json();
return data.data[0];
const data: IVod = await res.json();
return data;
} catch (e) {
console.error(`there was an error while fetching vod ${id}`);
console.error(e);
@ -195,10 +150,10 @@ export async function getRawVod(id: number, env: NodeJS.ProcessEnv): Promise<IRa
}
export async function getVod(id: number, env: NodeJS.ProcessEnv): Promise<IVod | null> {
const rawVod = await getRawVod(id, env);
return unmarshallVod(rawVod);
}
// export async function getVod(id: number, env: NodeJS.ProcessEnv): Promise<IVod | null> {
// const rawVod = await getRawVod(id, env);
// return unmarshallVod(rawVod);
// }
export async function getVods(page: number = 1, pageSize: number = 25, sortDesc = true): Promise<IVods> {
const query = qs.stringify(
@ -230,14 +185,9 @@ export async function getVods(page: number = 1, pageSize: number = 25, sortDesc
}
}
)
return fetch(`${strapiUrl}/api/vods?${query}`)
.then((res) => res.json())
.then((j: any) => (
{
data: j.data.map(unmarshallVod),
pagination: j.meta.pagination
}
))
const res = await fetch(`${strapiUrl}/api/vods?${query}`);
const vods = await res.json() as IVods
return vods;
}

View File

@ -34,8 +34,9 @@
]
},
"include": [
"**/*.ts",
"cli.ts.noexec", "test/tweets.test.js",
"src/**/*.ts",
"cli.ts.noexec",
"test/tweets.test.js",
],
"exclude": [
"node_modules",

View File

@ -1,13 +1,15 @@
{
"entryFile": "manager.ts",
"noImplicitAdditionalProperties": "throw-on-extras",
"controllerPathGlobs": ["src/**/*Controller.ts"],
"spec": {
"outputDirectory": "dist/futureporn-qa/swagger",
"specVersion": 3
},
"routes": {
"routesDir": "dist/futureporn-qa/routes"
},
"entryFile": "manager.ts",
"noImplicitAdditionalProperties": "throw-on-extras",
"controllerPathGlobs": [
"src/**/*Controller.ts"
],
"spec": {
"outputDirectory": "dist/futureporn-qa/swagger",
"specVersion": 3
},
"routes": {
"routesDir": "dist/futureporn-qa/routes",
"esm": true
}
}
}

View File

@ -1,4 +1,3 @@
import * as fs from 'fs';
import * as path from 'path';
import * as dotenv from 'dotenv'
import pkg from './package.json' assert {type: 'json'};
@ -6,12 +5,12 @@ import { loggerFactory } from './src/logger.js';
import { type Logger } from 'winston';
import { dirname } from 'path';
import { fileURLToPath } from 'url';
import { type ITask, loadTaskDefinitions } from './src/Task.js';
import { Worker, Job, Queue } from 'bullmq'
import { loadTaskDefinitions } from './src/Task.js';
import { Worker, Job } from 'bullmq'
import { default as Redis } from 'ioredis'
import { REDIS_HOST, REDIS_PORT, TASK_LIST, IPFS_CLUSTER_HTTP_API_MULTIADDR } from './src/env.js';
const connection = new Redis.default({
const connection = new Redis({
port: parseInt(REDIS_PORT),
host: REDIS_HOST,
maxRetriesPerRequest: null,
@ -56,7 +55,7 @@ async function main(logger: Logger, taskList: string[]) {
job: job,
connection: connection,
})
logger.log({ level: 'info', message: `🍻 Worker completed task ${task.name} (job ID ${job.id})`})
logger.log({ level: 'info', message: `🍻 Worker completed task ${task.name} (job ID ${job.id})` })
return res;
}, {
concurrency: 1,
@ -70,8 +69,9 @@ async function main(logger: Logger, taskList: string[]) {
logger.log({ level: 'error', message: `🔥 ${err.message}` })
}
})
worker.on('failed', (job: Job, error: Error) => {
logger.log({ level: 'warn', message: `💔 worker failed on job ${job.id}`});
worker.on('failed', (job: Job<any, any, string> | undefined) => {
if (!job) logger.log({ level: 'error', message: `job failed. job was undefined.`});
logger.log({ level: 'warn', message: `💔 worker failed on job ${job}` });
})
worker.run();
}