image caching improvements

2025-08-12 00:11:21 -08:00 · 2025-08-12 00:11:21 -08:00 · 90e8da3246
commit 90e8da3246
parent 93d67045a1
10 changed files with 174 additions and 20 deletions
--- a/services/our/package-lock.json
+++ b/services/our/package-lock.json
@ -11,6 +11,7 @@
 				"@aws-sdk/client-s3": "3.726.1",
 				"@aws-sdk/s3-request-presigner": "^3.844.0",
 				"@dotenvx/dotenvx": "^1.47.5",
+				"@fastify/caching": "^9.0.3",
 				"@fastify/flash": "^6.0.3",
 				"@fastify/formbody": "^8.0.2",
 				"@fastify/multipart": "^9.0.3",
@ -1951,6 +1952,27 @@
 			"integrity": "sha512-5DGmA8FTdB2XbDeEwc/5ZXBl6UbBAyBOOLlPuBnZ/N1SwdH9Ii+cOX3tBROlDgcTXxjOYnLMVoKk9+FXAw0CJw==",
 			"license": "MIT"
 		},
+		"node_modules/@fastify/caching": {
+			"version": "9.0.3",
+			"resolved": "https://registry.npmjs.org/@fastify/caching/-/caching-9.0.3.tgz",
+			"integrity": "sha512-5K/2shYpvWHWiSAs59SaCVBoFhHEF8Yz4TTiXZf8YWVDcxuIxw0Adn5eDQ7s132s7vwURNOnCKHBjUQSOI+PLA==",
+			"funding": [
+				{
+					"type": "github",
+					"url": "https://github.com/sponsors/fastify"
+				},
+				{
+					"type": "opencollective",
+					"url": "https://opencollective.com/fastify"
+				}
+			],
+			"license": "MIT",
+			"dependencies": {
+				"abstract-cache": "^1.0.1",
+				"fastify-plugin": "^5.0.0",
+				"uid-safe": "^2.1.5"
+			}
+		},
 		"node_modules/@fastify/cookie": {
 			"version": "11.0.2",
 			"resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-11.0.2.tgz",
@ -4880,6 +4902,17 @@
 				"url": "https://opencollective.com/vitest"
 			}
 		},
+		"node_modules/abstract-cache": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/abstract-cache/-/abstract-cache-1.0.1.tgz",
+			"integrity": "sha512-EfUeMhRUbG5bVVbrSY/ogLlFXoyfMAPxMlSP7wrEqH53d+59r2foVy9a5KjmprLKFLOfPQCNKEfpBN/nQ76chw==",
+			"license": "MIT",
+			"dependencies": {
+				"clone": "^2.1.1",
+				"lru_map": "^0.3.3",
+				"merge-options": "^1.0.0"
+			}
+		},
 		"node_modules/abstract-logging": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz",
@ -5743,6 +5776,15 @@
 				"node": ">=6"
 			}
 		},
+		"node_modules/clone": {
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
+			"integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=0.8"
+			}
+		},
 		"node_modules/clsx": {
 			"version": "2.1.1",
 			"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
@ -8064,6 +8106,15 @@
 				"node": ">=0.12.0"
 			}
 		},
+		"node_modules/is-plain-obj": {
+			"version": "1.1.0",
+			"resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
+			"integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
 		"node_modules/is-reference": {
 			"version": "3.0.3",
 			"resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz",
@ -8483,6 +8534,12 @@
 			"dev": true,
 			"license": "MIT"
 		},
+		"node_modules/lru_map": {
+			"version": "0.3.3",
+			"resolved": "https://registry.npmjs.org/lru_map/-/lru_map-0.3.3.tgz",
+			"integrity": "sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==",
+			"license": "MIT"
+		},
 		"node_modules/lru-cache": {
 			"version": "11.1.0",
 			"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz",
@ -8528,6 +8585,18 @@
 				"node": ">= 0.4"
 			}
 		},
+		"node_modules/merge-options": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/merge-options/-/merge-options-1.0.1.tgz",
+			"integrity": "sha512-iuPV41VWKWBIOpBsjoxjDZw8/GbSfZ2mk7N1453bwMrfzdrIk7EzBd+8UVR6rkw67th7xnk9Dytl3J+lHPdxvg==",
+			"license": "MIT",
+			"dependencies": {
+				"is-plain-obj": "^1.1"
+			},
+			"engines": {
+				"node": ">=4"
+			}
+		},
 		"node_modules/merge-stream": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@ -9759,6 +9828,15 @@
 			"integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==",
 			"license": "MIT"
 		},
+		"node_modules/random-bytes": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz",
+			"integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==",
+			"license": "MIT",
+			"engines": {
+				"node": ">= 0.8"
+			}
+		},
 		"node_modules/rate-limiter-flexible": {
 			"version": "7.1.1",
 			"resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-7.1.1.tgz",
@ -11346,6 +11424,18 @@
 				"node": ">=0.8.0"
 			}
 		},
+		"node_modules/uid-safe": {
+			"version": "2.1.5",
+			"resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz",
+			"integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==",
+			"license": "MIT",
+			"dependencies": {
+				"random-bytes": "~1.0.0"
+			},
+			"engines": {
+				"node": ">= 0.8"
+			}
+		},
 		"node_modules/uint8-util": {
 			"version": "2.2.5",
 			"resolved": "https://registry.npmjs.org/uint8-util/-/uint8-util-2.2.5.tgz",
--- a/services/our/package.json
+++ b/services/our/package.json
@ -42,6 +42,7 @@
 		"@aws-sdk/client-s3": "3.726.1",
 		"@aws-sdk/s3-request-presigner": "^3.844.0",
 		"@dotenvx/dotenvx": "^1.47.5",
+		"@fastify/caching": "^9.0.3",
 		"@fastify/flash": "^6.0.3",
 		"@fastify/formbody": "^8.0.2",
 		"@fastify/multipart": "^9.0.3",
@ -100,4 +101,4 @@
 	"prisma": {
 		"seed": "tsx prisma/seed.ts"
 	}
-}
+}
--- a/services/our/src/app.ts
+++ b/services/our/src/app.ts
@ -10,7 +10,6 @@ import hls from './plugins/hls.ts'
 import fastifyStatic from '@fastify/static'
 import fastifySecureSession from '@fastify/secure-session'
 import path, { basename } from 'node:path'
-// import fastifyMultipart from '@fastify/multipart'
 import fastifyFormbody from '@fastify/formbody'
 import fastifyView from "@fastify/view"
 import { env } from './config/env'
@ -18,8 +17,8 @@ import { constants } from './config/constants'
 import authRoutes from './plugins/auth'
 import Handlebars from 'handlebars'
 import graphileWorker from './plugins/graphileWorker'
-import fastifySwagger from "@fastify/swagger";
-import fastifySwaggerUi from "@fastify/swagger-ui";
+import fastifySwagger from "@fastify/swagger"
+import fastifySwaggerUi from "@fastify/swagger-ui"
 import { join } from 'node:path'
 import { format } from 'date-fns'
 import * as jdenticon from 'jdenticon'
@ -30,6 +29,9 @@ import { signUrl } from './utils/cdn'
 import { extractBasePath } from './utils/filesystem'
 import { truncate } from './utils/formatters.ts'
 import { icons } from './utils/icons.ts'
+import logger from './utils/logger.ts'
+import fastifyCaching from '@fastify/caching'
+

 export function buildApp() {
    const app = Fastify()
@ -59,7 +61,7 @@ export function buildApp() {
        return a !== b;
    });
    Handlebars.registerHelper('isEqual', function (a, b) {
-        // console.log(`isEqual a=${a} b=${b}`)
+        logger.trace(`isEqual a=${a} b=${b}`)
        return a == b
    });
    Handlebars.registerHelper('isModerator', function (user) {
@ -72,8 +74,8 @@ export function buildApp() {
        return new Handlebars.SafeString(text);
    });
    Handlebars.registerHelper('getCdnUrl', function (s3Key) {
-        // Before you remove this console.log, find a way to memoize this function!
-        console.log(`getCdnUrl called with CDN_ORIGIN=${env.CDN_ORIGIN} and CDN_TOKEN_SECRET=${env.CDN_TOKEN_SECRET}`)
+        // Before you remove this log, find a way to memoize this function!
+        logger.info(`getCdnUrl called with CDN_ORIGIN=${env.CDN_ORIGIN} and CDN_TOKEN_SECRET=${env.CDN_TOKEN_SECRET}`)
        return signUrl(`${env.CDN_ORIGIN}/${s3Key}`, {
            securityKey: env.CDN_TOKEN_SECRET,
            expirationTime: constants.timeUnits.sevenDaysInSeconds,
@ -91,7 +93,7 @@ export function buildApp() {
            isDirectory: true,
            expirationTime: constants.timeUnits.sevenDaysInSeconds,
        })
-        console.log(`pathAllowed=${pathAllowed} url=${url}`)
+        logger.debug(`pathAllowed=${pathAllowed} url=${url}`)
        return url
    })
    Handlebars.registerHelper('basename', function (url: string) {
@ -178,6 +180,12 @@ export function buildApp() {
        }
    })

+    app.register(fastifyCaching, {
+        expiresIn: 300,
+        privacy: 'private',
+        serverExpiresIn: 300,
+    })
+
    app.register(graphileWorker)
    app.register(prismaPlugin)
    app.register(hls)
--- a/services/our/src/plugins/index.ts
+++ b/services/our/src/plugins/index.ts
@ -4,13 +4,16 @@ import { type FastifyInstance, type FastifyReply, type FastifyRequest } from 'fa
 import { env } from '../config/env'
 import { constants } from '../config/constants'
 import { getTargetUser } from '../utils/authorization'
+import logger from '../utils/logger'

 const prisma = new PrismaClient().$extends(withAccelerate())


 export default async function indexRoutes(fastify: FastifyInstance): Promise<void> {
    fastify.get('/profile', async function (request, reply) {
-        const userId = request.session.get('userId')
+        const userId = request.session.get('userId');
+        logger.debug(`userId=${userId}`);
+        if (!userId) return reply.redirect('/auth/patreon');
        const user = await prisma.user.findUnique({
            where: { id: userId },
            include: {
@ -27,7 +30,7 @@ export default async function indexRoutes(fastify: FastifyInstance): Promise<voi
        const cdnOrigin = env.CDN_ORIGIN;
        const NODE_ENV = env.NODE_ENV;
        const userId = request.session.get('userId');
-        // console.log(`we are at the GET root (/) route, with userId=${userId}`);
+        logger.debug(`we are at the GET root (/) route, with userId=${userId}`);


        const vods = await prisma.vod.findMany({
@ -36,8 +39,8 @@ export default async function indexRoutes(fastify: FastifyInstance): Promise<voi
                createdAt: 'desc'
            }
        })
-        // console.log('vods as follows')
-        // console.log(vods)
+        logger.debug('vods as follows')
+        logger.debug(vods)

        const vtubers = await prisma.vtuber.findMany({
            take: 3,
@ -49,7 +52,7 @@ export default async function indexRoutes(fastify: FastifyInstance): Promise<voi
        // Guard: no user in session
        if (!userId) {
            const authPath = env.PATREON_AUTHORIZE_PATH
-            // console.log(`either patreon_user or patreon_user.id was falsy. userId=${userId}`)
+            logger.debug(`either patreon_user or patreon_user.id was falsy. userId=${userId}`)
            return reply.viewAsync("index.hbs", {
                user: { roles: [{ name: 'anon' }] },
                cdnOrigin,
@ -75,8 +78,8 @@ export default async function indexRoutes(fastify: FastifyInstance): Promise<voi
            }
        });

-        console.log('user as follows');
-        console.log(user);
+        logger.debug('user as follows');
+        logger.debug(user);



--- a/services/our/src/plugins/vods.ts
+++ b/services/our/src/plugins/vods.ts
@ -104,8 +104,6 @@ export default async function vodsRoutes(
      return reply.status(404).send({ error: 'VOD not found' });
    }

-
-
    return reply.viewAsync('vod.hbs', {
      vod,
      site: constants.site,
--- a/services/our/src/tasks/createVideoThumbnail.ts
+++ b/services/our/src/tasks/createVideoThumbnail.ts
@ -7,6 +7,7 @@ import { getS3Client, uploadFile } from "../utils/s3";
 import { nanoid } from "nanoid";
 import { getNanoSpawn } from "../utils/nanoSpawn";
 import { preparePython } from "../utils/python";
+import { generateS3Path } from "../utils/formatters";

 const prisma = new PrismaClient().$extends(withAccelerate());

@ -69,6 +70,14 @@ export default async function createVideoThumbnail(payload: any, helpers: Helper
  const vod = await prisma.vod.findFirstOrThrow({
    where: {
      id: vodId
+    },
+    include: {
+      vtubers: {
+        select: {
+          slug: true,
+          id: true,
+        }
+      }
    }
  })
  // * [x] load vod
@ -96,7 +105,9 @@ export default async function createVideoThumbnail(payload: any, helpers: Helper
  console.log(`thumbnailPath=${thumbnailPath}`)

  // * [x] generate thumbnail s3 key
-  const s3Key = `/thumb/${nanoid()}`
+  const slug = vod.vtubers[0].slug
+  if (!slug) throw new Error(`vtuber ${vod.vtubers[0].id} was missing slug`);
+  const s3Key = generateS3Path(slug, vod.streamDate, vod.id, `thumbnail.png`);


  // * [x] upload thumbnail to s3
--- a/services/our/src/tests/formatters.unit.test.ts
+++ b/services/our/src/tests/formatters.unit.test.ts
@ -0,0 +1,27 @@
+import { describe, it, expect, vi } from 'vitest';
+import { generateS3Path } from '../utils/formatters';
+
+
+
+describe('generateS3Path', () => {
+  it('generates correct path with slug, date, and filename', () => {
+    const slug = 'test-slug';
+    const date = new Date('2025-08-11T12:34:56Z');
+    const filename = 'video.mp4';
+    const vodId = 'VatL00tDBasHNo4eVSNVm'
+
+    const result = generateS3Path(slug, date, vodId, filename);
+
+    expect(result).toBe('fp/test-slug/2025/08/11/VatL00tDBasHNo4eVSNVm/video.mp4');
+  });
+
+  it('works with .png file ext', () => {
+    const slug = 'test-slug2';
+    const date = new Date('2024-07-01T12:34:56Z');
+    const filename = 'thumbnail.png';
+    const vodId = '-M-ETeTeeODng-4Vmj01D'
+
+    const result = generateS3Path(slug, date, vodId, filename);
+    expect(result).toBe('fp/test-slug2/2024/07/01/-M-ETeTeeODng-4Vmj01D/thumbnail.png');
+  })
+});
--- a/services/our/src/utils/cdn.ts
+++ b/services/our/src/utils/cdn.ts
@ -53,7 +53,6 @@ export function signUrl(
  let token = "";


-
  const expires = Math.floor(Date.now() / 1000) + expirationTime;
  const updatedUrl = addCountries(url, countriesAllowed, countriesBlocked);
  const parsedUrl = new URL(updatedUrl);
--- a/services/our/src/utils/formatters.ts
+++ b/services/our/src/utils/formatters.ts
@ -1,4 +1,5 @@
 import slugify from 'slugify'
+import { getYear, getMonth, getDate } from 'date-fns';

 export function toJsonSafe<T>(obj: T): T {
    return JSON.parse(JSON.stringify(obj))
@ -17,7 +18,23 @@ export function slug(s: string) {
 }


+
+
 export function truncate(text: string, n: number = 6) {
    if (typeof text !== 'string') return '';
    return text.length > n ? text.slice(0, n) + '…' : text;
 }
+
+
+function pad(n: number): string {
+    return n.toString().padStart(2, '0');
+}
+
+
+export function generateS3Path(slug: string, date: Date, vodId: string, filename: string): string {
+    const year = getYear(date);
+    const month = pad(getMonth(date) + 1);
+    const day = pad(getDate(date));
+
+    return `fp/${slug}/${year}/${month}/${day}/${vodId}/${filename}`;
+}
--- a/services/our/src/views/vod.hbs
+++ b/services/our/src/views/vod.hbs
@ -217,7 +217,7 @@

      <h4>Thumbnail Image</h4>
      {{#if vod.thumbnail}}
-      <img src="{{getCdnUrl vod.thumbnail}}" alt="{{this.vtuber.displayName}} thumbnail">
+      <img src="{{getCdnUrl vod.thumbnail}}" alt="{{vtuber.displayName}} thumbnail">
      <div class="mx-5"></div>
      {{else}}
      <article>