progress

ARCHITECHTURE.md

@@ -4,6 +4,8 @@ pnpm required for workspaces.
 
 Kubernetes for Development using Tiltfile
 
+kubefwd and entr for DNS in dev cluster
+
 dokku for Production, deployed with `git push`.
 
 (dokku is slowly being replaced by Kubernetes)

Makefile

@@ -30,6 +30,17 @@ tilt:
 	tilt up
 
 secrets:
+
+	kubectl --namespace futureporn delete secret scout --ignore-not-found
+	kubectl --namespace futureporn create secret generic scout \
+	--from-literal=recentsToken=${SCOUT_RECENTS_TOKEN} \
+	--from-literal=strapiApiKey=${SCOUT_STRAPI_API_KEY} \
+	--from-literal=imapServer=${SCOUT_IMAP_SERVER} \
+	--from-literal=imapPort=${SCOUT_IMAP_PORT} \
+	--from-literal=imapUsername=${SCOUT_IMAP_USERNAME} \
+	--from-literal=imapPassword=${SCOUT_IMAP_PASSWORD} \
+	--from-literal=imapAccessToken=${SCOUT_IMAP_ACCESS_TOKEN} \
+
 	kubectl --namespace futureporn delete secret link2cid --ignore-not-found
 	kubectl --namespace futureporn create secret generic link2cid \
 	--from-literal=apiKey=${LINK2CID_API_KEY}
@@ -38,7 +49,6 @@ secrets:
 	kubectl --namespace cert-manager create secret generic vultr \
 	--from-literal=apiKey=${VULTR_API_KEY}
 
-
 	kubectl --namespace futureporn delete secret vultr --ignore-not-found
 	kubectl --namespace futureporn create secret generic vultr \
 	--from-literal=containerRegistryUsername=${VULTR_CONTAINER_REGISTRY_USERNAME} \
@@ -58,7 +68,7 @@ secrets:
 	--from-literal=adminJwtSecret=${STRAPI_ADMIN_JWT_SECRET} \
 	--from-literal=apiTokenSalt=${STRAPI_API_TOKEN_SALT} \
 	--from-literal=appKeys=${STRAPI_APP_KEYS} \
-	--from-literal=databaseUrl=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} \
+	--from-literal=databaseUrl=postgres.futureporn.svc.cluster.local://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} \
 	--from-literal=jwtSecret=${STRAPI_JWT_SECRET} \
 	--from-literal=muxPlaybackRestrictionId=${MUX_PLAYBACK_RESTRICTION_ID} \
 	--from-literal=muxSigningKeyPrivateKey=${MUX_SIGNING_KEY_PRIVATE_KEY} \
@@ -75,14 +85,6 @@ secrets:
 
 
 
-
-# --from-literal=sessionSecret=$(SESSION_SECRET) \
-# --from-literal=twitchClientId=$(TWITCH_CLIENT_ID) \
-# --from-literal=twitchClientSecret=$(TWITCH_CLIENT_SECRET) \
-# --from-literal=gumroadClientId=$(GUMROAD_CLIENT_ID) \
-# --from-literal=gumroadClientSecret=$(GUMROAD_CLIENT_SECRET)
-
-
 define _script
 cat <<'EOF' | ctlptl apply -f -
 apiVersion: ctlptl.dev/v1alpha1

apps/base/windmill/windmill.yaml

@@ -24,7 +24,7 @@ spec:
         kind: HelmRepository
         name: bitnami
   values:
-    fullnameOverride: windmill-postgresql-cool
+    fullnameOverride: windmill-postgresql-uncool
     postgresql:
       auth:
         postgresPassword: windmill

apps/staging/chisel/chisel.yaml

@@ -0,0 +1,25 @@
+
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: GitRepository
+metadata:
+  name: chisel-operator
+  namespace: futureporn
+spec:
+  interval: 5m
+  url: https://github.com/FyraLabs/chisel-operator
+  ref:
+    branch: master
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: chisel-operator
+  namespace: futureporn
+spec:
+  interval: 10m
+  targetNamespace: futureporn
+  sourceRef:
+    kind: GitRepository
+    name: chisel-operator
+  path: "./kustomize"
+  prune: true

apps/staging/chisel/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: futureporn
+resources:
+  - chisel.yaml

charts/fp/templates-staging/capture.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: capture
+  namespace: futureporn
+spec:
+  selector:
+    app.kubernetes.io/name: capture
+  ports:
+    - name: capture
+      port: 80
+      targetPort: 5566
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capture
+  namespace: futureporn
+  labels:
+    app: capture
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: capture
+  template:
+    metadata:
+      labels:
+        app: capture
+    spec:
+      containers:
+      - name: capture
+        image: "{{ .Values.capture.containerName }}"
+        ports:
+          - containerPort: 5566
+

charts/fp/templates-staging/echo.yaml

@@ -0,0 +1,122 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: echo-deployment
+  namespace: futureporn
+  labels:
+    app.kubernetes.io/name: echo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: echo-server
+  template:
+    metadata:
+      labels:
+        app: echo-server
+    spec:
+      containers:
+        - name: echo-server
+          resources:
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          image: jmalloc/echo-server
+          ports:
+            - name: http-port
+              containerPort: 8080
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echo
+  namespace: futureporn
+  labels:
+    app.kubernetes.io/name: echo
+spec:
+  ports:
+    - name: http-port
+      port: 8080
+      targetPort: http-port
+      protocol: TCP
+  selector:
+    app: echo-server
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ngrok
+  namespace: futureporn
+  annotations:
+    kubernetes.io/ingress.class: ngrok
+    k8s.ngrok.com/namespace: futureporn
+    k8s.ngrok.com/service: ngrok
+spec:
+  ingressClassName: ngrok
+  tls:
+    - secretName: ngrok-tls
+      hosts:
+        - "{{ .Values.ngrok.hostname }}"
+  rules:
+    - host: "{{ .Values.ngrok.hostname }}"
+      http:
+        paths:
+        - path: /echo
+          pathType: Prefix
+          backend:
+            service:
+              name: echo
+              port:
+                number: 8080
+        - path: /game
+          pathType: Prefix
+          backend:
+            service:
+              name: game-2048
+              port:
+                number: 8080
+        # - path: /strapi
+        #   pathType: Prefix
+        #   backend:
+        #     service:
+        #       name: strapi
+        #       port:
+        #         number: 1337
+        # - path: /next
+        #   pathType: Prefix
+        #   backend:
+        #     service:
+        #       name: next
+        #       port:
+        #         number: 3000
+
+# ---
+# apiVersion: networking.k8s.io/v1
+# kind: Ingress
+# metadata:
+#   name: echo-ing
+#   namespace: futureporn
+# annotations:
+#   kubernetes.io/ingress.class: nginx
+#   cert-manager.io/cluster-issuer: letsencrypt-staging
+# spec:
+#   backend:
+#     serviceName: echo-service
+#     servicePort: 8080
+#   tls:
+#     - secretName: next-tls
+#       hosts:
+#         - echo.test
+#   rules:
+#     - host: echo.test
+#       http:
+#         paths:
+#         - path: /
+#           pathType: Prefix
+#           backend:
+#             service:
+#               name: echo-service
+#               port:
+#                 number: 8080

charts/fp/templates-staging/external-dns.yaml

@@ -1,3 +1,6 @@
+{{ if eq .Values.managedBy "Helm" }} 
+
+
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -60,3 +63,5 @@ spec:
             secretKeyRef:
               name: vultr
               key: apiKey
+
+{{ end }}

charts/fp/templates-staging/game.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: game-2048
+  namespace: futureporn
+spec:
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080
+  selector:
+    app: game-2048
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: game-2048
+  namespace: futureporn
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: game-2048
+  template:
+    metadata:
+      labels:
+        app: game-2048
+    spec:
+      containers:
+        - name: backend
+          image: mendhak/http-https-echo
+          ports:
+            - name: http
+              containerPort: 8080

charts/fp/templates-staging/ingress.yaml.noexec

@@ -0,0 +1,108 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: traefik-ingress-controller
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+    - extensions
+    resources:
+    - ingresses/status
+    verbs:
+    - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: traefik-ingress-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: traefik-ingress-controller
+subjects:
+- kind: ServiceAccount
+  name: traefik-ingress-controller
+  namespace: kube-system
+
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: traefik-ingress-controller
+  namespace: kube-system
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: traefik-ingress-controller
+  namespace: kube-system
+  labels:
+    k8s-app: traefik-ingress-lb
+spec:
+  selector:
+    matchLabels:
+      k8s-app: traefik-ingress-lb
+      name: traefik-ingress-lb
+  template:
+    metadata:
+      labels:
+        k8s-app: traefik-ingress-lb
+        name: traefik-ingress-lb
+    spec:
+      serviceAccountName: traefik-ingress-controller
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: traefik:v1.7
+        name: traefik-ingress-lb
+        ports:
+        - name: http
+          containerPort: 80
+          hostPort: 80
+        - name: admin
+          containerPort: 8080
+          hostPort: 8080
+        securityContext:
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
+        args:
+        - --api
+        - --kubernetes
+        - --logLevel=INFO
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: traefik-ingress-service
+  namespace: kube-system
+spec:
+  selector:
+    k8s-app: traefik-ingress-lb
+  ports:
+    - protocol: TCP
+      port: 80
+      name: web
+    - protocol: TCP
+      port: 8080
+      name: admin

charts/fp/templates-staging/ipfs-service.yaml

@@ -0,0 +1,70 @@
+{{ if eq .Values.managedBy "Helm" }} 
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ipfs-pod
+  namespace: default
+  labels:
+    app.kubernetes.io/name: ipfs
+spec:
+  containers:
+    - name: ipfs
+      image: ipfs/kubo
+      ports:
+        - containerPort: 5001
+        - containerPort: 8080
+      volumeMounts:
+        - name: ipfs-pvc
+          mountPath: /data/ipfs
+  restartPolicy: OnFailure
+  volumes:
+    - name: ipfs-pvc
+      persistentVolumeClaim:
+        claimName: ipfs-pvc
+
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ipfs
+  namespace: default
+  annotations:
+    meta.helm.sh/release-name: fp
+    meta.helm.sh/release-namespace: default
+  labels:
+    app.kubernetes.io/managed-by: {{ .Values.managedBy }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 40Gi
+  storageClassName: {{ .Values.storageClassName }}
+
+
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: ipfs-service
+  namespace: default
+  annotations:
+    meta.helm.sh/release-name: fp
+    meta.helm.sh/release-namespace: default
+  labels:
+    app.kubernetes.io/managed-by: {{ .Values.managedBy }}
+spec:
+  selector:
+    app.kubernetes.io/name: ipfs
+  ports:
+    - name: gateway
+      protocol: TCP
+      port: 8080
+      targetPort: 8080
+    - name: api
+      protocol: TCP
+      port: 5001
+      targetPort: 5001
+
+{{ end }}

charts/fp/templates-staging/link2cid.yaml.noexec

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: link2cid
-  namespace: default
+  namespace: futureporn
 spec:
   selector:
     app: link2cid
@@ -22,7 +22,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: link2cid
-  namespace: default
+  namespace: futureporn
 spec:
   selector:
     matchLabels:
@@ -54,10 +54,10 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: link2cid
-  namespace: default
+  namespace: futureporn
   annotations:
     meta.helm.sh/release-name: fp
-    meta.helm.sh/release-namespace: default
+    meta.helm.sh/release-namespace: futureporn
   labels:
     app.kubernetes.io/managed-by: {{ .Values.managedBy }}
 spec:
@@ -75,7 +75,7 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: link2cid-ingress
-  namespace: default
+  namespace: futureporn
   annotations:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/ssl-redirect: "true"

charts/fp/templates-staging/realtime.yaml

@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: realtime
+  namespace: futureporn
+spec:
+  selector:
+    app.kubernetes.io/name: realtime
+  ports:
+    - name: realtime
+      port: 80
+      targetPort: 5535
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: realtime
+  namespace: futureporn
+  labels:
+    app: realtime
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: realtime
+  template:
+    metadata:
+      labels:
+        app: realtime
+    spec:
+      containers:
+      - name: realtime
+        image: "{{ .Values.realtime.containerName }}"
+        ports:
+          - containerPort: 5535
+
+
+{{ if eq .Values.managedBy "Helm" }} 
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: realtime
+  namespace: futureporn
+annotations:
+  kubernetes.io/ingress.class: nginx
+  cert-manager.io/cluster-issuer: letsencrypt-staging
+spec:
+tls:
+  - secretName: realtime-tls
+    hosts:
+      - realtime.futureporn.net
+rules:
+  - host: realtime.futureporn.net
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+        service:
+          name: realtime
+          port:
+          number: 5535
+{{ end }}

charts/fp/templates-staging/scout.yaml

@@ -0,0 +1,104 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: scout
+  namespace: futureporn
+spec:
+  selector:
+    app.kubernetes.io/name: scout
+  ports:
+    - name: web
+      port: 3000
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scout
+  namespace: futureporn
+  labels:
+    app: scout
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: scout
+  template:
+    metadata:
+      labels:
+        app: scout
+    spec:
+      containers:
+      - name: scout
+        image: "{{ .Values.scout.containerName }}"
+        ports:
+          - containerPort: 5000
+        env:
+          - name: PUBSUB_SERVER_URL
+            value: "{{ .Values.scout.pubsubServerUrl }}"
+          - name: STRAPI_URL
+            value: https://strapi.futureporn.svc.cluster.local
+          - name: SCOUT_RECENTS_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: recentsToken
+          - name: SCOUT_IMAP_SERVER
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: imapServer
+          - name: SCOUT_IMAP_PORT
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: imapPort
+          - name: SCOUT_IMAP_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: imapUsername
+          - name: SCOUT_IMAP_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: imapPassword
+          - name: SCOUT_IMAP_ACCESS_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: imapAccessToken
+          - name: SCOUT_STRAPI_API_KEY
+            valueFrom:
+              secretKeyRef:
+                name: scout
+                key: strapiApiKey
+
+
+{{ if eq .Values.managedBy "Helm" }} 
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: scout
+  namespace: futureporn
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: letsencrypt-staging
+spec:
+tls:
+  - secretName: scout-tls
+    hosts:
+      - scout.sbtp.xyz
+rules:
+  - host: scout.sbtp.xyz
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+        service:
+          name: scout
+          port:
+          number: 3000
+{{ end }}

charts/fp/templates/ipfs-pod.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: ipfs-pod
-  namespace: default
-  labels:
-    app.kubernetes.io/name: ipfs
-spec:
-  containers:
-    - name: ipfs
-      image: ipfs/kubo
-      ports:
-        - containerPort: 5001
-        - containerPort: 8080
-      volumeMounts:
-        - name: ipfs-pvc
-          mountPath: /data/ipfs
-  restartPolicy: OnFailure
-  volumes:
-    - name: ipfs-pvc
-      persistentVolumeClaim:
-        claimName: ipfs-pvc

charts/fp/templates/ipfs-pvc.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: ipfs-pvc
-  namespace: default
-  annotations:
-    meta.helm.sh/release-name: fp
-    meta.helm.sh/release-namespace: default
-  labels:
-    app.kubernetes.io/managed-by: {{ .Values.managedBy }}
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 40Gi
-  storageClassName: {{ .Values.storageClassName }}
-

charts/fp/templates/ipfs-service.yaml

@@ -1,23 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: ipfs-service
-  namespace: default
-  annotations:
-    meta.helm.sh/release-name: fp
-    meta.helm.sh/release-namespace: default
-  labels:
-    app.kubernetes.io/managed-by: {{ .Values.managedBy }}
-spec:
-  selector:
-    app.kubernetes.io/name: ipfs
-  ports:
-    - name: gateway
-      protocol: TCP
-      port: 8080
-      targetPort: 8080
-    - name: api
-      protocol: TCP
-      port: 5001
-      targetPort: 5001
-

charts/fp/templates/letsencrypt-prod.yaml

@@ -1,22 +0,0 @@
-
----
-apiVersion: cert-manager.io/v1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-prod
-spec:
-  acme:
-    # server: https://acme-staging-v02.api.letsencrypt.org/directory
-    server: https://acme-v02.api.letsencrypt.org/directory
-    email: {{ .Values.adminEmail }}
-    privateKeySecretRef:
-      name: letsencrypt-prod
-    solvers:
-    - dns01:
-        webhook:
-          groupName: acme.vultr.com
-          solverName: vultr
-          config:
-            apiKeySecretRef:
-              key: apiKey
-              name: vultr

charts/fp/templates/letsencrypt.yaml

@@ -1,3 +1,4 @@
+{{ if eq .Values.managedBy "Helm" }} 
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
@@ -21,3 +22,28 @@ spec:
             apiKeySecretRef:
               key: apiKey
               name: vultr-credentials
+
+
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-prod
+spec:
+  acme:
+    # server: https://acme-staging-v02.api.letsencrypt.org/directory
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: {{ .Values.adminEmail }}
+    privateKeySecretRef:
+      name: letsencrypt-prod
+    solvers:
+    - dns01:
+        webhook:
+          groupName: acme.vultr.com
+          solverName: vultr
+          config:
+            apiKeySecretRef:
+              key: apiKey
+              name: vultr
+
+{{ end }}

charts/fp/templates/next.yaml

@@ -0,0 +1,64 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: next
+  namespace: futureporn
+spec:
+  selector:
+    app.kubernetes.io/name: next
+  ports:
+    - name: web
+      port: 3000
+      targetPort: 3000
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: next
+  namespace: futureporn
+  labels:
+    app.kubernetes.io/name: next
+spec:
+  containers:
+    - name: next
+      image: "{{ .Values.next.containerName }}"
+      env:
+        - name: HOSTNAME
+          value: 0.0.0.0
+      ports:
+        - containerPort: 3000
+      resources: {}
+  restartPolicy: OnFailure
+
+
+{{ if eq .Values.managedBy "Helm" }} 
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: next
+  namespace: futureporn
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: "{{ .Values.next.certIssuer }}"
+spec:
+  backend:
+    serviceName: next
+    servicePort: 3000
+  tls:
+    - secretName: next-tls
+      hosts:
+        - "{{ .Values.next.hostname }}"
+  rules:
+    - host: "{{ .Values.next.hostname }}"
+      http:
+        paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: next
+              port:
+                number: 3000
+{{ end }}

charts/fp/templates/ngrok.yaml

@@ -0,0 +1,37 @@
+{{ if eq .Values.managedBy "tilt" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ngrok
+  namespace: futureporn
+  annotations:
+    kubernetes.io/ingress.class: ngrok
+    k8s.ngrok.com/namespace: futureporn
+    k8s.ngrok.com/service: ngrok
+spec:
+  ingressClassName: ngrok
+  tls:
+    - secretName: ngrok-tls
+      hosts:
+        - "{{ .Values.ngrok.hostname }}"
+  rules:
+    - host: "{{ .Values.ngrok.hostname }}"
+      http:
+        paths:
+        - path: /next
+          pathType: Prefix
+          backend:
+            service:
+              name: next
+              port:
+                number: 3000
+        - path: /strapi
+          pathType: Prefix
+          backend:
+            service:
+              name: strapi
+              port:
+                number: 1337
+{{ end }}
+

charts/fp/templates/pgadmin.yaml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pgadmin
+  namespace: futureporn
+spec:
+  selector:
+    app.kubernetes.io/name: pgadmin
+  ports:
+    - name: web
+      protocol: TCP
+      port: 5050
+      targetPort: 5050
+status:
+  loadBalancer: {}
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pgadmin
+  namespace: futureporn
+  labels:
+    app.kubernetes.io/name: pgadmin
+spec:
+  containers:
+    - name: pgadmin
+      image: dpage/pgadmin4
+      ports:
+        - containerPort: 5050
+      resources:
+        limits:
+          cpu: 500m
+          memory: 1Gi
+      env:
+        - name: PGADMIN_LISTEN_PORT
+          value: '5050'
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgres
+              key: password
+        - name: PGADMIN_DEFAULT_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: pgadmin
+              key: defaultPassword
+        - name: PGADMIN_DEFAULT_EMAIL
+          valueFrom:
+            secretKeyRef:
+              name: pgadmin
+              key: defaultEmail           
+  restartPolicy: OnFailure

charts/fp/templates/postgres.yaml

@@ -0,0 +1,70 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  namespace: futureporn
+  name: postgres
+annotations: 
+  tilt.dev/down-policy: keep
+spec:
+  accessModes:
+    - ReadWriteOncePod
+  persistentVolumeReclaimPolicy: Retain
+  resources:
+    requests:
+      storage: 40Gi
+  storageClassName: {{ .Values.storageClassName }}
+
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: futureporn
+  name: postgres
+annotations: 
+  tilt.dev/down-policy: keep
+spec:
+  selector:
+    app.kubernetes.io/name: postgres
+  ports:
+    - name: db
+      protocol: TCP
+      port: 5432
+      targetPort: 5432
+status:
+  loadBalancer: {}
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: futureporn
+  name: postgres
+  labels:
+    app.kubernetes.io/name: postgres
+annotations: 
+  tilt.dev/down-policy: keep
+spec:
+  containers:
+    - name: postgres
+      image: postgres:16.0
+      env:
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: postgres
+              key: password
+      ports:
+        - containerPort: 5432
+      resources:
+        limits:
+          cpu: 500m
+          memory: 1Gi   
+      volumeMounts:
+        - name: postgres
+          mountPath: /data/postgres
+  restartPolicy: OnFailure
+  volumes:
+    - name: postgres
+      persistentVolumeClaim:
+        claimName: postgres

charts/fp/templates/roles.yaml

@@ -1,3 +1,5 @@
+{{ if eq .Values.managedBy "Helm" }} 
+
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -23,3 +25,4 @@ roleRef:
   name: cert-manager-webhook-vultr-secret-reader
   apiGroup: rbac.authorization.k8s.io
 
+{{ end }}

charts/fp/templates/strapi.yaml

@@ -0,0 +1,180 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: strapi
+  namespace: futureporn
+spec:
+  selector:
+    app.kubernetes.io/name: strapi
+  ports:
+    - name: web
+      port: 1337
+      targetPort: 1337
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: strapi
+  namespace: futureporn
+  labels:
+    app.kubernetes.io/name: strapi
+spec:
+  containers:
+    - name: strapi
+      image: "{{ .Values.strapi.containerName }}"
+      ports:
+        - containerPort: 1337
+      env:
+        - name: ADMIN_JWT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: adminJwtSecret
+        - name: API_TOKEN_SALT
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: apiTokenSalt
+        - name: APP_KEYS
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: appKeys
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: databaseUrl
+        - name: CDN_BUCKET_USC_URL
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: cdnBucketUscUrl
+        - name: DATABASE_CLIENT
+          value: postgres
+        - name: DATABASE_HOST
+          value: postgres.futureporn.svc.cluster.local
+        - name: DATABASE_NAME
+          value: futureporn-strapi
+        - name: JWT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: jwtSecret
+        - name: MUX_PLAYBACK_RESTRICTION_ID
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: muxPlaybackRestrictionId
+        - name: MUX_SIGNING_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: muxSigningKeyId
+        - name: MUX_SIGNING_KEY_PRIVATE_KEY
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: muxSigningKeyPrivateKey
+        - name: NODE_ENV
+          value: production
+        - name: S3_USC_BUCKET_APPLICATION_KEY
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: s3UscBucketApplicationKey
+        - name: S3_USC_BUCKET_ENDPOINT
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: s3UscBucketEndpoint
+        - name: S3_USC_BUCKET_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: s3UscBucketKeyId
+        - name: S3_USC_BUCKET_NAME
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: s3UscBucketName
+        - name: S3_USC_BUCKET_REGION
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: s3UscBucketRegion
+        - name: SENDGRID_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: sendgridApiKey
+        - name: STRAPI_URL
+          value: "{{ .Values.strapi.url }}"
+        - name: TRANSFER_TOKEN_SALT
+          valueFrom:
+            secretKeyRef:
+              name: strapi
+              key: transferTokenSalt
+        - name: PORT
+          value: "{{ .Values.strapi.port }}"
+      resources:
+        limits:
+          cpu: 500m
+          memory: 1Gi   
+  restartPolicy: OnFailure
+
+
+# ---
+# apiVersion: v1
+# kind: PersistentVolumeClaim
+# metadata:
+#   name: strapi
+#   namespace: futureporn
+#   annotations:
+#     meta.helm.sh/release-name: fp
+#     meta.helm.sh/release-namespace: futureporn
+#   labels:
+#     app.kubernetes.io/managed-by: {{ .Values.managedBy }}
+# spec:
+#   accessModes:
+#     - ReadWriteOnce
+#   resources:
+#     requests:
+#       storage: 100Gi
+#   storageClassName: {{ .Values.storageClassName }}
+
+
+{{ if eq .Values.managedBy "Helm" }}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: strapi
+  namespace: futureporn
+  annotations:
+    sbtp.xyz/managed-by: "{{ .Values.managedBy }}"
+    kubernetes.io/ingress.class: nginx
+    cert-manager.io/cluster-issuer: "{{ .Values.strapi.certIssuer }}"
+spec:
+  ingressClassName: "{{ .Values.strapi.ingressClassName }}"
+  backend:
+    serviceName: strapi
+    servicePort: 1337
+  tls:
+    - secretName: strapi-tls
+      hosts:
+        - "{{ .Values.strapi.hostname }}"
+  rules:
+    - host: "{{ .Values.strapi.hostname }}"
+      http:
+        paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: strapi
+              port:
+                number: 1337
+
+{{ end }}

charts/fp/templates/windmill-ingress.yaml

@@ -1,3 +1,5 @@
+{{ if eq .Values.managedBy "Helm" }} 
+
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -26,3 +28,5 @@ spec:
   - hosts:
     - windmill2.sbtp.xyz
     secretName: windmill-tls
+
+{{ end }}

charts/fp/values-dev.yaml

@@ -1,12 +1,26 @@
-# storageClassName: csi-hostpath-sc # used by minikube
+storageClassName: csi-hostpath-sc # used by minikube
-storageClassName: standard # used by Kind
+# storageClassName: standard # used by Kind
+managedBy: tilt
 link2cid:
   containerName: fp/link2cid
 next:
   containerName: fp/next
+  certIssuer: letsencrypt-staging
+  hostname: next.futureporn.svc.cluster.local
+capture:
+  containerName: fp/capture
+scout:
+  containerName: fp/scout
+  pubsubServerUrl: https://realtime.futureporn.svc.cluster.local/faye
 strapi:
   containerName: fp/strapi
   port: 1337
-  url: http://localhost:1337
+  url: https://strapi.futureporn.svc.cluster.local
-managedBy: Dildo
+  certIssuer: letsencrypt-staging
+  hostname: strapi.futureporn.svc.cluster.local
+  ingressClassName: ngrok
+ngrok:
+  hostname: prawn-sweeping-muskrat.ngrok-free.app
+realtime:
+  containerName: fp/realtime
 adminEmail: cj@futureporn.net

charts/fp/values-prod.yaml

@@ -1,12 +1,22 @@
 storageClassName: vultr-block-storage-hdd
 link2cid:
   containerName: gitea.futureporn.net/futureporn/link2cid:latest
+scout:
+  containerName: gitea.futureporn.net/futureporn/scout:latest
+  pubsubServerUrl: https://realtime.futureporn.net/faye
 next:
-  containerName: sjc.vultrcr.com/fpcontainers/next
+  containerName: gitea.futureporn.net/futureporn/next:latest
+  certIssuer: letsencrypt-staging
+  hostname: next.sbtp.xyz
+capture:
+  containerName: gitea.futureporn.net/futureporn/capture:latest
 strapi:
   containerName: sjc.vultrcr.com/fpcontainers/strapi
   port: 1337
   url: https://portal.futureporn.net
+  certIssuer: letsencrypt-prod
+  hostname: strapi.sbtp.xyz
+  ingressClassName: nginx
 managedBy: Helm
 adminEmail: cj@futureporn.net
 extraArgs: 

d.capture.dockerfile

@@ -0,0 +1,35 @@
+FROM node:18-alpine
+# Install dependencies only when needed
+# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
+RUN apk add --no-cache libc6-compat
+RUN corepack enable && corepack prepare pnpm@latest --activate
+
+# Enable `pnpm add --global` on Alpine Linux by setting
+# home location environment variable to a location already in $PATH
+# https://github.com/pnpm/pnpm/issues/784#issuecomment-1518582235
+ENV PNPM_HOME=/usr/local/bin
+
+# update and install latest dependencies, add dumb-init package
+# add a non root user
+RUN apk update && apk upgrade && apk add dumb-init ffmpeg make gcc g++ python3
+
+WORKDIR /app
+# Copy and install the dependencies for the project
+COPY ./packages/capture/package.json ./packages/capture/pnpm-lock.yaml ./
+
+# Copy all other project files to working directory
+COPY ./packages/capture .
+# Run the next build process and generate the artifacts
+RUN pnpm i;
+
+
+# expose 3000 on container
+EXPOSE 3000
+
+# set app host ,port and node env 
+ENV HOSTNAME=0.0.0.0 PORT=3000 NODE_ENV=production
+# start the app with dumb init to spawn the Node.js runtime process
+# with signal support
+CMD [ "dumb-init", "node", "index.js" ]
+
+

d.link2cid.dockerfile

@@ -0,0 +1,14 @@
+# Reference-- https://pnpm.io/docker
+
+FROM node:20-alpine AS base
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+WORKDIR /app
+COPY ./packages/link2cid/package.json /app
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod
+COPY ./packages/link2cid/index.js /app
+COPY ./packages/link2cid/src /app/src
+ENTRYPOINT ["pnpm"]
+CMD ["start"]
+

d.next.dockerfile

@@ -0,0 +1,65 @@
+## Important! Build context is the ROOT of the project.
+## this keeps the door open for future possibility of shared code between pnpm workspace packages
+
+FROM node:20-slim AS base
+
+FROM base AS deps
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+WORKDIR /app
+
+
+FROM deps AS install
+ARG NEXT_PUBLIC_SITE_URL=https://futureporn.net
+ARG NEXT_PUBLIC_STRAPI_URL=https://portal.futureporn.net
+ARG NEXT_PUBLIC_UPPY_COMPANION_URL=https://uppy.futureporn.net
+ENV NEXT_PUBLIC_SITE_URL ${NEXT_PUBLIC_SITE_URL}
+ENV NEXT_PUBLIC_STRAPI_URL ${NEXT_PUBLIC_STRAPI_URL}
+ENV NEXT_PUBLIC_UPPY_COMPANION_URL ${NEXT_PUBLIC_UPPY_COMPANION_URL}
+ENV NEXT_TELEMETRY_DISABLED 1
+COPY pnpm-lock.yaml ./
+RUN pnpm fetch
+COPY ./packages/next /app
+RUN --mount=type=cache,id=pnpm-store,target=/pnpm/store pnpm install
+
+
+FROM install AS dev
+CMD ["pnpm", "run", "dev"]
+
+
+
+FROM install AS build
+RUN pnpm run build
+# COPY --chown=node:node --from=install /app/package.json /app/pnpm-lock.yaml ./
+# RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
+# COPY --from=install /app /app # i think this is duplicate
+# can't get these to work because errors like "/prod/next/.next/standalone": not found
+# as if pnpm is not copying the build artifacts.
+# also this makes the build REALLY slow (adds ~10mins to build time)
+# RUN pnpm deploy --filter=@futureporn/next --prod /prod/next
+# RUN pnpm deploy --filter=@futureporn/link2cid --prod /prod/link2cid
+
+# FROM deps as release
+# # ENV NEXT_SHARP_PATH=/app/node_modules/sharp
+# ENV NODE_ENV=production
+# WORKDIR /app
+# COPY --from=build /app/public ./public
+# COPY --from=build /app/.next/standalone ./
+# COPY --from=build /app/.next/static ./.next/static
+# CMD [ "dumb-init", "node", "server.js" ]
+
+
+
+FROM deps AS next
+RUN apt-get update && apt-get install -y -qq --no-install-recommends dumb-init
+COPY --chown=node:node --from=build /app/package.json /app/pnpm-lock.yaml ./
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
+COPY --chown=node:node --from=build /app/public ./public
+COPY --chown=node:node --from=build /app/.next/standalone ./
+COPY --chown=node:node --from=build /app/.next/static ./.next/static
+ENV TZ=UTC
+ENV NODE_ENV=production
+ENV HOSTNAME="0.0.0.0"
+CMD [ "dumb-init", "node", "server.js" ]
+

d.realtime.dockerfile

@@ -0,0 +1,14 @@
+FROM node:20-alpine
+WORKDIR /app
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+RUN apk update
+
+ENV NODE_ENV=production
+COPY pnpm-lock.yaml ./
+RUN pnpm fetch
+COPY ./packages/realtime /app
+
+ENTRYPOINT ["pnpm"]
+CMD ["run", "start"]

d.scout.dockerfile

@@ -0,0 +1,17 @@
+FROM node:20-alpine
+WORKDIR /app
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+RUN apk update
+
+
+
+ENV NODE_ENV=production
+COPY pnpm-lock.yaml ./
+RUN pnpm fetch
+COPY ./packages/scout /app
+
+
+ENTRYPOINT ["pnpm"]
+CMD ["run", "start"]

d.strapi.dockerfile

@@ -0,0 +1,34 @@
+FROM node:20-alpine as base
+WORKDIR /app
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+RUN apk update && apk add --no-cache build-base gcc autoconf automake zlib-dev libpng-dev vips-dev libc6-compat git nasm bash gcompat
+
+FROM base AS install
+COPY ./packages/strapi/pnpm-lock.yaml ./packages/strapi/package.json ./
+RUN pnpm install --prod --shamefully-hoist && pnpm run build
+COPY ./packages/strapi .
+RUN chown -R node:node /app
+USER node
+
+
+# FROM base AS install
+# COPY ./packages/strapi/pnpm-lock.yaml ./
+# RUN pnpm fetch --prod
+# COPY ./packages/strapi .
+# RUN pnpm i --offline --prod --shamefully-hoist && pnpm run build
+# RUN chown -R node:node /app
+# USER node
+
+
+
+FROM install AS dev
+ENV NODE_ENV=development
+ENTRYPOINT ["pnpm"]
+CMD ["run", "dev"]
+
+FROM install AS release
+ENV NODE_ENV=production
+ENTRYPOINT ["pnpm"]
+CMD ["run", "start"]

packages/capture/.gitignore

@@ -0,0 +1,153 @@
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+
+# Created by https://www.toptal.com/developers/gitignore/api/node
+# Edit at https://www.toptal.com/developers/gitignore?templates=node
+
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+### Node Patch ###
+# Serverless Webpack directories
+.webpack/
+
+# Optional stylelint cache
+
+# SvelteKit build / generate output
+.svelte-kit
+
+# End of https://www.toptal.com/developers/gitignore/api/node

packages/capture/README.md

@@ -0,0 +1,25 @@
+# Capture
+
+## Dev notes
+
+### youtube-dl end of stream output
+
+```
+[https @ 0x5646887f1580] Opening 'https://edge11-lax.live.mmcdn.com/live-hls/amlst:hotfallingdevil-sd-fdf87e5b6c880e52d38e8c94f8ebf8728c980a91d56fb4ace13748ba59012336_trns_h264/chunklist_w881713853_b5128000_t64RlBTOjMwLjA=.m3u8' for reading
+[hls @ 0x564687dd0980] Skip ('#EXT-X-VERSION:4')
+[hls @ 0x564687dd0980] Skip ('#EXT-X-DISCONTINUITY-SEQUENCE:0')
+[hls @ 0x564687dd0980] Skip ('#EXT-X-PROGRAM-DATE-TIME:2023-01-31T17:48:45.947+00:00')
+[https @ 0x5646880bf880] Opening 'https://edge11-lax.live.mmcdn.com/live-hls/amlst:hotfallingdevil-sd-fdf87e5b6c880e52d38e8c94f8ebf8728c980a91d56fb4ace13748ba59012336_trns_h264/media_w881713853_b5128000_t64RlBTOjMwLjA=_18912.ts' for reading
+[https @ 0x564688097d00] Opening 'https://edge11-lax.live.mmcdn.com/live-hls/amlst:hotfallingdevil-sd-fdf87e5b6c880e52d38e8c94f8ebf8728c980a91d56fb4ace13748ba59012336_trns_h264/media_w881713853_b5128000_t64RlBTOjMwLjA=_18913.ts' for reading
+[https @ 0x5646887f1580] Opening 'https://edge11-lax.live.mmcdn.com/live-hls/amlst:hotfallingdevil-sd-fdf87e5b6c880e52d38e8c94f8ebf8728c980a91d56fb4ace13748ba59012336_trns_h264/chunklist_w881713853_b5128000_t64RlBTOjMwLjA=.m3u8' for reading
+[https @ 0x5646886e8580] HTTP error 403 Forbidden
+[hls @ 0x564687dd0980] keepalive request failed for 'https://edge11-lax.live.mmcdn.com/live-hls/amlst:hotfallingdevil-sd-fdf87e5b6c880e52d38e8c94f8ebf8728c980a91d56fb4ace13748ba59012336_trns_h264/chunklist_w881713853_b5128000_t64RlBTOjMwLjA=.m3u8' with error: 'Server returned 403 Forbidden (access denied)' when parsing playlist
+[https @ 0x5646886ccfc0] HTTP error 403 Forbidden
+[hls @ 0x564687dd0980] Failed to reload playlist 0
+[https @ 0x5646886bf680] HTTP error 403 Forbidden
+[hls @ 0x564687dd0980] Failed to reload playlist 0
+frame= 5355 fps= 31 q=-1.0 Lsize=   71404kB time=00:02:58.50 bitrate=3277.0kbits/s speed=1.02x    
+video:68484kB audio:2790kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.181873%
+[ffmpeg] Downloaded 73117881 bytes
+[download] 100% of 69.73MiB in 02:57
+```

packages/capture/index.js

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+
+// import Capture from './src/Capture.js'
+// import Video from './src/Video.js'
+
+import dotenv from 'dotenv'
+dotenv.config()
+import { createId } from '@paralleldrive/cuid2'
+import os from 'os'
+import fs from 'node:fs'
+import { loggerFactory } from "./src/logger.js"
+import { verifyStorage } from './src/disk.js'
+import faye from 'faye'
+import { record, assertDependencyDirectory, checkFFmpeg } from './src/record.js'
+import fastq from 'fastq'
+import pRetry from 'p-retry';
+import Fastify from 'fastify';
+
+
+
+// Create a map to store the work queues
+const workQueues = new Map();
+
+
+
+async function captureTask (args, cb) {
+  const { appContext, playlistUrl, roomName } = args;
+
+  try {
+    const downloadStream = async () => {
+      const rc = await record(appContext, playlistUrl, roomName)
+      if (rc !== 0) throw new Error('ffmpeg exited irregularly (return code was other than zero)')
+    }
+    await pRetry(downloadStream, {
+      retries: 3,
+      onFailedAttempt: error => {
+        appContext.logger.log({ level: 'error', message: `downloadStream attempt ${error.attemptNumber} failed. There are ${error.retriesLeft} retries left.` });
+      },
+    })
+  } catch (e) {
+    // we can get here if all retries are exhausted.
+    // this could be that the stream is over, the playlistUrl might be different, etc.
+    // we might have queued tasks so we don't want to crash.
+    appContext.logger.log({ level: 'error', message: `downloadStream exhausted all retries.` })
+    appContext.logger.log({ level: 'error', message: e })
+  }
+
+  verifyStorage(appContext)
+
+  appContext.logger.log({ level: 'info', message: 'Capture task complete'})
+  cb(null, null)
+}
+
+/**
+ * 
+ * Fastify is used to facilitate Docker health checks
+ * 
+ */
+async function initFastify(appContext) {
+  appContext.fastify = Fastify({
+    logger: true
+  })
+
+  // Declare a route
+  appContext.fastify.get('/health', function (_, reply) {
+    reply.send({ message: 'futureporn-capture sneed' });
+  })
+
+    // Run the server!
+  appContext.fastify.listen({ port: appContext.env.PORT }, function (err, address) {
+    if (err) {
+      appContext.fastify.log.error(err)
+      process.exit(1)
+    }
+  })
+}
+
+async function init () {
+
+  const appEnv = new Array(
+    'FUTUREPORN_WORKDIR', 
+    'PUBSUB_SERVER_URL', 
+    'DOWNLOADER_UA',
+    'PORT'
+  )
+
+  const logger = loggerFactory({
+    service: 'futureporn/capture'
+  })
+
+
+  const appContext = {
+    env: appEnv.reduce((acc, ev) => {
+      if (typeof process.env[ev] === 'undefined') throw new Error(`${ev} is undefined in env`);
+      acc[ev] = process.env[ev];
+      return acc;
+    }, {}),
+    logger,
+    pkg: JSON.parse(fs.readFileSync('./package.json', { encoding: 'utf-8'})),
+    workerId: `${os.hostname}-${createId()}`,
+    pubsub: new faye.Client(process.env.PUBSUB_SERVER_URL)
+  };
+
+  await initFastify(appContext);
+
+  assertDependencyDirectory(appContext)
+  await checkFFmpeg(appContext)
+  verifyStorage(appContext)
+
+  return appContext
+}
+
+
+async function main () {
+
+  const appContext = await init()
+
+
+
+  appContext.logger.log({ level: 'info', message: `capture version: ${appContext.pkg.version}` })
+  appContext.logger.log({ level: 'info', message: `my capture directory is ${appContext.env.FUTUREPORN_WORKDIR}` })
+  appContext.logger.log({ level: 'info', message: `listening for faye signals from ${appContext.env.PUBSUB_SERVER_URL}` })
+
+
+	// connect to realtime server
+  appContext.pubsub.subscribe('/signals', (message) => {
+    appContext.logger.log({ level: 'debug', message: JSON.stringify(message) })
+
+    if (
+      (message?.signal === 'start') &&
+      (message?.room) &&
+      (message?.url.startsWith('https://'))
+    ) {
+
+      const roomName = message.room;
+      const playlistUrl = message.url;
+
+      // Check if a work queue for the room already exists, otherwise create a new one
+      if (!workQueues.has(roomName)) {
+        workQueues.set(roomName, fastq(captureTask, 1));
+      }
+
+
+      // Push the task to the corresponding work queue
+      workQueues.get(roomName).push({ appContext, playlistUrl, roomName });
+    }
+
+  })
+
+
+
+}
+
+main()
+
+
+
+

packages/capture/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "futureporn-capture",
+  "version": "0.1.12",
+  "main": "index.js",
+  "license": "Unlicense",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "node --trace-warnings index",
+    "test": "FUTUREPORN_WORKDIR=/home/chris/Downloads mocha",
+    "integration": "FUTUREPORN_WORKDIR=/home/chris/Downloads mocha ./integration/**/*.test.js",
+    "dev": "FUTUREPORN_WORKDIR=/home/chris/Downloads nodemon index"
+  },
+  "dependencies": {
+    "@paralleldrive/cuid2": "^2.1.8",
+    "diskusage": "^1.1.3",
+    "dotenv": "^16.0.3",
+    "execa": "^6.1.0",
+    "fastify": "^4.12.0",
+    "fastq": "^1.15.0",
+    "faye": "^1.4.0",
+    "faye-websocket": "^0.11.4",
+    "fluent-ffmpeg": "^2.1.2",
+    "https": "^1.0.0",
+    "ioredis": "^5.2.4",
+    "minimatch": "^5.1.1",
+    "p-retry": "^5.1.2",
+    "postgres": "^3.3.3",
+    "rxjs": "^7.8.0",
+    "sql": "^0.78.0",
+    "winston": "^3.9.0",
+    "youtube-dl-wrap": "git+https://github.com/insanity54/youtube-dl-wrap.git"
+  },
+  "devDependencies": {
+    "chai": "^4.3.7",
+    "cheerio": "^1.0.0-rc.12",
+    "mocha": "^10.2.0",
+    "multiformats": "^11.0.1",
+    "node-abort-controller": "^3.0.1",
+    "node-fetch": "^3.3.0",
+    "nodemon": "^2.0.20",
+    "sinon": "^15.0.1",
+    "sinon-chai": "^3.7.0",
+    "sinon-test": "^3.1.5"
+  }
+}

packages/capture/src/Capture.js

@@ -0,0 +1,125 @@
+
+import Voddo from './Voddo.js'
+import {loggerFactory} from 'common/logger'
+
+const logger = loggerFactory({
+  service: 'futureporn/capture'
+})
+
+export default class Capture {
+
+  constructor(opts) {
+    this.date = opts?.date
+    this.sql = opts.sql
+    this.ipfs = opts.ipfs
+    this.idleTimeout = opts.idleTimeout || 1000*60*15
+    this.video = opts.video
+    this.voddo = opts.voddo
+    this.workerId = opts.workerId
+    return this
+  }
+
+
+  /**
+   * upload VOD to ipfs
+   * 
+   * @return {Promise}
+   * @resolves {String} cid
+   */
+  async upload (filename) {
+    const cid = await this.ipfs.upload(filename)
+    return cid
+  }
+
+
+
+  /**
+   * save Vod data to db
+   */
+  async save (cid, timestamp) {
+    logger.log({ level: 'debug', message: `saving ${cid} \n      w/ captureDate ${timestamp}` })
+    this.date = timestamp
+    return await this.sql`INSERT INTO vod ( "videoSrcHash", "captureDate" ) values (${cid}, ${timestamp}) returning *`
+  }
+
+
+  /**
+   * advertise the vod segment(s) we captured.
+   * futureporn/commander uses this data to elect one worker to upload the VOD
+   */
+  async advertise () {
+    const segments = await this.voddo.getRecordedSegments()
+    const streams = Voddo.groupStreamSegments(segments)
+    const workerId = this.workerId
+    logger.log({ level: 'debug', message: `Advertising our VOD streams(s) ${JSON.stringify({segments, streams, workerId})}` })
+    this.sql.notify('capture/vod/advertisement', JSON.stringify({segments, streams, workerId}))
+  }
+
+
+  listen () {
+    this.sql.listen('scout/stream/stop', (data) => {
+      logger.log({ level: 'debug', message: 'Scout said the stream has stopped. I will advertise the vod segment(s) I have.' })
+      this.advertise()
+    })
+
+    this.sql.listen('commander/vod/election', async (data) => {
+      if (data.workerId === this.workerId) {
+        logger.log({ level: 'debug', message: 'Commander elected me to process/upload' })
+        this.process(await this.voddo.getFilenames())
+      } else {
+        logger.log({ level: 'debug', message: `Commander elected ${data.workerId} to process/upload their vod segment(s)` })
+      }
+    })
+
+    return this
+  }
+
+
+  /**
+   * process video(s) after end of stream
+   *
+   * @param {String[]} filenames
+   * @returns {void}
+   */
+  async process (filenames) {
+    this.date = filenames[0].timestamp
+
+    logger.log({ level: 'debug', message: 'concatenation in progress...' })
+    const file = await this.video.concat(filenames)
+
+    logger.log({ level: 'debug', message: `uploading ${file}` })
+    const cid = await this.ipfs.upload(file)
+
+    logger.log({ level: 'debug', message: 'db save in progress' })
+    await this.save(cid, this.date)
+
+  }
+
+
+
+  /**
+   * download a livestream
+   * 
+   *   - initializes Voddo
+   *   - invokes this.process() as side effect
+   * 
+   * @return {void}
+   */
+  async download () {
+    this.voddo.on('start', (data) => {
+      logger.log({ level: 'debug', message: 'voddo started' })
+      logger.log({ level: 'debug', message: data })
+      this.sql.notify('capture/file', JSON.stringify(data))
+    })
+    this.voddo.on('stop', (report) => {
+      logger.log({ level: 'debug', message: `Got a stop event from Voddo` })
+    })
+    logger.log({ level: 'debug', message: 'starting voddo' })
+    this.voddo.start()
+  }
+
+}
+
+
+
+

packages/capture/src/Ipfs.js

@@ -0,0 +1,57 @@
+
+import {execa} from 'execa'
+import {loggerFactory} from 'common/logger'
+
+const logger = loggerFactory({
+  service: 'futureporn/capture'
+})
+
+export default class Ipfs {
+  constructor(opts) {
+    this.multiaddr = opts?.IPFS_CLUSTER_HTTP_API_MULTIADDR
+    this.username = opts?.IPFS_CLUSTER_HTTP_API_USERNAME
+    this.password = opts?.IPFS_CLUSTER_HTTP_API_PASSWORD
+    this.ctlExecutable = opts?.ctlExecutable || '/usr/local/bin/ipfs-cluster-ctl'
+    this.ipfsExecutable = opts?.ipfsExecutable || '/usr/local/bin/ipfs'
+  }
+  getArgs () {
+    let args = [
+      '--no-check-certificate',
+      '--host', this.multiaddr,
+      '--basic-auth', `${this.username}:${this.password}`
+    ]
+    return args
+  }
+  async upload (filename, expiryDuration = false) {
+    try {
+      let args = getArgs()
+
+      args = args.concat([
+        'add', 
+        '--quieter', 
+        '--cid-version', 1
+      ])
+
+      if (expiryDuration) {
+        args = args.concat(['--expire-in', expiryDuration])
+      }
+
+      args.push(filename)
+
+      const { stdout } = await execa(this.ctlExecutable, args)
+      return stdout
+    } catch (e) {
+      logger.log({ level: 'error', message: 'Error while adding file to ipfs' })
+      logger.log({ level: 'error', message: e })
+    }
+  }
+  async hash (filename) {
+    try {
+      const { stdout } = await execa(this.ipfsExecutable, ['add', '--quiet', '--cid-version=1', '--only-hash', filename])
+      return stdout
+    } catch (e) {
+      logger.log({ level: 'error', message: 'Error while hashing file' })
+      logger.log({ level: 'error', message: e })
+    }
+  }
+}

packages/capture/src/Video.js

@@ -0,0 +1,68 @@
+
+import { execa } from 'execa'
+import { tmpdir } from 'os'
+import path from 'node:path'
+import fs from 'node:fs'
+import os from 'node:os'
+
+export class VideoConcatError extends Error {
+  constructor (msg) {
+    super(msg || 'Failed to concatenate video')
+    this.name = 'VideoConcatError'
+  }
+}
+
+
+
+export default class Video {
+  constructor (opts) {
+    if (typeof opts.filePaths === 'undefined') throw new Error('Video must be called with opts.filePaths');
+    if (typeof opts.cwd === 'undefined') throw new Error('Video must be called with opts.cwd');
+    this.filePaths = opts.filePaths
+    this.cwd = opts.cwd
+    this.room = opts.room || 'projektmelody'
+    this.execa = opts.execa || execa
+  }
+
+
+
+  getFilesTxt () {
+    return this.filePaths
+      .sort((a, b) => a.timestamp - b.timestamp)
+      .map((d) => `file '${d.file}'`)
+      .join('\n')
+      .concat('\n')
+  }
+
+
+  getFilesFile () {
+    const p = path.join(this.cwd, 'files.txt')
+    fs.writeFileSync(
+      p, 
+      this.getFilesTxt(this.filePaths), 
+      { encoding: 'utf-8' }
+    )
+    return p
+  }
+
+  async concat () {
+    const target = path.join(this.cwd, `${this.room}-chaturbate-${new Date().valueOf()}.mp4`)
+
+    const { exitCode, killed, stdout, stderr } = await this.execa('ffmpeg', [
+      '-y',
+      '-f', 'concat',
+      '-safe', '0',
+      '-i', this.getFilesFile(this.filePaths),
+      '-c', 'copy',
+      target
+    ], {
+      cwd: this.cwd
+    });
+
+    if (exitCode !== 0 || killed !== false) {
+      throw new VideoConcatError(`exitCode:${exitCode}, killed:${killed}, stdout:${stdout}, stderr:${stderr}`);
+    }
+
+    return target
+  }
+}

packages/capture/src/Voddo.js

@@ -0,0 +1,243 @@
+import 'dotenv/config'
+import YoutubeDlWrap from "youtube-dl-wrap";
+import { EventEmitter } from 'node:events';
+import { AbortController } from "node-abort-controller";
+import { readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path'
+import ffmpeg from 'fluent-ffmpeg'
+import { loggerFactory } from 'common/logger'
+
+const logger = loggerFactory({
+  service: 'futureporn/capture'
+})
+const defaultStats = {segments:[],lastUpdatedAt:null}
+
+export default class Voddo extends EventEmitter {
+  constructor(opts) {
+    super()
+    this.courtesyTimer = setTimeout(() => {}, 0);
+    this.retryCount = 0;
+    this.url = opts.url;
+    this.format = opts.format || 'best';
+    this.cwd = opts.cwd;
+    this.ytdlee; // event emitter for ytdlwrap
+    this.stats = Object.assign({}, defaultStats);
+    this.abortController = new AbortController();
+    this.ytdl = opts.ytdl || new YoutubeDlWrap();
+    if (process.env.YOUTUBE_DL_BINARY) this.ytdl.setBinaryPath(process.env.YOUTUBE_DL_BINARY);
+  }
+
+  static async getVideoLength (filePath) {
+    return new Promise((resolve, reject) => {
+      ffmpeg.ffprobe(filePath, function(err, metadata) {
+        if (err) reject(err)
+        resolve(Math.floor(metadata.format.duration*1000))
+      });
+    })
+  }
+
+  // greets ChatGPT
+  static groupStreamSegments(segments, threshold = 1000*60*60) {
+    segments.sort((a, b) => a.startTime - b.startTime);
+    const streams = [];
+    let currentStream = [];
+
+    for (let i = 0; i < segments.length; i++) {
+      const currentSegment = segments[i];
+      const previousSegment = currentStream[currentStream.length - 1];
+
+      if (!previousSegment || currentSegment.startTime - previousSegment.endTime <= threshold) {
+        currentStream.push(currentSegment);
+      } else {
+        streams.push(currentStream);
+        currentStream = [currentSegment];
+      }
+    }
+
+    streams.push(currentStream);
+    return streams;
+  }
+
+
+
+
+
+
+
+  /**
+   * getRecordedStreams
+   * 
+   * get the metadata of the videos captured
+   */
+  async getRecordedSegments() {
+    let f = []
+    const files = await readdir(this.cwd).then((raw) => raw.filter((f) => /\.mp4$/.test(f) ))
+    for (const file of files) {
+      const filePath = join(this.cwd, file)
+      const s = await stat(filePath)
+      const videoDuration = await Voddo.getVideoLength(filePath)
+      const startTime = parseInt(s.ctimeMs)
+      const endTime = startTime+videoDuration
+      const size = s.size
+      f.push({
+        startTime,
+        endTime,
+        file,
+        size
+      })
+    }
+    this.stats.segments = f
+
+
+    return this.stats.segments
+  }
+
+  isDownloading() {
+    // if there are event emitter listeners for the progress event,
+    // we are probably downloading.
+    return (
+      this.ytdlee?.listeners('progress').length !== undefined
+    )
+  }
+
+  delayedStart() {
+    // only for testing
+    this.retryCount = 500
+    this.courtesyTimer = this.getCourtesyTimer(() => this.download())
+  }
+
+
+  start() {
+    // if download is in progress, do nothing
+    if (this.isDownloading()) {
+      logger.log({ level: 'debug', message: 'Doing nothing because a download is in progress.' })
+      return;
+    }
+
+    // if download is not in progress, start download immediately
+    // reset the retryCount so the backoff timer resets to 1s between attempts
+    this.retryCount = 0
+    clearTimeout(this.courtesyTimer)
+
+    // create new abort controller
+    //this.abortController = new AbortController() // @todo do i need this? Can't I reuse the existing this.abortController?
+
+    this.download()
+  }
+
+  stop() {
+    logger.log({ level: 'info', message: 'Received stop(). Stopping.' })
+    clearTimeout(this.courtesyTimer)
+    this.abortController.abort()
+  }
+
+  /** generate a report **/
+  getReport(errorMessage) {
+    let report = {}
+    report.stats = Object.assign({}, this.stats)
+    report.error = errorMessage
+    report.reason = (() => { 
+      if (errorMessage) return 'error';
+      else if (this.abortController.signal.aborted) return 'aborted';
+      else return 'close';
+    })()
+    // clear stats to prepare for next run
+    this.stats = Object.assign({}, defaultStats) 
+    return report
+  }
+
+  emitReport(report) {
+    logger.log({ level: 'debug', message: 'EMITTING REPORT' })
+    this.emit('stop', report)
+  }
+
+  getCourtesyTimer(callback) {
+    // 600000ms = 10m
+    const waitTime = Math.min(600000, (Math.pow(2, this.retryCount) * 1000));
+    this.retryCount += 1;
+    logger.log({ level: 'debug', message: `courtesyWait for ${waitTime/1000} seconds. (retryCount: ${this.retryCount})` })
+    return setTimeout(callback, waitTime)
+  }
+
+  download() {
+    const handleProgress = (progress) => {
+      logger.log({ level: 'debug', message:`  [*] progress event` })
+      this.stats.lastUpdatedAt = Date.now(),
+      this.stats.totalSize = progress.totalSize
+    }
+
+    const handleError = (error) => {
+      if (error?.message !== undefined && error.message.includes('Room is currently offline')) {
+        logger.log({ level: 'debug', message: 'Handled an expected \'Room is offline\' error' })
+
+      } else {
+        logger.log({ level: 'error', message: 'ytdl error' })
+        logger.log({ level: 'error', message: error.message })
+      }
+      this.ytdlee.off('progress', handleProgress)
+      this.ytdlee.off('handleYtdlEvent', handleYtdlEvent)
+
+      // restart the download after the courtesyTimeout
+      this.courtesyTimer = this.getCourtesyTimer(() => this.download())
+      this.emitReport(this.getReport(error.message))
+    }
+
+
+    const handleYtdlEvent = (type, data) => {
+      logger.log({ level: 'debug', message: `handleYtdlEvent type: ${type}, data: ${data}` })
+      logger.log({ level: 'debug', message: `handleYtdlEvent type: ${type}, data: ${data}` })
+      if (type === 'download' && data.includes('Destination:')) {
+        let filePath = /Destination:\s(.*)$/.exec(data)[1]
+        logger.log({ level: 'debug', message: `Destination file detected: ${filePath}` })
+        let datum = { file: filePath, timestamp: new Date().valueOf() }
+        let segments = this.stats.segments
+        segments.push(datum) && segments.length > 64 && segments.shift(); // limit the size of the segments array
+        this.emit('start', datum)
+      } else if (type === 'ffmpeg' && data.includes('bytes')) {
+        const bytes = /(\d*)\sbytes/.exec(data)[1]
+        logger.log({ level: 'debug', message: `ffmpeg reports ${bytes}`})
+        let mostRecentFile = this.stats.segments[this.stats.segments.length-1]
+        mostRecentFile['size'] = bytes
+        logger.log({ level: 'debug', message: mostRecentFile })
+      }
+    }
+
+    const handleClose = () => {
+      logger.log({ level: 'debug', message: 'got a close event. handling!' });
+
+      this.ytdlee.off('progress', handleProgress)
+      this.ytdlee.off('handleYtdlEvent', handleYtdlEvent)
+
+      // restart Voddo only if the close was not due to stop()
+      if (!this.abortController.signal.aborted) {
+        // restart the download after the courtesyTimeout
+        this.courtesyTimer = this.getCourtesyTimer(() => this.download())
+      }
+
+      this.emitReport(this.getReport())
+    }
+
+    logger.log({ level: 'debug', message: `Downloading url:${this.url} format:${this.format}` })
+    logger.log({ level: 'debug', message: JSON.stringify(this.ytdl) })
+
+    // sanity check. ensure cwd exists
+    stat(this.cwd, (err) => {
+      if (err) logger.log({ level: 'error', message: `Error while getting cwd stats of ${this.cwd} Does it exist?` })
+    })
+
+    this.ytdlee = this.ytdl.exec(
+      [this.url, '-f', this.format], 
+      { 
+        cwd: this.cwd
+      }, 
+      this.abortController.signal
+    );
+    this.ytdlee.on('progress', handleProgress);
+    this.ytdlee.on('youtubeDlEvent', handleYtdlEvent);
+    this.ytdlee.once('error', handleError);
+    this.ytdlee.once('close', handleClose);
+  }
+
+
+
+}

packages/capture/src/cb.js

@@ -0,0 +1,17 @@
+import cheerio from 'cheerio'
+import fetch from 'node-fetch'
+
+export async function getRandomRoom () {
+	const res = await fetch('https://chaturbate.com/')
+	const body = await res.text()
+	const $ = cheerio.load(body)
+	let roomsRaw = $('a[data-room]')
+	let rooms = []
+	$(roomsRaw).each((_, e) => {
+		rooms.push($(e).attr('href'))
+	})
+
+	// greets https://stackoverflow.com/a/4435017/1004931
+	var randomIndex = Math.floor(Math.random() * rooms.length);
+	return rooms[randomIndex].replaceAll('/', '')
+}

packages/capture/src/disk.js

@@ -0,0 +1,33 @@
+import disk from 'diskusage';
+
+
+export function verifyStorage (appContext) {
+  const mountPath = appContext.env.FUTUREPORN_WORKDIR
+  disk.check(mountPath, (err, info) => {
+    if (err) {
+      appContext.logger.log({ level: 'error', message: `Error retrieving disk usage for ${mountPath}: ${err}` });
+      return;
+    }
+
+    const totalSize = info.total;
+    const availableSize = info.available;
+    const freeSize = info.free;
+
+    appContext.logger.log({ 'level': 'info', message: `${mountPath} Disk Usage:` });
+    appContext.logger.log({ 'level': 'info', message: `Total: ${bytesToSize(totalSize)}` });
+    appContext.logger.log({ 'level': 'info', message: `Free: ${bytesToSize(freeSize)}` });
+    appContext.logger.log({ 'level': 'info', message: `Available: ${bytesToSize(availableSize)}` });
+
+    if (availableSize < 85899345920) appContext.logger.log({ 'level': 'warn', message: `⚠️ Available disk is getting low! ${bytesToSize(availableSize)}` });
+    else if (availableSize < 42949672960) appContext.logger.log({ 'level': 'error', message: `⚠️☠️ AVAILABLE DISK IS TOO LOW! ${bytesToSize(availableSize)}` });
+  });
+}
+
+
+// Helper function to convert bytes to human-readable format
+export function bytesToSize(bytes) {
+  const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+  if (bytes === 0) return '0 Bytes';
+  const i = Math.floor(Math.log2(bytes) / 10);
+  return `${(bytes / Math.pow(1024, i)).toFixed(2)} ${sizes[i]}`;
+}

packages/capture/src/logger.js

@@ -0,0 +1,25 @@
+import winston from 'winston'
+
+export const loggerFactory = (options) => {
+  const mergedOptions = Object.assign({}, {
+    level: 'info',
+    defaultMeta: { service: 'futureporn' },
+    format: winston.format.timestamp()
+  }, options)
+  const logger = winston.createLogger(mergedOptions);
+
+  if (process.env.NODE_ENV !== 'production') {
+    logger.add(new winston.transports.Console({
+      level: 'debug',
+      format: winston.format.simple()
+    }))
+  } else {
+    logger.add(new winston.transports.Console({
+      level: 'info',
+      format: winston.format.json()
+    }))
+  }
+
+  return logger
+}
+

packages/capture/src/record.js

@@ -0,0 +1,117 @@
+import { join } from 'path';
+import { spawn } from 'child_process';
+import fs from 'node:fs';
+
+export const getFilename = (appContext, roomName) => {
+  const name = `${roomName}_${new Date().toISOString()}.ts`
+  return join(appContext.env.FUTUREPORN_WORKDIR, 'recordings', name);
+}
+
+
+export const assertDirectory = (directoryPath) => {
+  if (fs.statSync(directoryPath, { throwIfNoEntry: false }) === undefined) fs.mkdirSync(directoryPath);
+}
+
+export const checkFFmpeg = async (appContext) => {
+  return new Promise((resolve, reject) => {
+    const childProcess = spawn('ffmpeg', ['-version']);
+
+    childProcess.on('error', (err) => {
+      appContext.logger.log({
+        level: 'error',
+        message: `ffmpeg -version failed, which likely means ffmpeg is not installed or not on $PATH`,
+      });
+      throw new Error('ffmpeg is missing')
+    });
+
+    childProcess.on('exit', (code) => {
+      if (code !== 0) reject(`'ffmpeg -version' exited with code ${code}`)
+      if (code === 0) {
+        appContext.logger.log({ level: 'info', message: `ffmpeg PRESENT.` });
+        resolve()
+      }
+    });
+  })
+};
+
+export const assertDependencyDirectory = (appContext) => {
+  // Extract the directory path from the filename
+  const directoryPath = join(appContext.env.FUTUREPORN_WORKDIR, 'recordings');
+  console.log(`asserting ${directoryPath} exists`)
+
+  // Check if the directory exists, and create it if it doesn't
+  if (!fs.existsSync(directoryPath)) {
+    fs.mkdirSync(directoryPath, { recursive: true });
+    console.log(`Created directory: ${directoryPath}`);
+  }
+}
+
+export const record = async (appContext, playlistUrl, roomName) => {
+  if (appContext === undefined) throw new Error('appContext undef');
+  if (playlistUrl === undefined) throw new Error('playlistUrl undef');
+  if (roomName === undefined) throw new Error('roomName undef');
+
+  const filename = getFilename(appContext, roomName);
+  console.log(`downloading to ${filename}`)
+
+  // example: `ffmpeg -headers "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0" 
+  //                  -i ${chunkPlaylist} 
+  //                  -c:v copy 
+  //                  -c:a copy 
+  //                  -movflags faststart 
+  //                  -y 
+  //                  -f mpegts 
+  //                  ./my-recording.ts`
+  const ffmpegProcess = spawn('ffmpeg', [
+    '-headers', `"User-Agent: ${appContext.env.DOWNLOADER_UA}"`, 
+    '-i', playlistUrl, 
+    '-c:v', 'copy',
+    '-c:a', 'copy',
+    '-movflags', 'faststart',
+    '-y',
+    '-f', 'mpegts', 
+    filename
+  ], {
+    stdio: 'inherit'
+  });
+
+
+
+  return new Promise((resolve, reject) => {
+    ffmpegProcess.once('exit', (code) => {
+      resolve(code)
+    })
+  })
+
+  // ffmpegProcess.on('data', (data) => {
+  //   console.log(data.toString());
+  // });
+
+
+  // Optional: Handle other events such as 'error', 'close', etc.
+  // @todo this needs to be handled outside this function
+  //       otherwise this function is not testable
+  // ffmpegProcess.on('exit', (code, signal) => {
+  //   // Retry the download using exponential backoff if the process exits for any reason
+  //   console.log(`ffmpeg exited with code ${code} and signal ${signal}`)
+  //   retryDownload(appContext, playlistUrl, roomName);
+  // });
+
+  // return ffmpegProcess;
+}
+
+
+const calculateExponentialBackoffDelay = (attemptNumber) => {
+  return Math.pow(2, attemptNumber) * 1000;
+};
+
+const retryDownload = (appContext, playlistUrl, roomName, attemptNumber = 1, maxAttempts = 3) => {
+  const delay = calculateExponentialBackoffDelay(attemptNumber);
+
+  appContext.logger.log({ level: 'debug', message: `Retrying download in ${delay / 1000} seconds...` });
+
+  setTimeout(() => {
+    console.log('Retrying download...');
+    record(appContext, playlistUrl, roomName, attemptNumber + 1);
+  }, delay);
+};

packages/capture/test/Capture.test.js

@@ -0,0 +1,147 @@
+
+import Video from '../src/Video.js'
+import Capture from '../src/Capture.js'
+import Ipfs from '../src/Ipfs.js'
+import chai, { expect } from 'chai'
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import path from 'node:path'
+import sinon from 'sinon'
+import sinonChai from 'sinon-chai'
+import { CID } from 'multiformats/cid'
+import Voddo from '../src/Voddo.js'
+import EventEmitter from 'node:events'
+import postgres from 'postgres'
+
+chai.use(sinonChai)
+
+const Timer = setTimeout(()=>{},0).constructor
+const fixtureDate = 1581117660000
+const cidFixture = 'bafybeid3mg5lzrvnmpfi5ftwhiupp7i5bgkmdo7dnlwrvklbv33telrrry'
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+describe('Capture', function () {
+
+  let clock
+
+  const sandbox = sinon.createSandbox()
+
+  beforeEach(() => {
+
+    clock = sandbox.useFakeTimers({
+      toFake: ["setTimeout", "setInterval"],
+      shouldAdvanceTime: false
+    });
+
+    // // const sql = postgres({
+    // //   idle_timeout: 1
+    // // })
+
+    // let pgStub = (opts) => {
+    //   let sql = (args) => {}
+    //   return sql
+    // }
+    const sqlRaw = postgres()
+    const sql = sandbox.stub(sqlRaw)
+    // sql.listen.resolves(fixtureDate)
+    // sql.notify.resolves(92834)
+    // sinon.stub(postgres, 'notify')
+    // sinon.createStubInstance(postgres)
+    // sql
+    //   .withArgs('INSERT INTO vod ( videoSrcHash, captureDate ) values (bafybeid3mg5lzrvnmpfi5ftwhiupp7i5bgkmdo7dnlwrvklbv33telrrry, 1581117660000) returning *')
+    //   .resolves({ msg: 'idk' })
+    // sinon.stub(sql, 'notify').returns()
+
+
+
+
+    // const ipfs = sandbox.createStubInstance(Ipfs)
+    // ipfs.upload.withArgs('/tmp/mycoolfile.mp4').resolves(cidFixture)
+    // capture = new Capture({
+    //   sql,
+    //   ipfs,
+    //   video,
+    //   voddo
+    // })
+    // sandbox.stub(capture, 'process').resolves()
+  })
+
+  afterEach(() => {
+    sandbox.restore()
+    clock.restore()
+  })
+
+
+
+  describe('upload', function () {
+    it('should upload a video to ipfs', async function () {
+
+      const sqlRaw = postgres()
+      const sql = sandbox.stub(sqlRaw)
+
+      const video = sandbox.stub()
+      const voddo = sandbox.createStubInstance(Voddo)
+      voddo.on.callThrough()
+      voddo.emit.callThrough()
+      voddo.listeners.callThrough()
+      voddo.listenerCount.callThrough()
+
+
+      voddo.start.callsFake(() => {
+        voddo.emit('start', { file: '/tmp/burrito.mp4', timestamp: 1 })
+      })
+
+      const ipfs = sandbox.createStubInstance(Ipfs)
+      ipfs.upload.withArgs('/tmp/mycoolfile.mp4').resolves(cidFixture)
+      const capture = new Capture({
+        sql,
+        ipfs,
+        video,
+        voddo
+      })
+
+      const cid = await capture.upload('/tmp/mycoolfile.mp4')
+      expect(() => CID.parse(cid), `The IPFS CID '${cid}' is invalid.`).to.not.throw()
+      expect(capture.ipfs.upload).calledOnce
+    })
+  })
+  describe('save', function () {
+    it('should save to db', async function () {
+
+      const sqlRaw = postgres()
+      const sql = sandbox.stub(sqlRaw)
+
+      const video = sandbox.stub()
+      const voddo = sandbox.createStubInstance(Voddo)
+      voddo.on.callThrough()
+      voddo.emit.callThrough()
+      voddo.listeners.callThrough()
+      voddo.listenerCount.callThrough()
+
+
+      voddo.start.callsFake(() => {
+        voddo.emit('start', { file: '/tmp/burrito.mp4', timestamp: 1 })
+      })
+
+      const ipfs = sandbox.createStubInstance(Ipfs)
+      ipfs.upload.withArgs('/tmp/mycoolfile.mp4').resolves(cidFixture)
+      const capture = new Capture({
+        sql,
+        ipfs,
+        video,
+        voddo
+      })
+      
+      // I can't stub sql`` because of that template string override so i'm just stubbing capture.save
+      // I think this is an evergreen test ¯\_(ツ)_/¯
+      sandbox.stub(capture, 'save').resolves([
+        { id: 1, cid: cidFixture, captureDate: fixtureDate }
+      ])
+      const vod = await capture.save(cidFixture, fixtureDate)
+    })
+  })
+
+
+
+
+})

packages/capture/test/Video.test.js

@@ -0,0 +1,86 @@
+
+import 'dotenv/config'
+import Video from '../src/Video.js'
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import path from 'node:path'
+import os from 'node:os'
+import fs from 'node:fs'
+import sinon from 'sinon'
+import sinonChai from 'sinon-chai'
+import chai, { expect } from 'chai'
+
+chai.use(sinonChai);
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+
+
+const dataFixture = [
+  {
+      timestamp: 1,
+      file: 'mock-stream0.mp4'
+  }, {
+      timestamp: 2,
+      file: 'mock-stream1.mp4'
+  }, {
+      timestamp: 3,
+      file: 'mock-stream2.mp4'
+  }
+]
+
+describe('Video', function () {
+
+  let video
+
+  before(() => {
+    // copy files to /tmp so we dont clutter the fixtures dir
+    // and simulate cwd being process.env.FUTUREPORN_TMP
+    dataFixture.forEach((d) => {
+      fs.copyFileSync(
+        path.join(__dirname, 'fixtures', d.file), 
+        path.join(os.tmpdir(), d.file)
+      )
+    })
+  })
+
+
+
+  beforeEach(() => {
+    video = new Video({
+      cwd: os.tmpdir(),
+      filePaths: dataFixture,
+      execa: sinon.fake.resolves({ exitCode: 0, killed: false, stdout: "i am so horni rn", stderr: null })
+    })
+  })
+
+  afterEach(function() {
+    console.log('>> sinon.restore! (afterEach)')
+    sinon.restore();
+  })
+
+
+  describe('getFilesTxt', function () {
+    it('should generate contents suitable for input to `ffmpeg -f concat`', function () {
+      const txt = video.getFilesTxt()
+      expect(txt).to.deep.equal("file 'mock-stream0.mp4'\nfile 'mock-stream1.mp4'\nfile 'mock-stream2.mp4'\n")
+    })
+  })
+
+  describe('concat', function () {
+    it('should join multiple videos into one', async function () {
+      const file = await video.concat()
+      expect(typeof file === 'string').to.be.true
+      expect(video.execa).calledOnce
+      expect(file).to.match(/\.mp4$/)
+    })
+  })
+
+  describe('getFilesFile', function () {
+    it('should create a files.txt and return the path', async function () {
+      const file = await video.getFilesFile()
+      expect(typeof file === 'string').to.be.true
+      expect(file).to.equal(path.join(os.tmpdir(), 'files.txt'))
+    })
+  })
+})

packages/capture/test/Voddo.test.js

@@ -0,0 +1,491 @@
+import 'dotenv/config'
+import Voddo from '../src/Voddo.js'
+import chai, { expect } from 'chai'
+import sinon from 'sinon'
+import YoutubeDlWrap from 'youtube-dl-wrap'
+import {
+  AbortController
+} from "node-abort-controller";
+import {
+  EventEmitter
+} from 'events'
+import debugFactory from 'debug'
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import sinonChai from 'sinon-chai'
+import sinonTest from "sinon-test";
+import path from 'path'
+
+chai.use(sinonChai);
+
+const test = sinonTest(sinon, {
+  toFake: ["setTimeout", "setInterval"],
+  shouldAdvanceTime: false
+});
+const debug = debugFactory('voddo')
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+
+
+
+
+describe('Voddo', function() {
+
+
+  describe('groupStreamSegments', function () {
+    it('should separate two stream data objects', function () {
+      const fixture = [{
+        "startTime": 1675386000000,
+        "file": "projektmelody 2023-02-02 17_00-projektmelody.mp4",
+        "size": 550799038,
+        "endTime": 1675391400000,
+      }, {
+        "startTime": 1675391405000,
+        "file": "projektmelody 2023-02-02 18_30-projektmelody.mp4",
+        "size": 6556534941,
+        "endTime": 1675396800000
+      }, {
+        "startTime": 1675368000000,
+        "file": "projektmelody 2023-02-02 12_00-projektmelody.mp4",
+        "size": 6556534941,
+        "endTime": 1675378800000
+      }]
+
+      const streams = Voddo.groupStreamSegments(fixture)
+      expect(streams).to.deep.equal([
+        [
+          {
+            "startTime": 1675368000000,
+            "file": "projektmelody 2023-02-02 12_00-projektmelody.mp4",
+            "size": 6556534941,
+            "endTime": 1675378800000
+          }
+        ],
+        [
+          {
+            "startTime": 1675386000000,
+            "file": "projektmelody 2023-02-02 17_00-projektmelody.mp4",
+            "size": 550799038,
+            "endTime": 1675391400000,
+          }, {
+            "startTime": 1675391405000,
+            "file": "projektmelody 2023-02-02 18_30-projektmelody.mp4",
+            "size": 6556534941,
+            "endTime": 1675396800000
+          }
+        ]
+      ])
+    })
+  })
+
+  
+  // let clock;
+
+  // beforeEach(function() {
+  //   clock = sinon.useFakeTimers({
+  //     toFake: ["setTimeout", "setInterval"],
+  //     shouldAdvanceTime: false
+  //   });
+  // })
+
+  // afterEach(() => {
+  //   sinon.restore()
+  // })
+
+  
+
+  // Something faulty with Voddo or sinon or mocha, not sure. 
+  // When running by itself, test succeeds. When running with 'should start and stop stream download',
+  // voddo.stats gets set to whatever that test sets it to. So bizarre, it's like the same Voddo class instance
+  // exists in two different tests even though they are named differently. 
+  // Even though they are not in global scope. Even though each was called with `new Voddo(...)`
+  // Doesn't matter if I wrap both in sinon-test. Same leaky problem.
+  // Doesn't matter if I sinon.restore() afterEach. Same leaky problem.
+  // Doesn't matter if I manually set up a sinon sandbox. Same leaky problem.
+  // Fuck event emitters. I love their utility but I don't know how the fuck they are supposed to be tested.
+  // Solution might just call for a rewrite of Voddo, or perhaps deleting Voddo in favor of Capture
+  // For now, I'm moving forward because Voddo works even though this test does not.
+  describe('getRecordedSegments', function() {
+    xit('should populate it\'s log if log is empty', async function () {
+      const voddo = new Voddo({
+        url: 'https://example.com',
+        cwd: join(__dirname, 'fixtures')
+      })
+      const streams = await voddo.getRecordedSegments()
+      console.log(streams)
+      expect(streams.length).to.equal(3)
+      expect(streams[0]).to.have.property('startTime')
+      expect(streams[0]).to.have.property('file')
+      expect(streams[0]).to.have.property('size')
+    })
+    xit('should use Voddo\'s stats history to get filenames of only the most recent stream', async function() {
+      const sb = sinon.createSandbox()
+      const viddo = new Voddo({
+        url: 'https://example.com',
+        cwd: '~/Downloads'
+      })
+      sb.stub(viddo, 'stats').value({
+        segments: [{
+          startTime: 1674147647000,
+          size: 192627,
+          file: 'projektmelody 2023-01-19 17_00-projektmelody.mp4'
+        }, {
+          startTime: 1674151247000,
+          size: 192627,
+          file: 'projektmelody 2023-01-19 18_00-projektmelody.mp4'
+        }, {
+          startTime: 1674154847000,
+          size: 192627,
+          file: 'projektmelody 2023-01-19 19_00-projektmelody.mp4'
+        }, {
+          file: 'projektmelody 2023-01-20 20_10-projektmelody.mp4',
+          size: 192627,
+          startTime: 1674245400000,
+        }, {
+          file: 'projektmelody 2023-01-20 21_10-projektmelody.mp4',
+          size: 192627,
+          startTime: 1674249000000,
+        }, {
+          file: 'projektmelody 2023-01-20 22_10-projektmelody.mp4',
+          size: 192627,
+          startTime: 1674252600000,
+        }]
+      })
+
+      const filenames = await viddo.getRecordedSegments()
+      sb.restore()
+      expect(filenames).to.have.lengthOf(3)
+      expect(filenames).to.deep.equal([{
+        file: 'projektmelody 2023-01-20 20_10-projektmelody.mp4',
+        size: 192627,
+        startTime: 1674245400000,
+      }, {
+        file: 'projektmelody 2023-01-20 21_10-projektmelody.mp4',
+        size: 192627,
+        startTime: 1674249000000,
+      }, {
+        file: 'projektmelody 2023-01-20 22_10-projektmelody.mp4',
+        size: 192627,
+        startTime: 1674252600000,
+      }])
+    })
+  })
+
+
+  xit('should keep a log of the files downloaded', function(done) {
+    const ee = new EventEmitter()
+
+
+    const ytdl = sinon.createStubInstance(YoutubeDlWrap)
+    ytdl.exec.returns(ee)
+
+
+    const times = [
+      1000, // start
+      1000 * 60 * 60 * 1, // stop
+      1000 * 60 * 60 * 1 + 1, // start
+      1000 * 60 * 60 * 2, // stop
+      1000 * 60 * 60 * 3 + 1, // start
+      1000 * 60 * 60 * 4 // stop
+    ]
+
+    clock.setTimeout(() => {
+      ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 21_10-projektmelody.mp4')
+    }, times[0])
+
+    clock.setTimeout(() => {
+      ee.emit('close')
+    }, times[1])
+
+    clock.setTimeout(() => {
+      ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 22_10-projektmelody.mp4')
+    }, times[2])
+
+    clock.setTimeout(() => {
+      ee.emit('close')
+    }, times[3])
+
+    clock.setTimeout(() => {
+      ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 23_10-projektmelody.mp4')
+    }, times[4])
+
+    clock.setTimeout(() => {
+      ee.emit('close')
+    }, times[5])
+
+
+    let url = `https://chaturbate.com/projektmelody`
+    let cwd = process.env.FUTUREPORN_WORKDIR || '/tmp'
+    const voddo = new Voddo({
+      url: url,
+      format: 'best',
+      cwd: cwd,
+      ytdl
+    })
+
+    voddo.once('start', (data) => {
+      expect(data).to.have.property('file')
+      expect(data).to.have.property('timestamp')
+
+      voddo.once('start', (data) => {
+        expect(data).to.have.property('file')
+        expect(data).to.have.property('timestamp')
+
+        voddo.once('start', (data) => {
+          expect(data).to.have.property('file')
+          expect(data).to.have.property('timestamp')
+
+          voddo.once('stop', function(report) {
+            debug(report)
+            expect(report).to.have.property('stats')
+            expect(report.stats).to.have.property('files')
+            expect(report.stats.files).to.have.lengthOf(3)
+            debug(report.stats.files)
+            expect(report.stats.files[0]).to.include({
+              file: 'projektmelody 2023-01-18 21_10-projektmelody.mp4'
+            })
+
+            expect(ytdl.exec).calledThrice
+
+            console.log('>>WE ARE DONE')
+            expect(this.clock.countTimers()).to.equal(0)
+            done()
+          })
+          clock.tick(times[5]) // stop
+
+        })
+        clock.tick(times[3]) // stop
+        clock.tick(times[4]) // start
+
+      })
+      clock.tick(times[1]) // stop
+      clock.tick(times[2]) // start
+
+    })
+
+
+    voddo.start()
+    expect(ytdl.exec).calledOnce
+
+    clock.tick(times[0])
+
+
+
+
+  })
+
+  xit('should keep a log of the files downloaded', function(done) {
+    this.timeout(5000)
+      // https://github.com/insanity54/futureporn/issues/13
+    const ytdlStub = sinon.createStubInstance(YoutubeDlWrap)
+    ytdlStub.exec
+      .onCall(0)
+      .callsFake(function(args, opts, aborter) {
+        let ee = new EventEmitter()
+        clock.setTimeout(() => {
+          ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 21_10-projektmelody.mp4')
+        }, 50)
+        clock.setTimeout(() => {
+          ee.emit('close')
+        }, 100)
+        return ee
+      })
+      .onCall(1)
+      .callsFake(function(args, opts, aborter) {
+        let ee = new EventEmitter()
+        clock.setTimeout(() => {
+          ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 22_10-projektmelody.mp4')
+        }, 50)
+        clock.setTimeout(() => {
+          ee.emit('close')
+        }, 100)
+        return ee
+      })
+      .onCall(2)
+      .callsFake(function(args, opts, aborter) {
+        let ee = new EventEmitter()
+        clock.setTimeout(() => {
+          ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 23_10-projektmelody.mp4')
+        }, 50)
+        clock.setTimeout(() => {
+          ee.emit('close')
+        }, 100)
+        return ee
+      })
+    let url = `https://chaturbate.com/projektmelody`
+    let cwd = process.env.FUTUREPORN_WORKDIR || '/tmp'
+
+    const voddo = new Voddo({
+      url: url,
+      format: 'best',
+      cwd: cwd,
+      ytdl: ytdlStub
+    })
+
+    // expect(clock.countTimers()).to.equal(0)
+    voddo.once('start', function(data) {
+      expect(data).to.have.property('file')
+      expect(data).to.have.property('timestamp')
+
+      clock.next()
+      clock.next()
+      voddo.once('start', function(data) {
+        expect(data).to.have.property('file')
+        expect(data).to.have.property('timestamp')
+
+        voddo.once('start', function(data) {
+          debug('fake start?')
+          expect(data).to.have.property('file')
+          expect(data).to.have.property('timestamp')
+
+          voddo.once('stop', function(report) {
+            debug(report)
+            expect(report).to.have.property('stats')
+            expect(report.stats).to.have.property('files')
+            expect(report.stats.files).to.have.lengthOf(3)
+            debug(report.stats.files)
+            expect(report.stats.files[0]).to.include({
+              file: 'projektmelody 2023-01-18 21_10-projektmelody.mp4'
+            })
+
+            sinon.assert.calledThrice(ytdlStub.exec)
+            expect(this.clock.countTimers()).to.equal(0)
+            done()
+          })
+
+
+        })
+      })
+    })
+
+    voddo.start()
+  })
+
+
+  it('should start and stop stream download', test(function(done) {
+
+    const sandbox = this
+
+    const ee = new EventEmitter()
+
+    const ytdl = this.createStubInstance(YoutubeDlWrap);
+    ytdl.exec.returns(ee)
+
+
+    const url = 'https://chaturbate.com/projektmelody'
+    const format = 'best'
+    const cwd = '/tmp'
+    const v = new Voddo({
+      url,
+      format,
+      cwd,
+      ytdl
+    })
+    console.log(v.stats)
+
+    v.once('stop', function(data) {
+      console.log('ffffff')
+      console.log(this)
+      expect(this.abortController.signal.aborted, 'abortController did not abort').to.be.true
+      expect(sandbox.clock.countTimers()).to.equal(0)
+      done()
+    })
+    v.once('start', function(data) {
+      console.log('STARRRRRT')
+      expect(data).to.have.property('file')
+      expect(data).to.have.property('timestamp')
+      expect(this).to.have.property('abortController')
+      console.log('ey cool, voddo started')
+    })
+    v.start()
+
+    const times = [
+      500,
+      1000,
+      2000
+    ]
+
+    this.clock.setTimeout(() => {
+      ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-18 21_10-projektmelody.mp4')
+    }, times[0])
+
+    this.clock.setTimeout(() => {
+      v.stop()
+    }, times[1])
+
+    this.clock.setTimeout(() => {
+      ee.emit('close')
+    }, times[2])
+
+    this.clock.tick(times[0]) // start
+    this.clock.tick(times[1]) // stop
+    this.clock.tick(times[2]) // close
+
+  }))
+
+
+  xit('should retry when a stream closes', function(done) {
+
+    const ytdlStub = sinon.createStubInstance(YoutubeDlWrap);
+    ytdlStub.exec
+      .onCall(0)
+      .callsFake(function(args, opts, aborter) {
+        debug('  [test] callsFake 0')
+        let ee = new EventEmitter()
+        setTimeout(() => {
+          console.log('should retry when a stream closes -- emission')
+          ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-17 19_39-projektmelody.mp4')
+        }, 100)
+        setTimeout(() => {
+          console.log('should retry when a stream closes -- emission')
+          // this simulates youtube-dl closing
+          // (NOT Voddo closing)
+          ee.emit('close')
+        }, 550)
+        return ee
+      })
+      .onCall(1)
+      .callsFake(function(args, opts, aborter) {
+        debug('  [test] callsFake 1')
+        let ee = new EventEmitter()
+        setTimeout(() => {
+          ee.emit('youtubeDlEvent', 'download', ' Destination: projektmelody 2023-01-17 19_45-projektmelody.mp4')
+        }, 100)
+        return ee
+      })
+    let url = `https://chaturbate.com/projektmelody`
+    let cwd = process.env.FUTUREPORN_WORKDIR || '/tmp'
+    let abortController = new AbortController()
+
+    const voddo = new Voddo({
+      url: url,
+      format: 'best',
+      cwd: cwd,
+      ytdl: ytdlStub
+    })
+
+    voddo.once('start', function(data) {
+      debug('  [test] voddo <<<<<-----')
+      expect(data).to.have.property('file')
+      expect(data).to.have.property('timestamp')
+
+      voddo.once('start', function(data) {
+        debug('  [test] restarted after dl close! (expected) <<<<<-----')
+
+        sinon.assert.calledTwice(ytdlStub.exec)
+        expect(this.clock.countTimers()).to.equal(0)
+        done()
+      })
+    })
+
+    voddo.start()
+
+    clock.next()
+    clock.next()
+    clock.next()
+    clock.next()
+    clock.next()
+
+  })
+
+})

packages/capture/test/integration/Capture.test.js

@@ -0,0 +1,123 @@
+import 'dotenv/config'
+import chai, { expect } from 'chai'
+import sinon from 'sinon'
+import sinonChai from 'sinon-chai'
+import { CID } from 'multiformats/cid'
+import EventEmitter from 'node:events'
+import { fileURLToPath } from 'url';
+import path from 'node:path'
+import postgres from 'postgres'
+import Capture from '../src/Capture.js'
+import Voddo from '../src/Voddo.js'
+import Video from '../src/Video.js'
+
+chai.use(sinonChai)
+
+if (typeof process.env.POSTGRES_PASSWORD === 'undefined') throw new Error('missing POSTGRES_PASSWORD');
+if (typeof process.env.POSTGRES_USERNAME === 'undefined') throw new Error('missing POSTGRES_USERNAME');
+
+
+const cidFixture = 'bafybeid3mg5lzrvnmpfi5ftwhiupp7i5bgkmdo7dnlwrvklbv33telrrry'
+const inputFixture = 'projektmelody 3021-10-16 06-16.mp4'
+const outputFixture = 'projektmelody-chaturbate-30211016T000000Z.mp4'
+const timestampFixture = 33191316900000
+
+describe('Capture integration', function () {
+
+  let clock
+
+  beforeEach(() => {
+    clock = sinon.useFakeTimers({
+      toFake: ["setTimeout", "setInterval"],
+      shouldAdvanceTime: false
+    });
+  })
+
+  afterEach(() => {
+    sinon.restore()
+    clock.restore()
+  })
+
+  it('end of stream behavior', async function() {
+    const ipfsClusterUpload = sinon.mock()
+      .withExactArgs(outputFixture)
+      .resolves(cidFixture)
+
+    const sql = postgres({
+      username: process.env.POSTGRES_USERNAME,
+      password: process.env.POSTGRES_PASSWORD,
+      host: process.env.POSTGRES_HOST,
+      database: 'futureporn',
+      idle_timeout: 1
+    })
+
+
+    const voddo = sinon.createStubInstance(Voddo)
+    voddo.on.callThrough()
+    voddo.off.callThrough()
+    voddo.emit.callThrough()
+    voddo.listeners.callThrough()
+    voddo.listenerCount.callThrough()
+    voddo.getFilenames.returns([{
+      timestamp: timestampFixture,
+      filename: inputFixture
+    }])
+
+    const video = sinon.createStubInstance(Video)
+    video.concat.resolves(outputFixture)
+
+    const capture = new Capture({
+      voddo,
+      sql,
+      ipfsClusterUpload,
+      video
+    })
+
+    capture.download()
+    voddo.emit('stop', {
+      reason: 'close',
+      stats: {
+        filenames: [
+          inputFixture
+        ]
+      }
+    })
+
+    clock.next() // actionTimer elapse
+
+
+    expect(clock.countTimers()).to.equal(0)
+    clock.restore()
+
+    // gotta wait to verify otherwise verification
+    // occurs before ipfsClusterUpload has a chance
+    // to be invoked. 
+    //
+    // (not sure why)
+    // 
+    // maybe we're waiting for the 
+    // concat promise to resolve?
+
+    await Promise.resolve(() => {
+      expect(ipfsClusterUpload).calledOnce
+    })
+
+    // Capture.save is called as a side effect
+    // of Capture.process
+    // which is called as a side effect of Capture.download
+    // so we have to wait for it to complete
+    // this is not ideal because there is potential
+    // to not wait long enough
+    await new Promise((resolve) => {
+      setTimeout(resolve, 1000)
+    })
+
+    const rows = await sql`SELECT "videoSrcHash" FROM vod WHERE "videoSrcHash" = ${cidFixture}`
+
+    expect(rows[0]).to.exist
+    expect(rows[0]).to.have.property('videoSrcHash', cidFixture)
+
+
+
+  })
+})

packages/capture/test/integration/Ipfs.test.js

@@ -0,0 +1,18 @@
+
+import Ipfs from '../src/Ipfs.js'
+import { expect } from 'chai'
+import path, { dirname } from 'path';
+import { fileURLToPath } from 'url';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const ipfsExecutable = '/home/chris/.local/bin/ipfs'
+
+describe('Ipfs', function() {
+  describe('hash', function () {
+    it('should hash a file and return the v1 CID', async function () {
+      const ipfs = new Ipfs({ ipfsExecutable })
+      const cid = await ipfs.hash(path.join(__dirname, '../test/fixtures/mock-stream0.mp4'))
+      expect(cid).to.equal('bafkreihfbftehabfrakhr6tmbx72inewwpayw6cypwgm6lbhbf7mxm7wni')
+    })
+  })
+})

packages/capture/test/integration/Voddo.test.js

@@ -0,0 +1,62 @@
+import 'dotenv/config'
+import Voddo from '../src/Voddo.js'
+import {
+  expect
+} from 'chai'
+import sinon from 'sinon'
+import YoutubeDlWrap from 'youtube-dl-wrap'
+import {
+  EventEmitter
+} from 'events'
+import { getRandomRoom } from '../src/cb.js'
+import path, { dirname } from 'path';
+import { fileURLToPath } from 'url';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+
+describe('voddo', function() {
+
+
+  describe('getVideoLength', function () {
+    it('should return the video length in ms', async function () {
+      const fixtureFile = path.join(__dirname, '..', 'test', 'fixtures', 'mock-stream0.mp4')
+      const length = await Voddo.getVideoLength(fixtureFile)
+      expect(length).to.equal(3819)
+    })
+  })
+
+  it('aborted stream', function(done) {
+    this.timeout(10000)
+
+    getRandomRoom().then((room) => {
+      console.log(room)
+      const abortController = new AbortController()
+
+      const url = `https://chaturbate.com/${room}`
+      const format = 'best'
+      const cwd = '/tmp'
+      const voddo = new Voddo({
+        url,
+        format,
+        cwd
+      })
+
+
+      voddo.once('stop', function(data) {
+        console.log('f in chat')
+        expect(voddo.stats.files[0]).to.have.property('size')
+        done()
+      })
+
+      voddo.start()
+
+      setTimeout(() => {
+        voddo.stop()
+      }, 5000)
+    })
+
+
+  })
+
+
+})

packages/capture/test/integration/record.test.js

@@ -0,0 +1,35 @@
+import { record, assertDependencyDirectory } from '../../src/record.js'
+import { getRandomRoom } from '../../src/cb.js'
+import path from 'node:path'
+import os from 'node:os'
+import { execa } from 'execa'
+
+describe('record', function() {
+  it('should record a file to disk', async function () {
+    this.timeout(1000*60)
+    const roomName = await getRandomRoom()
+    console.log(`roomName:${roomName}`)
+    const appContext = {
+      env: {
+        FUTUREPORN_WORKDIR: os.tmpdir(),
+        DOWNLOADER_UA: "Mozilla/5.0 (X11; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0"
+      },
+      logger: {
+        log: (msg) => { console.log(JSON.stringify(msg)) }
+      }
+    }
+    console.log(appContext)
+    const { stdout } = await execa('yt-dlp', ['-g', `https://chaturbate.com/${roomName}`])
+    const playlistUrl = stdout.trim()
+    console.log(`playlistUrl:${playlistUrl}`)
+    assertDependencyDirectory(appContext)
+    const ffmpegProc = record(appContext, playlistUrl, roomName)
+    // console.log(ffmpegProc)
+    return new Promise((resolve) => {
+      setTimeout(() => {
+        ffmpegProc.kill('SIGINT')
+        resolve()
+      }, 1000*10)
+    })
+  })
+})

packages/capture/test/integration/video.test.js

@@ -0,0 +1,33 @@
+
+import 'dotenv/config'
+import Video from '../src/Video.js'
+import { expect } from 'chai'
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import path from 'node:path'
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const dataFixture = [
+  {
+      timestamp: 1,
+      file: 'mock-stream0.mp4'
+  }, {
+      timestamp: 2,
+      file: 'mock-stream1.mp4'
+  }, {
+      timestamp: 3,
+      file: 'mock-stream2.mp4'
+  }
+]
+
+xdescribe('video', function () {
+  describe('concat', function () {
+    it('should combine several videos into one', async function() {
+      const cwd = path.join(__dirname, './fixtures')
+      const outputFile = await concat(dataFixture, {
+        cwd
+      })
+    })
+  })
+})

packages/next/app/archive/[cuid]/not-found.tsx

@@ -4,9 +4,9 @@ export default function NotFound() {
   return (
     <div className='section'>
       <h2 className='title is-2'>404 Not Found</h2>
-      <p>Could not find that stream.</p>
+      <p>Could not find that stream archive.</p>
 
-      <Link href="/s">Return to streams list</Link>
+      <Link href="/s">Return to archive list</Link>
     </div>
   )
 }

packages/next/app/archive/page.tsx

@@ -0,0 +1,91 @@
+import Pager from "@/components/pager";
+import StreamsList from "@/components/streams-list";
+import StreamsTable from '@/components/streams-table';
+import { getAllStreams, getStreamsForVtuber } from "@/lib/streams";
+// import { getAllVtubers } from "@/lib/vtubers";
+import { notFound } from "next/navigation";
+
+
+
+export default async function Page() {
+    // const vtubers = await getAllVtubers();
+    // const streams = await getAllStreams();
+    // const streams = await getStreamsForVtuber(1)
+    // const pageSize = 100;
+    // const page = 1;
+
+
+    // export interface IStream {
+    //     id: number;
+    //     attributes: {
+    //         date: string;
+    //         archiveStatus: 'good' | 'issue' | 'missing';
+    //         vods: IVodsResponse;
+    //         cuid: string;
+    //         vtuber: IVtuberResponse;
+    //         tweet: ITweetResponse;
+    //         isChaturbateStream: boolean;
+    //         isFanslyStream: boolean;
+    //     }
+    // }
+
+    // if (!vtubers) notFound();
+    // const streams = [
+    //     {
+    //         "firstName": "Tanner",
+    //         "lastName": "Linsley",
+    //         "age": 33,
+    //         "visits": 100,
+    //         "progress": 50,
+    //         "status": "Married",
+    //         "id": 5,
+    //         "attributes": {
+    //             date: '2023-10-10T15:18:20.003Z',
+    //             archiveStatus: 'missing',
+    //             isChaturbateStream: false,
+    //             isFanslyStream: true,
+    //             vods: {},
+    //             cuid: '2983482932384',
+    //             vtuber: {},
+    //             tweet: '',
+    //         }
+    //     },
+    //     {
+    //         "firstName": "Kevin",
+    //         "lastName": "Vandy",
+    //         "age": 27,
+    //         "visits": 200,
+    //         "progress": 100,
+    //         "status": "Single",
+    //         "id": 3,
+    //         "attributes": {
+    //             date: '2023-10-10T15:18:20.003Z',
+    //             archiveStatus: 'missing',
+    //             isChaturbateStream: true,
+    //             isFanslyStream: true,
+    //             vods: {},
+    //             cuid: '29823432384',
+    //             vtuber: {},
+    //             tweet: '',
+    //         }
+    //     }
+    // ]
+
+    return (
+        <div className="section">
+            {/* <pre>
+                <p>here are the streams object</p>
+                <code>
+                    {JSON.stringify(streams, null, 2)}
+                </code>
+            </pre> */}
+
+
+
+            <h1 className="title">Stream Archive</h1>
+            <StreamsTable />
+            {/* <StreamsList vtubers={vtubers} page={page} pageSize={pageSize} />
+            <Pager baseUrl="/streams" page={page} pageCount={vtubers.length/pageSize}/> */}
+        </div>
+    )
+}

packages/next/app/components/archive-progress.tsx

@@ -7,7 +7,7 @@ export interface IArchiveProgressProps {
 }
 
 export default async function ArchiveProgress ({ vtuber }: IArchiveProgressProps) {
-    const vods = await getVodsForVtuber(vtuber.id)
+    // const vods = await getVodsForVtuber(vtuber.id)
     // const streams = await getAllStreamsForVtuber(vtuber.id);
     // const goodStreams = await getAllStreamsForVtuber(vtuber.id, ['good']);
     // const issueStreams = await getAllStreamsForVtuber(vtuber.id, ['issue']);
@@ -16,17 +16,21 @@ export default async function ArchiveProgress ({ vtuber }: IArchiveProgressProps
 
     // // Check if totalStreams is not zero before calculating completedPercentage
     // const completedPercentage = (totalStreams !== 0) ? Math.round(eligibleStreams / totalStreams * 100) : 0;
-    // return (
+    const completedPercentage = 50
-    //     <div>
+    const totalStreams = 500
-    //         <p className="heading">{eligibleStreams}/{totalStreams} Streams Archived ({completedPercentage}%)</p>
+    const eligibleStreams = 50
-    //         <progress className="progress is-success" value={eligibleStreams} max={totalStreams}>{completedPercentage}%</progress>
-    //     </div>
-    // )
-    // @todo
-
     return (
         <div>
-            <i><p className="">{(vods) ? vods.data.length : 0} vods</p></i>
+            <p>@todo</p>
+            <p className="heading">{eligibleStreams}/{totalStreams} Streams Archived ({completedPercentage}%)</p>
+            <progress className="progress is-success" value={eligibleStreams} max={totalStreams}>{completedPercentage}%</progress>
         </div>
     )
+    // @todo
+
+    // return (
+    //     <div>
+    //         <i><p className="">{(vods) ? vods.data.length : 0} vods</p></i>
+    //     </div>
+    // )
 }

packages/next/app/components/footer.tsx

@@ -17,7 +17,7 @@ export default function Footer() {
                             <ul>
                                 <li><Link href="#top">&uarr; Top of page</Link></li>
                                 <li><Link href="/vt">Vtubers</Link></li>
-                                {/* <li><Link href="/streams">Stream Archive</Link></li> */}
+                                <li><Link href="/archive">Archive</Link></li>
                                 <li><Link href="/about">About</Link></li>
                                 <li><Link href="/faq">FAQ</Link></li>
                                 <li><Link href="/goals">Goals</Link></li>

packages/next/app/components/navbar.tsx

@@ -43,7 +43,7 @@ export default function Navbar() {
                 <div className={`navbar-menu ${isExpanded ? 'is-active' : ''}`} id="navMenu">
                     <div className='navbar-start'>
                         <Link className="navbar-item is-expanded" href="/vt">Vtubers</Link>
-                        {/* <Link className="navbar-item is-expanded" href="/streams">Stream Archive</Link> */}
+                        <Link className="navbar-item is-expanded" href="/archive">Archive</Link>
                         <Link className="navbar-item is-expanded" href="/about">About</Link>
                         <Link className="navbar-item is-expanded" href="/faq">FAQ</Link>
                         <Link className="navbar-item is-expanded" href="/goals">Goals</Link>

packages/next/app/components/stream-button.tsx

@@ -8,7 +8,7 @@ export function StreamButton({ stream }: { stream: IStream }) {
 
     return (
         <Link 
-            href={`/streams/${stream.attributes.cuid}`}
+            href={`/archive/${stream.attributes.cuid}`}
             className="button is-medium"
         >
             <span className="mr-2"><FontAwesomeIcon icon={faCalendar} className="fas fa-calendar" /></span><span>{new Date(stream.attributes.date).toLocaleDateString()}</span>

packages/next/app/components/stream-page.tsx

@@ -89,13 +89,22 @@ export default function StreamPage({ stream }: IStreamProps) {
     //     </pre>
 
     // </p>
-    // const platformsList = '???';
+    // const platformsList = [
-    const { isChaturbateInvite, isFanslyInvite } = stream.attributes.tweet.data.attributes;
+    //     stream.attributes.isChaturbateStream ? 'Chaturbate' : null,
-    const platformsArray = [
+    //     stream.attributes.isFanslyStream ? 'Fansly' : null
-        isChaturbateInvite ? 'Chaturbate' : null,
+    // ].filter(Boolean).join(', ');
-        isFanslyInvite ? 'Fansly' : null
+    // platformsList = platformsArray.length > 0 ? platformsArray.join(', ') : 'None';
-    ].filter(Boolean);
+    
-    const platformsList = platformsArray.length > 0 ? platformsArray.join(', ') : 'None';
+    // const platformsList = [
+    //     (stream.attributes.isChaturbateStream && 'CB'), 
+    //     (stream.attributes.isFanslyStream && 'Fansly')
+    // ].filter(Boolean).join(', ')
+
+    const platformsList = [
+        (stream.attributes.isChaturbateStream && 'CB'), 
+        (stream.attributes.isFanslyStream && 'Fansly')
+    ].filter(Boolean).join(', ') || '!!!';
+
     
 
     return (
@@ -110,23 +119,39 @@ export default function StreamPage({ stream }: IStreamProps) {
                 <div className="section columns is-multiline">
                     <div className="column is-half">
                         <div className="box">
-                            <h2 className="title is-3">Details</h2>
                             <div className="columns is-multiline">
                                 <div className="column is-full">
-                                    <span><b>Announcement</b>&nbsp;<span><Link target="_blank" href={stream.attributes.tweet.data.attributes.url}><FontAwesomeIcon icon={faXTwitter}></FontAwesomeIcon><FontAwesomeIcon icon={faExternalLinkAlt}></FontAwesomeIcon></Link></span></span><br></br>
+                                    <table className="table">
-                                    <span><b>Platform</b>&nbsp;</span><span>{platformsList}</span><br></br>
+                                        <thead>
-                                    <span><b>UTC Datetime</b>&nbsp;</span><time dateTime={date.toISOString()}>{date.toISOString()}</time><br></br>
+                                            <tr>
-                                    <span><b>Local Datetime</b>&nbsp;</span><span>{date.toLocaleDateString()} {date.toLocaleTimeString()}</span><br></br>
+                                                <th className="is-family-sans-serif">Description</th>
-                                    <span><b>Lunar Phase</b>&nbsp;</span><span>{Moon.lunarPhase(date)} {Moon.lunarPhaseEmoji(date, { hemisphere })}</span><br></br>
+                                                <th className="is-family-sans-serif">Details</th>
-                                    <br></br>
+                                            </tr>
-                                    {/* <select className="mt-5"
+                                        </thead>
-                                        value={selectedStatus}
+                                        <tbody>
-                                        onChange={e => setSelectedStatus(e.target.value as Status)}
+                                            {/* <tr>
-                                    >
+                                                <td>Announcement</td>
-                                        <option>good</option>
+                                                <td><Link target="_blank" href={stream.attributes.tweet.data.attributes.url}><FontAwesomeIcon icon={faXTwitter}></FontAwesomeIcon><FontAwesomeIcon icon={faExternalLinkAlt}></FontAwesomeIcon></Link></td>
-                                        <option>issue</option>
+                                            </tr> */}
-                                        <option>missing</option>
+                                            <tr>
-                                    </select> */}
+                                                <td>Platform</td>
+                                                <td>{platformsList}</td>
+                                            </tr>
+                                            <tr>
+                                                <td>UTC Datetime</td>
+                                                <td><time dateTime={date.toISOString()}>{date.toISOString()}</time></td>
+                                            </tr>
+                                            <tr>
+                                                <td>Local Datetime</td>
+                                                <td>{date.toLocaleDateString()} {date.toLocaleTimeString()}</td>
+                                            </tr>
+                                            <tr>
+                                                <td>Lunar Phase</td>
+                                                <td>{Moon.lunarPhase(date)} {Moon.lunarPhaseEmoji(date, { hemisphere })}</td>
+                                            </tr>
+                                        </tbody>
+                                    </table>
+
                                 </div>
                             </div>
                         </div>
@@ -150,6 +175,7 @@ export default function StreamPage({ stream }: IStreamProps) {
                 </div>
 
 
+                {stream.attributes.vods.data.length !== 0 &&
                 <div className="section">
                     <h1 className="title">VODs</h1>
                     <table className="table">
@@ -179,7 +205,7 @@ export default function StreamPage({ stream }: IStreamProps) {
                             ))}
                         </tbody>
                     </table>
-                </div>
+                </div>}
 
             </div>
 

packages/next/app/components/streams-table.tsx

@@ -1,318 +1,249 @@
 'use client'
+import React from 'react'
+import ReactDOM from 'react-dom/client'
+import Link from 'next/link'
+import {
+  keepPreviousData,
+  QueryClient,
+  useQuery,
+} from '@tanstack/react-query'
+
 
 import {
-    Column,
+  PaginationState,
-    Table as ReactTable,
+  useReactTable,
-    useReactTable,
+  getCoreRowModel,
-    ColumnFiltersState,
+  ColumnDef,
-    getCoreRowModel,
+  flexRender,
-    getFilteredRowModel,
-    getFacetedRowModel,
-    getFacetedUniqueValues,
-    getFacetedMinMaxValues,
-    getPaginationRowModel,
-    sortingFns,
-    getSortedRowModel,
-    FilterFn,
-    SortingFn,
-    ColumnDef,
-    flexRender,
-    FilterFns,
-    ColumnOrderState,
-    createColumnHelper,
 } from '@tanstack/react-table'
-import Image from 'next/image';
-import { useState } from "react";
-import { IStream } from "@/lib/streams";
-import { LocalizedDate } from './localized-date';
-import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
-import { faAngleLeft, faAngleRight, faAnglesLeft, faAnglesRight, faChevronCircleRight, faChevronRight } from '@fortawesome/free-solid-svg-icons';
-import Link from 'next/link';
-
-function Filter({
-    column,
-    table,
-}: {
-    column: Column<any, any>
-    table: ReactTable<any>
-}) {
-    const firstValue = table
-        .getPreFilteredRowModel()
-        .flatRows[0]?.getValue(column.id)
-
-    const columnFilterValue = column.getFilterValue()
 
 
+import { fetchStreamData, IStream } from '@/lib/streams'
 
-    if (typeof firstValue === 'number') {
+const queryClient = new QueryClient()
-        return (
-            <div className="flex space-x-2">
-                <input
-                    type="number"
-                    value={(columnFilterValue as [number, number])?.[0] ?? ''}
-                    onChange={e =>
-                        column.setFilterValue((old: [number, number]) => [
-                            e.target.value,
-                            old?.[1],
-                        ])
-                    }
-                    placeholder={`Min`}
-                    className="w-24 border shadow rounded"
-                />
-                <input
-                    type="number"
-                    value={(columnFilterValue as [number, number])?.[1] ?? ''}
-                    onChange={e =>
-                        column.setFilterValue((old: [number, number]) => [
-                            old?.[0],
-                            e.target.value,
-                        ])
-                    }
-                    placeholder={`Max`}
-                    className="w-24 border shadow rounded"
-                />
-            </div>
-            )
-        }
 
-    if (typeof firstValue === 'boolean') {
+function getStatusClass(value: string) {
-        return (
+  switch (value) {
-            <>
+    case 'issue':
-            <div className='select'>
+      return 'is-warning';
-                <select
+    case 'missing':
-                    onChange={(evt) => {          
+      return 'is-danger';
-                        if (evt.target.value === "any")
+    case 'good':
-                        column?.setFilterValue(null);
+      return 'is-success';
-                        if (evt.target.value === "yes")
+    default:
-                        column?.setFilterValue(true);
+      return '';
-                        if (evt.target.value === "no")
+  }
-                        column?.setFilterValue(false);
-                    }}
-                >
-                    <option>any</option>
-                    <option>yes</option>
-                    <option>no</option>
-                </select>
-            </div>
-        </>
-        )
-    }
-    
-    return (
-        <input
-            type="text"
-            value={(columnFilterValue ?? '') as string}
-            onChange={e => column.setFilterValue(e.target.value)}
-            placeholder={`Search...`}
-            className="input"
-        />
-    )
 }
 
-const archiveStatusClassName = (archiveStatus: string): string => {
+export default function StreamsTable() {
-    if (archiveStatus === 'missing') return 'is-danger';
+  const rerender = React.useReducer(() => ({}), {})[1]
-    if (archiveStatus === 'issue') return 'is-warning';
-    if (archiveStatus === 'good') return 'is-success';
-    return 'is-info';
-};
-
-export default function StreamsTable({ streams }: { streams: IStream[] }) {
 
 
-
+  // name
-    const columnHelper = createColumnHelper<IStream>()
+  // title
-
+  // platform
-
+  // date
-    const columns = [
+  // archiveStatus
-        columnHelper.accessor('attributes.cuid', {
+  const columns = React.useMemo<ColumnDef<IStream>[]>(
-            cell: info => <Link href={`/streams/${info.getValue()}`}>{info.getValue()}</Link>,
+    () => [
-            header: () => <span>ID</span>
+        {
-        }),
+            header: 'VTuber',
-        columnHelper.accessor('attributes.vtuber.data.attributes.image', {
+            accessorFn: d => d.attributes.vtuber.data?.attributes?.displayName,
-            cell: info => <figure className='image is-24x24'><Image className="is-rounded" width={24} height={24} alt="" src={info.getValue()}></Image></figure>,
-            header: () => <span></span>,
-            enableColumnFilter: false
-        }),
-        columnHelper.accessor('attributes.vtuber.data.attributes.displayName', {
-            cell: info => info.getValue(),
-            header: () => <span>VTuber</span>
-        }),
-        columnHelper.accessor('attributes.date', {
-            cell: info => <LocalizedDate date={new Date(info.getValue())}/>,
-            header: () => <span>Date</span>
-        }),
-        columnHelper.accessor('attributes.isChaturbateStream', {
-            id: 'isChaturbateStream',
-            cell: info => info.getValue() === true ? 'yes' : 'no',
-            header: () => <span>Chaturbate</span>
-        }),
-        columnHelper.accessor('attributes.isFanslyStream', {
-            id: 'isFanslyStream',
-            cell: info => info.getValue() === true ? 'yes' : 'no',
-            header: () => <span>Fansly</span>
-        }),
-        columnHelper.accessor('attributes.archiveStatus', {
-            cell: info => <div className={`tag ${archiveStatusClassName(info.getValue())}`} >{info.getValue()}</div>,
-            header: () => <span>Status</span>
-        })
-    ]
-
-    const [columnVisibility, setColumnVisibility] = useState({})
-    const [columnOrder, setColumnOrder] = useState<ColumnOrderState>([])
-    const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
-    const [data, setData] = useState(() => streams)
-
-    const table = useReactTable({
-        data,
-        columns,
-        getCoreRowModel: getCoreRowModel(),
-        getFilteredRowModel: getFilteredRowModel(),
-        getPaginationRowModel: getPaginationRowModel(),
-        state: {
-            columnVisibility,
-            columnOrder,
-            columnFilters
         },
-        onColumnOrderChange: setColumnOrder,
+        {
-        onColumnFiltersChange: setColumnFilters,
+            header: 'Date',
-    })
+            accessorFn: d => new Date(d.attributes.date2).toISOString().split('T').at(0),
+            cell: info => <Link href={`/archive/${info.row.original.attributes.cuid}`}>{info.getValue() as string}</Link>
+        },
+        {
+            header: 'Platform',
+            accessorFn: d => [
+              (d.attributes.isChaturbateStream && 'CB'), 
+              (d.attributes.isFanslyStream && 'Fansly')
+            ].filter(Boolean).join(', ') || '???'
+        },
+        {
+            header: 'Status',
+            accessorFn: d => {
+              if (!d.attributes.archiveStatus) return 'missing';
+              return d.attributes.archiveStatus
+            }
+        },
+    //   {
+    //     header: 'Name',
+    //     footer: props => props.column.id,
+    //     columns: [
+    //       {
+    //         accessorKey: 'firstName',
+    //         cell: info => info.getValue(),
+    //         footer: props => props.column.id,
+    //       },
+    //       {
+    //         accessorFn: row => row.lastName,
+    //         id: 'lastName',
+    //         cell: info => info.getValue(),
+    //         header: () => <span>Last Name</span>,
+    //         footer: props => props.column.id,
+    //       },
+    //     ],
+    //   },
+    ],
+    []
+  )
 
+  const [pagination, setPagination] = React.useState<PaginationState>({
+    pageIndex: 0,
+    pageSize: 50,
+  })
 
+  const dataQuery = useQuery({
+    queryKey: ['streams', pagination.pageIndex, pagination.pageSize],
+    queryFn: () => fetchStreamData(pagination),
+    placeholderData: keepPreviousData, // don't have 0 rows flash while changing pages/loading next page,
+    staleTime: 1000
+  }, queryClient)
 
-    return (
+  const defaultData = React.useMemo(() => [], [])
-        <>
 
+  const table = useReactTable({
+    data: dataQuery?.data?.rows ?? defaultData,
+    columns,
+    // pageCount: dataQuery.data?.pageCount ?? -1, //you can now pass in `rowCount` instead of pageCount and `pageCount` will be calculated internally (new in v8.13.0)
+    rowCount: dataQuery.data?.rowCount, // new in v8.13.0 - alternatively, just pass in `pageCount` directly
+    state: {
+      pagination,
+    },
+    onPaginationChange: setPagination,
+    getCoreRowModel: getCoreRowModel(),
+    manualPagination: true, //we're doing manual "server-side" pagination
+    // getPaginationRowModel: getPaginationRowModel(), // If only doing manual pagination, you don't need this
+    debugTable: true,
+  })
 
-            <div className="p-2">
+  return (
-                <div className="h-2" />
+    <div className="p-2">
-
+      <div className="h-2" />
-
+      <table className='table is-hoverable is-fullwidth'>
-
+        <thead>
-                <table className='table'>
+          {table.getHeaderGroups().map(headerGroup => (
-                    <thead>
+            <tr key={headerGroup.id}>
-                        {table.getHeaderGroups().map(headerGroup => (
+              {headerGroup.headers.map(header => {
-                            <tr key={headerGroup.id}>
+                return (
-                                {headerGroup.headers.map(header => {
+                  <th key={header.id} colSpan={header.colSpan}>
-                                    return (
+                    {header.isPlaceholder ? null : (
-                                        <th key={header.id} colSpan={header.colSpan}>
+                      <div>
-                                            {header.isPlaceholder ? null : (
+                        {flexRender(
-                                                <div>
+                          header.column.columnDef.header,
-                                                    {flexRender(
+                          header.getContext()
-                                                        header.column.columnDef.header,
+                        )}
-                                                        header.getContext()
+                      </div>
-                                                    )}
+                    )}
-                                                    {header.column.getCanFilter() ? (
+                  </th>
-                                                        <div>
+                )
-                                                            <Filter column={header.column} table={table} />
+              })}
-                                                        </div>
+            </tr>
-                                                    ) : null}
+          ))}
-                                                </div>
+        </thead>
-                                            )}
+        <tbody>
-                                        </th>
+          {table.getRowModel().rows.map(row => {
-                                    )
+            return (
-                                })}
+              <tr key={row.id}>
-                            </tr>
+                {row.getVisibleCells().map(cell => {
-                        ))}
+                  return (
-                    </thead>
+                    <td 
-                    <tbody>
+                      className={getStatusClass(cell.getValue() as string)} 
-                        {table.getRowModel().rows.map(row => {
+                      key={cell.id}
-                            return (
-                                <tr key={row.id}>
-                                    {row.getVisibleCells().map(cell => {
-                                        return (
-                                            <td key={cell.id}>
-                                                {flexRender(
-                                                    cell.column.columnDef.cell,
-                                                    cell.getContext()
-                                                )}
-                                            </td>
-                                        )
-                                    })}
-                                </tr>
-                            )
-                        })}
-                    </tbody>
-                </table>
-                <div className="columns is-multiline is-mobile" />
-                <div className="column is-12">
-                    <button
-                        className="button icon is-rounded is-medium p-1 m-1"
-                        onClick={() => table.setPageIndex(0)}
-                        disabled={!table.getCanPreviousPage()}
                     >
-                        <FontAwesomeIcon className='fa-solid fa-angles-left' icon={faAnglesLeft}></FontAwesomeIcon>
+                      {flexRender(
-                    </button>
+                        cell.column.columnDef.cell,
-                    <button
+                        cell.getContext()
-                        className="button icon is-rounded is-medium p-1 m-1"
+                      )}
-                        onClick={() => table.previousPage()}
+                    </td>
-                        disabled={!table.getCanPreviousPage()}
+                  )
-                    >
+                })}
-                        <FontAwesomeIcon className='fa-solid fa-angle-left' icon={faAngleLeft}></FontAwesomeIcon>
+              </tr>
-                    </button>
+            )
-                    <button
+          })}
-                        className="button icon is-rounded is-medium p-1 m-1"
+        </tbody>
-                        onClick={() => table.nextPage()}
+      </table>
-                        disabled={!table.getCanNextPage()}
-                    >
-                        <FontAwesomeIcon className="fa-solid fa-angle-right" icon={faAngleRight}></FontAwesomeIcon>
-                    </button>
-                    <button
-                        className="button icon is-medium is-rounded p-1 m-1"
-                        onClick={() => table.setPageIndex(table.getPageCount() - 1)}
-                        disabled={!table.getCanNextPage()}
-                    >
-                        <FontAwesomeIcon className='fa-solid fa-angles-right' icon={faAnglesRight}></FontAwesomeIcon>
-                    </button>
-                </div>
-                <div className='column is-2'>
       
-                    <div className=''>
+      <div className="columns is-mobile is-vcentered">
-                        <div className=''>
+        <div className='column is-half'>
-                            <span className='mr-1'>Page</span>
+          <button
-                            {table.getState().pagination.pageIndex + 1} of{' '}
+            className="button border rounded mx-1"
-                            {table.getPageCount()}
+            onClick={() => table.firstPage()}
-                        </div>
+            disabled={!table.getCanPreviousPage()}
-                    </div>
+          >
+            {'<<'}
+          </button>
+          <button
+            className="button border rounded mx-1"
+            onClick={() => table.previousPage()}
+            disabled={!table.getCanPreviousPage()}
+          >
+            {'<'}
+          </button>
+          <button
+            className="button border rounded mx-1"
+            onClick={() => table.nextPage()}
+            disabled={!table.getCanNextPage()}
+          >
+            {'>'}
+          </button>
+          <button
+            className="button border rounded mx-1"
+            onClick={() => table.lastPage()}
+            disabled={!table.getCanNextPage()}
+          >
+            {'>>'}
+          </button>
+        </div>
+        <div className='column is-half'>
+          <span>Page </span>
+            <strong>
+              {table.getState().pagination.pageIndex + 1} of{' '}
+              {table.getPageCount().toLocaleString()}
+            </strong>
+        </div>
+      </div>
 
-                    <div className=''>
+      {/* second row with page number input and pages-per-screen select */}
-                            <label className='label'>
+      <div className='columns is-mobile is-vcentered'>
-                                Go to page:
+        <div className='column is-2 '>
-                            </label>
+          <span className='is-text-centered'>Go to page:</span>
-                            <div className="control is-expanded">
+        </div>
-                                <input
+        <div className='column is-3'>
-                                    type="number"
+          <input
-                                    defaultValue={table.getState().pagination.pageIndex + 1}
+            type="number"
-                                    onChange={e => {
+            defaultValue={table.getState().pagination.pageIndex + 1}
-                                        const page = e.target.value ? Number(e.target.value) - 1 : 0
+            onChange={e => {
-                                        table.setPageIndex(page)
+              const page = e.target.value ? Number(e.target.value) - 1 : 0
-                                    }}
+              table.setPageIndex(page)
-                                    className="input p-1"
+            }}
-                                />
+            className="input"
-                            </div>
+          />
-                    </div>
+        </div>
-                </div>
+        <div className='column is-5'>
+          <div className="select">
+            <select
+              value={table.getState().pagination.pageSize}
+              onChange={e => {
+                table.setPageSize(Number(e.target.value))
+              }}
+            >
+            {[20, 50, 100].map(pageSize => (
+              <option key={pageSize} value={pageSize}>
+                Show {pageSize}
+              </option>
+            ))}
+          </select>
+          </div>
+        </div>
+      </div>
 
-                <div className='column is-2'>
-                    <div className="m-1">
-                        <span className='mr-1'>Page</span>
-                    </div>
-                    <div className='select'>
-                        <select
-                            value={table.getState().pagination.pageSize}
-                            onChange={e => {
-                                table.setPageSize(Number(e.target.value))
-                            }}
-                        >
-                            {[10, 20, 30, 40, 50].map(pageSize => (
-                                <option key={pageSize} value={pageSize}>
-                                    Show {pageSize}
-                                </option>
-                            ))}
-                        </select>
-                    </div>
-                </div>
-            </div>
 
-        </>
+
-    )
+      </div>
+    
+  )
 }

packages/next/app/components/tagger.tsx

@@ -156,9 +156,9 @@ export function Tagger({ vod, setTimestamps }: ITaggerProps): React.JSX.Element
         }
     }
 
-    if (!isAuthed) {
+    // if (!isAuthed) {
-        return <></>
+    //     return <></>
-    } else {
+    // } else {
         if (isEditor) {
             return (
                 <div className='card mt-2' style={{ width: '100%' }}>
@@ -234,7 +234,7 @@ export function Tagger({ vod, setTimestamps }: ITaggerProps): React.JSX.Element
                 </button>
             );
         }
-    }
+    // }
     
 
 }

packages/next/app/components/video-player.tsx

@@ -110,7 +110,6 @@ export const VideoPlayer = forwardRef(function VideoPlayer( props: IPlayerProps,
 
     return (
         <>
-            <p className='notification'>CDN1 (for Patrons only)</p>
             <MuxPlayer
                 onCanPlay={() => {
                     setIsPlayerReady(true)}

packages/next/app/components/video-source-selector.tsx

@@ -31,12 +31,16 @@ export function VideoSourceSelector({
             if (isEntitledToCDN) {
                 if (selectedVideoSource === 'Mux' && isMux) {
                     return 'Mux';
-                } else if (selectedVideoSource === 'B2' && isB2) {
-                    return 'B2';
                 }
             }
-            // If the user doesn't have entitlements or their preference is not available, default to IPFS
+
-            if (isIPFSSource) {
+            // if the user has B2 as their preference or they have no preference, use B2
+            if (selectedVideoSource === 'B2' || !selectedVideoSource) {
+                return 'B2'
+            }
+
+            // use IPFS only if the user has opted to use it
+            if (selectedVideoSource === 'IPFSSource' && isIPFSSource) {
                 return 'IPFSSource';
             } else if (isIPFS240) {
                 return 'IPFS240';
@@ -53,8 +57,16 @@ export function VideoSourceSelector({
             // Check if the saved preference is valid based on entitlements and available sources
             if (savedPreference === 'Mux' && isMux && isEntitledToCDN) {
                 setSelectedVideoSource('Mux');
-            } else if (savedPreference === 'B2' && isB2 && isEntitledToCDN) {
+            } else if (savedPreference === 'B2') {
                 setSelectedVideoSource('B2');
+            } else if (savedPreference === 'IPFSSource') {
+                setSelectedVideoSource('IPFSSource');
+            } else if (savedPreference === 'IPFS240') {
+                if (isIPFS240) {
+                    setSelectedVideoSource('IPFS240');
+                } else {
+                    setSelectedVideoSource('IPFSSource');
+                }
             } else {
                 // Determine the best video source if the saved preference is invalid or not available
                 const bestSource = determineBestVideoSource();
@@ -69,7 +81,7 @@ export function VideoSourceSelector({
     const handleSourceClick = (source: string) => {
         if (
             (source === 'Mux' && isMux && isEntitledToCDN) || 
-            (source === 'B2' && isB2 && isEntitledToCDN) || 
+            (source === 'B2' && isB2) || 
             (source === 'IPFSSource') || 
             (source === 'IPFS240')
         ) {
@@ -100,9 +112,9 @@ export function VideoSourceSelector({
                     </button>
                 </div>}
                 {(isB2) && <div className="nav-item">
-                    <button onClick={() => handleSourceClick('B2')} disabled={!isEntitledToCDN} className={`button ${selectedVideoSource === 'B2' && 'is-active'}`}>
+                    <button onClick={() => handleSourceClick('B2')} className={`button ${selectedVideoSource === 'B2' && 'is-active'}`}>
                         <span className="icon">
-                            <FontAwesomeIcon icon={faPatreon} className="fab fa-patreon" />
+                            <FontAwesomeIcon icon={faGlobe} className="fab fa-globe" />
                         </span>
                         <span>CDN 2</span>
                     </button>

packages/next/app/layout.tsx

@@ -7,12 +7,17 @@ import "@fortawesome/fontawesome-svg-core/styles.css";
 import { AuthProvider } from './components/auth';
 import type { Metadata } from 'next';
 import NotificationCenter from './components/notification-center';
-import UppyProvider from './uppy';
+// import {
+//   QueryClientProvider,
+//   QueryClient
+// } from '@tanstack/react-query'
 // import NextTopLoader from 'nextjs-toploader';
 // import Ipfs from './components/ipfs'; // slows down the page too much
 
 
 
+// const queryClient = new QueryClient()
+
 export const metadata: Metadata = {
   title: 'Futureporn.net',
   description: "The Galaxy's Best VTuber Hentai Site",
@@ -56,14 +61,14 @@ export default function RootLayout({
         shadow="0 0 10px #2299DD,0 0 5px #2299DD"
       /> */}
         <AuthProvider>
-          <UppyProvider>
+          {/* <QueryClientProvider client={queryClient}> */}
             <Navbar />
             <NotificationCenter />
             <div className="container">
               {children}
               <Footer />
             </div>
-          </UppyProvider>
+          {/* </QueryClientProvider> */}
         </AuthProvider>
       </body>
     </html>

packages/next/app/lib/fetch-api.ts

@@ -29,6 +29,6 @@ export default async function fetchAPI(
     
   } catch (error) {
     console.error(error);
-    throw new Error(`Error while fetching data from API.`);
+    throw new Error(`Error while fetching data from API. ${path}`);
   }
 }

packages/next/app/lib/streams.ts

@@ -1,5 +1,5 @@
 
-import { strapiUrl, siteUrl } from './constants';
+import { siteUrl, strapiUrl } from './constants';
 import { getSafeDate } from './dates';
 import { IVodsResponse } from './vods';
 import { IVtuber, IVtuberResponse } from './vtubers';
@@ -12,6 +12,7 @@ export interface IStream {
     id: number;
     attributes: {
         date: string;
+        date2: string;
         archiveStatus: 'good' | 'issue' | 'missing';
         vods: IVodsResponse;
         cuid: string;
@@ -35,7 +36,8 @@ export interface IStreamsResponse {
 
 const fetchStreamsOptions = {
     next: {
-        tags: ['streams']
+        tags: ['streams'],
+        revalidation: 1
     }
 }
 
@@ -162,6 +164,7 @@ export async function getStream(id: number): Promise<IStream> {
 
 
 export async function getAllStreams(archiveStatuses = ['missing', 'issue', 'good']): Promise<IStream[]> {
+    throw new Error('getAllStreams function is not performant. please use something more efficient.')
     const pageSize = 100; // Adjust this value as needed
     const sortDesc = true; // Adjust the sorting direction as needed
 
@@ -308,7 +311,50 @@ export async function getAllStreamsForVtuber(vtuberId: number, archiveStatuses =
     return allStreams;
 }
 
+/**
+ * Used as table data on /archive page.
+ * .pageIndex, pagination.pageSize
+ */
+
+export async function fetchStreamData({ pageIndex, pageSize }: { pageIndex: number, pageSize: number }) {
+    const offset = pageIndex * pageSize;
+    const query = qs.stringify({
+        populate: {
+            vtuber: {
+                fields: ['slug', 'displayName', 'publishedAt']
+            }
+        },
+        filters: {
+            vtuber: {
+                publishedAt: {
+                    $notNull: true
+                }
+            }
+        },
+        pagination: {
+            start: offset,
+            limit: pageSize,
+            withCount: true
+        }
+    })
+    const response = await fetch(
+        `${strapiUrl}/api/streams?${query}`
+        );
+        const json = await response.json();
+        console.log(json)
+        const d = {
+            rows: json.data,
+            pageCount: Math.ceil(json.meta.pagination.total / pageSize),
+            rowCount: json.meta.pagination.total,
+        }
+    // console.log(`fetchStreamData with pageIndex=${pageIndex}, pageSize=${pageSize}\n\n${JSON.stringify(d, null, 2)}`)
+    return d;
+}
+
+  
+
 export async function getStreamsForVtuber(vtuberId: number, page: number = 1, pageSize: number = 25, sortDesc = true): Promise<IStreamsResponse> {
+    console.log(`getStreamsForVtuber() with strapiUrl=${strapiUrl}`)
     const query = qs.stringify(
         {
             populate: {
@@ -334,8 +380,10 @@ export async function getStreamsForVtuber(vtuberId: number, page: number = 1, pa
             }
         }
     )
-    return fetch(`${strapiUrl}/api/streams?${query}`, fetchStreamsOptions)
+    const res = await fetch(`${strapiUrl}/api/streams?${query}`, fetchStreamsOptions)
-        .then((res) => res.json())
+    const data = await res.json()
+    console.log(data)
+    return data
 }
 
 

packages/next/app/streams/page.tsx

@@ -1,30 +0,0 @@
-import Pager from "@/components/pager";
-import StreamsList from "@/components/streams-list";
-import StreamsTable from '@/components/streams-table';
-import { getAllStreams } from "@/lib/streams";
-// import { getAllVtubers } from "@/lib/vtubers";
-import { notFound } from "next/navigation";
-
-export default async function Page() {
-    // const vtubers = await getAllVtubers();
-    const streams = await getAllStreams();
-    // const pageSize = 100;
-    // const page = 1;
-
-    // if (!vtubers) notFound();
-
-    return (
-        <div className="section">
-            {/* <pre>
-                <code>
-                    {JSON.stringify(vtubers, null, 2)}
-                </code>
-            </pre> */}
-
-            <h1 className="title">Stream Archive</h1>
-            <StreamsTable streams={streams} />
-            {/* <StreamsList vtubers={vtubers} page={page} pageSize={pageSize} />
-            <Pager baseUrl="/streams" page={page} pageCount={vtubers.length/pageSize}/> */}
-        </div>
-    )
-}

packages/next/package.json

@@ -22,6 +22,7 @@
     "@mux/mux-player-react": "^2.4.1",
     "@paralleldrive/cuid2": "^2.2.2",
     "@react-hookz/web": "^24.0.4",
+    "@tanstack/react-query": "^5.32.1",
     "@tanstack/react-table": "^8.15.3",
     "@types/lodash": "^4.17.0",
     "@types/qs": "^6.9.14",
@@ -53,12 +54,14 @@
     "prism-react-renderer": "^2.3.1",
     "qs": "^6.12.0",
     "react": "^18.2.0",
+    "react-data-table-component": "^7.5.4",
     "react-dom": "^18.2.0",
     "react-hook-form": "^7.51.2",
     "react-loading-skeleton": "^3.4.0",
     "react-toastify": "^9.1.3",
     "sharp": "^0.33.3",
     "slugify": "^1.6.6",
+    "styled-components": "5.3.3",
     "yup": "^1.4.0"
   },
   "devDependencies": {

packages/next/pnpm-lock.yaml

@@ -41,6 +41,9 @@ dependencies:
   '@react-hookz/web':
     specifier: ^24.0.4
     version: 24.0.4(react-dom@18.2.0)(react@18.2.0)
+  '@tanstack/react-query':
+    specifier: ^5.32.1
+    version: 5.32.1(react@18.2.0)
   '@tanstack/react-table':
     specifier: ^8.15.3
     version: 8.15.3(react-dom@18.2.0)(react@18.2.0)
@@ -115,7 +118,7 @@ dependencies:
     version: 13.1.0
   next:
     specifier: 14.0.4
-    version: 14.0.4(react-dom@18.2.0)(react@18.2.0)
+    version: 14.0.4(@babel/core@7.24.5)(react-dom@18.2.0)(react@18.2.0)
   next-goatcounter:
     specifier: ^1.0.5
     version: 1.0.5(next@14.0.4)(react-dom@18.2.0)(react@18.2.0)
@@ -134,6 +137,9 @@ dependencies:
   react:
     specifier: ^18.2.0
     version: 18.2.0
+  react-data-table-component:
+    specifier: ^7.5.4
+    version: 7.5.4(react@18.2.0)(styled-components@5.3.3)
   react-dom:
     specifier: ^18.2.0
     version: 18.2.0(react@18.2.0)
@@ -152,6 +158,9 @@ dependencies:
   slugify:
     specifier: ^1.6.6
     version: 1.6.6
+  styled-components:
+    specifier: 5.3.3
+    version: 5.3.3(@babel/core@7.24.5)(react-dom@18.2.0)(react-is@16.13.1)(react@18.2.0)
   yup:
     specifier: ^1.4.0
     version: 1.4.0
@@ -180,12 +189,234 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: true
 
+  /@ampproject/remapping@2.3.0:
+    resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==}
+    engines: {node: '>=6.0.0'}
+    dependencies:
+      '@jridgewell/gen-mapping': 0.3.5
+      '@jridgewell/trace-mapping': 0.3.25
+    dev: false
+
+  /@babel/code-frame@7.24.2:
+    resolution: {integrity: sha512-y5+tLQyV8pg3fsiln67BVLD1P13Eg4lh5RW9mF0zUuvLrv9uIQ4MCL+CRT+FTsBlBjcIan6PGsLcBN0m3ClUyQ==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/highlight': 7.24.5
+      picocolors: 1.0.0
+    dev: false
+
+  /@babel/compat-data@7.24.4:
+    resolution: {integrity: sha512-vg8Gih2MLK+kOkHJp4gBEIkyaIi00jgWot2D9QOmmfLC8jINSOzmCLta6Bvz/JSBCqnegV0L80jhxkol5GWNfQ==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
+  /@babel/core@7.24.5:
+    resolution: {integrity: sha512-tVQRucExLQ02Boi4vdPp49svNGcfL2GhdTCT9aldhXgCJVAI21EtRfBettiuLUwce/7r6bFdgs6JFkcdTiFttA==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@ampproject/remapping': 2.3.0
+      '@babel/code-frame': 7.24.2
+      '@babel/generator': 7.24.5
+      '@babel/helper-compilation-targets': 7.23.6
+      '@babel/helper-module-transforms': 7.24.5(@babel/core@7.24.5)
+      '@babel/helpers': 7.24.5
+      '@babel/parser': 7.24.5
+      '@babel/template': 7.24.0
+      '@babel/traverse': 7.24.5(supports-color@5.5.0)
+      '@babel/types': 7.24.5
+      convert-source-map: 2.0.0
+      debug: 4.3.4(supports-color@5.5.0)
+      gensync: 1.0.0-beta.2
+      json5: 2.2.3
+      semver: 6.3.1
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /@babel/generator@7.24.5:
+    resolution: {integrity: sha512-x32i4hEXvr+iI0NEoEfDKzlemF8AmtOP8CcrRaEcpzysWuoEb1KknpcvMsHKPONoKZiDuItklgWhB18xEhr9PA==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/types': 7.24.5
+      '@jridgewell/gen-mapping': 0.3.5
+      '@jridgewell/trace-mapping': 0.3.25
+      jsesc: 2.5.2
+    dev: false
+
+  /@babel/helper-annotate-as-pure@7.22.5:
+    resolution: {integrity: sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/helper-compilation-targets@7.23.6:
+    resolution: {integrity: sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/compat-data': 7.24.4
+      '@babel/helper-validator-option': 7.23.5
+      browserslist: 4.23.0
+      lru-cache: 5.1.1
+      semver: 6.3.1
+    dev: false
+
+  /@babel/helper-environment-visitor@7.22.20:
+    resolution: {integrity: sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
+  /@babel/helper-function-name@7.23.0:
+    resolution: {integrity: sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/template': 7.24.0
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/helper-hoist-variables@7.22.5:
+    resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/helper-module-imports@7.24.3:
+    resolution: {integrity: sha512-viKb0F9f2s0BCS22QSF308z/+1YWKV/76mwt61NBzS5izMzDPwdq1pTrzf+Li3npBWX9KdQbkeCt1jSAM7lZqg==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/helper-module-transforms@7.24.5(@babel/core@7.24.5):
+    resolution: {integrity: sha512-9GxeY8c2d2mdQUP1Dye0ks3VDyIMS98kt/llQ2nUId8IsWqTF0l1LkSX0/uP7l7MCDrzXS009Hyhe2gzTiGW8A==}
+    engines: {node: '>=6.9.0'}
+    peerDependencies:
+      '@babel/core': ^7.0.0
+    dependencies:
+      '@babel/core': 7.24.5
+      '@babel/helper-environment-visitor': 7.22.20
+      '@babel/helper-module-imports': 7.24.3
+      '@babel/helper-simple-access': 7.24.5
+      '@babel/helper-split-export-declaration': 7.24.5
+      '@babel/helper-validator-identifier': 7.24.5
+    dev: false
+
+  /@babel/helper-plugin-utils@7.24.5:
+    resolution: {integrity: sha512-xjNLDopRzW2o6ba0gKbkZq5YWEBaK3PCyTOY1K2P/O07LGMhMqlMXPxwN4S5/RhWuCobT8z0jrlKGlYmeR1OhQ==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
+  /@babel/helper-simple-access@7.24.5:
+    resolution: {integrity: sha512-uH3Hmf5q5n7n8mz7arjUlDOCbttY/DW4DYhE6FUsjKJ/oYC1kQQUvwEQWxRwUpX9qQKRXeqLwWxrqilMrf32sQ==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/helper-split-export-declaration@7.24.5:
+    resolution: {integrity: sha512-5CHncttXohrHk8GWOFCcCl4oRD9fKosWlIRgWm4ql9VYioKm52Mk2xsmoohvm7f3JoiLSM5ZgJuRaf5QZZYd3Q==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/helper-string-parser@7.24.1:
+    resolution: {integrity: sha512-2ofRCjnnA9y+wk8b9IAREroeUP02KHp431N2mhKniy2yKIDKpbrHv9eXwm8cBeWQYcJmzv5qKCu65P47eCF7CQ==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
+  /@babel/helper-validator-identifier@7.24.5:
+    resolution: {integrity: sha512-3q93SSKX2TWCG30M2G2kwaKeTYgEUp5Snjuj8qm729SObL6nbtUldAi37qbxkD5gg3xnBio+f9nqpSepGZMvxA==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
+  /@babel/helper-validator-option@7.23.5:
+    resolution: {integrity: sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
+  /@babel/helpers@7.24.5:
+    resolution: {integrity: sha512-CiQmBMMpMQHwM5m01YnrM6imUG1ebgYJ+fAIW4FZe6m4qHTPaRHti+R8cggAwkdz4oXhtO4/K9JWlh+8hIfR2Q==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/template': 7.24.0
+      '@babel/traverse': 7.24.5(supports-color@5.5.0)
+      '@babel/types': 7.24.5
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /@babel/highlight@7.24.5:
+    resolution: {integrity: sha512-8lLmua6AVh/8SLJRRVD6V8p73Hir9w5mJrhE+IPpILG31KKlI9iz5zmBYKcWPS59qSfgP9RaSBQSHHE81WKuEw==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/helper-validator-identifier': 7.24.5
+      chalk: 2.4.2
+      js-tokens: 4.0.0
+      picocolors: 1.0.0
+    dev: false
+
+  /@babel/parser@7.24.5:
+    resolution: {integrity: sha512-EOv5IK8arwh3LI47dz1b0tKUb/1uhHAnHJOrjgtQMIpu1uXd9mlFrJg9IUgGUgZ41Ch0K8REPTYpO7B76b4vJg==}
+    engines: {node: '>=6.0.0'}
+    hasBin: true
+    dependencies:
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/plugin-syntax-jsx@7.24.1(@babel/core@7.24.5):
+    resolution: {integrity: sha512-2eCtxZXf+kbkMIsXS4poTvT4Yu5rXiRa+9xGVT56raghjmBTKMpFNc9R4IDiB4emao9eO22Ox7CxuJG7BgExqA==}
+    engines: {node: '>=6.9.0'}
+    peerDependencies:
+      '@babel/core': ^7.0.0-0
+    dependencies:
+      '@babel/core': 7.24.5
+      '@babel/helper-plugin-utils': 7.24.5
+    dev: false
+
   /@babel/runtime@7.24.4:
     resolution: {integrity: sha512-dkxf7+hn8mFBwKjs9bvBlArzLVxVbS8usaPUDd5p2a9JCL9tB8OaOVN1isD4+Xyk4ns89/xeOmbQvgdK7IIVdA==}
     engines: {node: '>=6.9.0'}
     dependencies:
       regenerator-runtime: 0.14.1
 
+  /@babel/template@7.24.0:
+    resolution: {integrity: sha512-Bkf2q8lMB0AFpX0NFEqSbx1OkTHf0f+0j82mkw+ZpzBnkk7e9Ql0891vlfgi+kHwOk8tQjiQHpqh4LaSa0fKEA==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/code-frame': 7.24.2
+      '@babel/parser': 7.24.5
+      '@babel/types': 7.24.5
+    dev: false
+
+  /@babel/traverse@7.24.5(supports-color@5.5.0):
+    resolution: {integrity: sha512-7aaBLeDQ4zYcUFDUD41lJc1fG8+5IU9DaNSJAgal866FGvmD5EbWQgnEC6kO1gGLsX0esNkfnJSndbTXA3r7UA==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/code-frame': 7.24.2
+      '@babel/generator': 7.24.5
+      '@babel/helper-environment-visitor': 7.22.20
+      '@babel/helper-function-name': 7.23.0
+      '@babel/helper-hoist-variables': 7.22.5
+      '@babel/helper-split-export-declaration': 7.24.5
+      '@babel/parser': 7.24.5
+      '@babel/types': 7.24.5
+      debug: 4.3.4(supports-color@5.5.0)
+      globals: 11.12.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
+  /@babel/types@7.24.5:
+    resolution: {integrity: sha512-6mQNsaLeXTw0nxYUYu+NSa4Hx4BlF1x1x8/PMFbiR+GBSr+2DkECc69b8hgy2frEodNcvPffeH8YfWd3LI6jhQ==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      '@babel/helper-string-parser': 7.24.1
+      '@babel/helper-validator-identifier': 7.24.5
+      to-fast-properties: 2.0.0
+    dev: false
+
   /@emnapi/runtime@1.1.1:
     resolution: {integrity: sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==}
     requiresBuild: true
@@ -194,6 +425,24 @@ packages:
     dev: false
     optional: true
 
+  /@emotion/is-prop-valid@0.8.8:
+    resolution: {integrity: sha512-u5WtneEAr5IDG2Wv65yhunPSMLIpuKsbuOktRojfrEiEvRyC85LgPMZI63cr7NUqT8ZIGdSVg8ZKGxIug4lXcA==}
+    dependencies:
+      '@emotion/memoize': 0.7.4
+    dev: false
+
+  /@emotion/memoize@0.7.4:
+    resolution: {integrity: sha512-Ja/Vfqe3HpuzRsG1oBtWTHk2PGZ7GR+2Vz5iYGelAw8dx32K0y7PjVuxK6z1nMpZOqAFsRUPCkK1YjJ56qJlgw==}
+    dev: false
+
+  /@emotion/stylis@0.8.5:
+    resolution: {integrity: sha512-h6KtPihKFn3T9fuIrwvXXUOwlx3rfUvfZIcP5a6rh8Y7zjE3O06hT5Ss4S/YI1AYhuZ1kjaE/5EaOOI2NqSylQ==}
+    dev: false
+
+  /@emotion/unitless@0.7.5:
+    resolution: {integrity: sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg==}
+    dev: false
+
   /@eslint-community/eslint-utils@4.4.0(eslint@8.57.0):
     resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -214,7 +463,7 @@ packages:
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
     dependencies:
       ajv: 6.12.6
-      debug: 4.3.4
+      debug: 4.3.4(supports-color@5.5.0)
       espree: 9.6.1
       globals: 13.24.0
       ignore: 5.3.1
@@ -303,7 +552,7 @@ packages:
     engines: {node: '>=10.10.0'}
     dependencies:
       '@humanwhocodes/object-schema': 2.0.3
-      debug: 4.3.4
+      debug: 4.3.4(supports-color@5.5.0)
       minimatch: 3.1.2
     transitivePeerDependencies:
       - supports-color
@@ -506,6 +755,36 @@ packages:
     dev: false
     optional: true
 
+  /@jridgewell/gen-mapping@0.3.5:
+    resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==}
+    engines: {node: '>=6.0.0'}
+    dependencies:
+      '@jridgewell/set-array': 1.2.1
+      '@jridgewell/sourcemap-codec': 1.4.15
+      '@jridgewell/trace-mapping': 0.3.25
+    dev: false
+
+  /@jridgewell/resolve-uri@3.1.2:
+    resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==}
+    engines: {node: '>=6.0.0'}
+    dev: false
+
+  /@jridgewell/set-array@1.2.1:
+    resolution: {integrity: sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==}
+    engines: {node: '>=6.0.0'}
+    dev: false
+
+  /@jridgewell/sourcemap-codec@1.4.15:
+    resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==}
+    dev: false
+
+  /@jridgewell/trace-mapping@0.3.25:
+    resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==}
+    dependencies:
+      '@jridgewell/resolve-uri': 3.1.2
+      '@jridgewell/sourcemap-codec': 1.4.15
+    dev: false
+
   /@mux/blurhash@0.1.2:
     resolution: {integrity: sha512-fWLOXHS2l3CGyPHF4NSRLqZx9DDAz1WYC4YXD3du24xxibIKUyBYzg7PDtx54z5QaQ12ln5oPFvhH5LhaLzeZg==}
     dependencies:
@@ -712,6 +991,19 @@ packages:
       tslib: 2.6.2
     dev: false
 
+  /@tanstack/query-core@5.32.1:
+    resolution: {integrity: sha512-mCWa1wdGb1jiny4+qYegbSeadcFj+Nq65KFSs4A1DRveoIq7SrTwUhqu7hrB6d54cQH5x59DfJvxusn3w1Cj/g==}
+    dev: false
+
+  /@tanstack/react-query@5.32.1(react@18.2.0):
+    resolution: {integrity: sha512-+nXLMB0JK0XwTJ+lQt49DPNLrbSppni9N5W5yMR085yW3YaRKRUFhfVTER3TvQd1UycHpoGPFnt1gHiijXERAg==}
+    peerDependencies:
+      react: ^18.0.0
+    dependencies:
+      '@tanstack/query-core': 5.32.1
+      react: 18.2.0
+    dev: false
+
   /@tanstack/react-table@8.15.3(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-aocQ4WpWiAh7R+yxNp+DGQYXeVACh5lv2kk96DjYgFiHDCB0cOFoYMT/pM6eDOzeMXR9AvPoLeumTgq8/0qX+w==}
     engines: {node: '>=12'}
@@ -794,7 +1086,7 @@ packages:
       '@typescript-eslint/types': 6.21.0
       '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.3.3)
       '@typescript-eslint/visitor-keys': 6.21.0
-      debug: 4.3.4
+      debug: 4.3.4(supports-color@5.5.0)
       eslint: 8.57.0
       typescript: 5.3.3
     transitivePeerDependencies:
@@ -825,7 +1117,7 @@ packages:
     dependencies:
       '@typescript-eslint/types': 6.21.0
       '@typescript-eslint/visitor-keys': 6.21.0
-      debug: 4.3.4
+      debug: 4.3.4(supports-color@5.5.0)
       globby: 11.1.0
       is-glob: 4.0.3
       minimatch: 9.0.3
@@ -1198,6 +1490,13 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
+  /ansi-styles@3.2.1:
+    resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==}
+    engines: {node: '>=4'}
+    dependencies:
+      color-convert: 1.9.3
+    dev: false
+
   /ansi-styles@4.3.0:
     resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
     engines: {node: '>=8'}
@@ -1353,6 +1652,21 @@ packages:
       dequal: 2.0.3
     dev: true
 
+  /babel-plugin-styled-components@2.1.4(@babel/core@7.24.5)(styled-components@5.3.3):
+    resolution: {integrity: sha512-Xgp9g+A/cG47sUyRwwYxGM4bR/jDRg5N6it/8+HxCnbT5XNKSKDT9xm4oag/osgqjC2It/vH0yXsomOG6k558g==}
+    peerDependencies:
+      styled-components: '>= 2'
+    dependencies:
+      '@babel/helper-annotate-as-pure': 7.22.5
+      '@babel/helper-module-imports': 7.24.3
+      '@babel/plugin-syntax-jsx': 7.24.1(@babel/core@7.24.5)
+      lodash: 4.17.21
+      picomatch: 2.3.1
+      styled-components: 5.3.3(@babel/core@7.24.5)(react-dom@18.2.0)(react-is@16.13.1)(react@18.2.0)
+    transitivePeerDependencies:
+      - '@babel/core'
+    dev: false
+
   /balanced-match@1.0.2:
     resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
     dev: true
@@ -1397,6 +1711,17 @@ packages:
     dependencies:
       fill-range: 7.0.1
 
+  /browserslist@4.23.0:
+    resolution: {integrity: sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==}
+    engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7}
+    hasBin: true
+    dependencies:
+      caniuse-lite: 1.0.30001607
+      electron-to-chromium: 1.4.752
+      node-releases: 2.0.14
+      update-browserslist-db: 1.0.13(browserslist@4.23.0)
+    dev: false
+
   /buffer@5.7.1:
     resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
     dependencies:
@@ -1432,6 +1757,10 @@ packages:
     engines: {node: '>=6'}
     dev: true
 
+  /camelize@1.0.1:
+    resolution: {integrity: sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ==}
+    dev: false
+
   /caniuse-lite@1.0.30001607:
     resolution: {integrity: sha512-WcvhVRjXLKFB/kmOFVwELtMxyhq3iM/MvmXcyCe2PNf166c39mptscOc/45TTS96n2gpNV2z7+NakArTWZCQ3w==}
     dev: false
@@ -1442,6 +1771,15 @@ packages:
       custom-media-element: 1.2.3
     dev: false
 
+  /chalk@2.4.2:
+    resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
+    engines: {node: '>=4'}
+    dependencies:
+      ansi-styles: 3.2.1
+      escape-string-regexp: 1.0.5
+      supports-color: 5.5.0
+    dev: false
+
   /chalk@4.1.2:
     resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
     engines: {node: '>=10'}
@@ -1487,12 +1825,22 @@ packages:
     engines: {node: '>=6'}
     dev: false
 
+  /color-convert@1.9.3:
+    resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==}
+    dependencies:
+      color-name: 1.1.3
+    dev: false
+
   /color-convert@2.0.1:
     resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
     engines: {node: '>=7.0.0'}
     dependencies:
       color-name: 1.1.4
 
+  /color-name@1.1.3:
+    resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==}
+    dev: false
+
   /color-name@1.1.4:
     resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
 
@@ -1515,6 +1863,10 @@ packages:
     resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
     dev: true
 
+  /convert-source-map@2.0.0:
+    resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==}
+    dev: false
+
   /core-js@3.36.1:
     resolution: {integrity: sha512-BTvUrwxVBezj5SZ3f10ImnX2oRByMxql3EimVqMysepbC9EeMUOpLwdy6Eoili2x6E4kf+ZUB5k/+Jv55alPfA==}
     requiresBuild: true
@@ -1529,6 +1881,19 @@ packages:
       which: 2.0.2
     dev: true
 
+  /css-color-keywords@1.0.0:
+    resolution: {integrity: sha512-FyyrDHZKEjXDpNJYvVsV960FiqQyXc/LlYmsxl2BcdMb2WPx0OGRVgTg55rPSyLSNMqP52R9r8geSp7apN3Ofg==}
+    engines: {node: '>=4'}
+    dev: false
+
+  /css-to-react-native@3.2.0:
+    resolution: {integrity: sha512-e8RKaLXMOFii+02mOlqwjbD00KSEKqblnpO9e++1aXS1fPQOpS1YoqdVHBqPjHNoxeF2mimzVqawm2KCbEdtHQ==}
+    dependencies:
+      camelize: 1.0.1
+      css-color-keywords: 1.0.0
+      postcss-value-parser: 4.2.0
+    dev: false
+
   /csstype@3.1.3:
     resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
     dev: false
@@ -1602,7 +1967,7 @@ packages:
       ms: 2.1.3
     dev: true
 
-  /debug@4.3.4:
+  /debug@4.3.4(supports-color@5.5.0):
     resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
     engines: {node: '>=6.0'}
     peerDependencies:
@@ -1612,7 +1977,7 @@ packages:
         optional: true
     dependencies:
       ms: 2.1.2
-    dev: true
+      supports-color: 5.5.0
 
   /decompress-response@6.0.0:
     resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
@@ -1630,6 +1995,11 @@ packages:
     resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==}
     dev: true
 
+  /deepmerge@4.3.1:
+    resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   /define-data-property@1.1.4:
     resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==}
     engines: {node: '>= 0.4'}
@@ -1678,6 +2048,10 @@ packages:
       esutils: 2.0.3
     dev: true
 
+  /electron-to-chromium@1.4.752:
+    resolution: {integrity: sha512-P3QJreYI/AUTcfBVrC4zy9KvnZWekViThgQMX/VpJ+IsOBbcX5JFpORM4qWapwWQ+agb2nYAOyn/4PMXOk0m2Q==}
+    dev: false
+
   /emoji-regex@9.2.2:
     resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
     dev: true
@@ -1809,6 +2183,16 @@ packages:
       is-symbol: 1.0.4
     dev: true
 
+  /escalade@3.1.2:
+    resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}
+    engines: {node: '>=6'}
+    dev: false
+
+  /escape-string-regexp@1.0.5:
+    resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==}
+    engines: {node: '>=0.8.0'}
+    dev: false
+
   /escape-string-regexp@4.0.0:
     resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
     engines: {node: '>=10'}
@@ -1856,7 +2240,7 @@ packages:
       eslint: '*'
       eslint-plugin-import: '*'
     dependencies:
-      debug: 4.3.4
+      debug: 4.3.4(supports-color@5.5.0)
       enhanced-resolve: 5.16.0
       eslint: 8.57.0
       eslint-module-utils: 2.8.1(@typescript-eslint/parser@6.21.0)(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0)
@@ -2027,7 +2411,7 @@ packages:
       ajv: 6.12.6
       chalk: 4.1.2
       cross-spawn: 7.0.3
-      debug: 4.3.4
+      debug: 4.3.4(supports-color@5.5.0)
       doctrine: 3.0.0
       escape-string-regexp: 4.0.0
       eslint-scope: 7.2.2
@@ -2226,6 +2610,11 @@ packages:
     resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
     dev: true
 
+  /gensync@1.0.0-beta.2:
+    resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==}
+    engines: {node: '>=6.9.0'}
+    dev: false
+
   /get-intrinsic@1.2.4:
     resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
     engines: {node: '>= 0.4'}
@@ -2294,6 +2683,11 @@ packages:
       path-is-absolute: 1.0.1
     dev: true
 
+  /globals@11.12.0:
+    resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==}
+    engines: {node: '>=4'}
+    dev: false
+
   /globals@13.24.0:
     resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==}
     engines: {node: '>=8'}
@@ -2346,6 +2740,10 @@ packages:
     resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==}
     dev: true
 
+  /has-flag@3.0.0:
+    resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
+    engines: {node: '>=4'}
+
   /has-flag@4.0.0:
     resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
     engines: {node: '>=8'}
@@ -2385,6 +2783,12 @@ packages:
     resolution: {integrity: sha512-Hnyf7ojTBtXHeOW1/t6wCBJSiK1WpoKF9yg7juxldDx8u3iswrkPt2wbOA/1NiwU4j27DSIVoIEJRAhcdMef/A==}
     dev: false
 
+  /hoist-non-react-statics@3.3.2:
+    resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==}
+    dependencies:
+      react-is: 16.13.1
+    dev: false
+
   /ieee754@1.2.1:
     resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
     dev: false
@@ -2658,6 +3062,12 @@ packages:
       argparse: 2.0.1
     dev: true
 
+  /jsesc@2.5.2:
+    resolution: {integrity: sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==}
+    engines: {node: '>=4'}
+    hasBin: true
+    dev: false
+
   /json-buffer@3.0.1:
     resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==}
     dev: true
@@ -2677,6 +3087,12 @@ packages:
       minimist: 1.2.8
     dev: true
 
+  /json5@2.2.3:
+    resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
+    engines: {node: '>=6'}
+    hasBin: true
+    dev: false
+
   /jsx-ast-utils@3.3.5:
     resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==}
     engines: {node: '>=4.0'}
@@ -2742,6 +3158,12 @@ packages:
     dependencies:
       js-tokens: 4.0.0
 
+  /lru-cache@5.1.1:
+    resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}
+    dependencies:
+      yallist: 3.1.1
+    dev: false
+
   /lru-cache@6.0.0:
     resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
     engines: {node: '>=10'}
@@ -2810,7 +3232,6 @@ packages:
 
   /ms@2.1.2:
     resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
-    dev: true
 
   /ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@@ -2855,12 +3276,12 @@ packages:
       react: '>18.0.0'
       react-dom: '>18.0.0'
     dependencies:
-      next: 14.0.4(react-dom@18.2.0)(react@18.2.0)
+      next: 14.0.4(@babel/core@7.24.5)(react-dom@18.2.0)(react@18.2.0)
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)
     dev: false
 
-  /next@14.0.4(react-dom@18.2.0)(react@18.2.0):
+  /next@14.0.4(@babel/core@7.24.5)(react-dom@18.2.0)(react@18.2.0):
     resolution: {integrity: sha512-qbwypnM7327SadwFtxXnQdGiKpkuhaRLE2uq62/nRul9cj9KhQ5LhHmlziTNqUidZotw/Q1I9OjirBROdUJNgA==}
     engines: {node: '>=18.17.0'}
     hasBin: true
@@ -2883,7 +3304,7 @@ packages:
       postcss: 8.4.31
       react: 18.2.0
       react-dom: 18.2.0(react@18.2.0)
-      styled-jsx: 5.1.1(react@18.2.0)
+      styled-jsx: 5.1.1(@babel/core@7.24.5)(react@18.2.0)
       watchpack: 2.4.0
     optionalDependencies:
       '@next/swc-darwin-arm64': 14.0.4
@@ -2908,7 +3329,7 @@ packages:
       react-dom: '>= 16.0.0'
     dependencies:
       '@types/nprogress': 0.2.3
-      next: 14.0.4(react-dom@18.2.0)(react@18.2.0)
+      next: 14.0.4(@babel/core@7.24.5)(react-dom@18.2.0)(react@18.2.0)
       nprogress: 0.2.0
       prop-types: 15.8.1
       react: 18.2.0
@@ -2926,6 +3347,10 @@ packages:
     resolution: {integrity: sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA==}
     dev: false
 
+  /node-releases@2.0.14:
+    resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==}
+    dev: false
+
   /normalize-path@3.0.0:
     resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
     engines: {node: '>=0.10.0'}
@@ -3110,6 +3535,10 @@ packages:
     engines: {node: '>= 0.4'}
     dev: true
 
+  /postcss-value-parser@4.2.0:
+    resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==}
+    dev: false
+
   /postcss@8.4.31:
     resolution: {integrity: sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==}
     engines: {node: ^10 || ^12 || >=14}
@@ -3205,6 +3634,17 @@ packages:
       strip-json-comments: 2.0.1
     dev: false
 
+  /react-data-table-component@7.5.4(react@18.2.0)(styled-components@5.3.3):
+    resolution: {integrity: sha512-6DGVj3urJZfEEMuP652fSjxdRVKeyb+9d0YounVc+MX8jwoyXQW6KO10eyZqElE9QtVrKrCeJxR7vht9yxyJiw==}
+    peerDependencies:
+      react: '>= 16.8.3'
+      styled-components: '>= 4'
+    dependencies:
+      deepmerge: 4.3.1
+      react: 18.2.0
+      styled-components: 5.3.3(@babel/core@7.24.5)(react-dom@18.2.0)(react-is@16.13.1)(react@18.2.0)
+    dev: false
+
   /react-dom@18.2.0(react@18.2.0):
     resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==}
     peerDependencies:
@@ -3399,7 +3839,6 @@ packages:
   /semver@6.3.1:
     resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
     hasBin: true
-    dev: true
 
   /semver@7.6.0:
     resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==}
@@ -3429,6 +3868,10 @@ packages:
       has-property-descriptors: 1.0.2
     dev: true
 
+  /shallowequal@1.1.0:
+    resolution: {integrity: sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==}
+    dev: false
+
   /sharp@0.30.7:
     resolution: {integrity: sha512-G+MY2YW33jgflKPTXXptVO28HvNOo9G3j0MybYAHeEmby+QuD2U98dT6ueht9cv/XDqZspSpIhoSW+BAKJ7Hig==}
     engines: {node: '>=12.13.0'}
@@ -3615,7 +4058,32 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
-  /styled-jsx@5.1.1(react@18.2.0):
+  /styled-components@5.3.3(@babel/core@7.24.5)(react-dom@18.2.0)(react-is@16.13.1)(react@18.2.0):
+    resolution: {integrity: sha512-++4iHwBM7ZN+x6DtPPWkCI4vdtwumQ+inA/DdAsqYd4SVgUKJie5vXyzotA00ttcFdQkCng7zc6grwlfIfw+lw==}
+    engines: {node: '>=10'}
+    peerDependencies:
+      react: '>= 16.8.0'
+      react-dom: '>= 16.8.0'
+      react-is: '>= 16.8.0'
+    dependencies:
+      '@babel/helper-module-imports': 7.24.3
+      '@babel/traverse': 7.24.5(supports-color@5.5.0)
+      '@emotion/is-prop-valid': 0.8.8
+      '@emotion/stylis': 0.8.5
+      '@emotion/unitless': 0.7.5
+      babel-plugin-styled-components: 2.1.4(@babel/core@7.24.5)(styled-components@5.3.3)
+      css-to-react-native: 3.2.0
+      hoist-non-react-statics: 3.3.2
+      react: 18.2.0
+      react-dom: 18.2.0(react@18.2.0)
+      react-is: 16.13.1
+      shallowequal: 1.1.0
+      supports-color: 5.5.0
+    transitivePeerDependencies:
+      - '@babel/core'
+    dev: false
+
+  /styled-jsx@5.1.1(@babel/core@7.24.5)(react@18.2.0):
     resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==}
     engines: {node: '>= 12.0.0'}
     peerDependencies:
@@ -3628,10 +4096,17 @@ packages:
       babel-plugin-macros:
         optional: true
     dependencies:
+      '@babel/core': 7.24.5
       client-only: 0.0.1
       react: 18.2.0
     dev: false
 
+  /supports-color@5.5.0:
+    resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==}
+    engines: {node: '>=4'}
+    dependencies:
+      has-flag: 3.0.0
+
   /supports-color@7.2.0:
     resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
     engines: {node: '>=8'}
@@ -3677,6 +4152,11 @@ packages:
     resolution: {integrity: sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q==}
     dev: false
 
+  /to-fast-properties@2.0.0:
+    resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==}
+    engines: {node: '>=4'}
+    dev: false
+
   /to-regex-range@5.0.1:
     resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
     engines: {node: '>=8.0'}
@@ -3800,6 +4280,17 @@ packages:
     resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
     dev: true
 
+  /update-browserslist-db@1.0.13(browserslist@4.23.0):
+    resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==}
+    hasBin: true
+    peerDependencies:
+      browserslist: '>= 4.21.0'
+    dependencies:
+      browserslist: 4.23.0
+      escalade: 3.1.2
+      picocolors: 1.0.0
+    dev: false
+
   /uri-js@4.4.1:
     resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
     dependencies:
@@ -3893,6 +4384,10 @@ packages:
       sax: 1.3.0
     dev: false
 
+  /yallist@3.1.1:
+    resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==}
+    dev: false
+
   /yallist@4.0.0:
     resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
 

packages/realtime/index.js

@@ -0,0 +1,44 @@
+const http = require('node:http');
+const faye = require('faye');
+const url = require('url');
+const pkg = require('./package.json')
+require('dotenv').config()
+
+var server = http.createServer(),
+    bayeux = new faye.NodeAdapter({ mount: '/faye' });
+
+
+bayeux.on('subscribe', function (clientId, channel) {
+    console.log(`bayeux client ${clientId} subscribed to ${channel}`)
+})
+
+const client = bayeux.getClient()
+
+if (process.env.NODE_ENV === 'development') {
+    function publish(msg) {
+        client.publish('/signals', {
+            text: msg
+        })
+    }
+
+    setInterval(() => {
+        publish('Hello mocha!')
+    }, 1000);
+}
+
+
+
+server.on('request', (req, res) => {
+    const parsedUrl = url.parse(req.url, true);
+    if (parsedUrl.pathname === '/metrics') {
+        // Handle the /metrics route
+        res.writeHead(200, { 'Content-Type': 'text/plain' });
+        res.end(`version=${pkg.version}`);
+    }
+});
+
+bayeux.attach(server);
+const port = process.env.PORT || 5000;
+console.log(`listening on ${port}`);
+server.listen(port);
+

packages/realtime/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "futureporn-realtime",
+  "version": "1.1.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node index.js",
+    "dev": "nodemon index.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "Unlicense",
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "faye": "^1.4.0"
+  },
+  "devDependencies": {
+    "chai": "^4.4.1",
+    "mocha": "^10.4.0"
+  }
+}

packages/realtime/pnpm-lock.yaml

@@ -0,0 +1,652 @@
+lockfileVersion: '6.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+dependencies:
+  dotenv:
+    specifier: ^16.4.5
+    version: 16.4.5
+  faye:
+    specifier: ^1.4.0
+    version: 1.4.0
+
+devDependencies:
+  chai:
+    specifier: ^4.4.1
+    version: 4.4.1
+  mocha:
+    specifier: ^10.4.0
+    version: 10.4.0
+
+packages:
+
+  /ansi-colors@4.1.1:
+    resolution: {integrity: sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==}
+    engines: {node: '>=6'}
+    dev: true
+
+  /ansi-regex@5.0.1:
+    resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /ansi-styles@4.3.0:
+    resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
+    engines: {node: '>=8'}
+    dependencies:
+      color-convert: 2.0.1
+    dev: true
+
+  /anymatch@3.1.3:
+    resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
+    engines: {node: '>= 8'}
+    dependencies:
+      normalize-path: 3.0.0
+      picomatch: 2.3.1
+    dev: true
+
+  /argparse@2.0.1:
+    resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
+    dev: true
+
+  /asap@2.0.6:
+    resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}
+    dev: false
+
+  /assertion-error@1.1.0:
+    resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==}
+    dev: true
+
+  /balanced-match@1.0.2:
+    resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
+    dev: true
+
+  /binary-extensions@2.3.0:
+    resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /brace-expansion@2.0.1:
+    resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==}
+    dependencies:
+      balanced-match: 1.0.2
+    dev: true
+
+  /braces@3.0.2:
+    resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==}
+    engines: {node: '>=8'}
+    dependencies:
+      fill-range: 7.0.1
+    dev: true
+
+  /browser-stdout@1.3.1:
+    resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==}
+    dev: true
+
+  /camelcase@6.3.0:
+    resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==}
+    engines: {node: '>=10'}
+    dev: true
+
+  /chai@4.4.1:
+    resolution: {integrity: sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==}
+    engines: {node: '>=4'}
+    dependencies:
+      assertion-error: 1.1.0
+      check-error: 1.0.3
+      deep-eql: 4.1.3
+      get-func-name: 2.0.2
+      loupe: 2.3.7
+      pathval: 1.1.1
+      type-detect: 4.0.8
+    dev: true
+
+  /chalk@4.1.2:
+    resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
+    engines: {node: '>=10'}
+    dependencies:
+      ansi-styles: 4.3.0
+      supports-color: 7.2.0
+    dev: true
+
+  /check-error@1.0.3:
+    resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==}
+    dependencies:
+      get-func-name: 2.0.2
+    dev: true
+
+  /chokidar@3.5.3:
+    resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==}
+    engines: {node: '>= 8.10.0'}
+    dependencies:
+      anymatch: 3.1.3
+      braces: 3.0.2
+      glob-parent: 5.1.2
+      is-binary-path: 2.1.0
+      is-glob: 4.0.3
+      normalize-path: 3.0.0
+      readdirp: 3.6.0
+    optionalDependencies:
+      fsevents: 2.3.3
+    dev: true
+
+  /cliui@7.0.4:
+    resolution: {integrity: sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==}
+    dependencies:
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+      wrap-ansi: 7.0.0
+    dev: true
+
+  /color-convert@2.0.1:
+    resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
+    engines: {node: '>=7.0.0'}
+    dependencies:
+      color-name: 1.1.4
+    dev: true
+
+  /color-name@1.1.4:
+    resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
+    dev: true
+
+  /csprng@0.1.2:
+    resolution: {integrity: sha512-D3WAbvvgUVIqSxUfdvLeGjuotsB32bvfVPd+AaaTWMtyUeC9zgCnw5xs94no89yFLVsafvY9dMZEhTwsY/ZecA==}
+    engines: {node: '>=0.6.0'}
+    dependencies:
+      sequin: 0.1.1
+    dev: false
+
+  /debug@4.3.4(supports-color@8.1.1):
+    resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+    dependencies:
+      ms: 2.1.2
+      supports-color: 8.1.1
+    dev: true
+
+  /decamelize@4.0.0:
+    resolution: {integrity: sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ==}
+    engines: {node: '>=10'}
+    dev: true
+
+  /deep-eql@4.1.3:
+    resolution: {integrity: sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==}
+    engines: {node: '>=6'}
+    dependencies:
+      type-detect: 4.0.8
+    dev: true
+
+  /diff@5.0.0:
+    resolution: {integrity: sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==}
+    engines: {node: '>=0.3.1'}
+    dev: true
+
+  /dotenv@16.4.5:
+    resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
+    engines: {node: '>=12'}
+    dev: false
+
+  /emoji-regex@8.0.0:
+    resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
+    dev: true
+
+  /escalade@3.1.2:
+    resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}
+    engines: {node: '>=6'}
+    dev: true
+
+  /escape-string-regexp@4.0.0:
+    resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}
+    engines: {node: '>=10'}
+    dev: true
+
+  /faye-websocket@0.11.4:
+    resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==}
+    engines: {node: '>=0.8.0'}
+    dependencies:
+      websocket-driver: 0.7.4
+    dev: false
+
+  /faye@1.4.0:
+    resolution: {integrity: sha512-kRrIg4be8VNYhycS2PY//hpBJSzZPr/DBbcy9VWelhZMW3KhyLkQR0HL0k0MNpmVoNFF4EdfMFkNAWjTP65g6w==}
+    engines: {node: '>=0.8.0'}
+    dependencies:
+      asap: 2.0.6
+      csprng: 0.1.2
+      faye-websocket: 0.11.4
+      safe-buffer: 5.2.1
+      tough-cookie: 4.1.4
+      tunnel-agent: 0.6.0
+    dev: false
+
+  /fill-range@7.0.1:
+    resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==}
+    engines: {node: '>=8'}
+    dependencies:
+      to-regex-range: 5.0.1
+    dev: true
+
+  /find-up@5.0.0:
+    resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==}
+    engines: {node: '>=10'}
+    dependencies:
+      locate-path: 6.0.0
+      path-exists: 4.0.0
+    dev: true
+
+  /flat@5.0.2:
+    resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==}
+    hasBin: true
+    dev: true
+
+  /fs.realpath@1.0.0:
+    resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
+    dev: true
+
+  /fsevents@2.3.3:
+    resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
+    engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /get-caller-file@2.0.5:
+    resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
+    engines: {node: 6.* || 8.* || >= 10.*}
+    dev: true
+
+  /get-func-name@2.0.2:
+    resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==}
+    dev: true
+
+  /glob-parent@5.1.2:
+    resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
+    engines: {node: '>= 6'}
+    dependencies:
+      is-glob: 4.0.3
+    dev: true
+
+  /glob@8.1.0:
+    resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==}
+    engines: {node: '>=12'}
+    dependencies:
+      fs.realpath: 1.0.0
+      inflight: 1.0.6
+      inherits: 2.0.4
+      minimatch: 5.0.1
+      once: 1.4.0
+    dev: true
+
+  /has-flag@4.0.0:
+    resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /he@1.2.0:
+    resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}
+    hasBin: true
+    dev: true
+
+  /http-parser-js@0.5.8:
+    resolution: {integrity: sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==}
+    dev: false
+
+  /inflight@1.0.6:
+    resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==}
+    dependencies:
+      once: 1.4.0
+      wrappy: 1.0.2
+    dev: true
+
+  /inherits@2.0.4:
+    resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
+    dev: true
+
+  /is-binary-path@2.1.0:
+    resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
+    engines: {node: '>=8'}
+    dependencies:
+      binary-extensions: 2.3.0
+    dev: true
+
+  /is-extglob@2.1.1:
+    resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
+    engines: {node: '>=0.10.0'}
+    dev: true
+
+  /is-fullwidth-code-point@3.0.0:
+    resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /is-glob@4.0.3:
+    resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==}
+    engines: {node: '>=0.10.0'}
+    dependencies:
+      is-extglob: 2.1.1
+    dev: true
+
+  /is-number@7.0.0:
+    resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==}
+    engines: {node: '>=0.12.0'}
+    dev: true
+
+  /is-plain-obj@2.1.0:
+    resolution: {integrity: sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /is-unicode-supported@0.1.0:
+    resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==}
+    engines: {node: '>=10'}
+    dev: true
+
+  /js-yaml@4.1.0:
+    resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}
+    hasBin: true
+    dependencies:
+      argparse: 2.0.1
+    dev: true
+
+  /locate-path@6.0.0:
+    resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==}
+    engines: {node: '>=10'}
+    dependencies:
+      p-locate: 5.0.0
+    dev: true
+
+  /log-symbols@4.1.0:
+    resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==}
+    engines: {node: '>=10'}
+    dependencies:
+      chalk: 4.1.2
+      is-unicode-supported: 0.1.0
+    dev: true
+
+  /loupe@2.3.7:
+    resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==}
+    dependencies:
+      get-func-name: 2.0.2
+    dev: true
+
+  /minimatch@5.0.1:
+    resolution: {integrity: sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==}
+    engines: {node: '>=10'}
+    dependencies:
+      brace-expansion: 2.0.1
+    dev: true
+
+  /mocha@10.4.0:
+    resolution: {integrity: sha512-eqhGB8JKapEYcC4ytX/xrzKforgEc3j1pGlAXVy3eRwrtAy5/nIfT1SvgGzfN0XZZxeLq0aQWkOUAmqIJiv+bA==}
+    engines: {node: '>= 14.0.0'}
+    hasBin: true
+    dependencies:
+      ansi-colors: 4.1.1
+      browser-stdout: 1.3.1
+      chokidar: 3.5.3
+      debug: 4.3.4(supports-color@8.1.1)
+      diff: 5.0.0
+      escape-string-regexp: 4.0.0
+      find-up: 5.0.0
+      glob: 8.1.0
+      he: 1.2.0
+      js-yaml: 4.1.0
+      log-symbols: 4.1.0
+      minimatch: 5.0.1
+      ms: 2.1.3
+      serialize-javascript: 6.0.0
+      strip-json-comments: 3.1.1
+      supports-color: 8.1.1
+      workerpool: 6.2.1
+      yargs: 16.2.0
+      yargs-parser: 20.2.4
+      yargs-unparser: 2.0.0
+    dev: true
+
+  /ms@2.1.2:
+    resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
+    dev: true
+
+  /ms@2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+    dev: true
+
+  /normalize-path@3.0.0:
+    resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==}
+    engines: {node: '>=0.10.0'}
+    dev: true
+
+  /once@1.4.0:
+    resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
+    dependencies:
+      wrappy: 1.0.2
+    dev: true
+
+  /p-limit@3.1.0:
+    resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==}
+    engines: {node: '>=10'}
+    dependencies:
+      yocto-queue: 0.1.0
+    dev: true
+
+  /p-locate@5.0.0:
+    resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
+    engines: {node: '>=10'}
+    dependencies:
+      p-limit: 3.1.0
+    dev: true
+
+  /path-exists@4.0.0:
+    resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /pathval@1.1.1:
+    resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==}
+    dev: true
+
+  /picomatch@2.3.1:
+    resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
+    engines: {node: '>=8.6'}
+    dev: true
+
+  /psl@1.9.0:
+    resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==}
+    dev: false
+
+  /punycode@2.3.1:
+    resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
+    engines: {node: '>=6'}
+    dev: false
+
+  /querystringify@2.2.0:
+    resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==}
+    dev: false
+
+  /randombytes@2.1.0:
+    resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}
+    dependencies:
+      safe-buffer: 5.2.1
+    dev: true
+
+  /readdirp@3.6.0:
+    resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
+    engines: {node: '>=8.10.0'}
+    dependencies:
+      picomatch: 2.3.1
+    dev: true
+
+  /require-directory@2.1.1:
+    resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
+    engines: {node: '>=0.10.0'}
+    dev: true
+
+  /requires-port@1.0.0:
+    resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==}
+    dev: false
+
+  /safe-buffer@5.2.1:
+    resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
+
+  /sequin@0.1.1:
+    resolution: {integrity: sha512-hJWMZRwP75ocoBM+1/YaCsvS0j5MTPeBHJkS2/wruehl9xwtX30HlDF1Gt6UZ8HHHY8SJa2/IL+jo+JJCd59rA==}
+    engines: {node: '>=0.4.0'}
+    dev: false
+
+  /serialize-javascript@6.0.0:
+    resolution: {integrity: sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==}
+    dependencies:
+      randombytes: 2.1.0
+    dev: true
+
+  /string-width@4.2.3:
+    resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
+    engines: {node: '>=8'}
+    dependencies:
+      emoji-regex: 8.0.0
+      is-fullwidth-code-point: 3.0.0
+      strip-ansi: 6.0.1
+    dev: true
+
+  /strip-ansi@6.0.1:
+    resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
+    engines: {node: '>=8'}
+    dependencies:
+      ansi-regex: 5.0.1
+    dev: true
+
+  /strip-json-comments@3.1.1:
+    resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==}
+    engines: {node: '>=8'}
+    dev: true
+
+  /supports-color@7.2.0:
+    resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
+    engines: {node: '>=8'}
+    dependencies:
+      has-flag: 4.0.0
+    dev: true
+
+  /supports-color@8.1.1:
+    resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==}
+    engines: {node: '>=10'}
+    dependencies:
+      has-flag: 4.0.0
+    dev: true
+
+  /to-regex-range@5.0.1:
+    resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
+    engines: {node: '>=8.0'}
+    dependencies:
+      is-number: 7.0.0
+    dev: true
+
+  /tough-cookie@4.1.4:
+    resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==}
+    engines: {node: '>=6'}
+    dependencies:
+      psl: 1.9.0
+      punycode: 2.3.1
+      universalify: 0.2.0
+      url-parse: 1.5.10
+    dev: false
+
+  /tunnel-agent@0.6.0:
+    resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
+    dependencies:
+      safe-buffer: 5.2.1
+    dev: false
+
+  /type-detect@4.0.8:
+    resolution: {integrity: sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==}
+    engines: {node: '>=4'}
+    dev: true
+
+  /universalify@0.2.0:
+    resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==}
+    engines: {node: '>= 4.0.0'}
+    dev: false
+
+  /url-parse@1.5.10:
+    resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==}
+    dependencies:
+      querystringify: 2.2.0
+      requires-port: 1.0.0
+    dev: false
+
+  /websocket-driver@0.7.4:
+    resolution: {integrity: sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==}
+    engines: {node: '>=0.8.0'}
+    dependencies:
+      http-parser-js: 0.5.8
+      safe-buffer: 5.2.1
+      websocket-extensions: 0.1.4
+    dev: false
+
+  /websocket-extensions@0.1.4:
+    resolution: {integrity: sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==}
+    engines: {node: '>=0.8.0'}
+    dev: false
+
+  /workerpool@6.2.1:
+    resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==}
+    dev: true
+
+  /wrap-ansi@7.0.0:
+    resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==}
+    engines: {node: '>=10'}
+    dependencies:
+      ansi-styles: 4.3.0
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+    dev: true
+
+  /wrappy@1.0.2:
+    resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
+    dev: true
+
+  /y18n@5.0.8:
+    resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
+    engines: {node: '>=10'}
+    dev: true
+
+  /yargs-parser@20.2.4:
+    resolution: {integrity: sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==}
+    engines: {node: '>=10'}
+    dev: true
+
+  /yargs-unparser@2.0.0:
+    resolution: {integrity: sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA==}
+    engines: {node: '>=10'}
+    dependencies:
+      camelcase: 6.3.0
+      decamelize: 4.0.0
+      flat: 5.0.2
+      is-plain-obj: 2.1.0
+    dev: true
+
+  /yargs@16.2.0:
+    resolution: {integrity: sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==}
+    engines: {node: '>=10'}
+    dependencies:
+      cliui: 7.0.4
+      escalade: 3.1.2
+      get-caller-file: 2.0.5
+      require-directory: 2.1.1
+      string-width: 4.2.3
+      y18n: 5.0.8
+      yargs-parser: 20.2.4
+    dev: true
+
+  /yocto-queue@0.1.0:
+    resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
+    engines: {node: '>=10'}
+    dev: true

packages/realtime/test/test.js

@@ -0,0 +1,49 @@
+const expect = require('chai').expect
+const Faye = require('faye')
+
+// for reference of _state contants,
+// @see https://github.com/faye/faye/blob/60141e8d3942a88ba3de6a9b3aef9503bc1bb1e6/src/protocol/client.js#L178
+const UNCONNECTED = 1
+const CONNECTING = 2
+const CONNECTED = 3
+const DISCONNECTED = 4
+
+describe('faye', function () {
+    describe('sending event', function () {
+        let client = new Faye.Client('http://localhost:3535/faye');
+        // let client = new Faye.Client('https://realtime.futureporn.net/faye');
+        client.connect()
+        it('should send a signal', function (done) {
+            const pub = client.publish('/signals', {
+                text: 'hello worldy 123!'
+            })
+            pub.callback(function () {
+                expect(client._state).to.equal(CONNECTED)
+                done()
+            })
+            pub.errback(function (e) {
+                throw new Error(e)
+            })
+        })
+        it('should receive a signal', function (done) {
+            const sub = client.subscribe('/signals', function(message) {
+                console.log('Got a message: ' + message.text);
+                expect(message.text).to.equal('Hello mocha!')
+                done()
+            });
+            sub.callback(function () {
+                expect(client._state).to.equal(CONNECTED)
+            })
+            sub.errback(function (e) {
+                throw new Error(e)
+            })
+        })
+        this.afterEach(function () {
+            expect(client._state).to.equal(CONNECTED)
+        })
+        this.afterAll(function () {
+            client.disconnect()
+            expect(client._state).to.equal(DISCONNECTED)
+        })
+    })
+})

packages/scout/.dockerignore

@@ -0,0 +1,9 @@
+.dockerignore
+.gitignore
+*~
+node_modules
+npm-debug.log
+README.md
+.git
+LICENSE
+.nvmrc

packages/scout/.mocharc.json

@@ -0,0 +1,4 @@
+{
+    "extension": ["js"],
+    "spec": "src/**/*.spec.js"
+}

packages/scout/README.md

@@ -0,0 +1,21 @@
+# scout
+
+Watches an e-mail inbox for going live notifications
+
+Support for
+  * [ ] Chaturbate
+  * [ ] Fansly
+
+## Design requirements
+
+  * [ ] get watched channels list from Strapi
+  * [ ] every 3 mins, watch/unwatch based on channels list
+    * [ ] watch important sources for go-live notifications      
+        * [ ] CB tab
+        * [ ] email inbox
+  * [ ] alerts realtime server when watched room goes live
+  * [ ] logs chat messages
+  * [ ] throws errors when unable to connect
+  * [ ] runs browser headless
+  * [ ] runs in the cloud
+  * [ ] runs in k8s cluster

packages/scout/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "futureporn-scout",
+  "type": "module",
+  "version": "3.0.0",
+  "description": "detect when a stream goes live",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha",
+    "start:email": "node ./src/index.email.js",
+    "start:browser": "node ./src/index.browser.js",
+    "start": "concurrently \"npm:start:email\""
+  },
+  "keywords": [],
+  "author": "@CJ_Clippy",
+  "license": "Unlicense",
+  "dependencies": {
+    "cheerio": "1.0.0-rc.12",
+    "concurrently": "^8.2.2",
+    "dotenv": "^16.4.5",
+    "fastq": "^1.17.1",
+    "faye": "^1.4.0",
+    "imapflow": "^1.0.160",
+    "limiter": "^2.0.1",
+    "mailparser": "^3.7.1",
+    "puppeteer": "^22.7.1",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-repl": "^2.3.3",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "xpath": "^0.0.34"
+  },
+  "devDependencies": {
+    "chai": "^5.1.0",
+    "mocha": "^10.4.0"
+  },
+  "engines": {
+    "node": "^20"
+  }
+}

packages/scout/src/fixtures/.gitignore

@@ -0,0 +1 @@
+*.txt

packages/scout/src/imap.js

@@ -0,0 +1,130 @@
+
+import { ImapFlow } from 'imapflow';
+import EventEmitter from 'node:events';
+import 'dotenv/config';
+import { simpleParser } from 'mailparser';
+import { RateLimiter } from 'limiter';
+
+
+
+if (!process.env.SCOUT_IMAP_SERVER) throw new Error('SCOUT_IMAP_SERVER is missing from env');
+if (!process.env.SCOUT_IMAP_PORT) throw new Error('SCOUT_IMAP_PORT is missing from env');
+if (!process.env.SCOUT_IMAP_USERNAME) throw new Error('SCOUT_IMAP_USERNAME is missing from env');
+if (!process.env.SCOUT_IMAP_PASSWORD) throw new Error('SCOUT_IMAP_PASSWORD is missing from env');
+
+const limiter = new RateLimiter({ tokensPerInterval: 1, interval: 3000 });
+
+// https://stackoverflow.com/a/49428486/1004931
+function streamToString(stream) {
+    const chunks = [];
+    return new Promise((resolve, reject) => {
+        stream.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
+        stream.on('error', (err) => reject(err));
+        stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
+    })
+}
+
+
+export class Email extends EventEmitter {
+
+
+    constructor() {
+        super()
+        this.client = null
+    }
+
+    async archiveMessage(uid) {
+        await limiter.removeTokens(1);
+        await this.client.messageDelete(uid, { uid: true })
+    }
+
+    async connect() {
+        this.client = new ImapFlow({
+            host: process.env.SCOUT_IMAP_SERVER,
+            port: process.env.SCOUT_IMAP_PORT,
+            secure: true,
+            auth: {
+                user: process.env.SCOUT_IMAP_USERNAME,
+                pass: process.env.SCOUT_IMAP_PASSWORD
+            }
+        });
+
+        this.registerEventListeners()
+        await this.client.connect()
+        const stat = await this.getStatus()
+        if (stat.messages > 0) {
+            await this.emitAllMessages()
+        }
+    }
+
+    async reconnect() {
+        console.log('    RECONNECTING...')
+        delete this.client
+        await this.connect()
+    }
+
+    async getStatus() {
+        let lock = await this.client.getMailboxLock('INBOX');
+        let status;
+        try {
+            status = await this.client.status('INBOX', { messages: true });
+        } finally {
+            lock.release()
+        }
+        return status
+    }
+
+    async loadMessage(uid) {
+        console.log(`    💾 loading message uid=${uid}`)
+        let lock = await this.client.getMailboxLock('INBOX');
+        let dl, body
+        try {
+            dl = await this.client.download(uid, undefined, { uid: true })
+            body = await streamToString(dl.content)
+        } finally {
+            lock.release()
+        }
+        return body
+    }
+
+    async emitAllMessages() {
+        console.log('emitAllMessages is running')
+        let lock = await this.client.getMailboxLock('INBOX');
+        try {
+            for await (let message of this.client.fetch('1:*', { envelope: true })) {
+                // it is tempting to call this.client.download here, but that is not possible while the mailbox is locked.
+                // client.download must be called outside of this lock
+                // console.log('here is a message')
+                console.log(JSON.stringify(message, null, 2))
+                this.emit('message', message)
+            }
+        } finally {
+            lock.release();
+        }
+    }
+
+    registerEventListeners() {
+        console.log(`    > REGISTERING EVENT LISTENERS <`)
+        this.client.once('end', () => this.reconnect())
+        this.client.on('exists', (evt) => {
+            // console.log(`exists event! count=${evt.count} prevCount=${evt.prevCount}`)
+            // console.log(evt)
+            if (evt.path === 'INBOX') {
+                this.emitAllMessages()
+            }
+        })
+    }
+
+}
+
+
+
+
+
+
+// // Select and lock a mailbox. Throws if mailbox does not exist
+// console.log('get lock')
+
+
+// // log out and close connection
+// // await client.logout();

packages/scout/src/index.browser.js

@@ -0,0 +1,174 @@
+// watches chaturbate for go live 
+
+import puppeteer from 'puppeteer-extra';
+import { dirname } from 'path';
+import { fileURLToPath } from 'url';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth'
+import delay from 'delay'
+import ReplPlugin from 'puppeteer-extra-plugin-repl'
+import fsp from 'fs/promises';
+import repl from 'puppeteer-extra-plugin-repl';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const browserDataDir = __dirname+'/futureporn-scout-datadir';
+
+
+const searchInputSelector = 'body > app-root > div > div.site-wrapper.nav-bar-visible.nav-bar-top-visible > div > app-explore-route > div > app-account-explore-route > div > input'
+const followButtonSelector = "xpath///app-account-follow-button/div/xd-localization-string[contains(., 'Follow')]"
+const onlineIndicator = 'a.online-indicator.is-live'
+
+// @todo Get this value from Strapi
+const channels = [
+    'avabrooks'
+]
+
+
+const scrollDown = async (page) => {
+  console.log('scrolling down')
+  await page.keyboard.press('PageDown')
+  await page.evaluate(async () => {
+    window.scrollBy(0, 500);
+  });
+}
+
+const handleOnlineChannel = async (page) => {
+  const href = await page.evaluate(async () => {
+    const onlineIndicator = 'a.online-indicator.is-live'
+    const href = document.querySelector(onlineIndicator).href
+    return href
+  })
+  console.log(`${href} is online!`)
+  await fsp.appendFile('./data.csv', `${new Date().toISOString()},${href}\n`)
+}
+
+/**
+ * Open up a tab to the channel.
+ * Intercept websockets events which tell us when the channel is online
+ */
+const monitor = async (browser, channel) => {
+    if (!browser) throw new Error('monitor requires page arg');
+    if (!channel) throw new Error('monitor requires channel arg');
+    console.log(`monitoring ${channel}`)
+    const page = await browser.newPage();
+
+    await page.setRequestInterception(true);
+    
+    const url = await new Promise((resolve) => {
+        page.on('request', interceptRequest => {
+            const url = interceptRequest.url();
+            console.log(url)
+            if (url.startsWith('wss://realtime')) {
+                console.log(`request! ${interceptRequest.url()} ${JSON.stringify(interceptRequest.headers(), null, 2)}`);
+                resolve(interceptRequest.url());
+            }
+            interceptRequest.continue();
+        });
+  
+        page.goto(`https://chaturbate.com/${channel}`);
+    })
+
+    await page.repl()
+}
+
+const dash = async (page, channel) => {
+  console.log('check')
+  await page.goto(`https://chaturbate.com/${channel}`)
+
+  
+  // look for is-live indicator
+  try {
+    console.log('waiting for home page')
+    await page.waitForSelector('div.stories-scroll-container', { timeout: 15000 })
+    console.log('waiting for online indicators')
+    await page.waitForSelector(onlineIndicator, { timeout: 10000 })
+    console.log(`FOUND online channel! ${JSON.stringify(onlineIndicator,  null, 2)}`);
+    handleOnlineChannel(page)
+    delay(1000)
+  } catch (e) {
+    console.error(e)
+    console.log('error on the dash. lets move on.')
+  }
+}
+
+const discover = async (page) => {
+
+  // Navigate the page to a URL
+  await page.goto('https://fansly.com/explore/discover');
+
+
+  console.log('wait for search input')
+  await page.waitForSelector(searchInputSelector)
+  
+  // console.log('click search input')
+  // await page.click(searchInputSelector)
+
+  console.log('type a random letter')
+  const letter = (() => {
+    const letters = 'abcdefghijklmnopqrstuvwxyz'
+    const randomIndex = Math.floor(Math.random() * letters.length);
+    return letters[randomIndex];
+  })()
+  await page.type(searchInputSelector, letter)
+
+
+
+  // // Start an interactive REPL here with the `page` instance.
+  // await page.repl()
+  // // Afterwards start REPL with the `browser` instance.
+  // await browser.repl()
+
+  let go = true
+  setTimeout(() => {
+    go = false
+  }, 1000*30*1) // ~2 minutes of following
+
+  while (go) {
+    try {
+      console.log('Wait for Follow/Unfollow buttons')
+      try {
+        await page.waitForSelector(followButtonSelector, { timeout: 1000 })
+        await page.click(followButtonSelector, { timeout: 1000 })
+      } catch {}
+
+      // look for is-live indicator
+      try {
+        await page.waitForSelector(onlineIndicator, { timeout: 100 })
+        console.log(`FOUND online channel! ${JSON.stringify(onlineIndicator,  null, 2)}`);
+        handleOnlineChannel(page)
+      } catch (e) {
+        // console.log('online channel not found')
+      }
+
+      scrollDown(page)
+      await page.keyboard.press('PageDown')
+    } catch (e) {
+      console.error('Error while waiting for follow button')
+      console.error(e)
+    }
+    await delay(15000)
+  }
+}
+
+(async () => {
+  // Launch the browser and open a new blank page
+  puppeteer.use(StealthPlugin())
+  puppeteer.use(ReplPlugin())
+  const browser = await puppeteer.launch({
+    headless: false,
+    args: [
+        `--user-data-dir=${browserDataDir}`
+    ]
+  });
+
+  
+  for (const ch of channels) {
+    await monitor(browser, ch)
+    // await dash(page)
+    // await discover(page) // discover & follow
+    // await dash(page)     // dashboard & see who's live
+  }
+
+  
+
+
+})();

packages/scout/src/index.email.js

@@ -0,0 +1,45 @@
+'use strict'
+
+/**
+ * watches an e-mail inbox for going live notifications
+ */
+
+import { checkEmail } from './parsers.js'
+import { signalRealtime, createStreamInDb, pushToRecents } from './signals.js'
+import { Email } from './imap.js'
+import fastq from "fastq";
+
+
+const q = fastq.promise(handleMessage, 1)
+
+async function handleMessage({email, msg}) {
+    try {
+        console.log('    ✏️ loading message')
+        const body = await email.loadMessage(msg.uid)
+
+        console.log('    ✏️ checking e-mail')
+        const { isMatch, url, platform, channel, displayName, date } = (await checkEmail(body))
+
+        console.log('    ✏️ signalling realtime')
+        if (isMatch) await signalRealtime(url);
+
+        // console.log('    ✏️ creating stream entry in db')
+        // if (isMatch) await createStreamInDb({ platform, channel, date });
+
+        console.log('    ✏️ pushToRecents')
+        if (platform === 'Fansly') await pushToRecents({ date, url, channel, displayName });
+
+        console.log('    ✏️ archiving e-mail')
+        await email.archiveMessage(msg.uid)
+    } catch (e) {
+        // console.error('error encoutered')
+        console.error(`    An error was encountered while handling the following e-mail message.\n${JSON.stringify(msg, null, 2)}\nError as follows.\n${e}`)
+    }
+}
+
+
+(async () => {
+    const email = new Email()
+    email.on('message', (msg) => q.push({email, msg}))
+    await email.connect()
+})()

packages/scout/src/parsers.js

@@ -0,0 +1,76 @@
+
+import { simpleParser } from 'mailparser';
+import { load } from 'cheerio'
+
+const definitions = [
+    {
+        platform: 'Chaturbate',
+        selectors: {
+            channel: 'td[id*="onlinemessage"] a:nth-child(1)'
+        },
+        from: 'follownotify@bk.chaturbate.com',
+        template: 'https://chaturbate.com/:channel'
+    },
+    { 
+        platform: 'Fansly',
+        selectors: {
+            url: ($) => $("a[href*='/live/']").attr('href'),
+            displayName: 'div[class*="message-col"] div:nth-child(5)'
+        },
+        from: 'no-reply@fansly.com',
+        template: 'https://fansly.com/live/:channel',
+        regex: /https:\/\/fansly.com\/live\/([a-zA-Z0-9_]+)/
+    }
+]
+
+function render(template, values) {
+    // console.log(`values=${values}`)
+    // console.log(values)
+    return template.replace(/:([a-zA-Z0-9_]+)/g, (match, key) => {
+        // Replace :channel with the corresponding property from the values object
+        return values[key] || match;
+    });
+}
+
+
+/**
+ * checkEmail
+ * 
+ * Check an e-mail for go-live notification content.
+ * 
+ * @param {String}      body   -- raw mail body
+ * @returns {Object}    result
+ * @returns {Boolean}   result.isMatch true if e-mail contains a go-live notification
+ * @returns {String}    result.channel
+ */
+export async function checkEmail (body) {
+
+    const mail = await simpleParser(body)
+
+
+    let res = {}
+    let def = definitions.find((def) => def.from === mail.from.value[0].address)
+    if (!def) return { isMatch: false, channel: null, platform: null, url: null };
+    res.isMatch = true
+
+    // Step 0, get values from e-mail metadata
+    res.platform = def.platform
+    res.date = new Date(mail.date).toISOString()
+    
+    // Step 1, get values using CSS selectors
+    const $ = load(mail.html)
+    for (const s in def.selectors) {
+        res[s] = (def.selectors[s] instanceof Object) ? def.selectors[s]($) : $(def.selectors[s]).text()
+    }
+
+    // Step 2, get values using regex & templates
+    res.channel = (() => {
+        if (res.channel) return res.channel;
+        if (def.regex && res.url) return def.regex.exec(res.url).at(1);
+    })()
+
+    res.url = res.url || render(def.template, {channel: res.channel})
+
+
+    return res
+}

packages/scout/src/parsers.spec.js

@@ -0,0 +1,30 @@
+'use strict'
+
+import { expect } from 'chai'
+import { checkEmail } from './parsers.js'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+const __dirname = import.meta.dirname;
+
+describe('parsers', function () {
+    describe('checkEmail', function () {
+        it('should detect fansly e-mails', async function () {
+            const mailBody = await fs.readFile(path.join(__dirname, './fixtures/fansly.fixture.txt'), { encoding: 'utf8' })
+            const { isMatch, channel, platform, url, date } = await checkEmail(mailBody)
+            expect(isMatch).to.equal(true, 'a Fansly heuristic was not found')
+            expect(platform).to.equal('Fansly')
+            expect(channel).to.equal('SkiaObsidian')
+            expect(url).to.equal('https://fansly.com/live/SkiaObsidian')
+            expect(date).to.equal('2024-05-05T03:04:33.000Z')
+        })
+        it('should detect cb e-mails', async function () {
+            const mailBody = await fs.readFile(path.join(__dirname, './fixtures/chaturbate.fixture.txt'), { encoding: 'utf8' })
+            const { isMatch, channel, platform, url, date } = await checkEmail(mailBody)
+            expect(isMatch).to.equal(true, 'a CB heuristic was not found')
+            expect(platform).to.equal('Chaturbate')
+            expect(channel).to.equal('skyeanette')
+            expect(url).to.equal('https://chaturbate.com/skyeanette')
+            expect(date).to.equal('2023-07-24T01:08:28.000Z')
+        })
+    })
+})

packages/scout/src/signals.js

@@ -0,0 +1,67 @@
+
+import 'dotenv/config'
+import Faye from 'faye'
+
+if (!process.env.PUBSUB_SERVER_URL) throw new Error('PUBSUB_SERVER_URL is missing from env');
+if (!process.env.SCOUT_STRAPI_API_KEY) throw new Error('SCOUT_STRAPI_API_KEY is missing from env');
+if (!process.env.STRAPI_URL) throw new Error('STRAPI_URL is missing from env');
+if (!process.env.SCOUT_RECENTS_TOKEN) throw new Error('SCOUT_RECENTS_TOKEN is undefined in env');
+
+
+const faye = new Faye.Client(process.env.PUBSUB_SERVER_URL);
+
+export async function signalRealtime (url) {
+    faye.publish('/signals', { 
+        signal: 'startV2',
+        url: url
+    })
+}
+
+
+/**
+ * Create a database record which shows this stream exists
+ */
+export async function createStreamInDb ({ platform, channel, date }) {
+    const res = await fetch(`${process.env.STRAPI_URL}/api/streams`, {
+        method: 'POST',
+        headers: {
+            'authorization': `Bearer ${process.env.SCOUT_STRAPI_API_KEY}`
+        },
+        body: JSON.stringify({
+            data: {
+                isFanslyStream: (platform === 'Fansly') ? true : false,
+                isChaturbateStream: (platform === 'Chaturbate') ? true : false,
+                archiveStatus: 'missing',
+                date: date
+            }
+        })
+    })
+}
+
+
+
+
+
+/**
+ * this function is for whoisonline.sbtp.xyz
+ * 
+ * this is a short-term thing, feel free to delete when it's no longer useful
+ */
+export async function pushToRecents ({ url, channel, displayName, date }) {
+    const lastSeen = new Date().toISOString()
+    console.log(`pushToRecents with url=${url} channel=${channel} displayName=${displayName} date=${date}`)
+    const res = await fetch('https://whoisonline.sbtp.xyz/recents', {
+        method: 'POST',
+        headers: {
+            'authorization': `Bearer ${process.env.SCOUT_RECENTS_TOKEN}`
+        },
+        body: JSON.stringify({
+            data: {
+                url,
+                channel,
+                displayName,
+                date
+            }
+        })
+    })
+}

packages/strapi/package.json

@@ -80,6 +80,7 @@
   "strapi": {
     "uuid": false
   },
+  "packageManager": "pnpm@8.10.5",
   "engines": {
     "node": "20.x.x",
     "npm": ">=6.0.0"

t.wip.tiltfile

@@ -0,0 +1,173 @@
+# Tiltfile for working with Next and Strapi locally
+
+
+# load('ext://cert_manager', 'deploy_cert_manager')
+# deploy_cert_manager()
+load('ext://dotenv', 'dotenv')
+dotenv(fn='.env')
+
+
+load('ext://helm_remote', 'helm_remote')
+helm_remote(
+    'kubernetes-ingress-controller',
+    repo_name='kubernetes-ingress-controller',
+    repo_url='https://ngrok.github.io/kubernetes-ingress-controller',
+    namespace='futureporn',
+    create_namespace='false',
+    set=[
+        'credentials.apiKey=%s' % os.getenv('NGROK_API_KEY'),
+        'credentials.authtoken=%s' % os.getenv('NGROK_AUTHTOKEN')
+    ]
+)
+
+k8s_yaml(helm(
+    './charts/fp',
+    values=['./charts/fp/values-dev.yaml'],
+))
+
+
+
+# # docker_build('fp/link2cid', './packages/link2cid')
+# docker_build(
+#     'fp/strapi',
+#     '.',
+#     only=["./packages/strapi"],
+#     dockerfile='d.strapi.dockerfile',
+#     target='release',
+#     live_update=[
+#         sync('./packages/strapi', '/app')
+#     ]
+# )
+
+load('ext://uibutton', 'cmd_button')
+# @todo in the future we can add a button for seeding the db from scratch.
+#       this would be good for onboarding devs
+#       step 1 would be creating the db
+#
+
+# step 2 is ???
+#
+#  set -eu
+#  # get k8s pod name from tilt resource name
+#  POD_NAME="$(tilt get kubernetesdiscovery "$resource" -ojsonpath='{.status.pods[0].name}')"
+#  kubectl exec "$POD_NAME" -- $command
+
+
+pg_seed_script = '''
+kubectl -n futureporn exec postgres -- psql -U postgres --command "CREATE DATABASE futureporn_db WITH OWNER = postgres ENCODING = 'UTF8' LOCALE_PROVIDER = 'libc' CONNECTION LIMIT = -1 IS_TEMPLATE = False;"
+'''
+cmd_button('postgres:seed',
+        argv=['sh', '-c', pg_seed_script],
+        resource='postgres',
+        icon_name='dataset',
+        text='seed db with sample data',
+)
+cmd_button('postgres:restore',
+        argv=['sh', '-c', 'cd letters && yarn install'],
+        resource='postgres',
+        icon_name='cloud_download',
+        text='restore db from backup',
+)
+pgadmin_import_script = '''
+    # @todo this script is meant for first-time setup of the cluster
+    #       it sets up the UI to have a futureporn postgres server connection
+    #
+    # @see https://www.pgadmin.org/docs/pgadmin4/development/import_export_servers.html#json-format
+    #
+
+    # futureporn.pgadmin.json
+    {
+        "Servers": {
+            "1": {
+                "Name": "futureporn",
+                "Group": "Servers",
+                "Host": "postgres.futureporn.svc.cluster.local",
+                "Port": 5432,
+                "MaintenanceDB": "postgres",
+                "Username": "postgres",
+                "UseSSHTunnel": 0,
+                "TunnelPort": "22",
+                "TunnelAuthentication": 0,
+                "KerberosAuthentication": false,
+                "ConnectionParameters": {
+                    "sslmode": "prefer",
+                    "connect_timeout": 10,
+                    "sslcert": "<STORAGE_DIR>/.postgresql/postgresql.crt",
+                    "sslkey": "<STORAGE_DIR>/.postgresql/postgresql.key"
+                }
+            }
+        }
+    }
+    kubectl -n futureporn exec pgadmin -- /usr/bin/python /pgadmin/setup.py load-servers futureporn.pgadmin.json
+'''
+cmd_button('pgadmin:server',
+        argv=['sh', '-c', pgadmin_import_script],
+        resource='pgadmin',
+        icon_name='dataset',
+        text='create connection'
+)
+
+
+## Uncomment the following for fp/next in dev mode
+## this is useful for changing the UI and seeing results
+docker_build(
+    'fp/next',
+    '.',
+    only=['./pnpm-lock.yaml', './package.json', './packages/next'],
+    dockerfile='d.next.dockerfile',
+    target='dev',
+    build_args={
+        'NEXT_PUBLIC_STRAPI_URL': 'http://strapi.futureporn.svc.cluster.local:1337'
+    },
+    live_update=[
+        sync('./packages/next', '/app')
+    ]
+)
+
+
+docker_build(
+    'fp/scout',
+    '.',
+    only=['./pnpm-lock.yaml', './package.json', './packages/scout'],
+    dockerfile='d.scout.dockerfile',
+    live_update=[
+        sync('./packages/scout', '/app')
+    ]
+)
+k8s_resource(
+    workload='kubernetes-ingress-controller-manager',
+    links=[
+      link(os.getenv('NGROK_URL'), 'Endpoint')
+    ],
+    labels='ngrok'
+)
+
+# k8s_resource(
+#     workload='echo-deployment',
+#     port_forwards=['8080'],
+#     labels='debug'
+# )
+# k8s_resource(
+#     workload='game-2048',
+#     port_forwards=['8081:8080'],
+#     labels='debug'
+# )
+
+k8s_resource(
+    workload='next',
+    port_forwards=['3000']
+)
+k8s_resource(
+    workload='strapi',
+    port_forwards=['1337'],
+    links=[
+      link('http://localhost:1337/admin', 'Strapi Admin UI')
+    ]
+)
+k8s_resource(
+    workload='postgres',
+)
+k8s_resource(
+    workload='pgadmin',
+    port_forwards=['5050']
+)