diff --git a/.cache_ggshield b/.cache_ggshield index 37a9bf5..3dda68e 100644 --- a/.cache_ggshield +++ b/.cache_ggshield @@ -1 +1 @@ -{"last_found_secrets": [{"match": "f5c0ea409f9fc16f713ef47c76d4376c3f6ae0d0124f358f7e314dc81879b999", "name": "Generic High Entropy Secret - commit://staged/apps/bright/torrentfile-0.9.1/PKG-INFO"}]} \ No newline at end of file +{"last_found_secrets": [{"match": "a5cd2a1a54ccc453d07fae38adafb7b0791b4824afad85d962266e48a54be75a", "name": "Base64 Basic Authentication - commit://staged/services/tracker-helper/test/app.test.ts"}]} \ No newline at end of file diff --git a/.gitea/workflows/builder.yaml b/.gitea/workflows/builder.yaml index c2c2ac4..efc76c5 100644 --- a/.gitea/workflows/builder.yaml +++ b/.gitea/workflows/builder.yaml @@ -23,6 +23,21 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - uses: mr-smithers-excellent/docker-build-push@v6 + name: Build futureporn/tracker-helper + with: + image: futureporn/tracker-helper + tags: latest + registry: gitea.futureporn.net + directory: ./services/tracker-helper + dockerfile: Dockerfile + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + env: + WL_CREDENTIALS: ${{ secrets.WL_CREDENTIALS }} + WL_FIFO_PATH: /tmp/adder.fifo + WL_FILE_PATH: /usr/src/app/test/fixtures/whitelist + - uses: mr-smithers-excellent/docker-build-push@v6 name: Build futureporn/bright with: diff --git a/.gitea/workflows/tests.yaml b/.gitea/workflows/tests.yaml index 73917d1..299af7d 100644 --- a/.gitea/workflows/tests.yaml +++ b/.gitea/workflows/tests.yaml @@ -9,23 +9,6 @@ on: - cron: "6 */12 * * *" jobs: - # test_javascript: - # runs-on: ubuntu-latest - # environment: docker - # steps: - # - name: Check out code - # uses: actions/checkout@v3 - - # - name: Setup pnpm - # uses: pnpm/action-setup@v4 - # with: - # run_install: | - # - recursive: true - # args: [--frozen-lockfile, --strict-peer-dependencies] - - # - name: Unit test all packages - # run: pnpm test -r - test_phoenix: name: Tests & Checks runs-on: ubuntu-22.04 @@ -50,7 +33,6 @@ jobs: SITE_URL: https://futureporn.net SECRET_KEY_BASE: ${{ secrets.SECRET_KEY_BASE }} - # @blocking @see https://gitea.com/gitea/act_runner/issues/506 services: db: image: postgres:16 @@ -61,31 +43,49 @@ jobs: POSTGRES_USER: ${{ vars.DB_USER }} POSTGRES_PASSWORD: ${{ secrets.DB_PASS }} + tracker-helper: + image: gitea.futureporn.net/futureporn/tracker-helper:latest + ports: + - 3000:3000 + env: + WL_FIFO_PATH: /etc/opentracker/adder.fifo + WL_FILE_PATH: /etc/opentracker/whitelist + WL_CREDENTIALS: ${{ secrets.WL_CREDENTIALS }} + volumes: + - /tmp/futureporn/opentracker:/etc/opentracker + opentracker: image: gitea.futureporn.net/futureporn/opentracker:latest ports: - - 8666:8666 + - 6969:6969 env: - WHITELIST_USERNAME: ${{ secrets.WHITELIST_USERNAME }} - WHITELIST_PASSWORD_CADDY: ${{ secrets.WHITELIST_PASSWORD_CADDY }} - # @todo delete WHITELIST_PASSWORD ASAP. we're waiting for futureporn/opentracker to update - # We've corrected it to WHITELIST_PASSWORD_CADDY in the Caddyfile, - # but the change isn't live in the container yet. - WHITELIST_PASSWORD: ${{ secrets.WHITELIST_PASSWORD_CADDY }} WHITELIST_FEED_URL: https://bright.futureporn.net/torrents + volumes: + - /tmp/futureporn/opentracker:/etc/opentracker steps: - - name: Install apt packages - run: apt-get update && apt-get install -y iputils-ping postgresql + - name: wait a few seconds + run: sleep 30 - - name: Check opentracker pingability - run: ping -c 3 opentracker + - name: Debug services + run: docker ps -a + + # - name: Install apt packages + # run: apt-get update && apt-get install -y iputils-ping postgresql + + - name: tracker-helper service check (localhost) + run: curl http://localhost:3000/health + + - name: tracker-helper service check + run: curl http://tracker-helper:3000/health + + # - name: Check opentracker pingability + # run: ping -c 3 opentracker - name: opentracker service check - run: curl http://opentracker:8666/stats - - - name: opentracker whitelister service check - run: curl http://opentracker:8666/whitelister + run: | + getent hosts opentracker + curl -v http://opentracker:6969/stats - name: Check postgres pingability run: ping -c 3 db diff --git a/apps/README.md b/apps/README.md index 66c97a4..a2d366c 100644 --- a/apps/README.md +++ b/apps/README.md @@ -1,4 +1,4 @@ This directory is for complete applications that are NOT node.js applications. For node apps, see ../services -For node app dependencies, see ../packages +For node app common dependencies, see ../packages diff --git a/dockerfiles/opentracker.dockerfile b/apps/opentracker/Dockerfile similarity index 57% rename from dockerfiles/opentracker.dockerfile rename to apps/opentracker/Dockerfile index 7b89027..2b1273d 100644 --- a/dockerfiles/opentracker.dockerfile +++ b/apps/opentracker/Dockerfile @@ -4,6 +4,10 @@ # FROM gcc:14 AS compile-stage +ARG TINI_VERSION=v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-static /tini +RUN chmod +x /tini + RUN apt update ; \ apt install cvs -y @@ -15,6 +19,8 @@ RUN adduser \ WORKDIR /usr/src + + # Run libowfat compilation in separated layer to benefit from docker layer cache RUN cvs -d :pserver:cvs@cvs.fefe.de:/cvs -z9 co libowfat ; \ git clone git://erdgeist.org/opentracker ; \ @@ -34,7 +40,6 @@ RUN cd /usr/src/opentracker ; \ FEATURES+=-DWANT_DYNAMIC_ACCESSLIST \ ;\ bash -c 'mkdir -pv /tmp/stage/{etc/opentracker,bin}' ; \ - bash -c 'touch /tmp/stage/etc/opentracker/{white,black}list' ; \ cp -v opentracker.conf.sample /tmp/stage/etc/opentracker/opentracker.conf ; \ # Opentrack configuration file sed -ri \ @@ -42,39 +47,34 @@ RUN cd /usr/src/opentracker ; \ -e 's!(.*)(access.whitelist)(.*)!\2 /etc/opentracker/whitelist!g;' \ /tmp/stage/etc/opentracker/opentracker.conf ; \ install -m 755 opentracker.debug /tmp/stage/bin ; \ - make DESTDIR=/tmp/stage BINDIR="/bin" install + make DESTDIR=/tmp/stage BINDIR="/bin" install ; \ + mkfifo /tmp/stage/etc/opentracker/adder.fifo FROM alpine -RUN apk add --no-cache curl bash socat libstdc++ libc6-compat nodejs npm -ARG S6_OVERLAY_VERSION=v3.2.0.2 - - +COPY --from=compile-stage /tini / COPY --from=compile-stage /tmp/stage / COPY --from=compile-stage /etc/passwd /etc/passwd -COPY ./packages/opentracker/opentracker.conf /etc/opentracker/opentracker.conf -COPY ./packages/opentracker/root/ / +COPY ./opentracker.conf /etc/opentracker/opentracker.conf + +RUN chown -R 6969:6969 /etc/opentracker ; \ + chmod 0664 /etc/opentracker/whitelist ; \ + chmod 0664 /etc/opentracker/adder.fifo WORKDIR /etc/opentracker -## install caddy so we can use basic-auth on the /whitelist endpoint. -## caddy is run via s6-overlay -## we can remove Caddy once kamal-proxy is more mature @see https://github.com/basecamp/kamal-proxy/issues/48 -COPY --from=caddy:alpine /usr/bin/caddy /usr/bin/caddy +USER 6969 +RUN touch /etc/opentracker/whitelist +RUN ls -lash /etc/opentracker/ + +EXPOSE 6969/udp +EXPOSE 6969/tcp -# EXPOSE 6969/udp -# EXPOSE 6969/tcp -EXPOSE 8666/udp -EXPOSE 8666/tcp +HEALTHCHECK --interval=5s --timeout=3s --retries=5 \ + CMD curl -f http://localhost:6969/stats || exit 1 -## use s6-overlay -ADD https://github.com/just-containers/s6-overlay/releases/download/${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp -RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz -ADD https://github.com/just-containers/s6-overlay/releases/download/${S6_OVERLAY_VERSION}/s6-overlay-x86_64.tar.xz /tmp -RUN tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz -ENTRYPOINT ["/init"] -# CMD ["/etc/s6-overlay/s6-rc.d/svc-opentracker/run"] # IDK if this is correct -# USER 6969 # I think we can instead drop privs via s6 \ No newline at end of file +ENTRYPOINT ["/tini", "--", "/bin/opentracker"] +CMD ["-f", "/etc/opentracker/opentracker.conf"] \ No newline at end of file diff --git a/packages/opentracker/opentracker.conf b/apps/opentracker/opentracker.conf similarity index 99% rename from packages/opentracker/opentracker.conf rename to apps/opentracker/opentracker.conf index 918dc7b..d6127e9 100644 --- a/packages/opentracker/opentracker.conf +++ b/apps/opentracker/opentracker.conf @@ -64,7 +64,7 @@ access.whitelist /etc/opentracker/whitelist # The semantic of the respective dynamic changeset depends on whether # WANT_ACCESSLIST_WHITE or WANT_ACCESSLIST_BLACK is enabled. # -access.fifo_add /var/run/opentracker/adder.fifo +access.fifo_add /etc/opentracker/adder.fifo # # Any info_hash (format see above) written to the fifo_delete file will # be kept on a dynamic delete-changeset, removed from the dynamic diff --git a/packages/opentracker/whitelist b/apps/opentracker/whitelist similarity index 100% rename from packages/opentracker/whitelist rename to apps/opentracker/whitelist diff --git a/config/deploy.yml b/config/deploy.yml index ac1aea6..82d9dac 100644 --- a/config/deploy.yml +++ b/config/deploy.yml @@ -1,5 +1,3 @@ - - # Name of your application. Used to uniquely configure containers. service: futureporn @@ -11,7 +9,6 @@ servers: web: - 45.76.57.101 - # Enable SSL auto certification via Let's Encrypt and allow for multiple apps on a single web server. # Remove this section when using multiple web servers and ensure you terminate SSL at your load balancer. # @@ -49,7 +46,6 @@ builder: # args: # RUBY_VERSION: <%= File.read('.ruby-version').strip %> - env: clear: PORT: 4000 @@ -93,11 +89,28 @@ ssh: # Use accessory services (secrets come from .kamal/secrets). # accessories: + tracker-helper: + image: gitea.futureporn.net/futureporn/opentracker:latest + host: 45.76.57.101 + port: "127.0.0.1:3000:3000" + env: + secret: + - WL_CREDENTIALS + proxy: + ssl: true + forward_headers: true + app_port: 3000 + host: tracker-helper.futureporn.net + healthcheck: + path: /health + volumes: + - opentracker-etc:/etc/opentracker + - opentracker-var:/var/run/opentracker opentracker: image: gitea.futureporn.net/futureporn/opentracker:latest host: 45.76.57.101 - port: "127.0.0.1:8666:8666" + port: "127.0.0.1:6969:6969" env: clear: WHITELIST_FEED_URL: https://bright.futureporn.net/torrents @@ -107,7 +120,7 @@ accessories: proxy: ssl: true forward_headers: true - app_port: 8666 + app_port: 6969 host: tracker.futureporn.net healthcheck: path: /stats @@ -115,7 +128,6 @@ accessories: - opentracker-etc:/etc/opentracker - opentracker-var:/var/run/opentracker - qbittorrent: image: lscr.io/linuxserver/qbittorrent:latest host: 45.76.57.101 @@ -137,7 +149,6 @@ accessories: volumes: - /root/.cache/futureporn:/root/.cache/futureporn - db: image: postgres:15 host: 45.76.57.101 @@ -174,4 +185,4 @@ accessories: app_port: 5050 host: pgadmin.futureporn.net healthcheck: - path: /login \ No newline at end of file + path: /login diff --git a/docker-compose.yml b/docker-compose.yml index 753fac8..090037d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,14 +10,10 @@ services: env_file: - .kamal/secrets.development ports: - - "8666:8666/tcp" - - "8666:8666/udp" - # - "6969:6969/tcp" # accessed via caddy at port 80 - # - "6969:6969/udp" # accessed via caddy at port 80 - # - "8666:8666/tcp" # accessed via caddy at port 80 + - "6969:6969/tcp" + - "6969:6969/udp" volumes: - opentracker-etc:/etc/opentracker - - opentracker-var:/var/run/opentracker # qbittorrent: # build: @@ -36,20 +32,6 @@ services: # ports: # - "8181:8181/tcp" - # ## socat for exposing opentracker's named pipe (adder.fifo) to the docker network - # ## we use the named pipe to update the list of whitelisted torrents without having to reload the entire (huge) whitelist - # opentracker-socat: - # build: - # context: . - # dockerfile: dockerfiles/opentracker-socat.dockerfile - # container_name: opentracker-socat - # ports: - # - '8666:8666/tcp' - # volumes: - # # we use this volume to share adder.fifo - # - opentracker-var:/var/run/opentracker - # depends_on: - # - opentracker # bright: @@ -136,5 +118,4 @@ volumes: pg_data: redis_data: cache: - opentracker-var: opentracker-etc: \ No newline at end of file diff --git a/packages/opentracker/root/etc/caddy/Caddyfile b/packages/opentracker/root/etc/caddy/Caddyfile deleted file mode 100644 index 83889c6..0000000 --- a/packages/opentracker/root/etc/caddy/Caddyfile +++ /dev/null @@ -1,18 +0,0 @@ -{ - auto_https off - admin off - http_port 8666 -} - - - -:8666 { - reverse_proxy 127.0.0.1:6969 - - route /whitelist* { - basic_auth /whitelist { - {$WHITELIST_USERNAME} {$WHITELIST_PASSWORD_CADDY} - } - reverse_proxy 127.0.0.1:3001 - } -} \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/dependencies.d/base b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/dependencies.d/base deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/script b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/script deleted file mode 100755 index 53c2689..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/script +++ /dev/null @@ -1,9 +0,0 @@ -#!/command/with-contenv sh - - -mkdir -p /var/run/opentracker - -## Create FIFO only if it doesn't already exist -if [ ! -p /var/run/opentracker/adder.fifo ]; then - mkfifo -m a+rw /var/run/opentracker/adder.fifo -fi diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/type b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/type deleted file mode 100644 index 3d92b15..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/type +++ /dev/null @@ -1 +0,0 @@ -oneshot \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/up b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/up deleted file mode 100644 index ec63558..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/init-opentracker/up +++ /dev/null @@ -1 +0,0 @@ -/etc/s6-overlay/s6-rc.d/init-opentracker/script \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/dependencies.d/base b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/dependencies.d/base deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/run b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/run deleted file mode 100644 index 06afbb1..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/run +++ /dev/null @@ -1,15 +0,0 @@ -#!/command/with-contenv sh - - - -# curl -sS --request GET --url "${WHITELIST_FEED_URL}" --header 'accept: text/plain' -o /etc/opentracker/whitelist - - -echo "Loading torrent whitelist from ${WHITELIST_FEED_URL}" -curl -sS --request GET --url "${WHITELIST_FEED_URL}" --header 'accept: text/plain' -o /etc/opentracker/whitelist - -echo "Starting caddy" -exec 2>&1 -exec /usr/bin/caddy run --config /etc/caddy/Caddyfile - - diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/type b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/type deleted file mode 100644 index 1780f9f..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-caddy/type +++ /dev/null @@ -1 +0,0 @@ -longrun \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/dependencies.d/init-opentracker b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/dependencies.d/init-opentracker deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/finish b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/finish deleted file mode 100644 index ce74d4b..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/finish +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -echo "svc-opentracker was killed by 'docker stop'" -echo "@see https://github.com/just-containers/s6-overlay#setting-the-exit-code-of-the-container-to-the-exit-code-of-your-main-service" - -if test "$1" -eq 256 ; then - e=$((128 + $2)) -else - e="$1" -fi - -echo "$e" > /run/s6-linux-init-container-results/exitcode \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/run b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/run deleted file mode 100644 index 343e24b..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/run +++ /dev/null @@ -1,15 +0,0 @@ -#!/command/with-contenv sh - - -if [ -z "$WHITELIST_FEED_URL" ]; then - echo "Error: WHITELIST_FEED_URL is not set" >&2 - exit 1 -fi - - -echo "Loading whitelist from ${WHITELIST_FEED_URL}" -curl -sS --request GET --header "accept: text/plain" --url "$WHITELIST_FEED_URL" -o /etc/opentracker/whitelist || echo "Warning: Failed to fetch whitelist, using existing whitelist file." - - -s6-setuidgid farmhand /bin/opentracker -f /etc/opentracker/opentracker.conf - diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/type b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/type deleted file mode 100644 index 1780f9f..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-opentracker/type +++ /dev/null @@ -1 +0,0 @@ -longrun \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-whitelister/run b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-whitelister/run deleted file mode 100644 index 88af5b4..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-whitelister/run +++ /dev/null @@ -1,7 +0,0 @@ -#!/command/with-contenv sh - -export PORT=3001 - - -exec s6-setuidgid farmhand node /etc/whitelister/index.js - diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-whitelister/type b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-whitelister/type deleted file mode 100644 index 1780f9f..0000000 --- a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/svc-whitelister/type +++ /dev/null @@ -1 +0,0 @@ -longrun \ No newline at end of file diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/init-opentracker b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/init-opentracker deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-caddy b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-caddy deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-opentracker b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-opentracker deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-whitelister b/packages/opentracker/root/etc/s6-overlay/s6-rc.d/user/contents.d/svc-whitelister deleted file mode 100644 index e69de29..0000000 diff --git a/packages/opentracker/root/etc/whitelister/index.js b/packages/opentracker/root/etc/whitelister/index.js deleted file mode 100644 index 54b3620..0000000 --- a/packages/opentracker/root/etc/whitelister/index.js +++ /dev/null @@ -1,50 +0,0 @@ -const http = require('http'); -const fs = require('fs'); -const packagejson = require('./package.json'); - -const port = process.env.PORT || 3000; -const fifoPath = '/var/run/opentracker/adder.fifo'; - -console.log(`[${new Date().toISOString()}] Starting whitelister server ${packagejson.version} on port ${port}`); - -const server = http.createServer((req, res) => { - const requestStart = Date.now(); - console.log(`[${new Date().toISOString()}] Incoming request: ${req.method} ${req.url}`); - - if (req.method === 'POST') { - let body = ''; - - req.on('data', chunk => { - body += chunk; - }); - - req.on('end', () => { - console.log(`[${new Date().toISOString()}] Received POST body: ${body}`); - - fs.appendFile(fifoPath, body + '\n', err => { - if (err) { - console.error(`[${new Date().toISOString()}] ERROR writing to FIFO: ${err.message}`); - res.statusCode = 500; - res.end('Error writing to FIFO'); - return; - } - - console.log(`[${new Date().toISOString()}] Successfully added to whitelist`); - res.statusCode = 200; - res.end('Successfully added to whitelist'); - }); - }); - } else { - console.warn(`[${new Date().toISOString()}] Method not allowed: ${req.method} ${req.url}`); - res.statusCode = 405; - res.end('Method Not Allowed'); - } - - res.on('finish', () => { - console.log(`[${new Date().toISOString()}] Response sent. Status: ${res.statusCode}. Duration: ${Date.now() - requestStart}ms`); - }); -}); - -server.listen(port, () => { - console.log(`[${new Date().toISOString()}] Whitelister server ${packagejson.version} running on http://localhost:${port}`); -}); diff --git a/packages/opentracker/root/etc/whitelister/package.json b/packages/opentracker/root/etc/whitelister/package.json deleted file mode 100644 index 73e9427..0000000 --- a/packages/opentracker/root/etc/whitelister/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "whitelister", - "version": "1.0.0", - "description": "Add entries to opentracker's whitelist FIFO via HTTP", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "@CJ_Clippy", - "license": "unlicense" -} diff --git a/services/tracker-helper/.dockerignore b/services/tracker-helper/.dockerignore new file mode 100644 index 0000000..9b49524 --- /dev/null +++ b/services/tracker-helper/.dockerignore @@ -0,0 +1,15 @@ +node_modules +Dockerfile* +docker-compose* +.dockerignore +.git +.gitignore +README.md +LICENSE +.vscode +Makefile +helm-charts +.env +.editorconfig +.idea +coverage* \ No newline at end of file diff --git a/services/tracker-helper/.gitignore b/services/tracker-helper/.gitignore new file mode 100644 index 0000000..9b1ee42 --- /dev/null +++ b/services/tracker-helper/.gitignore @@ -0,0 +1,175 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/services/tracker-helper/Dockerfile b/services/tracker-helper/Dockerfile new file mode 100644 index 0000000..5aaac52 --- /dev/null +++ b/services/tracker-helper/Dockerfile @@ -0,0 +1,39 @@ +# use the official Bun image +# see all versions at https://hub.docker.com/r/oven/bun/tags +FROM oven/bun:1 AS base +WORKDIR /usr/src/app + +# install dependencies into temp directory +# this will cache them and speed up future builds +FROM base AS install +RUN mkdir -p /temp/dev +COPY package.json bun.lockb /temp/dev/ +RUN cd /temp/dev && bun install --frozen-lockfile + +# install with --production (exclude devDependencies) +RUN mkdir -p /temp/prod +COPY package.json bun.lockb /temp/prod/ +RUN cd /temp/prod && bun install --frozen-lockfile --production + +# copy node_modules from temp directory +# then copy all (non-ignored) project files into the image +FROM base AS prerelease +COPY --from=install /temp/dev/node_modules node_modules +COPY . . + +# [optional] tests & build +ENV NODE_ENV=production WL_FILE_PATH=/usr/src/app/test/fixtures/whitelist WL_FIFO_PATH=/tmp/adder.fifo +RUN --mount=type=secret,id=WL_CREDENTIALS,env=WL_CREDENTIALS,required=true \ + bun test + +# copy production dependencies and source code into final image +FROM base AS release +COPY --from=install /temp/prod/node_modules node_modules +COPY --from=prerelease /usr/src/app/index.ts . +COPY --from=prerelease /usr/src/app/app.ts . +COPY --from=prerelease /usr/src/app/package.json . + +# run the app +USER bun +EXPOSE 3000/tcp +ENTRYPOINT [ "bun", "run", "index.ts" ] \ No newline at end of file diff --git a/services/tracker-helper/README.md b/services/tracker-helper/README.md new file mode 100644 index 0000000..cc130f8 --- /dev/null +++ b/services/tracker-helper/README.md @@ -0,0 +1,30 @@ +# tracker-helper + +Helper service for opentracker. Adds opentracker whitelisting via HTTP. + +## setup + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.1.42. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. + + +## building a docker image + +tracker-helper unit & integration tests are run during the docker build. That step requires WL_CREDENTIALS env variable, as well as WL_FIFO_PATH and WL_FILE_PATH. WL_CREDENTIALS must be set to admin:admin during that test. Not really a secret at that stage, but to avoid docker complaining about "CREDENTIALS" in env, we pass it as a build `--secret`. The other two env vars are loaded from `secrets.testing`. + + dotenvx run -f ../../.kamal/secrets.testing -- docker build --secret id=WL_CREDENTIALS -t gitea.futureporn.net/futureporn/tracker-helper:latest . + +When validating the container before pushing to production, it can be run as follows + + dotenvx run -f ../../.kamal/secrets.production -- docker run -it --init --rm -p 3000:3000 -e WL_CREDENTIALS -e WL_FILE_PATH -e WL_FIFO_PATH fp/tracker-helper \ No newline at end of file diff --git a/services/tracker-helper/app.ts b/services/tracker-helper/app.ts new file mode 100644 index 0000000..4d64d23 --- /dev/null +++ b/services/tracker-helper/app.ts @@ -0,0 +1,75 @@ +import { Elysia, t, type Context } from 'elysia' +import { version } from './package.json'; +import { basicAuth } from '@eelkevdbos/elysia-basic-auth' + + + +const whitelistFilePath = process.env.WL_FILE_PATH || "/etc/opentracker/whitelist" +const adderFifoFilePath = process.env.WL_FIFO_PATH || "/var/run/opentracker/adder.fifo" + +const authOpts = { + scope: [ + "/whitelist", + "/version" + ], + credentials: { + env: 'WL_CREDENTIALS' + } +} + +const startupChecks = function startupChecks() { + + if (!process.env.WL_CREDENTIALS) { + const msg = `WL_CREDENTIALS is missing in env!` + if (process.env.NODE_ENV === "test") { + console.warn(msg) + } else { + throw new Error(msg) + } + } + + if (!process.env.WL_FILE_PATH) { + console.warn(`WL_FILE_PATH is missing in env. Using default ${whitelistFilePath}`) + } + + if (!process.env.WL_FIFO_PATH) { + console.warn(`WL_FIFO_PATH is missing in env. Using default ${adderFifoFilePath}`) + } + + // throw if the whitelist file doesn't exist + Bun.file(whitelistFilePath); + +} + + + +const getWhitelist = function getWhitelist(ctx: Context) { + const wl = Bun.file(whitelistFilePath); // relative to cwd + console.debug(`read from whitelist file at ${whitelistFilePath}. size=${wl.size}, type=${wl.type}`) + return wl.text() +} + +const postWhitelist = async function postWhitelist(ctx: Context) { + const body = ctx.body + console.log(`Whitelister is appending ${body} to fifo at ${adderFifoFilePath}`) + const fifo = Bun.file(adderFifoFilePath) + Bun.write(fifo, body + "\n") + console.log(`${body} was sent to the FIFO at ${adderFifoFilePath}`) + return body +} + + +startupChecks() + + +const app = new Elysia() + .use(basicAuth(authOpts)) + .get('/health', () => 'OK') + .get('/version', () => `version ${version} `) + .get('/whitelist', getWhitelist) + .post('/whitelist', postWhitelist, { + body: t.String() + }) + + +export default app diff --git a/services/tracker-helper/bun.lockb b/services/tracker-helper/bun.lockb new file mode 100755 index 0000000..9bb23f7 Binary files /dev/null and b/services/tracker-helper/bun.lockb differ diff --git a/services/tracker-helper/fifo-helper.ts b/services/tracker-helper/fifo-helper.ts new file mode 100644 index 0000000..718d6c3 --- /dev/null +++ b/services/tracker-helper/fifo-helper.ts @@ -0,0 +1,10 @@ +// fifo-helper.ts +async function main() { + const fifo = Bun.file("/tmp/adder.fifo") + // Bun.write(fifo, "testing 123\n") + const txt = await fifo.text() + console.log(txt) + +} + +main() diff --git a/services/tracker-helper/index.ts b/services/tracker-helper/index.ts new file mode 100644 index 0000000..14a53b1 --- /dev/null +++ b/services/tracker-helper/index.ts @@ -0,0 +1,4 @@ +import app from './app.ts' + +const port = process.env.PORT || 3000 +app.listen(port) \ No newline at end of file diff --git a/services/tracker-helper/package.json b/services/tracker-helper/package.json new file mode 100644 index 0000000..f51a2df --- /dev/null +++ b/services/tracker-helper/package.json @@ -0,0 +1,23 @@ +{ + "name": "tracker-helper", + "description": "Opentracker helper service. Adds info_hash whitelisting via HTTP", + "module": "index.ts", + "type": "module", + "version": "1.0.0", + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "dependencies": { + "@eelkevdbos/elysia-basic-auth": "^2.0.1", + "@elysiajs/eden": "^1.2.0", + "elysia": "^1.2.12" + }, + "scripts": { + "docker.build": "dotenvx run -f ../../.kamal/secrets.testing -- docker build --secret id=WL_CREDENTIALS -t gitea.futureporn.net/futureporn/tracker-helper:latest .", + "docker.run": "dotenvx run -f ../../.kamal/secrets.development -- docker run -e WL_CREDENTIALS -p 3000:3000 -t gitea.futureporn.net/futureporn/tracker-helper:latest", + "docker.push": "docker push gitea.futureporn.net/futureporn/tracker-helper:latest" + } +} \ No newline at end of file diff --git a/services/tracker-helper/pnpm-lock.yaml b/services/tracker-helper/pnpm-lock.yaml new file mode 100644 index 0000000..6e8b4be --- /dev/null +++ b/services/tracker-helper/pnpm-lock.yaml @@ -0,0 +1,127 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@eelkevdbos/elysia-basic-auth': + specifier: ^2.0.1 + version: 2.0.1(elysia@1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3))(typescript@5.7.3) + '@elysiajs/eden': + specifier: ^1.2.0 + version: 1.2.0(elysia@1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3)) + elysia: + specifier: ^1.2.12 + version: 1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3) + typescript: + specifier: ^5.0.0 + version: 5.7.3 + devDependencies: + '@types/bun': + specifier: latest + version: 1.2.2 + +packages: + + '@eelkevdbos/elysia-basic-auth@2.0.1': + resolution: {integrity: sha512-k2XOnq+0wg3RhLRhweADtiM9xLGNERV1iToMDJAT6naJfCviufRwlCDgOUiHaQDRR82u4F6c7XAtTFAMBVIATw==} + peerDependencies: + elysia: ^1.0.15 + typescript: ^5.0.0 + + '@elysiajs/eden@1.2.0': + resolution: {integrity: sha512-MpV45ahuF+iFZUg4tyJbLr9qxzY99m8clJVgQrDrz7Qh6eOKQ8MY6vjYMj3Wh21pTIRHPHzOLhVorRGby1/Owg==} + peerDependencies: + elysia: '>= 1.2.0' + + '@sinclair/typebox@0.34.22': + resolution: {integrity: sha512-0avTcz3XUm6mMcq5tQRoEnxyvmr3uanplFepD+a/TiDzOBZ0Us5bsShW41xOO2kST7AYv4xiCsmE5Ag02yOPfQ==} + + '@types/bun@1.2.2': + resolution: {integrity: sha512-tr74gdku+AEDN5ergNiBnplr7hpDp3V1h7fqI2GcR/rsUaM39jpSeKH0TFibRvU0KwniRx5POgaYnaXbk0hU+w==} + + '@types/node@22.13.4': + resolution: {integrity: sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==} + + '@types/ws@8.5.14': + resolution: {integrity: sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==} + + bun-types@1.2.2: + resolution: {integrity: sha512-RCbMH5elr9gjgDGDhkTTugA21XtJAy/9jkKe/G3WR2q17VPGhcquf9Sir6uay9iW+7P/BV0CAHA1XlHXMAVKHg==} + + cookie@1.0.2: + resolution: {integrity: sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==} + engines: {node: '>=18'} + + elysia@1.2.12: + resolution: {integrity: sha512-X1bZo09qe8/Poa/5tz08Y+sE/77B/wLwnA5xDDENU3FCrsUtYJuBVcy6BPXGRCgnJ1fPQpc0Ov2ZU5MYJXluTg==} + peerDependencies: + '@sinclair/typebox': '>= 0.34.0' + openapi-types: '>= 12.0.0' + typescript: '>= 5.0.0' + peerDependenciesMeta: + openapi-types: + optional: true + typescript: + optional: true + + memoirist@0.3.0: + resolution: {integrity: sha512-wR+4chMgVPq+T6OOsk40u9Wlpw1Pjx66NMNiYxCQQ4EUJ7jDs3D9kTCeKdBOkvAiqXlHLVJlvYL01PvIJ1MPNg==} + + typescript@5.7.3: + resolution: {integrity: sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.20.0: + resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} + +snapshots: + + '@eelkevdbos/elysia-basic-auth@2.0.1(elysia@1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3))(typescript@5.7.3)': + dependencies: + elysia: 1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3) + typescript: 5.7.3 + + '@elysiajs/eden@1.2.0(elysia@1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3))': + dependencies: + elysia: 1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3) + + '@sinclair/typebox@0.34.22': {} + + '@types/bun@1.2.2': + dependencies: + bun-types: 1.2.2 + + '@types/node@22.13.4': + dependencies: + undici-types: 6.20.0 + + '@types/ws@8.5.14': + dependencies: + '@types/node': 22.13.4 + + bun-types@1.2.2: + dependencies: + '@types/node': 22.13.4 + '@types/ws': 8.5.14 + + cookie@1.0.2: {} + + elysia@1.2.12(@sinclair/typebox@0.34.22)(typescript@5.7.3): + dependencies: + '@sinclair/typebox': 0.34.22 + cookie: 1.0.2 + memoirist: 0.3.0 + optionalDependencies: + typescript: 5.7.3 + + memoirist@0.3.0: {} + + typescript@5.7.3: {} + + undici-types@6.20.0: {} diff --git a/services/tracker-helper/test/app.test.ts b/services/tracker-helper/test/app.test.ts new file mode 100644 index 0000000..10f82d4 --- /dev/null +++ b/services/tracker-helper/test/app.test.ts @@ -0,0 +1,90 @@ +import { + describe + , expect + , it +} from 'bun:test' +import { + Elysia +} from 'elysia' +import { + treaty +} from '@elysiajs/eden' +import app from '../app.ts' + +if (!process.env.WL_FIFO_PATH) throw new Error("WL_FIFO_PATH is missing in env."); +if (!process.env.WL_CREDENTIALS) throw new Error("WL_CREDENTIALS is missing in env."); + + +function getCredentialsFromEnv(envValue?: string): { username: string; password: string } { + if (!envValue) throw new Error("WL_CREDENTIALS is not set"); + + const firstCredential = envValue.split(";")[0]; // Get the first username:password pair + const [username, password] = firstCredential.split(":"); + + if (!username || !password) throw new Error("Invalid credentials format"); + + return { username, password }; +} +const { username, password } = getCredentialsFromEnv(process.env.WL_CREDENTIALS) +const opts = { + headers: { + authorization: "Basic " + btoa(username + ':' + password) + } +} +const api = treaty(app) + + +describe + ('Elysia', () => { + it('return a health response', async () => { + const { data, status } = await api.health.get() + expect(status).toBe(200) + expect(data).toBe("OK") + }) + + it('return a version', async () => { + const { data, status } = await api.version.get(opts) + expect(status).toBe(200) + expect(data).toContain("version") + }) + + it('return a whitelist', async () => { + const { data, status } = await api.whitelist.get(opts) + expect(status).toBe(200) + expect(data).toContain("07b4516336e4afe9232c73bc312642590a7d7e95") + }) + + it('writes a new info_hash to a fifo', async () => { + const fifoFilePath = process.env.WL_FIFO_PATH! + const fifo = Bun.file(fifoFilePath) + const fifoExists = await fifo.exists(); + + + // create fifo if it doesn't exist + if (!fifoExists) { + await Bun.spawn(["mkfifo", fifoFilePath]).exited; + } + + // Start a process to read from the FIFO + const reader = Bun.spawn(["cat", fifoFilePath], { stdout: "pipe" }); + + const { data, status } = await api.whitelist.post("3aa5ad5e62eaffd148cff3dbe93ff2e1e9cbcf01", opts) + + const text = await new Response(reader.stdout).text(); + + expect(text).toBe("3aa5ad5e62eaffd148cff3dbe93ff2e1e9cbcf01\n") + expect(status).toBe(200) + expect(data).toContain("3aa5ad5e62eaffd148cff3dbe93ff2e1e9cbcf01") + }) + + it('returns 401 when username/password is missing from GET /whitelist ', async () => { + const { status } = await api.whitelist.get() + expect(status).toBe(401) + }) + + it('returns 401 when username/password is missing from POST /whitelist ', async () => { + const { status } = await api.whitelist.post() + expect(status).toBe(401) + }) + + }) diff --git a/services/tracker-helper/test/fixtures/whitelist b/services/tracker-helper/test/fixtures/whitelist new file mode 100644 index 0000000..42b2130 --- /dev/null +++ b/services/tracker-helper/test/fixtures/whitelist @@ -0,0 +1 @@ +07b4516336e4afe9232c73bc312642590a7d7e95 \ No newline at end of file diff --git a/services/tracker-helper/tsconfig.json b/services/tracker-helper/tsconfig.json new file mode 100644 index 0000000..238655f --- /dev/null +++ b/services/tracker-helper/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +}