rewrite

packages/link2cid/index.js

@@ -1,301 +1,9 @@
-'use strict';
+require('dotenv').config()
+const app = require('./src/app.js')
+const port = process.env.PORT || 3000
+const version = require('./package.json').version
 
-require('dotenv').config();
-const express = require('express');
-const bodyParser = require('body-parser');
-const cors = require('cors');
-const fs = require('fs');
-const fsp = require('fs/promises');
-const { openAsBlob } = require('node:fs');
-const { rm, stat } = require('fs/promises');
-const os = require('os');
-const path = require('path');
-const SseStream = require('ssestream').default;
-const { Transform, Readable } = require('node:stream');
-const { pipeline } = require('node:stream/promises');
-const { differenceInSeconds } = require('date-fns');
 
-const cidRegex = /Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}/;
+app.listen(port, () => {
-const app = express();
+    console.log(`link2cid ${version} listening on port ${port}`)
-app.use(cors());
-app.use(bodyParser.json());
-app.use(bodyParser.urlencoded({ extended: false }));
-
-// environment variables
-const port = process.env.PORT || 3000;
-const ipfsUrl = process.env.IPFS_URL || 'http://localhost:5001';
-if (!process.env.API_KEY) throw new Error('API_KEY was missing in env');
-if (!process.env.PORT) throw new Error('PORT is missing in env');
-
-async function ipfsHealthCheck() {
-    const url = ipfsUrl+'/version'
-    // console.log(`ipfsHealthCheck at url=${url}`)
-    try {
-        const res = await fetch(url, {
-            method: 'GET'
 })
-        const body = await res.text()
-        if (!body.includes('Version')) throw new Error('response from ipfs did not contain Version')
-    } catch (e) {
-        console.error('failure while checking IPFS connection.')
-        console.error(e)
-    }
-}
-
-// greetz https://stackoverflow.com/a/51302466/1004931
-async function downloadFile(url, filePath, sse) {
-    console.log(`downloading url=${url} to filePath=${filePath}`);
-    const res = await fetch(url);
-    const fileSize = res.headers.get('content-length');
-    const fileStream = fs.createWriteStream(filePath, { flags: 'wx' });
-
-    let downloadedBytes = 0;
-    const logInterval = 1 * 1024 * 1024; // 1MB in bytes
-
-    const progressLogger = new Transform({
-        transform(chunk, encoding, callback) {
-            downloadedBytes += chunk.length;
-
-            if (downloadedBytes % logInterval < chunk.length) {
-                console.log(`${downloadedBytes / (1024 * 1024)} MB processed`);
-                const progress = (downloadedBytes / fileSize) * 100;
-                console.log(`Download Progress: ${progress.toFixed(2)}%`);
-                sse.write({
-                    event: 'dlProgress',
-                    data: `${Math.floor(progress)}`
-                });
-            }
-
-            this.push(chunk);
-            callback();
-        }
-    });
-
-
-    await pipeline(
-        res.body,
-        progressLogger,
-        fileStream
-    )
-
-    console.log('download finished');
-    
-    // verify the file
-    // If we don't, we get text error messages sent to kubo which gets added and it's a bad time.
-    console.log(`fileSize=${fileSize}. downloadedBytes=${downloadedBytes}`);
-    if (fileSize != downloadedBytes) throw new Error('downloadedBytes did not match fileSize');
-}
-
-
-async function healthRes(_, res) {
-    const version = await getPackageVersion();
-    res.json({ error: false, message: `*link2cid ${version} pisses on the floor*` });
-}
-
-async function getPackageVersion() {
-    const packageJsonFile = await fsp.readFile(path.join(__dirname, 'package.json'), { encoding: 'utf-8' });
-    const json = JSON.parse(packageJsonFile);
-    return json.version;
-}
-
-/**
- * 
- * We use this to upload files and get progress notifications
- * 
- */
-async function streamingPostFetch(
-    url,
-    formData,
-    basename,
-    sse,
-    filesize
-) {
-    console.log(`streamingPostFetch with url=${url}, formData=${formData.get('file')}, basename=${basename}, sse=${sse}, filesize=${filesize}`);
-
-    try {
-        const res = await fetch(url, {
-            method: 'POST',
-            body: formData
-        });
-
-        if (!res.ok) {
-            throw new Error(`HTTP error! Status-- ${res.status}`);
-        }
-
-        const reader = res.body?.getReader();
-        if (!reader) {
-            throw new Error('Failed to get reader from response body');
-        }
-
-
-        while (true) {
-            const { done, value } = await reader.read();
-
-            const chunk = new TextDecoder().decode(value);
-            const lines = chunk.split('\n');
-            for (const line of lines) {
-                const trimmedLine = line.trim()
-                if (!!trimmedLine) {
-                    console.log(trimmedLine);
-                    const json = JSON.parse(trimmedLine);
-                    // console.log(`comparing json.Name=${json.Name} with basename=${basename}`);
-                    sse.write({
-                        event: 'addProgress',
-                        data: `${Math.floor(json?.Size / filesize * 100)}`
-                    })
-                    if (json.Name === basename && json.Hash && json.Size) {
-                        // this is the last chunk
-                        return json;
-                    }
-                }
-            }
-
-            if (done) {
-                throw new Error('Response reader finished before receiving a CID which indicates a failiure.');
-            }
-
-        }
-    } catch (error) {
-        console.error('An error occurred:', error);
-        throw error;
-    }
-}
-
-
-function authenticate(req, res, next) {
-    const apiKey = req.query?.token;
-    if (!apiKey) {
-        const msg = `authorization 'token' was missing from query`;
-        console.error(msg);
-        return res.status(401).json({ error: true, message: msg });
-    }
-    if (apiKey !== process.env.API_KEY) {
-        const msg = 'INCORRECT API_KEY (wrong token)';
-        console.error(msg);
-        return res.status(403).json({ error: true, message: msg });
-    } else {
-        next();
-    }
-}
-
-
-async function getFormStuff(filePath) {
-    const url = `${ipfsUrl}/api/v0/add?progress=false&cid-version=1&pin=true`;
-    const blob = await openAsBlob(filePath);
-    const basename = path.basename(filePath);
-    const filesize = (await stat(filePath)).size;
-    const formData = new FormData();
-    return {
-        url,
-        blob,
-        basename,
-        filesize,
-        formData
-    }
-}
-
-
-/**
- * Add a file from URL to IPFS.
- * 
- * uses SSE to send progress reports as the script
- * downloads the file to disk and then does `ipfs add`
- * finally returning a CID
- * 
- * events:
- *   - heartbeat
- *   - dlProgress
- *   - addProgress
- *   - end
- */
-async function addHandler(req, res) {
-    console.log(`/add`)
-    let url;
-    const urlStr = req.query.url;
-    if (!urlStr) return res.status(400).json({
-        error: 'url was missing from query'
-    });
-    try {
-        url = new URL(urlStr);
-    } catch (e) {
-        return res.status(400).json({
-            error: e?.message
-        })
-    }
-    const timestamp = new Date().valueOf();
-    const fileName = `${timestamp}-${url.pathname.split('/').at(-1)}`;
-    const destinationFilePath = path.join(os.tmpdir(), fileName);
-
-    console.log(`fileName=${fileName}, destinationFilePath=${destinationFilePath}`);
-
-    const sse = new SseStream(req);
-    sse.pipe(res);
-
-    let hbStartTime = new Date();
-    const heartbeat = setInterval(() => {
-        sse.write({
-            event: 'heartbeat',
-            data: `${differenceInSeconds(new Date(), hbStartTime)}`
-        });
-    }, 15000);
-
-    res.on('close', () => {
-        console.log('Connection closed.');
-        clearTimeout(heartbeat);
-        sse.unpipe(res);
-    });
-
-    console.log(`Downloading '${urlStr}' to destinationFilePath=${destinationFilePath}`);
-    await downloadFile(urlStr, destinationFilePath, sse);
-
-
-    sse.write({
-        event: 'dlProgress',
-        data: '100'
-    })
-
-    console.log(`'ipfs add' the file ${destinationFilePath}`);
-    const { url: kuboUrl, blob, basename, filesize, formData } = await getFormStuff(destinationFilePath);
-
-    formData.append('file', blob, basename);
-
-    let cid;
-    try {
-        const output = await streamingPostFetch(kuboUrl, formData, basename, sse, filesize);
-        console.log(`streamingPostFetch output as follows.`);
-        console.log(output);
-        if (!output?.Hash) throw new Error('No CID was received from remote IPFS node.');
-        if (!output?.Size) throw new Error(`'ipfs add' was missing Size in its output.`);
-        // if (output.Size !== filesize) throw new Error(`input and output sizes did not match. Expected output.Size ${output.Size} to equal ${filesize}.`);
-        // console.log(`filesize=${filesize} output.Size=${output.Size}`);
-        cid = output.Hash;
-
-        console.log('cleanup');
-        await rm(destinationFilePath);
-
-        console.log('end SSE');
-        clearTimeout(heartbeat);
-    } catch (e) {
-        return sse.end({
-            event: 'end',
-            error: true,
-            message: e
-        })
-    }
-    return sse.end({
-        event: 'end',
-        data: cid
-    })
-}
-
-app.get('/', authenticate, healthRes);
-app.get('/health', healthRes);
-app.get('/add', authenticate, addHandler);
-
-app.listen(port, async () => {
-    setInterval(async () => {
-        await ipfsHealthCheck()
-    }, 10000)
-    const version = await getPackageVersion();
-    console.log(`link2cid ${version} listening on port ${port}`);
-});

packages/link2cid/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@futureporn/link2cid",
-  "version": "3.2.0",
+  "version": "4.2.0",
-  "description": "REST API for adding files to IPFS",
+  "description": "REST API for adding files via URL to IPFS",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "mocha \"./src/**/*.spec.js\"",
     "dev": "pnpm nodemon ./index.js",
     "start": "node index.js"
   },
@@ -15,16 +15,17 @@
   "author": "",
   "license": "Unlicense",
   "dependencies": {
+    "@paralleldrive/cuid2": "^2.2.2",
     "@types/express": "^4.17.21",
+    "better-queue": "^3.8.12",
     "body-parser": "^1.20.2",
     "cors": "^2.8.5",
-    "date-fns": "^3.0.5",
     "dotenv": "^16.3.1",
-    "express": "^4.18.2",
+    "express": "^4.18.2"
-    "jsonwebtoken": "^9.0.2",
-    "ssestream": "^1.1.0"
   },
   "devDependencies": {
-    "nodemon": "^3.0.3"
+    "chai": "^5.1.0",
+    "nodemon": "^3.0.3",
+    "supertest": "^6.3.4"
   }
 }

packages/link2cid/pnpm-lock.yaml

@@ -5,33 +5,49 @@ settings:
   excludeLinksFromLockfile: false
 
 dependencies:
+  '@paralleldrive/cuid2':
+    specifier: ^2.2.2
+    version: 2.2.2
   '@types/express':
     specifier: ^4.17.21
     version: 4.17.21
+  better-queue:
+    specifier: ^3.8.12
+    version: 3.8.12
   body-parser:
     specifier: ^1.20.2
     version: 1.20.2
   cors:
     specifier: ^2.8.5
     version: 2.8.5
-  date-fns:
-    specifier: ^3.0.5
-    version: 3.0.5
   dotenv:
     specifier: ^16.3.1
     version: 16.3.1
   express:
     specifier: ^4.18.2
     version: 4.18.2
-  jsonwebtoken:
+
-    specifier: ^9.0.2
+devDependencies:
-    version: 9.0.2
+  chai:
-  ssestream:
+    specifier: ^5.1.0
-    specifier: ^1.1.0
+    version: 5.1.0
-    version: 1.1.0
+  supertest:
+    specifier: ^6.3.4
+    version: 6.3.4
 
 packages:
 
+  /@noble/hashes@1.4.0:
+    resolution: {integrity: sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==}
+    engines: {node: '>= 16'}
+    dev: false
+
+  /@paralleldrive/cuid2@2.2.2:
+    resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==}
+    dependencies:
+      '@noble/hashes': 1.4.0
+    dev: false
+
   /@types/body-parser@1.19.5:
     resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}
     dependencies:
@@ -116,6 +132,31 @@ packages:
     resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==}
     dev: false
 
+  /asap@2.0.6:
+    resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}
+    dev: true
+
+  /assertion-error@2.0.1:
+    resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}
+    engines: {node: '>=12'}
+    dev: true
+
+  /asynckit@0.4.0:
+    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+    dev: true
+
+  /better-queue-memory@1.0.4:
+    resolution: {integrity: sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA==}
+    dev: false
+
+  /better-queue@3.8.12:
+    resolution: {integrity: sha512-D9KZ+Us+2AyaCz693/9AyjTg0s8hEmkiM/MB3i09cs4MdK1KgTSGJluXRYmOulR69oLZVo2XDFtqsExDt8oiLA==}
+    dependencies:
+      better-queue-memory: 1.0.4
+      node-eta: 0.9.0
+      uuid: 9.0.1
+    dev: false
+
   /body-parser@1.20.1:
     resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==}
     engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -156,10 +197,6 @@ packages:
       - supports-color
     dev: false
 
-  /buffer-equal-constant-time@1.0.1:
-    resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}
-    dev: false
-
   /bytes@3.1.2:
     resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}
     engines: {node: '>= 0.8'}
@@ -171,7 +208,33 @@ packages:
       function-bind: 1.1.2
       get-intrinsic: 1.2.2
       set-function-length: 1.1.1
-    dev: false
+
+  /chai@5.1.0:
+    resolution: {integrity: sha512-kDZ7MZyM6Q1DhR9jy7dalKohXQ2yrlXkk59CR52aRKxJrobmlBNqnFQxX9xOX8w+4mz8SYlKJa/7D7ddltFXCw==}
+    engines: {node: '>=12'}
+    dependencies:
+      assertion-error: 2.0.1
+      check-error: 2.0.0
+      deep-eql: 5.0.1
+      loupe: 3.1.0
+      pathval: 2.0.0
+    dev: true
+
+  /check-error@2.0.0:
+    resolution: {integrity: sha512-tjLAOBHKVxtPoHe/SA7kNOMvhCRdCJ3vETdeY0RuAc9popf+hyaSV6ZEg9hr4cpWF7jmo/JSWEnLDrnijS9Tog==}
+    engines: {node: '>= 16'}
+    dev: true
+
+  /combined-stream@1.0.8:
+    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      delayed-stream: 1.0.0
+    dev: true
+
+  /component-emitter@1.3.1:
+    resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}
+    dev: true
 
   /content-disposition@0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
@@ -194,6 +257,10 @@ packages:
     engines: {node: '>= 0.6'}
     dev: false
 
+  /cookiejar@2.1.4:
+    resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==}
+    dev: true
+
   /cors@2.8.5:
     resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==}
     engines: {node: '>= 0.10'}
@@ -202,10 +269,6 @@ packages:
       vary: 1.1.2
     dev: false
 
-  /date-fns@3.0.5:
-    resolution: {integrity: sha512-Q4Tq5c5s/Zl/zbgdWf6pejn9ru7UwdIlLfvEEg1hVsQNQ7LKt76qIduagIT9OPK7+JCv1mAKherdU6bOqGYDnw==}
-    dev: false
-
   /debug@2.6.9:
     resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
     peerDependencies:
@@ -217,6 +280,23 @@ packages:
       ms: 2.0.0
     dev: false
 
+  /debug@4.3.4:
+    resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+    dependencies:
+      ms: 2.1.2
+    dev: true
+
+  /deep-eql@5.0.1:
+    resolution: {integrity: sha512-nwQCf6ne2gez3o1MxWifqkciwt0zhl0LO1/UwVu4uMBuPmflWM4oQ70XMqHqnBJA+nhzncaqL9HVL6KkHJ28lw==}
+    engines: {node: '>=6'}
+    dev: true
+
   /define-data-property@1.1.1:
     resolution: {integrity: sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==}
     engines: {node: '>= 0.4'}
@@ -224,7 +304,11 @@ packages:
       get-intrinsic: 1.2.2
       gopd: 1.0.1
       has-property-descriptors: 1.0.1
-    dev: false
+
+  /delayed-stream@1.0.0:
+    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+    engines: {node: '>=0.4.0'}
+    dev: true
 
   /depd@2.0.0:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
@@ -236,17 +320,18 @@ packages:
     engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
     dev: false
 
+  /dezalgo@1.0.4:
+    resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==}
+    dependencies:
+      asap: 2.0.6
+      wrappy: 1.0.2
+    dev: true
+
   /dotenv@16.3.1:
     resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==}
     engines: {node: '>=12'}
     dev: false
 
-  /ecdsa-sig-formatter@1.0.11:
-    resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
-    dependencies:
-      safe-buffer: 5.2.1
-    dev: false
-
   /ee-first@1.1.1:
     resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
     dev: false
@@ -304,6 +389,10 @@ packages:
       - supports-color
     dev: false
 
+  /fast-safe-stringify@2.1.1:
+    resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
+    dev: true
+
   /finalhandler@1.2.0:
     resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==}
     engines: {node: '>= 0.8'}
@@ -319,6 +408,24 @@ packages:
       - supports-color
     dev: false
 
+  /form-data@4.0.0:
+    resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
+    engines: {node: '>= 6'}
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      mime-types: 2.1.35
+    dev: true
+
+  /formidable@2.1.2:
+    resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==}
+    dependencies:
+      dezalgo: 1.0.4
+      hexoid: 1.0.0
+      once: 1.4.0
+      qs: 6.11.0
+    dev: true
+
   /forwarded@0.2.0:
     resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
     engines: {node: '>= 0.6'}
@@ -331,7 +438,10 @@ packages:
 
   /function-bind@1.1.2:
     resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
-    dev: false
+
+  /get-func-name@2.0.2:
+    resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==}
+    dev: true
 
   /get-intrinsic@1.2.2:
     resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==}
@@ -340,36 +450,35 @@ packages:
       has-proto: 1.0.1
       has-symbols: 1.0.3
       hasown: 2.0.0
-    dev: false
 
   /gopd@1.0.1:
     resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
     dependencies:
       get-intrinsic: 1.2.2
-    dev: false
 
   /has-property-descriptors@1.0.1:
     resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==}
     dependencies:
       get-intrinsic: 1.2.2
-    dev: false
 
   /has-proto@1.0.1:
     resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==}
     engines: {node: '>= 0.4'}
-    dev: false
 
   /has-symbols@1.0.3:
     resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==}
     engines: {node: '>= 0.4'}
-    dev: false
 
   /hasown@2.0.0:
     resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==}
     engines: {node: '>= 0.4'}
     dependencies:
       function-bind: 1.1.2
-    dev: false
+
+  /hexoid@1.0.0:
+    resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==}
+    engines: {node: '>=8'}
+    dev: true
 
   /http-errors@2.0.0:
     resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
@@ -398,71 +507,18 @@ packages:
     engines: {node: '>= 0.10'}
     dev: false
 
-  /jsonwebtoken@9.0.2:
+  /loupe@3.1.0:
-    resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==}
+    resolution: {integrity: sha512-qKl+FrLXUhFuHUoDJG7f8P8gEMHq9NFS0c6ghXG1J0rldmZFQZoNVv/vyirE9qwCIhWZDsvEFd1sbFu3GvRQFg==}
-    engines: {node: '>=12', npm: '>=6'}
     dependencies:
-      jws: 3.2.2
+      get-func-name: 2.0.2
-      lodash.includes: 4.3.0
+    dev: true
-      lodash.isboolean: 3.0.3
-      lodash.isinteger: 4.0.4
-      lodash.isnumber: 3.0.3
-      lodash.isplainobject: 4.0.6
-      lodash.isstring: 4.0.1
-      lodash.once: 4.1.1
-      ms: 2.1.3
-      semver: 7.5.4
-    dev: false
-
-  /jwa@1.4.1:
-    resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==}
-    dependencies:
-      buffer-equal-constant-time: 1.0.1
-      ecdsa-sig-formatter: 1.0.11
-      safe-buffer: 5.2.1
-    dev: false
-
-  /jws@3.2.2:
-    resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==}
-    dependencies:
-      jwa: 1.4.1
-      safe-buffer: 5.2.1
-    dev: false
-
-  /lodash.includes@4.3.0:
-    resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==}
-    dev: false
-
-  /lodash.isboolean@3.0.3:
-    resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
-    dev: false
-
-  /lodash.isinteger@4.0.4:
-    resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==}
-    dev: false
-
-  /lodash.isnumber@3.0.3:
-    resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==}
-    dev: false
-
-  /lodash.isplainobject@4.0.6:
-    resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==}
-    dev: false
-
-  /lodash.isstring@4.0.1:
-    resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==}
-    dev: false
-
-  /lodash.once@4.1.1:
-    resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==}
-    dev: false
 
   /lru-cache@6.0.0:
     resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
     engines: {node: '>=10'}
     dependencies:
       yallist: 4.0.0
-    dev: false
+    dev: true
 
   /media-typer@0.3.0:
     resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
@@ -476,19 +532,16 @@ packages:
   /methods@1.1.2:
     resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
     engines: {node: '>= 0.6'}
-    dev: false
 
   /mime-db@1.52.0:
     resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
     engines: {node: '>= 0.6'}
-    dev: false
 
   /mime-types@2.1.35:
     resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
     engines: {node: '>= 0.6'}
     dependencies:
       mime-db: 1.52.0
-    dev: false
 
   /mime@1.6.0:
     resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==}
@@ -496,10 +549,20 @@ packages:
     hasBin: true
     dev: false
 
+  /mime@2.6.0:
+    resolution: {integrity: sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==}
+    engines: {node: '>=4.0.0'}
+    hasBin: true
+    dev: true
+
   /ms@2.0.0:
     resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}
     dev: false
 
+  /ms@2.1.2:
+    resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
+    dev: true
+
   /ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
     dev: false
@@ -509,6 +572,10 @@ packages:
     engines: {node: '>= 0.6'}
     dev: false
 
+  /node-eta@0.9.0:
+    resolution: {integrity: sha512-mTCTZk29tmX1OGfVkPt63H3c3VqXrI2Kvua98S7iUIB/Gbp0MNw05YtUomxQIxnnKMyRIIuY9izPcFixzhSBrA==}
+    dev: false
+
   /object-assign@4.1.1:
     resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
     engines: {node: '>=0.10.0'}
@@ -516,7 +583,6 @@ packages:
 
   /object-inspect@1.13.1:
     resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==}
-    dev: false
 
   /on-finished@2.4.1:
     resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
@@ -525,6 +591,12 @@ packages:
       ee-first: 1.1.1
     dev: false
 
+  /once@1.4.0:
+    resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
+    dependencies:
+      wrappy: 1.0.2
+    dev: true
+
   /parseurl@1.3.3:
     resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}
     engines: {node: '>= 0.8'}
@@ -534,6 +606,11 @@ packages:
     resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==}
     dev: false
 
+  /pathval@2.0.0:
+    resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==}
+    engines: {node: '>= 14.16'}
+    dev: true
+
   /proxy-addr@2.0.7:
     resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
     engines: {node: '>= 0.10'}
@@ -547,7 +624,6 @@ packages:
     engines: {node: '>=0.6'}
     dependencies:
       side-channel: 1.0.4
-    dev: false
 
   /range-parser@1.2.1:
     resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
@@ -588,7 +664,7 @@ packages:
     hasBin: true
     dependencies:
       lru-cache: 6.0.0
-    dev: false
+    dev: true
 
   /send@0.18.0:
     resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==}
@@ -631,7 +707,6 @@ packages:
       get-intrinsic: 1.2.2
       gopd: 1.0.1
       has-property-descriptors: 1.0.1
-    dev: false
 
   /setprototypeof@1.2.0:
     resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
@@ -643,17 +718,40 @@ packages:
       call-bind: 1.0.5
       get-intrinsic: 1.2.2
       object-inspect: 1.13.1
-    dev: false
-
-  /ssestream@1.1.0:
-    resolution: {integrity: sha512-UOS3JTuGqGEOH89mfHFwVOJNH2+JX9ebIWuw6WBQXpkVOxbdoY3RMliSHzshL4XVYJJrcul5NkuvDFCzgYu1Lw==}
-    dev: false
 
   /statuses@2.0.1:
     resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
     engines: {node: '>= 0.8'}
     dev: false
 
+  /superagent@8.1.2:
+    resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==}
+    engines: {node: '>=6.4.0 <13 || >=14'}
+    dependencies:
+      component-emitter: 1.3.1
+      cookiejar: 2.1.4
+      debug: 4.3.4
+      fast-safe-stringify: 2.1.1
+      form-data: 4.0.0
+      formidable: 2.1.2
+      methods: 1.1.2
+      mime: 2.6.0
+      qs: 6.11.0
+      semver: 7.5.4
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
+  /supertest@6.3.4:
+    resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==}
+    engines: {node: '>=6.4.0'}
+    dependencies:
+      methods: 1.1.2
+      superagent: 8.1.2
+    transitivePeerDependencies:
+      - supports-color
+    dev: true
+
   /toidentifier@1.0.1:
     resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
     engines: {node: '>=0.6'}
@@ -681,11 +779,20 @@ packages:
     engines: {node: '>= 0.4.0'}
     dev: false
 
+  /uuid@9.0.1:
+    resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}
+    hasBin: true
+    dev: false
+
   /vary@1.1.2:
     resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
     engines: {node: '>= 0.8'}
     dev: false
 
+  /wrappy@1.0.2:
+    resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
+    dev: true
+
   /yallist@4.0.0:
     resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
-    dev: false
+    dev: true

packages/link2cid/src/app.js

@@ -0,0 +1,33 @@
+'use strict';
+
+require('dotenv').config();
+const express = require('express');
+const bodyParser = require('body-parser');
+const cors = require('cors');
+const fs = require('fs');
+const fsp = require('fs/promises');
+const { openAsBlob } = require('node:fs');
+const { rm, stat } = require('fs/promises');
+const os = require('os');
+const path = require('path');
+const { authenticate } = require('./middleware/auth.js')
+const { createTask, readTask, deleteTask } = require('./models/task.js')
+const readHeath = require('./models/health.js')
+const store = require('./middleware/store.js');
+const queue = require('./middleware/queue.js');
+
+const app = express();
+app.use(store);
+app.use(queue);
+app.use(cors());
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: false }));
+
+
+app.post('/task', authenticate, createTask)
+app.get('/task', readTask)
+app.delete('/task', authenticate, deleteTask)
+app.get('/health', readHeath)
+
+
+module.exports = app

packages/link2cid/src/app.spec.js

@@ -0,0 +1,109 @@
+
+const app = require('./app.js')
+const request = require('supertest')
+const qs = require('querystring')
+require('dotenv').config()
+
+describe('app', function () {
+    it('should exist', function (done) {
+        if (!app?.mountpath) throw new Error('app doesnt exist');
+        done()
+    })
+
+
+    describe('/health', function () {
+        it('should be publicly readable', function (done) {
+            request(app)
+                .get('/health')
+                .set('Accept', 'text/html')
+                .expect('Content-Type', /text/)
+                .expect(/piss/)
+                .expect(200, done)
+        })
+    })
+
+    describe('/task', function () {
+        describe('POST', function () {
+            it('should create a task', function (done) {
+                request(app)
+                    .post('/task')
+                    .set('Authorization', `Bearer ${process.env.API_KEY}`)
+                    .set('Accept', 'application/json')
+                    .send({
+                        url: 'https://futureporn-b2.b-cdn.net/projekt-melody.jpg'
+                    })
+                    .expect('Content-Type', /json/)
+                    .expect((res) => {
+                        if (!res.body?.data) throw new Error('response body was missing data')
+                        if (!res.body?.data?.id) throw new Error('response body was missing id')
+                        return true
+                    })
+                    .expect(200, done)
+            })
+        })
+        describe('GET', function () {
+            it('should show all tasks specifications', async function () {
+                await request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/my.jpg' })
+                await request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/your.png' })
+                return request(app)
+                    .get(`/task`)
+                    .set('Authorization', `Bearer ${process.env.API_KEY}`)
+                    .set('Accept', 'application/json')
+                    .expect('Content-Type', /json/)
+                    .expect((res) => {
+                        if (!res?.body?.data) throw new Error('there was no data in response')
+                    })
+                    .expect(200)
+            })
+            it('should accept task id as query param and return task specification', function (done) {
+                const seed = request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/z.jpg' })
+                seed.then((res) => {
+                    
+                    const query = qs.stringify({
+                        id: res.body.data.id
+                    })
+                    request(app)
+                        .get(`/task?${query}`)
+                        .set('Accept', 'application/json')
+                        .expect('Content-Type', /json/)
+                        .expect((res) => {
+                            if (res?.body?.error) throw new Error('there was an error in the response: '+res.body?.message)
+                            if (!res?.body?.data?.url) throw new Error('data.url was missing')
+                            if (!res?.body?.data?.createdAt) throw new Error('data.createdAt was missing')
+                            return true
+                        })
+                        .expect(200, done)
+                })
+            })
+            it('should show all tasks by default', function (done) {
+                request(app)
+                    .get('/task')
+                    .set('Accept', 'application/json')
+                    .expect('Content-Type', /json/)
+                    .expect((res) => {
+                        if (res.body?.error) throw new Error('there was an error in the response'+res.error)
+                        if (!res.body?.data) throw new Error('data was missing')
+                        return true
+                    })
+                    .expect(200, done)
+
+            })
+        })
+        describe('DELETE', function () {
+            const query = qs.stringify({
+                id: 'awejf9wiejf9we'
+            })
+            it('should delete a single task', function (done) {
+                request(app)
+                    .delete(`/task?${query}`)
+                    .set('Authorization', `Bearer ${process.env.API_KEY}`)
+                    .set('Accept', 'application/json')
+                    .expect('Content-Type', /json/)
+                    .expect(200, done);
+            });
+        })
+    })
+
+
+
+})

packages/link2cid/src/fixtures/hello-worlds.txt

@@ -0,0 +1 @@
+hello worlds

packages/link2cid/src/middleware/auth.js

@@ -0,0 +1,16 @@
+
+module.exports.authenticate = function authenticate(req, res, next) {
+    const bearerToken = req.headers?.authorization.split(' ').at(1);
+    if (!bearerToken) {
+        const msg = `authorization bearer token was missing from request headers`;
+        console.error(msg);
+        return res.status(401).json({ error: true, message: msg });
+    }
+    if (bearerToken !== process.env.API_KEY) {
+        const msg = 'INCORRECT API_KEY (wrong token)';
+        console.error(msg);
+        return res.status(403).json({ error: true, message: msg });
+    } else {
+        next();
+    }
+}

packages/link2cid/src/middleware/queue.js

@@ -0,0 +1,21 @@
+const Queue = require('better-queue');
+const taskProcess = require('../utils/taskProcess.js');
+
+const options = {
+    id: 'id',
+    maxRetries: 3,
+    concurrent: 1 
+    // @todo better-queue has batching and concurrency. might be useful to implement in the future 
+    // @see https://github.com/diamondio/better-queue?tab=readme-ov-file#queue-management
+}
+
+let q = new Queue(taskProcess, options)
+
+
+// Middleware function to attach db to request
+const queueMiddleware = (req, res, next) => {
+    req.queue = q;
+    next();
+};
+
+module.exports = queueMiddleware

packages/link2cid/src/middleware/store.js

@@ -0,0 +1,10 @@
+const store = {
+    tasks: {}
+}
+
+const storeMiddleware = (req, res, next) => {
+    req.store = store
+    next();
+};
+
+module.exports = storeMiddleware;

packages/link2cid/src/models/health.js

@@ -0,0 +1,3 @@
+module.exports = function readHealth (req, res) {
+    return res.send('**link2cid pisses on the floor**')
+}

packages/link2cid/src/models/task.js

@@ -0,0 +1,59 @@
+const { createId } = require('@paralleldrive/cuid2');
+const { getTmpFilePath } = require('../utils/paths.js');
+const fsp = require('fs/promises');
+
+
+module.exports.createTask = function createTask (req, res) {
+
+    const url = req.body.url
+    const task = {
+        id: createId(),
+        url: url,
+        filePath: getTmpFilePath(url),
+        fileSize: null,
+        createdAt: new Date().toISOString(),
+        cid: null,
+        downloadProgress: null,
+        addProgress: null
+    }
+
+    if (!req?.body?.url) return res.status(400).json({ error: true, message: 'request body was missing a url' });
+
+    req.store.tasks[task.id] = task;
+    req.queue.push(task, function (err, result) {
+        if (err) throw err;
+        console.log('the following is the result of the queued task being complete')
+        console.log(result)
+    })
+
+    return res.json({ error: false, data: task })
+
+}
+module.exports.readTask = function readTask (req, res) {
+    const id = req?.query?.id
+
+    // If we get an id in the query, show the one task.
+    // Otherwise, we show all tasks.
+    if (!!id) {
+        const task = req.store.tasks[id]
+        if (!task) return res.json({ error: true, message: 'there was no task in the store with that id' });
+        return res.json({ error: false, data: task })
+    } else {
+        const tasks = req.store.tasks
+        return res.json({ error: false, data: tasks })
+    }
+
+}
+module.exports.deleteTask = async function deleteTask (req, res) {
+    const id = req?.query?.id;
+    const task = req.store.tasks[id];
+    
+    try {
+        if (task?.filePath) await fsp.unlink(task.filePath);
+    } catch (err) {}
+    delete req.store.tasks[id];
+    return res.json({ error: false, message: 'task deleted' });
+    if (err) return res.json({ error: true, message: err });
+}
+
+

packages/link2cid/src/utils/download.js

@@ -0,0 +1,20 @@
+const fs = require("fs");
+const { Readable } = require('stream');
+const { finished } = require('stream/promises');
+const path = require("path");
+
+
+/**
+ * Download a file at url to local disk filePath
+ * @param {String} url 
+ * @param {String} filePath
+ * 
+ * greetz https://stackoverflow.com/a/51302466/1004931
+ */
+const download = (async (url, filePath) => {
+    const res = await fetch(url);
+    const fileStream = fs.createWriteStream(filePath, { flags: 'wx' });
+    await finished(Readable.fromWeb(res.body).pipe(fileStream));
+});
+
+module.exports = download;

packages/link2cid/src/utils/download.spec.js

@@ -0,0 +1,12 @@
+const download = require('./download.js');
+const fsp = require('fs/promises');
+
+describe('download', function () {
+    it('should download a file from url', async function () {
+        const testFilePath = '/tmp/pmel.jpg'
+        try {
+            await fsp.unlink(testFilePath)
+        } catch (e) {}
+        await download('https://futureporn-b2.b-cdn.net/projekt-melody.jpg', testFilePath)
+    })
+})

packages/link2cid/src/utils/ipfsAdd.js

@@ -0,0 +1,100 @@
+
+require('dotenv').config();
+const { openAsBlob } = require('node:fs');
+const { rm, stat } = require('fs/promises');
+const path = require('path');
+
+if (!process.env.IPFS_URL) throw new Error('IPFS_URL was missing in env');
+
+
+async function streamingPostFetch(
+    url,
+    formData,
+    basename,
+    filesize
+) {
+    // console.log(`streamingPostFetch with url=${url}, formData=${formData.get('file')}, basename=${basename}, filesize=${filesize}`);
+
+    try {
+        const res = await fetch(url, {
+            method: 'POST',
+            body: formData
+        });
+
+        if (!res.ok) {
+            throw new Error(`HTTP error! Status-- ${res.status}`);
+        }
+
+        const reader = res.body?.getReader();
+        if (!reader) {
+            throw new Error('Failed to get reader from response body');
+        }
+
+
+        while (true) {
+            const { done, value } = await reader.read();
+
+            const chunk = new TextDecoder().decode(value);
+            const lines = chunk.split('\n');
+            for (const line of lines) {
+                const trimmedLine = line.trim()
+                if (!!trimmedLine) {
+                    // console.log(trimmedLine);
+                    const json = JSON.parse(trimmedLine);
+                    // console.log(`comparing json.Name=${json.Name} with basename=${basename}`);
+                    if (json.Name === basename && json.Hash && json.Size) {
+                        // this is the last chunk
+                        return json;
+                    }
+                }
+            }
+
+            if (done) {
+                throw new Error('Response reader finished before receiving a CID which indicates a failiure.');
+            }
+
+        }
+    } catch (error) {
+        console.error('An error occurred:', error);
+        throw error;
+    }
+}
+
+
+async function getFormStuff(filePath) {
+    const url = `${process.env.IPFS_URL}/api/v0/add?progress=false&cid-version=1&pin=true`;
+    const blob = await openAsBlob(filePath);
+    const basename = path.basename(filePath);
+    const filesize = (await stat(filePath)).size;
+    const formData = new FormData();
+    return {
+        url,
+        blob,
+        basename,
+        filesize,
+        formData
+    }
+}
+
+
+/**
+ * @param {String} filePath 
+ * @returns {String} CID
+ */
+const ipfsAdd = async function (filePath) {
+    
+    
+    const { url: kuboUrl, blob, basename, filesize, formData } = await getFormStuff(filePath);
+    
+
+    formData.append('file', blob, basename);
+    const output = await streamingPostFetch(kuboUrl, formData, basename, filesize);
+    if (!output?.Hash) throw new Error('No CID was received from remote IPFS node.');
+    const cid = output.Hash;
+
+
+    return cid
+}
+
+module.exports = ipfsAdd;
+

packages/link2cid/src/utils/ipfsAdd.spec.js

@@ -0,0 +1,13 @@
+const ipfsAdd = require('./ipfsAdd.js')
+const path = require('path');
+
+describe('ipfs', function () {
+    describe('ipfsAdd', function () {
+        it('should add a file from disk to ipfs and return a {string} CID', async function () {
+            const expectedCid = 'bafkreibxh3ly47pr3emvrqtax6ieq2ybom4ywyil3yurxnlwirtcvb5pfi'
+            const file = path.join(__dirname, '..', 'fixtures', 'hello-worlds.txt')
+            const cid = await ipfsAdd(file, { cidVersion: 1 })
+            if (cid !== expectedCid) throw new Error(`expected ${cid} to match ${expectedCid}`)
+        })
+    })
+})

packages/link2cid/src/utils/paths.js

@@ -0,0 +1,12 @@
+
+const os = require('os');
+const path = require('path');
+
+const getTmpFilePath = function (url) {
+    const timestamp = new Date().valueOf()
+    return path.join(os.tmpdir(), timestamp+'-'+path.basename(url))
+}
+
+module.exports = {
+    getTmpFilePath
+}

packages/link2cid/src/utils/paths.spec.js

@@ -0,0 +1,11 @@
+const paths = require('./paths.js')
+
+describe('paths', function () {
+    describe('getTmpFilePath', function () {
+        it('should accept a url and receive a /tmp/<datestamp><basename> path on disk', function () {
+            const url = 'https://example.com/my.jpg'
+            const p = paths.getTmpFilePath(url)
+            if (!/\/tmp\/\d+-my\.jpg/.test(p)) throw new Error(`expected ${p} to use format /tmp/<datestamp><basename>`)
+        })
+    })
+})

packages/link2cid/src/utils/taskProcess.js

@@ -0,0 +1,25 @@
+const os = require('os');
+const path = require('path');
+const downloadFile = require('./download');
+const ipfsAdd = require('./ipfsAdd');
+
+
+const taskProcess = async function (taskSpec, cb) {
+
+    console.log('downloading')
+    this.progressTask(1, 3, "downloading")
+    await downloadFile(taskSpec.url, taskSpec.filePath)
+
+    console.log('adding')
+    this.progressTask(2, 3, "adding")
+
+    const cid = await ipfsAdd(taskSpec.filePath)
+
+    taskSpec.cid = cid
+
+    cb(null, taskSpec)
+
+}
+
+
+module.exports = taskProcess;