diff --git a/packages/link2cid/index.js b/packages/link2cid/index.js index 6f9b021..46751a6 100644 --- a/packages/link2cid/index.js +++ b/packages/link2cid/index.js @@ -1,301 +1,9 @@ -'use strict'; - -require('dotenv').config(); -const express = require('express'); -const bodyParser = require('body-parser'); -const cors = require('cors'); -const fs = require('fs'); -const fsp = require('fs/promises'); -const { openAsBlob } = require('node:fs'); -const { rm, stat } = require('fs/promises'); -const os = require('os'); -const path = require('path'); -const SseStream = require('ssestream').default; -const { Transform, Readable } = require('node:stream'); -const { pipeline } = require('node:stream/promises'); -const { differenceInSeconds } = require('date-fns'); - -const cidRegex = /Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}/; -const app = express(); -app.use(cors()); -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: false })); - -// environment variables -const port = process.env.PORT || 3000; -const ipfsUrl = process.env.IPFS_URL || 'http://localhost:5001'; -if (!process.env.API_KEY) throw new Error('API_KEY was missing in env'); -if (!process.env.PORT) throw new Error('PORT is missing in env'); - -async function ipfsHealthCheck() { - const url = ipfsUrl+'/version' - // console.log(`ipfsHealthCheck at url=${url}`) - try { - const res = await fetch(url, { - method: 'GET' - }) - const body = await res.text() - if (!body.includes('Version')) throw new Error('response from ipfs did not contain Version') - } catch (e) { - console.error('failure while checking IPFS connection.') - console.error(e) - } -} - -// greetz https://stackoverflow.com/a/51302466/1004931 -async function downloadFile(url, filePath, sse) { - console.log(`downloading url=${url} to filePath=${filePath}`); - const res = await fetch(url); - const fileSize = res.headers.get('content-length'); - const fileStream = fs.createWriteStream(filePath, { flags: 'wx' }); - - let downloadedBytes = 0; - const logInterval = 1 * 1024 * 1024; // 1MB in bytes - - const progressLogger = new Transform({ - transform(chunk, encoding, callback) { - downloadedBytes += chunk.length; - - if (downloadedBytes % logInterval < chunk.length) { - console.log(`${downloadedBytes / (1024 * 1024)} MB processed`); - const progress = (downloadedBytes / fileSize) * 100; - console.log(`Download Progress: ${progress.toFixed(2)}%`); - sse.write({ - event: 'dlProgress', - data: `${Math.floor(progress)}` - }); - } - - this.push(chunk); - callback(); - } - }); +require('dotenv').config() +const app = require('./src/app.js') +const port = process.env.PORT || 3000 +const version = require('./package.json').version - await pipeline( - res.body, - progressLogger, - fileStream - ) - - console.log('download finished'); - - // verify the file - // If we don't, we get text error messages sent to kubo which gets added and it's a bad time. - console.log(`fileSize=${fileSize}. downloadedBytes=${downloadedBytes}`); - if (fileSize != downloadedBytes) throw new Error('downloadedBytes did not match fileSize'); -} - - -async function healthRes(_, res) { - const version = await getPackageVersion(); - res.json({ error: false, message: `*link2cid ${version} pisses on the floor*` }); -} - -async function getPackageVersion() { - const packageJsonFile = await fsp.readFile(path.join(__dirname, 'package.json'), { encoding: 'utf-8' }); - const json = JSON.parse(packageJsonFile); - return json.version; -} - -/** - * - * We use this to upload files and get progress notifications - * - */ -async function streamingPostFetch( - url, - formData, - basename, - sse, - filesize -) { - console.log(`streamingPostFetch with url=${url}, formData=${formData.get('file')}, basename=${basename}, sse=${sse}, filesize=${filesize}`); - - try { - const res = await fetch(url, { - method: 'POST', - body: formData - }); - - if (!res.ok) { - throw new Error(`HTTP error! Status-- ${res.status}`); - } - - const reader = res.body?.getReader(); - if (!reader) { - throw new Error('Failed to get reader from response body'); - } - - - while (true) { - const { done, value } = await reader.read(); - - const chunk = new TextDecoder().decode(value); - const lines = chunk.split('\n'); - for (const line of lines) { - const trimmedLine = line.trim() - if (!!trimmedLine) { - console.log(trimmedLine); - const json = JSON.parse(trimmedLine); - // console.log(`comparing json.Name=${json.Name} with basename=${basename}`); - sse.write({ - event: 'addProgress', - data: `${Math.floor(json?.Size / filesize * 100)}` - }) - if (json.Name === basename && json.Hash && json.Size) { - // this is the last chunk - return json; - } - } - } - - if (done) { - throw new Error('Response reader finished before receiving a CID which indicates a failiure.'); - } - - } - } catch (error) { - console.error('An error occurred:', error); - throw error; - } -} - - -function authenticate(req, res, next) { - const apiKey = req.query?.token; - if (!apiKey) { - const msg = `authorization 'token' was missing from query`; - console.error(msg); - return res.status(401).json({ error: true, message: msg }); - } - if (apiKey !== process.env.API_KEY) { - const msg = 'INCORRECT API_KEY (wrong token)'; - console.error(msg); - return res.status(403).json({ error: true, message: msg }); - } else { - next(); - } -} - - -async function getFormStuff(filePath) { - const url = `${ipfsUrl}/api/v0/add?progress=false&cid-version=1&pin=true`; - const blob = await openAsBlob(filePath); - const basename = path.basename(filePath); - const filesize = (await stat(filePath)).size; - const formData = new FormData(); - return { - url, - blob, - basename, - filesize, - formData - } -} - - -/** - * Add a file from URL to IPFS. - * - * uses SSE to send progress reports as the script - * downloads the file to disk and then does `ipfs add` - * finally returning a CID - * - * events: - * - heartbeat - * - dlProgress - * - addProgress - * - end - */ -async function addHandler(req, res) { - console.log(`/add`) - let url; - const urlStr = req.query.url; - if (!urlStr) return res.status(400).json({ - error: 'url was missing from query' - }); - try { - url = new URL(urlStr); - } catch (e) { - return res.status(400).json({ - error: e?.message - }) - } - const timestamp = new Date().valueOf(); - const fileName = `${timestamp}-${url.pathname.split('/').at(-1)}`; - const destinationFilePath = path.join(os.tmpdir(), fileName); - - console.log(`fileName=${fileName}, destinationFilePath=${destinationFilePath}`); - - const sse = new SseStream(req); - sse.pipe(res); - - let hbStartTime = new Date(); - const heartbeat = setInterval(() => { - sse.write({ - event: 'heartbeat', - data: `${differenceInSeconds(new Date(), hbStartTime)}` - }); - }, 15000); - - res.on('close', () => { - console.log('Connection closed.'); - clearTimeout(heartbeat); - sse.unpipe(res); - }); - - console.log(`Downloading '${urlStr}' to destinationFilePath=${destinationFilePath}`); - await downloadFile(urlStr, destinationFilePath, sse); - - - sse.write({ - event: 'dlProgress', - data: '100' - }) - - console.log(`'ipfs add' the file ${destinationFilePath}`); - const { url: kuboUrl, blob, basename, filesize, formData } = await getFormStuff(destinationFilePath); - - formData.append('file', blob, basename); - - let cid; - try { - const output = await streamingPostFetch(kuboUrl, formData, basename, sse, filesize); - console.log(`streamingPostFetch output as follows.`); - console.log(output); - if (!output?.Hash) throw new Error('No CID was received from remote IPFS node.'); - if (!output?.Size) throw new Error(`'ipfs add' was missing Size in its output.`); - // if (output.Size !== filesize) throw new Error(`input and output sizes did not match. Expected output.Size ${output.Size} to equal ${filesize}.`); - // console.log(`filesize=${filesize} output.Size=${output.Size}`); - cid = output.Hash; - - console.log('cleanup'); - await rm(destinationFilePath); - - console.log('end SSE'); - clearTimeout(heartbeat); - } catch (e) { - return sse.end({ - event: 'end', - error: true, - message: e - }) - } - return sse.end({ - event: 'end', - data: cid - }) -} - -app.get('/', authenticate, healthRes); -app.get('/health', healthRes); -app.get('/add', authenticate, addHandler); - -app.listen(port, async () => { - setInterval(async () => { - await ipfsHealthCheck() - }, 10000) - const version = await getPackageVersion(); - console.log(`link2cid ${version} listening on port ${port}`); -}); +app.listen(port, () => { + console.log(`link2cid ${version} listening on port ${port}`) +}) diff --git a/packages/link2cid/package.json b/packages/link2cid/package.json index de17ac5..622784c 100644 --- a/packages/link2cid/package.json +++ b/packages/link2cid/package.json @@ -1,10 +1,10 @@ { "name": "@futureporn/link2cid", - "version": "3.2.0", - "description": "REST API for adding files to IPFS", + "version": "4.2.0", + "description": "REST API for adding files via URL to IPFS", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", + "test": "mocha \"./src/**/*.spec.js\"", "dev": "pnpm nodemon ./index.js", "start": "node index.js" }, @@ -15,16 +15,17 @@ "author": "", "license": "Unlicense", "dependencies": { + "@paralleldrive/cuid2": "^2.2.2", "@types/express": "^4.17.21", + "better-queue": "^3.8.12", "body-parser": "^1.20.2", "cors": "^2.8.5", - "date-fns": "^3.0.5", "dotenv": "^16.3.1", - "express": "^4.18.2", - "jsonwebtoken": "^9.0.2", - "ssestream": "^1.1.0" + "express": "^4.18.2" }, "devDependencies": { - "nodemon": "^3.0.3" + "chai": "^5.1.0", + "nodemon": "^3.0.3", + "supertest": "^6.3.4" } } diff --git a/packages/link2cid/pnpm-lock.yaml b/packages/link2cid/pnpm-lock.yaml index 78985b7..a78265f 100644 --- a/packages/link2cid/pnpm-lock.yaml +++ b/packages/link2cid/pnpm-lock.yaml @@ -5,33 +5,49 @@ settings: excludeLinksFromLockfile: false dependencies: + '@paralleldrive/cuid2': + specifier: ^2.2.2 + version: 2.2.2 '@types/express': specifier: ^4.17.21 version: 4.17.21 + better-queue: + specifier: ^3.8.12 + version: 3.8.12 body-parser: specifier: ^1.20.2 version: 1.20.2 cors: specifier: ^2.8.5 version: 2.8.5 - date-fns: - specifier: ^3.0.5 - version: 3.0.5 dotenv: specifier: ^16.3.1 version: 16.3.1 express: specifier: ^4.18.2 version: 4.18.2 - jsonwebtoken: - specifier: ^9.0.2 - version: 9.0.2 - ssestream: - specifier: ^1.1.0 - version: 1.1.0 + +devDependencies: + chai: + specifier: ^5.1.0 + version: 5.1.0 + supertest: + specifier: ^6.3.4 + version: 6.3.4 packages: + /@noble/hashes@1.4.0: + resolution: {integrity: sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==} + engines: {node: '>= 16'} + dev: false + + /@paralleldrive/cuid2@2.2.2: + resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==} + dependencies: + '@noble/hashes': 1.4.0 + dev: false + /@types/body-parser@1.19.5: resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} dependencies: @@ -116,6 +132,31 @@ packages: resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} dev: false + /asap@2.0.6: + resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==} + dev: true + + /assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + dev: true + + /asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true + + /better-queue-memory@1.0.4: + resolution: {integrity: sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA==} + dev: false + + /better-queue@3.8.12: + resolution: {integrity: sha512-D9KZ+Us+2AyaCz693/9AyjTg0s8hEmkiM/MB3i09cs4MdK1KgTSGJluXRYmOulR69oLZVo2XDFtqsExDt8oiLA==} + dependencies: + better-queue-memory: 1.0.4 + node-eta: 0.9.0 + uuid: 9.0.1 + dev: false + /body-parser@1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -156,10 +197,6 @@ packages: - supports-color dev: false - /buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - dev: false - /bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -171,7 +208,33 @@ packages: function-bind: 1.1.2 get-intrinsic: 1.2.2 set-function-length: 1.1.1 - dev: false + + /chai@5.1.0: + resolution: {integrity: sha512-kDZ7MZyM6Q1DhR9jy7dalKohXQ2yrlXkk59CR52aRKxJrobmlBNqnFQxX9xOX8w+4mz8SYlKJa/7D7ddltFXCw==} + engines: {node: '>=12'} + dependencies: + assertion-error: 2.0.1 + check-error: 2.0.0 + deep-eql: 5.0.1 + loupe: 3.1.0 + pathval: 2.0.0 + dev: true + + /check-error@2.0.0: + resolution: {integrity: sha512-tjLAOBHKVxtPoHe/SA7kNOMvhCRdCJ3vETdeY0RuAc9popf+hyaSV6ZEg9hr4cpWF7jmo/JSWEnLDrnijS9Tog==} + engines: {node: '>= 16'} + dev: true + + /combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + dependencies: + delayed-stream: 1.0.0 + dev: true + + /component-emitter@1.3.1: + resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==} + dev: true /content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} @@ -194,6 +257,10 @@ packages: engines: {node: '>= 0.6'} dev: false + /cookiejar@2.1.4: + resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==} + dev: true + /cors@2.8.5: resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} engines: {node: '>= 0.10'} @@ -202,10 +269,6 @@ packages: vary: 1.1.2 dev: false - /date-fns@3.0.5: - resolution: {integrity: sha512-Q4Tq5c5s/Zl/zbgdWf6pejn9ru7UwdIlLfvEEg1hVsQNQ7LKt76qIduagIT9OPK7+JCv1mAKherdU6bOqGYDnw==} - dev: false - /debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -217,6 +280,23 @@ packages: ms: 2.0.0 dev: false + /debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + dev: true + + /deep-eql@5.0.1: + resolution: {integrity: sha512-nwQCf6ne2gez3o1MxWifqkciwt0zhl0LO1/UwVu4uMBuPmflWM4oQ70XMqHqnBJA+nhzncaqL9HVL6KkHJ28lw==} + engines: {node: '>=6'} + dev: true + /define-data-property@1.1.1: resolution: {integrity: sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==} engines: {node: '>= 0.4'} @@ -224,7 +304,11 @@ packages: get-intrinsic: 1.2.2 gopd: 1.0.1 has-property-descriptors: 1.0.1 - dev: false + + /delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + dev: true /depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} @@ -236,17 +320,18 @@ packages: engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} dev: false + /dezalgo@1.0.4: + resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} + dependencies: + asap: 2.0.6 + wrappy: 1.0.2 + dev: true + /dotenv@16.3.1: resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==} engines: {node: '>=12'} dev: false - /ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - dependencies: - safe-buffer: 5.2.1 - dev: false - /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} dev: false @@ -304,6 +389,10 @@ packages: - supports-color dev: false + /fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + dev: true + /finalhandler@1.2.0: resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} engines: {node: '>= 0.8'} @@ -319,6 +408,24 @@ packages: - supports-color dev: false + /form-data@4.0.0: + resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: true + + /formidable@2.1.2: + resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==} + dependencies: + dezalgo: 1.0.4 + hexoid: 1.0.0 + once: 1.4.0 + qs: 6.11.0 + dev: true + /forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -331,7 +438,10 @@ packages: /function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - dev: false + + /get-func-name@2.0.2: + resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==} + dev: true /get-intrinsic@1.2.2: resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} @@ -340,36 +450,35 @@ packages: has-proto: 1.0.1 has-symbols: 1.0.3 hasown: 2.0.0 - dev: false /gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} dependencies: get-intrinsic: 1.2.2 - dev: false /has-property-descriptors@1.0.1: resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==} dependencies: get-intrinsic: 1.2.2 - dev: false /has-proto@1.0.1: resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} engines: {node: '>= 0.4'} - dev: false /has-symbols@1.0.3: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} engines: {node: '>= 0.4'} - dev: false /hasown@2.0.0: resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} engines: {node: '>= 0.4'} dependencies: function-bind: 1.1.2 - dev: false + + /hexoid@1.0.0: + resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==} + engines: {node: '>=8'} + dev: true /http-errors@2.0.0: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} @@ -398,71 +507,18 @@ packages: engines: {node: '>= 0.10'} dev: false - /jsonwebtoken@9.0.2: - resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} - engines: {node: '>=12', npm: '>=6'} + /loupe@3.1.0: + resolution: {integrity: sha512-qKl+FrLXUhFuHUoDJG7f8P8gEMHq9NFS0c6ghXG1J0rldmZFQZoNVv/vyirE9qwCIhWZDsvEFd1sbFu3GvRQFg==} dependencies: - jws: 3.2.2 - lodash.includes: 4.3.0 - lodash.isboolean: 3.0.3 - lodash.isinteger: 4.0.4 - lodash.isnumber: 3.0.3 - lodash.isplainobject: 4.0.6 - lodash.isstring: 4.0.1 - lodash.once: 4.1.1 - ms: 2.1.3 - semver: 7.5.4 - dev: false - - /jwa@1.4.1: - resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==} - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - dev: false - - /jws@3.2.2: - resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==} - dependencies: - jwa: 1.4.1 - safe-buffer: 5.2.1 - dev: false - - /lodash.includes@4.3.0: - resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} - dev: false - - /lodash.isboolean@3.0.3: - resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} - dev: false - - /lodash.isinteger@4.0.4: - resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} - dev: false - - /lodash.isnumber@3.0.3: - resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} - dev: false - - /lodash.isplainobject@4.0.6: - resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} - dev: false - - /lodash.isstring@4.0.1: - resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} - dev: false - - /lodash.once@4.1.1: - resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} - dev: false + get-func-name: 2.0.2 + dev: true /lru-cache@6.0.0: resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} engines: {node: '>=10'} dependencies: yallist: 4.0.0 - dev: false + dev: true /media-typer@0.3.0: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} @@ -476,19 +532,16 @@ packages: /methods@1.1.2: resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} engines: {node: '>= 0.6'} - dev: false /mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} - dev: false /mime-types@2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} dependencies: mime-db: 1.52.0 - dev: false /mime@1.6.0: resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} @@ -496,10 +549,20 @@ packages: hasBin: true dev: false + /mime@2.6.0: + resolution: {integrity: sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==} + engines: {node: '>=4.0.0'} + hasBin: true + dev: true + /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} dev: false + /ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + dev: true + /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} dev: false @@ -509,6 +572,10 @@ packages: engines: {node: '>= 0.6'} dev: false + /node-eta@0.9.0: + resolution: {integrity: sha512-mTCTZk29tmX1OGfVkPt63H3c3VqXrI2Kvua98S7iUIB/Gbp0MNw05YtUomxQIxnnKMyRIIuY9izPcFixzhSBrA==} + dev: false + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -516,7 +583,6 @@ packages: /object-inspect@1.13.1: resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} - dev: false /on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} @@ -525,6 +591,12 @@ packages: ee-first: 1.1.1 dev: false + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: true + /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -534,6 +606,11 @@ packages: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} dev: false + /pathval@2.0.0: + resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==} + engines: {node: '>= 14.16'} + dev: true + /proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -547,7 +624,6 @@ packages: engines: {node: '>=0.6'} dependencies: side-channel: 1.0.4 - dev: false /range-parser@1.2.1: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} @@ -588,7 +664,7 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 - dev: false + dev: true /send@0.18.0: resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} @@ -631,7 +707,6 @@ packages: get-intrinsic: 1.2.2 gopd: 1.0.1 has-property-descriptors: 1.0.1 - dev: false /setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} @@ -643,17 +718,40 @@ packages: call-bind: 1.0.5 get-intrinsic: 1.2.2 object-inspect: 1.13.1 - dev: false - - /ssestream@1.1.0: - resolution: {integrity: sha512-UOS3JTuGqGEOH89mfHFwVOJNH2+JX9ebIWuw6WBQXpkVOxbdoY3RMliSHzshL4XVYJJrcul5NkuvDFCzgYu1Lw==} - dev: false /statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} dev: false + /superagent@8.1.2: + resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==} + engines: {node: '>=6.4.0 <13 || >=14'} + dependencies: + component-emitter: 1.3.1 + cookiejar: 2.1.4 + debug: 4.3.4 + fast-safe-stringify: 2.1.1 + form-data: 4.0.0 + formidable: 2.1.2 + methods: 1.1.2 + mime: 2.6.0 + qs: 6.11.0 + semver: 7.5.4 + transitivePeerDependencies: + - supports-color + dev: true + + /supertest@6.3.4: + resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==} + engines: {node: '>=6.4.0'} + dependencies: + methods: 1.1.2 + superagent: 8.1.2 + transitivePeerDependencies: + - supports-color + dev: true + /toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -681,11 +779,20 @@ packages: engines: {node: '>= 0.4.0'} dev: false + /uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true + dev: false + /vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} dev: false + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: true + /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: false + dev: true diff --git a/packages/link2cid/src/app.js b/packages/link2cid/src/app.js new file mode 100644 index 0000000..467199d --- /dev/null +++ b/packages/link2cid/src/app.js @@ -0,0 +1,33 @@ +'use strict'; + +require('dotenv').config(); +const express = require('express'); +const bodyParser = require('body-parser'); +const cors = require('cors'); +const fs = require('fs'); +const fsp = require('fs/promises'); +const { openAsBlob } = require('node:fs'); +const { rm, stat } = require('fs/promises'); +const os = require('os'); +const path = require('path'); +const { authenticate } = require('./middleware/auth.js') +const { createTask, readTask, deleteTask } = require('./models/task.js') +const readHeath = require('./models/health.js') +const store = require('./middleware/store.js'); +const queue = require('./middleware/queue.js'); + +const app = express(); +app.use(store); +app.use(queue); +app.use(cors()); +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ extended: false })); + + +app.post('/task', authenticate, createTask) +app.get('/task', readTask) +app.delete('/task', authenticate, deleteTask) +app.get('/health', readHeath) + + +module.exports = app diff --git a/packages/link2cid/src/app.spec.js b/packages/link2cid/src/app.spec.js new file mode 100644 index 0000000..7fd544a --- /dev/null +++ b/packages/link2cid/src/app.spec.js @@ -0,0 +1,109 @@ + +const app = require('./app.js') +const request = require('supertest') +const qs = require('querystring') +require('dotenv').config() + +describe('app', function () { + it('should exist', function (done) { + if (!app?.mountpath) throw new Error('app doesnt exist'); + done() + }) + + + describe('/health', function () { + it('should be publicly readable', function (done) { + request(app) + .get('/health') + .set('Accept', 'text/html') + .expect('Content-Type', /text/) + .expect(/piss/) + .expect(200, done) + }) + }) + + describe('/task', function () { + describe('POST', function () { + it('should create a task', function (done) { + request(app) + .post('/task') + .set('Authorization', `Bearer ${process.env.API_KEY}`) + .set('Accept', 'application/json') + .send({ + url: 'https://futureporn-b2.b-cdn.net/projekt-melody.jpg' + }) + .expect('Content-Type', /json/) + .expect((res) => { + if (!res.body?.data) throw new Error('response body was missing data') + if (!res.body?.data?.id) throw new Error('response body was missing id') + return true + }) + .expect(200, done) + }) + }) + describe('GET', function () { + it('should show all tasks specifications', async function () { + await request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/my.jpg' }) + await request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/your.png' }) + return request(app) + .get(`/task`) + .set('Authorization', `Bearer ${process.env.API_KEY}`) + .set('Accept', 'application/json') + .expect('Content-Type', /json/) + .expect((res) => { + if (!res?.body?.data) throw new Error('there was no data in response') + }) + .expect(200) + }) + it('should accept task id as query param and return task specification', function (done) { + const seed = request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/z.jpg' }) + seed.then((res) => { + + const query = qs.stringify({ + id: res.body.data.id + }) + request(app) + .get(`/task?${query}`) + .set('Accept', 'application/json') + .expect('Content-Type', /json/) + .expect((res) => { + if (res?.body?.error) throw new Error('there was an error in the response: '+res.body?.message) + if (!res?.body?.data?.url) throw new Error('data.url was missing') + if (!res?.body?.data?.createdAt) throw new Error('data.createdAt was missing') + return true + }) + .expect(200, done) + }) + }) + it('should show all tasks by default', function (done) { + request(app) + .get('/task') + .set('Accept', 'application/json') + .expect('Content-Type', /json/) + .expect((res) => { + if (res.body?.error) throw new Error('there was an error in the response'+res.error) + if (!res.body?.data) throw new Error('data was missing') + return true + }) + .expect(200, done) + + }) + }) + describe('DELETE', function () { + const query = qs.stringify({ + id: 'awejf9wiejf9we' + }) + it('should delete a single task', function (done) { + request(app) + .delete(`/task?${query}`) + .set('Authorization', `Bearer ${process.env.API_KEY}`) + .set('Accept', 'application/json') + .expect('Content-Type', /json/) + .expect(200, done); + }); + }) + }) + + + +}) \ No newline at end of file diff --git a/packages/link2cid/src/fixtures/hello-worlds.txt b/packages/link2cid/src/fixtures/hello-worlds.txt new file mode 100644 index 0000000..24a8d5e --- /dev/null +++ b/packages/link2cid/src/fixtures/hello-worlds.txt @@ -0,0 +1 @@ +hello worlds diff --git a/packages/link2cid/src/middleware/auth.js b/packages/link2cid/src/middleware/auth.js new file mode 100644 index 0000000..14c17ac --- /dev/null +++ b/packages/link2cid/src/middleware/auth.js @@ -0,0 +1,16 @@ + +module.exports.authenticate = function authenticate(req, res, next) { + const bearerToken = req.headers?.authorization.split(' ').at(1); + if (!bearerToken) { + const msg = `authorization bearer token was missing from request headers`; + console.error(msg); + return res.status(401).json({ error: true, message: msg }); + } + if (bearerToken !== process.env.API_KEY) { + const msg = 'INCORRECT API_KEY (wrong token)'; + console.error(msg); + return res.status(403).json({ error: true, message: msg }); + } else { + next(); + } +} diff --git a/packages/link2cid/src/middleware/queue.js b/packages/link2cid/src/middleware/queue.js new file mode 100644 index 0000000..8ac30a3 --- /dev/null +++ b/packages/link2cid/src/middleware/queue.js @@ -0,0 +1,21 @@ +const Queue = require('better-queue'); +const taskProcess = require('../utils/taskProcess.js'); + +const options = { + id: 'id', + maxRetries: 3, + concurrent: 1 + // @todo better-queue has batching and concurrency. might be useful to implement in the future + // @see https://github.com/diamondio/better-queue?tab=readme-ov-file#queue-management +} + +let q = new Queue(taskProcess, options) + + +// Middleware function to attach db to request +const queueMiddleware = (req, res, next) => { + req.queue = q; + next(); +}; + +module.exports = queueMiddleware \ No newline at end of file diff --git a/packages/link2cid/src/middleware/store.js b/packages/link2cid/src/middleware/store.js new file mode 100644 index 0000000..9a9f630 --- /dev/null +++ b/packages/link2cid/src/middleware/store.js @@ -0,0 +1,10 @@ +const store = { + tasks: {} +} + +const storeMiddleware = (req, res, next) => { + req.store = store + next(); +}; + +module.exports = storeMiddleware; \ No newline at end of file diff --git a/packages/link2cid/src/models/health.js b/packages/link2cid/src/models/health.js new file mode 100644 index 0000000..08f3369 --- /dev/null +++ b/packages/link2cid/src/models/health.js @@ -0,0 +1,3 @@ +module.exports = function readHealth (req, res) { + return res.send('**link2cid pisses on the floor**') +} diff --git a/packages/link2cid/src/models/task.js b/packages/link2cid/src/models/task.js new file mode 100644 index 0000000..3e6eda7 --- /dev/null +++ b/packages/link2cid/src/models/task.js @@ -0,0 +1,59 @@ +const { createId } = require('@paralleldrive/cuid2'); +const { getTmpFilePath } = require('../utils/paths.js'); +const fsp = require('fs/promises'); + + +module.exports.createTask = function createTask (req, res) { + + const url = req.body.url + const task = { + id: createId(), + url: url, + filePath: getTmpFilePath(url), + fileSize: null, + createdAt: new Date().toISOString(), + cid: null, + downloadProgress: null, + addProgress: null + } + + if (!req?.body?.url) return res.status(400).json({ error: true, message: 'request body was missing a url' }); + + req.store.tasks[task.id] = task; + req.queue.push(task, function (err, result) { + if (err) throw err; + console.log('the following is the result of the queued task being complete') + console.log(result) + }) + + return res.json({ error: false, data: task }) + +} +module.exports.readTask = function readTask (req, res) { + const id = req?.query?.id + + // If we get an id in the query, show the one task. + // Otherwise, we show all tasks. + if (!!id) { + const task = req.store.tasks[id] + if (!task) return res.json({ error: true, message: 'there was no task in the store with that id' }); + return res.json({ error: false, data: task }) + } else { + const tasks = req.store.tasks + return res.json({ error: false, data: tasks }) + } + +} +module.exports.deleteTask = async function deleteTask (req, res) { + const id = req?.query?.id; + const task = req.store.tasks[id]; + + try { + if (task?.filePath) await fsp.unlink(task.filePath); + } catch (err) {} + delete req.store.tasks[id]; + return res.json({ error: false, message: 'task deleted' }); + if (err) return res.json({ error: true, message: err }); +} + + diff --git a/packages/link2cid/src/utils/download.js b/packages/link2cid/src/utils/download.js new file mode 100644 index 0000000..7c1a9a1 --- /dev/null +++ b/packages/link2cid/src/utils/download.js @@ -0,0 +1,20 @@ +const fs = require("fs"); +const { Readable } = require('stream'); +const { finished } = require('stream/promises'); +const path = require("path"); + + +/** + * Download a file at url to local disk filePath + * @param {String} url + * @param {String} filePath + * + * greetz https://stackoverflow.com/a/51302466/1004931 + */ +const download = (async (url, filePath) => { + const res = await fetch(url); + const fileStream = fs.createWriteStream(filePath, { flags: 'wx' }); + await finished(Readable.fromWeb(res.body).pipe(fileStream)); +}); + +module.exports = download; \ No newline at end of file diff --git a/packages/link2cid/src/utils/download.spec.js b/packages/link2cid/src/utils/download.spec.js new file mode 100644 index 0000000..131059e --- /dev/null +++ b/packages/link2cid/src/utils/download.spec.js @@ -0,0 +1,12 @@ +const download = require('./download.js'); +const fsp = require('fs/promises'); + +describe('download', function () { + it('should download a file from url', async function () { + const testFilePath = '/tmp/pmel.jpg' + try { + await fsp.unlink(testFilePath) + } catch (e) {} + await download('https://futureporn-b2.b-cdn.net/projekt-melody.jpg', testFilePath) + }) +}) \ No newline at end of file diff --git a/packages/link2cid/src/utils/ipfsAdd.js b/packages/link2cid/src/utils/ipfsAdd.js new file mode 100644 index 0000000..f5cb513 --- /dev/null +++ b/packages/link2cid/src/utils/ipfsAdd.js @@ -0,0 +1,100 @@ + +require('dotenv').config(); +const { openAsBlob } = require('node:fs'); +const { rm, stat } = require('fs/promises'); +const path = require('path'); + +if (!process.env.IPFS_URL) throw new Error('IPFS_URL was missing in env'); + + +async function streamingPostFetch( + url, + formData, + basename, + filesize +) { + // console.log(`streamingPostFetch with url=${url}, formData=${formData.get('file')}, basename=${basename}, filesize=${filesize}`); + + try { + const res = await fetch(url, { + method: 'POST', + body: formData + }); + + if (!res.ok) { + throw new Error(`HTTP error! Status-- ${res.status}`); + } + + const reader = res.body?.getReader(); + if (!reader) { + throw new Error('Failed to get reader from response body'); + } + + + while (true) { + const { done, value } = await reader.read(); + + const chunk = new TextDecoder().decode(value); + const lines = chunk.split('\n'); + for (const line of lines) { + const trimmedLine = line.trim() + if (!!trimmedLine) { + // console.log(trimmedLine); + const json = JSON.parse(trimmedLine); + // console.log(`comparing json.Name=${json.Name} with basename=${basename}`); + if (json.Name === basename && json.Hash && json.Size) { + // this is the last chunk + return json; + } + } + } + + if (done) { + throw new Error('Response reader finished before receiving a CID which indicates a failiure.'); + } + + } + } catch (error) { + console.error('An error occurred:', error); + throw error; + } +} + + +async function getFormStuff(filePath) { + const url = `${process.env.IPFS_URL}/api/v0/add?progress=false&cid-version=1&pin=true`; + const blob = await openAsBlob(filePath); + const basename = path.basename(filePath); + const filesize = (await stat(filePath)).size; + const formData = new FormData(); + return { + url, + blob, + basename, + filesize, + formData + } +} + + +/** + * @param {String} filePath + * @returns {String} CID + */ +const ipfsAdd = async function (filePath) { + + + const { url: kuboUrl, blob, basename, filesize, formData } = await getFormStuff(filePath); + + + formData.append('file', blob, basename); + const output = await streamingPostFetch(kuboUrl, formData, basename, filesize); + if (!output?.Hash) throw new Error('No CID was received from remote IPFS node.'); + const cid = output.Hash; + + + return cid +} + +module.exports = ipfsAdd; + diff --git a/packages/link2cid/src/utils/ipfsAdd.spec.js b/packages/link2cid/src/utils/ipfsAdd.spec.js new file mode 100644 index 0000000..d7ea393 --- /dev/null +++ b/packages/link2cid/src/utils/ipfsAdd.spec.js @@ -0,0 +1,13 @@ +const ipfsAdd = require('./ipfsAdd.js') +const path = require('path'); + +describe('ipfs', function () { + describe('ipfsAdd', function () { + it('should add a file from disk to ipfs and return a {string} CID', async function () { + const expectedCid = 'bafkreibxh3ly47pr3emvrqtax6ieq2ybom4ywyil3yurxnlwirtcvb5pfi' + const file = path.join(__dirname, '..', 'fixtures', 'hello-worlds.txt') + const cid = await ipfsAdd(file, { cidVersion: 1 }) + if (cid !== expectedCid) throw new Error(`expected ${cid} to match ${expectedCid}`) + }) + }) +}) \ No newline at end of file diff --git a/packages/link2cid/src/utils/paths.js b/packages/link2cid/src/utils/paths.js new file mode 100644 index 0000000..fd8c5de --- /dev/null +++ b/packages/link2cid/src/utils/paths.js @@ -0,0 +1,12 @@ + +const os = require('os'); +const path = require('path'); + +const getTmpFilePath = function (url) { + const timestamp = new Date().valueOf() + return path.join(os.tmpdir(), timestamp+'-'+path.basename(url)) +} + +module.exports = { + getTmpFilePath +} \ No newline at end of file diff --git a/packages/link2cid/src/utils/paths.spec.js b/packages/link2cid/src/utils/paths.spec.js new file mode 100644 index 0000000..0b7d920 --- /dev/null +++ b/packages/link2cid/src/utils/paths.spec.js @@ -0,0 +1,11 @@ +const paths = require('./paths.js') + +describe('paths', function () { + describe('getTmpFilePath', function () { + it('should accept a url and receive a /tmp/ path on disk', function () { + const url = 'https://example.com/my.jpg' + const p = paths.getTmpFilePath(url) + if (!/\/tmp\/\d+-my\.jpg/.test(p)) throw new Error(`expected ${p} to use format /tmp/`) + }) + }) +}) \ No newline at end of file diff --git a/packages/link2cid/src/utils/taskProcess.js b/packages/link2cid/src/utils/taskProcess.js new file mode 100644 index 0000000..734f7f2 --- /dev/null +++ b/packages/link2cid/src/utils/taskProcess.js @@ -0,0 +1,25 @@ +const os = require('os'); +const path = require('path'); +const downloadFile = require('./download'); +const ipfsAdd = require('./ipfsAdd'); + + +const taskProcess = async function (taskSpec, cb) { + + console.log('downloading') + this.progressTask(1, 3, "downloading") + await downloadFile(taskSpec.url, taskSpec.filePath) + + console.log('adding') + this.progressTask(2, 3, "adding") + + const cid = await ipfsAdd(taskSpec.filePath) + + taskSpec.cid = cid + + cb(null, taskSpec) + +} + + +module.exports = taskProcess; \ No newline at end of file