rewrite
ci / build (push) Failing after 7m39s Details

This commit is contained in:
Chris Grimmett 2024-04-24 21:42:03 +00:00
parent e967320dcf
commit fb13c3ab1b
18 changed files with 671 additions and 410 deletions

View File

@ -1,301 +1,9 @@
'use strict';
require('dotenv').config()
const app = require('./src/app.js')
const port = process.env.PORT || 3000
const version = require('./package.json').version
require('dotenv').config();
const express = require('express');
const bodyParser = require('body-parser');
const cors = require('cors');
const fs = require('fs');
const fsp = require('fs/promises');
const { openAsBlob } = require('node:fs');
const { rm, stat } = require('fs/promises');
const os = require('os');
const path = require('path');
const SseStream = require('ssestream').default;
const { Transform, Readable } = require('node:stream');
const { pipeline } = require('node:stream/promises');
const { differenceInSeconds } = require('date-fns');
const cidRegex = /Qm[1-9A-HJ-NP-Za-km-z]{44,}|b[A-Za-z2-7]{58,}|B[A-Z2-7]{58,}|z[1-9A-HJ-NP-Za-km-z]{48,}|F[0-9A-F]{50,}/;
const app = express();
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
// environment variables
const port = process.env.PORT || 3000;
const ipfsUrl = process.env.IPFS_URL || 'http://localhost:5001';
if (!process.env.API_KEY) throw new Error('API_KEY was missing in env');
if (!process.env.PORT) throw new Error('PORT is missing in env');
async function ipfsHealthCheck() {
const url = ipfsUrl+'/version'
// console.log(`ipfsHealthCheck at url=${url}`)
try {
const res = await fetch(url, {
method: 'GET'
app.listen(port, () => {
console.log(`link2cid ${version} listening on port ${port}`)
})
const body = await res.text()
if (!body.includes('Version')) throw new Error('response from ipfs did not contain Version')
} catch (e) {
console.error('failure while checking IPFS connection.')
console.error(e)
}
}
// greetz https://stackoverflow.com/a/51302466/1004931
async function downloadFile(url, filePath, sse) {
console.log(`downloading url=${url} to filePath=${filePath}`);
const res = await fetch(url);
const fileSize = res.headers.get('content-length');
const fileStream = fs.createWriteStream(filePath, { flags: 'wx' });
let downloadedBytes = 0;
const logInterval = 1 * 1024 * 1024; // 1MB in bytes
const progressLogger = new Transform({
transform(chunk, encoding, callback) {
downloadedBytes += chunk.length;
if (downloadedBytes % logInterval < chunk.length) {
console.log(`${downloadedBytes / (1024 * 1024)} MB processed`);
const progress = (downloadedBytes / fileSize) * 100;
console.log(`Download Progress: ${progress.toFixed(2)}%`);
sse.write({
event: 'dlProgress',
data: `${Math.floor(progress)}`
});
}
this.push(chunk);
callback();
}
});
await pipeline(
res.body,
progressLogger,
fileStream
)
console.log('download finished');
// verify the file
// If we don't, we get text error messages sent to kubo which gets added and it's a bad time.
console.log(`fileSize=${fileSize}. downloadedBytes=${downloadedBytes}`);
if (fileSize != downloadedBytes) throw new Error('downloadedBytes did not match fileSize');
}
async function healthRes(_, res) {
const version = await getPackageVersion();
res.json({ error: false, message: `*link2cid ${version} pisses on the floor*` });
}
async function getPackageVersion() {
const packageJsonFile = await fsp.readFile(path.join(__dirname, 'package.json'), { encoding: 'utf-8' });
const json = JSON.parse(packageJsonFile);
return json.version;
}
/**
*
* We use this to upload files and get progress notifications
*
*/
async function streamingPostFetch(
url,
formData,
basename,
sse,
filesize
) {
console.log(`streamingPostFetch with url=${url}, formData=${formData.get('file')}, basename=${basename}, sse=${sse}, filesize=${filesize}`);
try {
const res = await fetch(url, {
method: 'POST',
body: formData
});
if (!res.ok) {
throw new Error(`HTTP error! Status-- ${res.status}`);
}
const reader = res.body?.getReader();
if (!reader) {
throw new Error('Failed to get reader from response body');
}
while (true) {
const { done, value } = await reader.read();
const chunk = new TextDecoder().decode(value);
const lines = chunk.split('\n');
for (const line of lines) {
const trimmedLine = line.trim()
if (!!trimmedLine) {
console.log(trimmedLine);
const json = JSON.parse(trimmedLine);
// console.log(`comparing json.Name=${json.Name} with basename=${basename}`);
sse.write({
event: 'addProgress',
data: `${Math.floor(json?.Size / filesize * 100)}`
})
if (json.Name === basename && json.Hash && json.Size) {
// this is the last chunk
return json;
}
}
}
if (done) {
throw new Error('Response reader finished before receiving a CID which indicates a failiure.');
}
}
} catch (error) {
console.error('An error occurred:', error);
throw error;
}
}
function authenticate(req, res, next) {
const apiKey = req.query?.token;
if (!apiKey) {
const msg = `authorization 'token' was missing from query`;
console.error(msg);
return res.status(401).json({ error: true, message: msg });
}
if (apiKey !== process.env.API_KEY) {
const msg = 'INCORRECT API_KEY (wrong token)';
console.error(msg);
return res.status(403).json({ error: true, message: msg });
} else {
next();
}
}
async function getFormStuff(filePath) {
const url = `${ipfsUrl}/api/v0/add?progress=false&cid-version=1&pin=true`;
const blob = await openAsBlob(filePath);
const basename = path.basename(filePath);
const filesize = (await stat(filePath)).size;
const formData = new FormData();
return {
url,
blob,
basename,
filesize,
formData
}
}
/**
* Add a file from URL to IPFS.
*
* uses SSE to send progress reports as the script
* downloads the file to disk and then does `ipfs add`
* finally returning a CID
*
* events:
* - heartbeat
* - dlProgress
* - addProgress
* - end
*/
async function addHandler(req, res) {
console.log(`/add`)
let url;
const urlStr = req.query.url;
if (!urlStr) return res.status(400).json({
error: 'url was missing from query'
});
try {
url = new URL(urlStr);
} catch (e) {
return res.status(400).json({
error: e?.message
})
}
const timestamp = new Date().valueOf();
const fileName = `${timestamp}-${url.pathname.split('/').at(-1)}`;
const destinationFilePath = path.join(os.tmpdir(), fileName);
console.log(`fileName=${fileName}, destinationFilePath=${destinationFilePath}`);
const sse = new SseStream(req);
sse.pipe(res);
let hbStartTime = new Date();
const heartbeat = setInterval(() => {
sse.write({
event: 'heartbeat',
data: `${differenceInSeconds(new Date(), hbStartTime)}`
});
}, 15000);
res.on('close', () => {
console.log('Connection closed.');
clearTimeout(heartbeat);
sse.unpipe(res);
});
console.log(`Downloading '${urlStr}' to destinationFilePath=${destinationFilePath}`);
await downloadFile(urlStr, destinationFilePath, sse);
sse.write({
event: 'dlProgress',
data: '100'
})
console.log(`'ipfs add' the file ${destinationFilePath}`);
const { url: kuboUrl, blob, basename, filesize, formData } = await getFormStuff(destinationFilePath);
formData.append('file', blob, basename);
let cid;
try {
const output = await streamingPostFetch(kuboUrl, formData, basename, sse, filesize);
console.log(`streamingPostFetch output as follows.`);
console.log(output);
if (!output?.Hash) throw new Error('No CID was received from remote IPFS node.');
if (!output?.Size) throw new Error(`'ipfs add' was missing Size in its output.`);
// if (output.Size !== filesize) throw new Error(`input and output sizes did not match. Expected output.Size ${output.Size} to equal ${filesize}.`);
// console.log(`filesize=${filesize} output.Size=${output.Size}`);
cid = output.Hash;
console.log('cleanup');
await rm(destinationFilePath);
console.log('end SSE');
clearTimeout(heartbeat);
} catch (e) {
return sse.end({
event: 'end',
error: true,
message: e
})
}
return sse.end({
event: 'end',
data: cid
})
}
app.get('/', authenticate, healthRes);
app.get('/health', healthRes);
app.get('/add', authenticate, addHandler);
app.listen(port, async () => {
setInterval(async () => {
await ipfsHealthCheck()
}, 10000)
const version = await getPackageVersion();
console.log(`link2cid ${version} listening on port ${port}`);
});

View File

@ -1,10 +1,10 @@
{
"name": "@futureporn/link2cid",
"version": "3.2.0",
"description": "REST API for adding files to IPFS",
"version": "4.2.0",
"description": "REST API for adding files via URL to IPFS",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"test": "mocha \"./src/**/*.spec.js\"",
"dev": "pnpm nodemon ./index.js",
"start": "node index.js"
},
@ -15,16 +15,17 @@
"author": "",
"license": "Unlicense",
"dependencies": {
"@paralleldrive/cuid2": "^2.2.2",
"@types/express": "^4.17.21",
"better-queue": "^3.8.12",
"body-parser": "^1.20.2",
"cors": "^2.8.5",
"date-fns": "^3.0.5",
"dotenv": "^16.3.1",
"express": "^4.18.2",
"jsonwebtoken": "^9.0.2",
"ssestream": "^1.1.0"
"express": "^4.18.2"
},
"devDependencies": {
"nodemon": "^3.0.3"
"chai": "^5.1.0",
"nodemon": "^3.0.3",
"supertest": "^6.3.4"
}
}

View File

@ -5,33 +5,49 @@ settings:
excludeLinksFromLockfile: false
dependencies:
'@paralleldrive/cuid2':
specifier: ^2.2.2
version: 2.2.2
'@types/express':
specifier: ^4.17.21
version: 4.17.21
better-queue:
specifier: ^3.8.12
version: 3.8.12
body-parser:
specifier: ^1.20.2
version: 1.20.2
cors:
specifier: ^2.8.5
version: 2.8.5
date-fns:
specifier: ^3.0.5
version: 3.0.5
dotenv:
specifier: ^16.3.1
version: 16.3.1
express:
specifier: ^4.18.2
version: 4.18.2
jsonwebtoken:
specifier: ^9.0.2
version: 9.0.2
ssestream:
specifier: ^1.1.0
version: 1.1.0
devDependencies:
chai:
specifier: ^5.1.0
version: 5.1.0
supertest:
specifier: ^6.3.4
version: 6.3.4
packages:
/@noble/hashes@1.4.0:
resolution: {integrity: sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==}
engines: {node: '>= 16'}
dev: false
/@paralleldrive/cuid2@2.2.2:
resolution: {integrity: sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==}
dependencies:
'@noble/hashes': 1.4.0
dev: false
/@types/body-parser@1.19.5:
resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}
dependencies:
@ -116,6 +132,31 @@ packages:
resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==}
dev: false
/asap@2.0.6:
resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}
dev: true
/assertion-error@2.0.1:
resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}
engines: {node: '>=12'}
dev: true
/asynckit@0.4.0:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
dev: true
/better-queue-memory@1.0.4:
resolution: {integrity: sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA==}
dev: false
/better-queue@3.8.12:
resolution: {integrity: sha512-D9KZ+Us+2AyaCz693/9AyjTg0s8hEmkiM/MB3i09cs4MdK1KgTSGJluXRYmOulR69oLZVo2XDFtqsExDt8oiLA==}
dependencies:
better-queue-memory: 1.0.4
node-eta: 0.9.0
uuid: 9.0.1
dev: false
/body-parser@1.20.1:
resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@ -156,10 +197,6 @@ packages:
- supports-color
dev: false
/buffer-equal-constant-time@1.0.1:
resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}
dev: false
/bytes@3.1.2:
resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==}
engines: {node: '>= 0.8'}
@ -171,7 +208,33 @@ packages:
function-bind: 1.1.2
get-intrinsic: 1.2.2
set-function-length: 1.1.1
dev: false
/chai@5.1.0:
resolution: {integrity: sha512-kDZ7MZyM6Q1DhR9jy7dalKohXQ2yrlXkk59CR52aRKxJrobmlBNqnFQxX9xOX8w+4mz8SYlKJa/7D7ddltFXCw==}
engines: {node: '>=12'}
dependencies:
assertion-error: 2.0.1
check-error: 2.0.0
deep-eql: 5.0.1
loupe: 3.1.0
pathval: 2.0.0
dev: true
/check-error@2.0.0:
resolution: {integrity: sha512-tjLAOBHKVxtPoHe/SA7kNOMvhCRdCJ3vETdeY0RuAc9popf+hyaSV6ZEg9hr4cpWF7jmo/JSWEnLDrnijS9Tog==}
engines: {node: '>= 16'}
dev: true
/combined-stream@1.0.8:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
engines: {node: '>= 0.8'}
dependencies:
delayed-stream: 1.0.0
dev: true
/component-emitter@1.3.1:
resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}
dev: true
/content-disposition@0.5.4:
resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
@ -194,6 +257,10 @@ packages:
engines: {node: '>= 0.6'}
dev: false
/cookiejar@2.1.4:
resolution: {integrity: sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==}
dev: true
/cors@2.8.5:
resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==}
engines: {node: '>= 0.10'}
@ -202,10 +269,6 @@ packages:
vary: 1.1.2
dev: false
/date-fns@3.0.5:
resolution: {integrity: sha512-Q4Tq5c5s/Zl/zbgdWf6pejn9ru7UwdIlLfvEEg1hVsQNQ7LKt76qIduagIT9OPK7+JCv1mAKherdU6bOqGYDnw==}
dev: false
/debug@2.6.9:
resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
peerDependencies:
@ -217,6 +280,23 @@ packages:
ms: 2.0.0
dev: false
/debug@4.3.4:
resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
engines: {node: '>=6.0'}
peerDependencies:
supports-color: '*'
peerDependenciesMeta:
supports-color:
optional: true
dependencies:
ms: 2.1.2
dev: true
/deep-eql@5.0.1:
resolution: {integrity: sha512-nwQCf6ne2gez3o1MxWifqkciwt0zhl0LO1/UwVu4uMBuPmflWM4oQ70XMqHqnBJA+nhzncaqL9HVL6KkHJ28lw==}
engines: {node: '>=6'}
dev: true
/define-data-property@1.1.1:
resolution: {integrity: sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==}
engines: {node: '>= 0.4'}
@ -224,7 +304,11 @@ packages:
get-intrinsic: 1.2.2
gopd: 1.0.1
has-property-descriptors: 1.0.1
dev: false
/delayed-stream@1.0.0:
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
dev: true
/depd@2.0.0:
resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
@ -236,17 +320,18 @@ packages:
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
dev: false
/dezalgo@1.0.4:
resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==}
dependencies:
asap: 2.0.6
wrappy: 1.0.2
dev: true
/dotenv@16.3.1:
resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==}
engines: {node: '>=12'}
dev: false
/ecdsa-sig-formatter@1.0.11:
resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
dependencies:
safe-buffer: 5.2.1
dev: false
/ee-first@1.1.1:
resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==}
dev: false
@ -304,6 +389,10 @@ packages:
- supports-color
dev: false
/fast-safe-stringify@2.1.1:
resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==}
dev: true
/finalhandler@1.2.0:
resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==}
engines: {node: '>= 0.8'}
@ -319,6 +408,24 @@ packages:
- supports-color
dev: false
/form-data@4.0.0:
resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
engines: {node: '>= 6'}
dependencies:
asynckit: 0.4.0
combined-stream: 1.0.8
mime-types: 2.1.35
dev: true
/formidable@2.1.2:
resolution: {integrity: sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==}
dependencies:
dezalgo: 1.0.4
hexoid: 1.0.0
once: 1.4.0
qs: 6.11.0
dev: true
/forwarded@0.2.0:
resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==}
engines: {node: '>= 0.6'}
@ -331,7 +438,10 @@ packages:
/function-bind@1.1.2:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
dev: false
/get-func-name@2.0.2:
resolution: {integrity: sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==}
dev: true
/get-intrinsic@1.2.2:
resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==}
@ -340,36 +450,35 @@ packages:
has-proto: 1.0.1
has-symbols: 1.0.3
hasown: 2.0.0
dev: false
/gopd@1.0.1:
resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
dependencies:
get-intrinsic: 1.2.2
dev: false
/has-property-descriptors@1.0.1:
resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==}
dependencies:
get-intrinsic: 1.2.2
dev: false
/has-proto@1.0.1:
resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==}
engines: {node: '>= 0.4'}
dev: false
/has-symbols@1.0.3:
resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==}
engines: {node: '>= 0.4'}
dev: false
/hasown@2.0.0:
resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==}
engines: {node: '>= 0.4'}
dependencies:
function-bind: 1.1.2
dev: false
/hexoid@1.0.0:
resolution: {integrity: sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==}
engines: {node: '>=8'}
dev: true
/http-errors@2.0.0:
resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
@ -398,71 +507,18 @@ packages:
engines: {node: '>= 0.10'}
dev: false
/jsonwebtoken@9.0.2:
resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==}
engines: {node: '>=12', npm: '>=6'}
/loupe@3.1.0:
resolution: {integrity: sha512-qKl+FrLXUhFuHUoDJG7f8P8gEMHq9NFS0c6ghXG1J0rldmZFQZoNVv/vyirE9qwCIhWZDsvEFd1sbFu3GvRQFg==}
dependencies:
jws: 3.2.2
lodash.includes: 4.3.0
lodash.isboolean: 3.0.3
lodash.isinteger: 4.0.4
lodash.isnumber: 3.0.3
lodash.isplainobject: 4.0.6
lodash.isstring: 4.0.1
lodash.once: 4.1.1
ms: 2.1.3
semver: 7.5.4
dev: false
/jwa@1.4.1:
resolution: {integrity: sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==}
dependencies:
buffer-equal-constant-time: 1.0.1
ecdsa-sig-formatter: 1.0.11
safe-buffer: 5.2.1
dev: false
/jws@3.2.2:
resolution: {integrity: sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==}
dependencies:
jwa: 1.4.1
safe-buffer: 5.2.1
dev: false
/lodash.includes@4.3.0:
resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==}
dev: false
/lodash.isboolean@3.0.3:
resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
dev: false
/lodash.isinteger@4.0.4:
resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==}
dev: false
/lodash.isnumber@3.0.3:
resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==}
dev: false
/lodash.isplainobject@4.0.6:
resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==}
dev: false
/lodash.isstring@4.0.1:
resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==}
dev: false
/lodash.once@4.1.1:
resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==}
dev: false
get-func-name: 2.0.2
dev: true
/lru-cache@6.0.0:
resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
engines: {node: '>=10'}
dependencies:
yallist: 4.0.0
dev: false
dev: true
/media-typer@0.3.0:
resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
@ -476,19 +532,16 @@ packages:
/methods@1.1.2:
resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}
engines: {node: '>= 0.6'}
dev: false
/mime-db@1.52.0:
resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
engines: {node: '>= 0.6'}
dev: false
/mime-types@2.1.35:
resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
engines: {node: '>= 0.6'}
dependencies:
mime-db: 1.52.0
dev: false
/mime@1.6.0:
resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==}
@ -496,10 +549,20 @@ packages:
hasBin: true
dev: false
/mime@2.6.0:
resolution: {integrity: sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==}
engines: {node: '>=4.0.0'}
hasBin: true
dev: true
/ms@2.0.0:
resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}
dev: false
/ms@2.1.2:
resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
dev: true
/ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
dev: false
@ -509,6 +572,10 @@ packages:
engines: {node: '>= 0.6'}
dev: false
/node-eta@0.9.0:
resolution: {integrity: sha512-mTCTZk29tmX1OGfVkPt63H3c3VqXrI2Kvua98S7iUIB/Gbp0MNw05YtUomxQIxnnKMyRIIuY9izPcFixzhSBrA==}
dev: false
/object-assign@4.1.1:
resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
engines: {node: '>=0.10.0'}
@ -516,7 +583,6 @@ packages:
/object-inspect@1.13.1:
resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==}
dev: false
/on-finished@2.4.1:
resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==}
@ -525,6 +591,12 @@ packages:
ee-first: 1.1.1
dev: false
/once@1.4.0:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
dependencies:
wrappy: 1.0.2
dev: true
/parseurl@1.3.3:
resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}
engines: {node: '>= 0.8'}
@ -534,6 +606,11 @@ packages:
resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==}
dev: false
/pathval@2.0.0:
resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==}
engines: {node: '>= 14.16'}
dev: true
/proxy-addr@2.0.7:
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
@ -547,7 +624,6 @@ packages:
engines: {node: '>=0.6'}
dependencies:
side-channel: 1.0.4
dev: false
/range-parser@1.2.1:
resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==}
@ -588,7 +664,7 @@ packages:
hasBin: true
dependencies:
lru-cache: 6.0.0
dev: false
dev: true
/send@0.18.0:
resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==}
@ -631,7 +707,6 @@ packages:
get-intrinsic: 1.2.2
gopd: 1.0.1
has-property-descriptors: 1.0.1
dev: false
/setprototypeof@1.2.0:
resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
@ -643,17 +718,40 @@ packages:
call-bind: 1.0.5
get-intrinsic: 1.2.2
object-inspect: 1.13.1
dev: false
/ssestream@1.1.0:
resolution: {integrity: sha512-UOS3JTuGqGEOH89mfHFwVOJNH2+JX9ebIWuw6WBQXpkVOxbdoY3RMliSHzshL4XVYJJrcul5NkuvDFCzgYu1Lw==}
dev: false
/statuses@2.0.1:
resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
engines: {node: '>= 0.8'}
dev: false
/superagent@8.1.2:
resolution: {integrity: sha512-6WTxW1EB6yCxV5VFOIPQruWGHqc3yI7hEmZK6h+pyk69Lk/Ut7rLUY6W/ONF2MjBuGjvmMiIpsrVJ2vjrHlslA==}
engines: {node: '>=6.4.0 <13 || >=14'}
dependencies:
component-emitter: 1.3.1
cookiejar: 2.1.4
debug: 4.3.4
fast-safe-stringify: 2.1.1
form-data: 4.0.0
formidable: 2.1.2
methods: 1.1.2
mime: 2.6.0
qs: 6.11.0
semver: 7.5.4
transitivePeerDependencies:
- supports-color
dev: true
/supertest@6.3.4:
resolution: {integrity: sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==}
engines: {node: '>=6.4.0'}
dependencies:
methods: 1.1.2
superagent: 8.1.2
transitivePeerDependencies:
- supports-color
dev: true
/toidentifier@1.0.1:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
@ -681,11 +779,20 @@ packages:
engines: {node: '>= 0.4.0'}
dev: false
/uuid@9.0.1:
resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}
hasBin: true
dev: false
/vary@1.1.2:
resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==}
engines: {node: '>= 0.8'}
dev: false
/wrappy@1.0.2:
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
dev: true
/yallist@4.0.0:
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
dev: false
dev: true

View File

@ -0,0 +1,33 @@
'use strict';
require('dotenv').config();
const express = require('express');
const bodyParser = require('body-parser');
const cors = require('cors');
const fs = require('fs');
const fsp = require('fs/promises');
const { openAsBlob } = require('node:fs');
const { rm, stat } = require('fs/promises');
const os = require('os');
const path = require('path');
const { authenticate } = require('./middleware/auth.js')
const { createTask, readTask, deleteTask } = require('./models/task.js')
const readHeath = require('./models/health.js')
const store = require('./middleware/store.js');
const queue = require('./middleware/queue.js');
const app = express();
app.use(store);
app.use(queue);
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.post('/task', authenticate, createTask)
app.get('/task', readTask)
app.delete('/task', authenticate, deleteTask)
app.get('/health', readHeath)
module.exports = app

View File

@ -0,0 +1,109 @@
const app = require('./app.js')
const request = require('supertest')
const qs = require('querystring')
require('dotenv').config()
describe('app', function () {
it('should exist', function (done) {
if (!app?.mountpath) throw new Error('app doesnt exist');
done()
})
describe('/health', function () {
it('should be publicly readable', function (done) {
request(app)
.get('/health')
.set('Accept', 'text/html')
.expect('Content-Type', /text/)
.expect(/piss/)
.expect(200, done)
})
})
describe('/task', function () {
describe('POST', function () {
it('should create a task', function (done) {
request(app)
.post('/task')
.set('Authorization', `Bearer ${process.env.API_KEY}`)
.set('Accept', 'application/json')
.send({
url: 'https://futureporn-b2.b-cdn.net/projekt-melody.jpg'
})
.expect('Content-Type', /json/)
.expect((res) => {
if (!res.body?.data) throw new Error('response body was missing data')
if (!res.body?.data?.id) throw new Error('response body was missing id')
return true
})
.expect(200, done)
})
})
describe('GET', function () {
it('should show all tasks specifications', async function () {
await request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/my.jpg' })
await request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/your.png' })
return request(app)
.get(`/task`)
.set('Authorization', `Bearer ${process.env.API_KEY}`)
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.expect((res) => {
if (!res?.body?.data) throw new Error('there was no data in response')
})
.expect(200)
})
it('should accept task id as query param and return task specification', function (done) {
const seed = request(app).post('/task').set('Authorization', `Bearer ${process.env.API_KEY}`).send({ url: 'https://example.com/z.jpg' })
seed.then((res) => {
const query = qs.stringify({
id: res.body.data.id
})
request(app)
.get(`/task?${query}`)
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.expect((res) => {
if (res?.body?.error) throw new Error('there was an error in the response: '+res.body?.message)
if (!res?.body?.data?.url) throw new Error('data.url was missing')
if (!res?.body?.data?.createdAt) throw new Error('data.createdAt was missing')
return true
})
.expect(200, done)
})
})
it('should show all tasks by default', function (done) {
request(app)
.get('/task')
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.expect((res) => {
if (res.body?.error) throw new Error('there was an error in the response'+res.error)
if (!res.body?.data) throw new Error('data was missing')
return true
})
.expect(200, done)
})
})
describe('DELETE', function () {
const query = qs.stringify({
id: 'awejf9wiejf9we'
})
it('should delete a single task', function (done) {
request(app)
.delete(`/task?${query}`)
.set('Authorization', `Bearer ${process.env.API_KEY}`)
.set('Accept', 'application/json')
.expect('Content-Type', /json/)
.expect(200, done);
});
})
})
})

View File

@ -0,0 +1 @@
hello worlds

View File

@ -0,0 +1,16 @@
module.exports.authenticate = function authenticate(req, res, next) {
const bearerToken = req.headers?.authorization.split(' ').at(1);
if (!bearerToken) {
const msg = `authorization bearer token was missing from request headers`;
console.error(msg);
return res.status(401).json({ error: true, message: msg });
}
if (bearerToken !== process.env.API_KEY) {
const msg = 'INCORRECT API_KEY (wrong token)';
console.error(msg);
return res.status(403).json({ error: true, message: msg });
} else {
next();
}
}

View File

@ -0,0 +1,21 @@
const Queue = require('better-queue');
const taskProcess = require('../utils/taskProcess.js');
const options = {
id: 'id',
maxRetries: 3,
concurrent: 1
// @todo better-queue has batching and concurrency. might be useful to implement in the future
// @see https://github.com/diamondio/better-queue?tab=readme-ov-file#queue-management
}
let q = new Queue(taskProcess, options)
// Middleware function to attach db to request
const queueMiddleware = (req, res, next) => {
req.queue = q;
next();
};
module.exports = queueMiddleware

View File

@ -0,0 +1,10 @@
const store = {
tasks: {}
}
const storeMiddleware = (req, res, next) => {
req.store = store
next();
};
module.exports = storeMiddleware;

View File

@ -0,0 +1,3 @@
module.exports = function readHealth (req, res) {
return res.send('**link2cid pisses on the floor**')
}

View File

@ -0,0 +1,59 @@
const { createId } = require('@paralleldrive/cuid2');
const { getTmpFilePath } = require('../utils/paths.js');
const fsp = require('fs/promises');
module.exports.createTask = function createTask (req, res) {
const url = req.body.url
const task = {
id: createId(),
url: url,
filePath: getTmpFilePath(url),
fileSize: null,
createdAt: new Date().toISOString(),
cid: null,
downloadProgress: null,
addProgress: null
}
if (!req?.body?.url) return res.status(400).json({ error: true, message: 'request body was missing a url' });
req.store.tasks[task.id] = task;
req.queue.push(task, function (err, result) {
if (err) throw err;
console.log('the following is the result of the queued task being complete')
console.log(result)
})
return res.json({ error: false, data: task })
}
module.exports.readTask = function readTask (req, res) {
const id = req?.query?.id
// If we get an id in the query, show the one task.
// Otherwise, we show all tasks.
if (!!id) {
const task = req.store.tasks[id]
if (!task) return res.json({ error: true, message: 'there was no task in the store with that id' });
return res.json({ error: false, data: task })
} else {
const tasks = req.store.tasks
return res.json({ error: false, data: tasks })
}
}
module.exports.deleteTask = async function deleteTask (req, res) {
const id = req?.query?.id;
const task = req.store.tasks[id];
try {
if (task?.filePath) await fsp.unlink(task.filePath);
} catch (err) {}
delete req.store.tasks[id];
return res.json({ error: false, message: 'task deleted' });
if (err) return res.json({ error: true, message: err });
}

View File

@ -0,0 +1,20 @@
const fs = require("fs");
const { Readable } = require('stream');
const { finished } = require('stream/promises');
const path = require("path");
/**
* Download a file at url to local disk filePath
* @param {String} url
* @param {String} filePath
*
* greetz https://stackoverflow.com/a/51302466/1004931
*/
const download = (async (url, filePath) => {
const res = await fetch(url);
const fileStream = fs.createWriteStream(filePath, { flags: 'wx' });
await finished(Readable.fromWeb(res.body).pipe(fileStream));
});
module.exports = download;

View File

@ -0,0 +1,12 @@
const download = require('./download.js');
const fsp = require('fs/promises');
describe('download', function () {
it('should download a file from url', async function () {
const testFilePath = '/tmp/pmel.jpg'
try {
await fsp.unlink(testFilePath)
} catch (e) {}
await download('https://futureporn-b2.b-cdn.net/projekt-melody.jpg', testFilePath)
})
})

View File

@ -0,0 +1,100 @@
require('dotenv').config();
const { openAsBlob } = require('node:fs');
const { rm, stat } = require('fs/promises');
const path = require('path');
if (!process.env.IPFS_URL) throw new Error('IPFS_URL was missing in env');
async function streamingPostFetch(
url,
formData,
basename,
filesize
) {
// console.log(`streamingPostFetch with url=${url}, formData=${formData.get('file')}, basename=${basename}, filesize=${filesize}`);
try {
const res = await fetch(url, {
method: 'POST',
body: formData
});
if (!res.ok) {
throw new Error(`HTTP error! Status-- ${res.status}`);
}
const reader = res.body?.getReader();
if (!reader) {
throw new Error('Failed to get reader from response body');
}
while (true) {
const { done, value } = await reader.read();
const chunk = new TextDecoder().decode(value);
const lines = chunk.split('\n');
for (const line of lines) {
const trimmedLine = line.trim()
if (!!trimmedLine) {
// console.log(trimmedLine);
const json = JSON.parse(trimmedLine);
// console.log(`comparing json.Name=${json.Name} with basename=${basename}`);
if (json.Name === basename && json.Hash && json.Size) {
// this is the last chunk
return json;
}
}
}
if (done) {
throw new Error('Response reader finished before receiving a CID which indicates a failiure.');
}
}
} catch (error) {
console.error('An error occurred:', error);
throw error;
}
}
async function getFormStuff(filePath) {
const url = `${process.env.IPFS_URL}/api/v0/add?progress=false&cid-version=1&pin=true`;
const blob = await openAsBlob(filePath);
const basename = path.basename(filePath);
const filesize = (await stat(filePath)).size;
const formData = new FormData();
return {
url,
blob,
basename,
filesize,
formData
}
}
/**
* @param {String} filePath
* @returns {String} CID
*/
const ipfsAdd = async function (filePath) {
const { url: kuboUrl, blob, basename, filesize, formData } = await getFormStuff(filePath);
formData.append('file', blob, basename);
const output = await streamingPostFetch(kuboUrl, formData, basename, filesize);
if (!output?.Hash) throw new Error('No CID was received from remote IPFS node.');
const cid = output.Hash;
return cid
}
module.exports = ipfsAdd;

View File

@ -0,0 +1,13 @@
const ipfsAdd = require('./ipfsAdd.js')
const path = require('path');
describe('ipfs', function () {
describe('ipfsAdd', function () {
it('should add a file from disk to ipfs and return a {string} CID', async function () {
const expectedCid = 'bafkreibxh3ly47pr3emvrqtax6ieq2ybom4ywyil3yurxnlwirtcvb5pfi'
const file = path.join(__dirname, '..', 'fixtures', 'hello-worlds.txt')
const cid = await ipfsAdd(file, { cidVersion: 1 })
if (cid !== expectedCid) throw new Error(`expected ${cid} to match ${expectedCid}`)
})
})
})

View File

@ -0,0 +1,12 @@
const os = require('os');
const path = require('path');
const getTmpFilePath = function (url) {
const timestamp = new Date().valueOf()
return path.join(os.tmpdir(), timestamp+'-'+path.basename(url))
}
module.exports = {
getTmpFilePath
}

View File

@ -0,0 +1,11 @@
const paths = require('./paths.js')
describe('paths', function () {
describe('getTmpFilePath', function () {
it('should accept a url and receive a /tmp/<datestamp><basename> path on disk', function () {
const url = 'https://example.com/my.jpg'
const p = paths.getTmpFilePath(url)
if (!/\/tmp\/\d+-my\.jpg/.test(p)) throw new Error(`expected ${p} to use format /tmp/<datestamp><basename>`)
})
})
})

View File

@ -0,0 +1,25 @@
const os = require('os');
const path = require('path');
const downloadFile = require('./download');
const ipfsAdd = require('./ipfsAdd');
const taskProcess = async function (taskSpec, cb) {
console.log('downloading')
this.progressTask(1, 3, "downloading")
await downloadFile(taskSpec.url, taskSpec.filePath)
console.log('adding')
this.progressTask(2, 3, "adding")
const cid = await ipfsAdd(taskSpec.filePath)
taskSpec.cid = cid
cb(null, taskSpec)
}
module.exports = taskProcess;