74 lines
2.0 KiB
Bash
74 lines
2.0 KiB
Bash
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
############################################################
|
|
# Script: pinall.sh
|
|
#
|
|
# Description:
|
|
# Downloads all files listed in an input file to a temporary
|
|
# directory (only if not already present), then pins them
|
|
# all to a local IPFS node. Cleans up temp files only if
|
|
# all downloads and pins succeeded.
|
|
#
|
|
############################################################
|
|
|
|
# Usage check
|
|
if [ $# -ne 1 ]; then
|
|
echo "Usage: $0 <file-with-s3-keys>"
|
|
exit 1
|
|
fi
|
|
|
|
FILELIST=$1
|
|
TMPDIR="/mnt/blockstorage/pinalltmp"
|
|
|
|
# Ensure tmp directory exists
|
|
mkdir -p "$TMPDIR"
|
|
|
|
ipfs id
|
|
echo "Using IPFS_PATH=$IPFS_PATH"
|
|
|
|
# Track overall success
|
|
ALL_OK=true
|
|
|
|
# First pass: download files if not already present
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Starting downloads..."
|
|
while IFS= read -r KEY; do
|
|
[[ -z "$KEY" || "$KEY" =~ ^# ]] && continue
|
|
|
|
TMPFILE="$TMPDIR/$KEY"
|
|
|
|
if [ -f "$TMPFILE" ]; then
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] File already exists, skipping: $KEY"
|
|
continue
|
|
fi
|
|
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Downloading $KEY to $TMPFILE..."
|
|
if b2 file download "b2://futureporn/$KEY" "$TMPFILE"; then
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Download complete: $KEY"
|
|
else
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Download failed: $KEY"
|
|
ALL_OK=false
|
|
fi
|
|
done < "$FILELIST"
|
|
|
|
# Second pass: pin all files
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Starting IPFS pinning..."
|
|
for FILE in "$TMPDIR"/*; do
|
|
[[ ! -f "$FILE" ]] && continue
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Adding $(basename "$FILE") to IPFS..."
|
|
if ! ipfs add --cid-version=1 "$FILE"; then
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] IPFS add failed for $(basename "$FILE")"
|
|
ALL_OK=false
|
|
fi
|
|
done
|
|
|
|
# Cleanup only if all succeeded
|
|
if [ "$ALL_OK" = true ]; then
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] All downloads and pins succeeded. Cleaning up temporary files..."
|
|
rm -rf "$TMPDIR"/*
|
|
else
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Some operations failed. Leaving temporary files for inspection."
|
|
fi
|
|
|
|
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Script finished."
|