From 04f8502ed430cf74085162f9fedfda0ed5dc0936 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 23 Jun 2025 15:57:00 +0100 Subject: [PATCH 1/4] split ipfs scripts --- scripts/check-ipfs.sh | 81 +++++++++++++++++++++++++++++ scripts/{ipfs.sh => pin-to-ipfs.sh} | 52 +++++++----------- 2 files changed, 99 insertions(+), 34 deletions(-) create mode 100755 scripts/check-ipfs.sh rename scripts/{ipfs.sh => pin-to-ipfs.sh} (82%) diff --git a/scripts/check-ipfs.sh b/scripts/check-ipfs.sh new file mode 100755 index 0000000..96dcfe9 --- /dev/null +++ b/scripts/check-ipfs.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +###################################################### + +# Can change if you want! + +# Gateways to check if file is already hosted on IPFS +DEFAULT_GATEWAY_1="https://ipfs.io/ipfs/" +DEFAULT_GATEWAY_2="https://dweb.link/ipfs/" +DEFAULT_GATEWAY_3="https://gateway.pinata.cloud/ipfs/" + +###################################################### + +# check if user has ipfs cli installed +if ! command -v ipfs >/dev/null 2>&1; then + echo "Error: ipfs cli is not installed or not in your PATH." >&2 + exit 1 +fi + +# Usage message +usage() { + echo "Usage: $0 " + echo "Check if a file is discoverable via free IPFS gateways" + echo " " + echo "Options:" + echo " Path to your file." + exit 1 +} + +# Initialize variables with defaults +input_path="" + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + usage + ;; + *) + if [ -z "$input_path" ]; then + input_path="$1" + fi + shift + ;; + esac +done + +# Generate CID from the given file +echo "Generating CID for the file..." + +# use ipfs add to generate a CID +# use CIDv1 +ipfs_cid=$(ipfs add -Q --cid-version 1 "$input_path") +echo "CID: $ipfs_cid" + +check_file_on_gateway() { + local gateway="$1" + local cid="$2" + echo " " + echo "Checking ${gateway}..." + if curl --silent --fail "${gateway}${cid}" >/dev/null; then + echo "File is accessible on IPFS via ${gateway}${cid}" + return 0 + else + echo "File not found at: ${gateway}${cid}" + return 1 + fi +} + +# If file can be found via gateways then exit +echo "Checking if file is already hosted on IPFS..." +if check_file_on_gateway "$DEFAULT_GATEWAY_1" "$ipfs_cid"; then + exit 0 +fi +echo " " +if check_file_on_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid"; then + exit 0 +fi + +echo " " +echo "File is cannot be found via gateways. Exiting." diff --git a/scripts/ipfs.sh b/scripts/pin-to-ipfs.sh similarity index 82% rename from scripts/ipfs.sh rename to scripts/pin-to-ipfs.sh index 17acff6..92aa192 100755 --- a/scripts/ipfs.sh +++ b/scripts/pin-to-ipfs.sh @@ -4,13 +4,13 @@ # Can change if you want! -# used to by pass waiting for gateway checks -JUST_PIN="false" -JUST_CHECK="false" +# Default behavior is to not check if file is discoverable on IPFS +CHECK_TOO="false" # Gateways to check if file is already hosted on IPFS DEFAULT_GATEWAY_1="https://ipfs.io/ipfs/" -DEFAULT_GATEWAY_2="https://gateway.pinata.cloud/ipfs/" +DEFAULT_GATEWAY_2="https://dweb.link/ipfs/" +DEFAULT_GATEWAY_3="https://gateway.pinata.cloud/ipfs/" # Pinning services to host the file on IPFS DEFAULT_HOST_ON_LOCAL_NODE="true" @@ -31,24 +31,22 @@ fi # Usage message usage() { - echo "Usage: $0 [--just-pin] [--just-check] [--no-local] [--no-pinata] [--no-blockfrost] [--no-nmkr]" - echo "Check if a file is on IPFS, and also pin it locally and via Blockfrost and NMKR." + echo "Usage: $0 [--check-too] [--no-local] [--no-pinata] [--no-blockfrost] [--no-nmkr]" + echo "Pin a file to local IPFS node and pin via Blockfrost, NMKR and Pinata. Optionally check if file is already discoverable on IPFS." echo " " echo "Options:" echo " Path to your file." - echo " --just-pin Don't look for the file, just pin it (default: $JUST_PIN)" - echo " --just-check Only look for the file don't try to pin it (default: $JUST_CHECK)" - echo " --no-local Don't try to pin file on local ipfs node? (default: $DEFAULT_HOST_ON_LOCAL_NODE)" - echo " --no-pinata Don't try to pin file on pinata service? (default: $DEFAULT_HOST_ON_PINATA)" - echo " --no-blockfrost Don't try to pin file on blockfrost service? (default: $DEFAULT_HOST_ON_BLOCKFROST)" - echo " --no-nmkr Don't try to pin file on NMKR service? (default: $DEFAULT_HOST_ON_NMKR_STORAGE)" + echo " --check-too Run a check if file is discoverable on ipfs, only pin if not discoverable (default: $CHECK_TOO)" + echo " --no-local Don't try to pin file on local ipfs node (default: $DEFAULT_HOST_ON_LOCAL_NODE)" + echo " --no-pinata Don't try to pin file on pinata service (default: $DEFAULT_HOST_ON_PINATA)" + echo " --no-blockfrost Don't try to pin file on blockfrost service (default: $DEFAULT_HOST_ON_BLOCKFROST)" + echo " --no-nmkr Don't try to pin file on NMKR service (default: $DEFAULT_HOST_ON_NMKR)" exit 1 } # Initialize variables with defaults input_path="" -just_pin="$JUST_PIN" -just_check="$JUST_CHECK" +check_discoverable="$CHECK_TOO" local_host="$DEFAULT_HOST_ON_LOCAL_NODE" pinata_host="$DEFAULT_HOST_ON_PINATA" blockfrost_host="$DEFAULT_HOST_ON_BLOCKFROST" @@ -57,12 +55,8 @@ nmkr_host="$DEFAULT_HOST_ON_NMKR" # Parse command line arguments while [[ $# -gt 0 ]]; do case $1 in - --just-pin) - just_pin="true" - shift - ;; - --just-check) - just_check="true" + --check-too) + check_discoverable="true" shift ;; --no-local) @@ -101,10 +95,6 @@ echo "Generating CID for the file..." ipfs_cid=$(ipfs add -Q --cid-version 1 "$input_path") echo "CID: $ipfs_cid" -# check two gateways if file can be accessed -echo " " -echo "Checking if file is already hosted on IPFS..." - check_file_on_gateway() { local gateway="$1" local cid="$2" @@ -119,14 +109,14 @@ check_file_on_gateway() { fi } -# If file can be found via gateways then exit -if [ "$just_pin" = "false" ]; then +# If user wants to check if file is discoverable on IPFS +if [ "$check_discoverable" = "true" ]; then echo "Checking if file is already hosted on IPFS..." - if check_file_on_gateway "$DEFAULT_GATEWAY_1" "$ipfs_cid" "TIMEOUT"; then + if check_file_on_gateway "$DEFAULT_GATEWAY_1" "$ipfs_cid"; then echo "File is already hosted on IPFS. No need to pin anywhere else." exit 0 fi - if check_file_on_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid" "TIMEOUT"; then + if check_file_on_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid"; then echo "File is already hosted on IPFS. No need to pin anywhere else." exit 0 fi @@ -134,12 +124,6 @@ else echo "Skipping check of file on ipfs..." fi -# If just checking then exit -if [ "$just_check" = "true" ]; then - echo "File is not hosted on IPFS, but you requested to just check. Exiting." - exit 0 -fi - # If file is not accessible then pin it!! echo " " echo "File is not hosted on IPFS, so pinning it..." From d3b94a6a243f84580fdc90ccf483eb130e9eba4e Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 23 Jun 2025 15:58:38 +0100 Subject: [PATCH 2/4] add third gateway --- scripts/check-ipfs.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/check-ipfs.sh b/scripts/check-ipfs.sh index 96dcfe9..7ec4d95 100755 --- a/scripts/check-ipfs.sh +++ b/scripts/check-ipfs.sh @@ -76,6 +76,10 @@ echo " " if check_file_on_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid"; then exit 0 fi +echo " " +if check_file_on_gateway "$DEFAULT_GATEWAY_3" "$ipfs_cid"; then + exit 0 +fi echo " " echo "File is cannot be found via gateways. Exiting." From 03a8777e0f230b811d4b08d60fe36983213395e8 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 23 Jun 2025 16:31:12 +0100 Subject: [PATCH 3/4] improve split, remove duplicated code --- scripts/check-ipfs.sh | 32 ++++++++++++++++++++------------ scripts/pin-to-ipfs.sh | 29 ++--------------------------- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/scripts/check-ipfs.sh b/scripts/check-ipfs.sh index 7ec4d95..853fc62 100755 --- a/scripts/check-ipfs.sh +++ b/scripts/check-ipfs.sh @@ -4,10 +4,19 @@ # Can change if you want! +# Timeout for curl requests in seconds +TIMEOUT=5 + # Gateways to check if file is already hosted on IPFS DEFAULT_GATEWAY_1="https://ipfs.io/ipfs/" -DEFAULT_GATEWAY_2="https://dweb.link/ipfs/" -DEFAULT_GATEWAY_3="https://gateway.pinata.cloud/ipfs/" +DEFAULT_GATEWAY_2="https://gateway.pinata.cloud/ipfs/" +# DEFAULT_GATEWAY_3="https://w3s.link/ipfs" + +# Other gateways like Dweb.link, 4everland, w3s.link don't seem to work well with curl +# in future we can fix them by reading the https header and checking if the file is content is returned + +# for more gateways see: +# https://ipfs.github.io/public-gateway-checker/ ###################################################### @@ -53,12 +62,13 @@ echo "Generating CID for the file..." ipfs_cid=$(ipfs add -Q --cid-version 1 "$input_path") echo "CID: $ipfs_cid" -check_file_on_gateway() { +check_file_via_gateway() { local gateway="$1" local cid="$2" + local timeout="$3" echo " " echo "Checking ${gateway}..." - if curl --silent --fail "${gateway}${cid}" >/dev/null; then + if curl --silent --fail --max-time $timeout "${gateway}${cid}" >/dev/null; then echo "File is accessible on IPFS via ${gateway}${cid}" return 0 else @@ -68,18 +78,16 @@ check_file_on_gateway() { } # If file can be found via gateways then exit -echo "Checking if file is already hosted on IPFS..." -if check_file_on_gateway "$DEFAULT_GATEWAY_1" "$ipfs_cid"; then - exit 0 -fi -echo " " -if check_file_on_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid"; then +if check_file_via_gateway "$DEFAULT_GATEWAY_1" "$ipfs_cid" "$TIMEOUT"; then exit 0 fi -echo " " -if check_file_on_gateway "$DEFAULT_GATEWAY_3" "$ipfs_cid"; then +if check_file_via_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid" "$TIMEOUT"; then exit 0 fi +# todo: add more gateways + +# If file cannot be found via gateways then exit echo " " echo "File is cannot be found via gateways. Exiting." +exit 0 diff --git a/scripts/pin-to-ipfs.sh b/scripts/pin-to-ipfs.sh index 92aa192..e7de0fc 100755 --- a/scripts/pin-to-ipfs.sh +++ b/scripts/pin-to-ipfs.sh @@ -7,11 +7,6 @@ # Default behavior is to not check if file is discoverable on IPFS CHECK_TOO="false" -# Gateways to check if file is already hosted on IPFS -DEFAULT_GATEWAY_1="https://ipfs.io/ipfs/" -DEFAULT_GATEWAY_2="https://dweb.link/ipfs/" -DEFAULT_GATEWAY_3="https://gateway.pinata.cloud/ipfs/" - # Pinning services to host the file on IPFS DEFAULT_HOST_ON_LOCAL_NODE="true" DEFAULT_HOST_ON_NMKR="true" @@ -95,31 +90,11 @@ echo "Generating CID for the file..." ipfs_cid=$(ipfs add -Q --cid-version 1 "$input_path") echo "CID: $ipfs_cid" -check_file_on_gateway() { - local gateway="$1" - local cid="$2" - local timeout="$3" - echo "Checking ${gateway}..." - if curl --silent --fail "${gateway}${cid}" >/dev/null; then - echo "File is accessible on IPFS via ${gateway}${cid}" - return 0 - else - echo "File not found at: ${gateway}${cid}" - return 1 - fi -} - # If user wants to check if file is discoverable on IPFS if [ "$check_discoverable" = "true" ]; then echo "Checking if file is already hosted on IPFS..." - if check_file_on_gateway "$DEFAULT_GATEWAY_1" "$ipfs_cid"; then - echo "File is already hosted on IPFS. No need to pin anywhere else." - exit 0 - fi - if check_file_on_gateway "$DEFAULT_GATEWAY_2" "$ipfs_cid"; then - echo "File is already hosted on IPFS. No need to pin anywhere else." - exit 0 - fi + echo "Using ./scripts/check-ipfs.sh script to check if file is discoverable on IPFS..." + ./scripts/check-ipfs.sh "$input_path" else echo "Skipping check of file on ipfs..." fi From 0445d0a8107f29ed87284380940b3cec547941e2 Mon Sep 17 00:00:00 2001 From: Ryan Williams Date: Mon, 23 Jun 2025 16:39:58 +0100 Subject: [PATCH 4/4] use exit codes to communicate between scripts --- scripts/check-ipfs.sh | 2 +- scripts/pin-to-ipfs.sh | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/check-ipfs.sh b/scripts/check-ipfs.sh index 853fc62..5b223f8 100755 --- a/scripts/check-ipfs.sh +++ b/scripts/check-ipfs.sh @@ -90,4 +90,4 @@ fi # If file cannot be found via gateways then exit echo " " echo "File is cannot be found via gateways. Exiting." -exit 0 +exit 1 diff --git a/scripts/pin-to-ipfs.sh b/scripts/pin-to-ipfs.sh index e7de0fc..3182aad 100755 --- a/scripts/pin-to-ipfs.sh +++ b/scripts/pin-to-ipfs.sh @@ -92,16 +92,22 @@ echo "CID: $ipfs_cid" # If user wants to check if file is discoverable on IPFS if [ "$check_discoverable" = "true" ]; then - echo "Checking if file is already hosted on IPFS..." echo "Using ./scripts/check-ipfs.sh script to check if file is discoverable on IPFS..." - ./scripts/check-ipfs.sh "$input_path" + # check if file is discoverable on IPFS + if ! ./scripts/check-ipfs.sh "$input_path"; then + echo "File is not discoverable on IPFS. Proceeding to pin it." + else + echo "File is already discoverable on IPFS. No need to pin it." + exit 0 + fi + else echo "Skipping check of file on ipfs..." fi -# If file is not accessible then pin it!! echo " " echo "File is not hosted on IPFS, so pinning it..." +echo " " # Pin on local node if [ "$local_host" = "true" ]; then