From 80be9f29bcae9f4aa387905b4c836f1a6597103e Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 5 Dec 2025 17:49:37 -0500 Subject: [PATCH 1/8] Add support for rdb-cli to replace rdb. --- data-loading/Dockerfile | 12 +++++++++++- data-loading/requirements.txt | 3 +-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/data-loading/Dockerfile b/data-loading/Dockerfile index 2463dcf..74bdb47 100644 --- a/data-loading/Dockerfile +++ b/data-loading/Dockerfile @@ -8,6 +8,7 @@ ARG DATA=/data RUN apt-get update RUN apt-get -y upgrade RUN apt-get -y install wget python3-venv python3-pip +RUN apt-get -y install git gcc # Create nru user RUN mkdir -p ${ROOT} @@ -15,6 +16,12 @@ RUN useradd nru -d ${ROOT} RUN chown nru ${ROOT} WORKDIR ${ROOT} +# We need to install rdb-cli, which we need to -- lolsob -- compile from source. So let's do that. +RUN git clone https://github.com/redis/librdb.git +WORKDIR ${ROOT}/librdb +RUN make install +WORKDIR ${ROOT} + # Set up a volume for data. VOLUME ${DATA} RUN mkdir -p ${DATA} @@ -32,5 +39,8 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" COPY --chown=nru requirements.txt ${ROOT} RUN pip3 install -r requirements.txt +# Make sure rdb-cli has been set up. +# RUN rdb-cli + # Entrypoint should be bash. -ENTRYPOINT /bin/bash +ENTRYPOINT ["/bin/bash"] diff --git a/data-loading/requirements.txt b/data-loading/requirements.txt index ed096b4..8b13789 100644 --- a/data-loading/requirements.txt +++ b/data-loading/requirements.txt @@ -1,2 +1 @@ -rdbtools -python-lzf + From e4c43a47a47f28dc8cd885c17810e9d9185f4162 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 5 Dec 2025 18:20:52 -0500 Subject: [PATCH 2/8] Added script to convert RDB to resp. --- data-loading/Dockerfile | 3 +++ data-loading/rdb-to-resp.sh | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 data-loading/rdb-to-resp.sh diff --git a/data-loading/Dockerfile b/data-loading/Dockerfile index 74bdb47..6a57f75 100644 --- a/data-loading/Dockerfile +++ b/data-loading/Dockerfile @@ -39,6 +39,9 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" COPY --chown=nru requirements.txt ${ROOT} RUN pip3 install -r requirements.txt +# Copy scripts. +COPY --chown=nru rdb-to-resp.sh ${ROOT} + # Make sure rdb-cli has been set up. # RUN rdb-cli diff --git a/data-loading/rdb-to-resp.sh b/data-loading/rdb-to-resp.sh new file mode 100644 index 0000000..d22e960 --- /dev/null +++ b/data-loading/rdb-to-resp.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# A Bash script to convert an RDB file into a stream of RESP commands. +# +# bash rdb-to-resp.sh RDB_FILENAME +set -euo pipefail + +# Check the number of arguments +if [ "$#" -lt 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +rdb-cli "$1" -l stdout -s 1000 -d 0 resp \ No newline at end of file From aedeebcd36a28484f2bcaa5ec161c06a09a3b791 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 5 Dec 2025 18:25:48 -0500 Subject: [PATCH 3/8] Added Redis version argument. --- data-loading/rdb-to-resp.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data-loading/rdb-to-resp.sh b/data-loading/rdb-to-resp.sh index d22e960..f00fa42 100644 --- a/data-loading/rdb-to-resp.sh +++ b/data-loading/rdb-to-resp.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # # A Bash script to convert an RDB file into a stream of RESP commands. +# Uses https://github.com/redis/librdb?tab=readme-ov-file#rdb-cli-usage # -# bash rdb-to-resp.sh RDB_FILENAME +# bash rdb-to-resp.sh RDB_FILENAME [REDIS_VERSION] set -euo pipefail # Check the number of arguments @@ -10,5 +11,6 @@ if [ "$#" -lt 1 ]; then echo "Usage: $0 " >&2 exit 1 fi +REDIS_VERSION="${2:-6.2}" # Default to a Redis version of 6.2 -rdb-cli "$1" -l stdout -s 1000 -d 0 resp \ No newline at end of file +rdb-cli "$1" --show-progress 1000 resp --target-redis-ver "${REDIS_VERSION}" \ No newline at end of file From 6975319b5cb33979d6cb82f3708aafe758a013a3 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Fri, 5 Dec 2025 18:29:57 -0500 Subject: [PATCH 4/8] Got rid of logging/progress options. We could write logs into the Docker, but I can't figure out how to send that to stderr. --- data-loading/rdb-to-resp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-loading/rdb-to-resp.sh b/data-loading/rdb-to-resp.sh index f00fa42..a81bbab 100644 --- a/data-loading/rdb-to-resp.sh +++ b/data-loading/rdb-to-resp.sh @@ -13,4 +13,4 @@ if [ "$#" -lt 1 ]; then fi REDIS_VERSION="${2:-6.2}" # Default to a Redis version of 6.2 -rdb-cli "$1" --show-progress 1000 resp --target-redis-ver "${REDIS_VERSION}" \ No newline at end of file +rdb-cli "$1" resp --target-redis-ver "${REDIS_VERSION}" \ No newline at end of file From 4d1f0028c41016ad3d4247cfc19c2fae55a3ee75 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 8 Dec 2025 13:49:11 -0500 Subject: [PATCH 5/8] Turned on on:push trigger for testing. --- .github/workflows/release-loading.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-loading.yml b/.github/workflows/release-loading.yml index f6bcf1f..8051e67 100644 --- a/.github/workflows/release-loading.yml +++ b/.github/workflows/release-loading.yml @@ -1,6 +1,7 @@ name: 'Publish nodenorm-loading to GitHub Packages' on: + push: release: types: [published] From 9d0230273884ef54c90b95c3fb41110ddbcf83ed Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 8 Dec 2025 13:55:00 -0500 Subject: [PATCH 6/8] Updated Redis in NodeNorm Loader to latest. --- data-loading/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-loading/Dockerfile b/data-loading/Dockerfile index 6a57f75..ca71a78 100644 --- a/data-loading/Dockerfile +++ b/data-loading/Dockerfile @@ -1,5 +1,5 @@ # Use Redis so we can redis-cli. -FROM redis:6 +FROM redis:latest # Configuration ARG ROOT=/home/nru From 3bb61815eb6a43d1424bd6867e61f72781b1e036 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 8 Dec 2025 13:55:40 -0500 Subject: [PATCH 7/8] Added executable bit to rdb-to-resp.sh. --- data-loading/rdb-to-resp.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 data-loading/rdb-to-resp.sh diff --git a/data-loading/rdb-to-resp.sh b/data-loading/rdb-to-resp.sh old mode 100644 new mode 100755 From f6d5d8f4a8f753e95ba9bf57b8ecf79f782a15b7 Mon Sep 17 00:00:00 2001 From: Gaurav Vaidya Date: Mon, 8 Dec 2025 14:12:49 -0500 Subject: [PATCH 8/8] Removed on:push trigger after testing. --- .github/workflows/release-loading.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release-loading.yml b/.github/workflows/release-loading.yml index 8051e67..f6bcf1f 100644 --- a/.github/workflows/release-loading.yml +++ b/.github/workflows/release-loading.yml @@ -1,7 +1,6 @@ name: 'Publish nodenorm-loading to GitHub Packages' on: - push: release: types: [published]