diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b40c49e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +.gitignore +docker/Makefile.appchain +docker/node/ +docker/*.sh diff --git a/docker/Makefile b/docker/Makefile index eeb9f66..93d8960 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,23 +1,14 @@ -# appchain, voting -pki?=voting - warped?=true mixes=3 +auths=3 gateways=1 serviceNodes=1 -# only used by pki=voting -auths=3 UserForwardPayloadLength=30000 distro=alpine - -# pki=appchain requires debian (katzenpost alpine docker image does not have bash) -ifeq ($(pki), appchain) - distro = debian -endif - -net_name=mixnet +net_name=voting_mixnet +docker_compose_yml=$(net_name)/docker-compose.yml sh=$(shell if echo ${distro}|grep -q alpine; then echo sh; else echo bash; fi) cache_dir=cache log_level=DEBUG @@ -34,7 +25,6 @@ docker_run_sh=$(docker) run ${docker_args} $(mount_net_name) $(mount_opt) --rm k katzenpost_dir?=/tmp/katzenpost.opt katzenpost_version?=$(shell grep -E '^ github.com/katzenpost/katzenpost ' ../go.mod | awk '{print $$2}') net_dir=$(katzenpost_dir)/docker/$(net_name) -docker_compose_yml=$(net_dir)/docker-compose.yml # export variables to the environment for consumption by invoked Makefile(s) export @@ -62,7 +52,7 @@ help: @$(MAKE) -e -C $(katzenpost_dir)/docker $@ .PHONY: custom-binaries -custom-binaries: $(net_dir)/http_proxy.$(distro) $(net_dir)/pki.$(distro) +custom-binaries: $(net_dir)/http_proxy.$(distro) .PHONY: custom-config custom-config: @@ -79,27 +69,6 @@ clone-katzenpost: $(katzenpost_dir); \ fi -# this genconfig target is intended to be run within the katzenpost docker container -# for pki=appchain -.PHONY: genconfig -genconfig: - cd ../genconfig/cmd/genconfig && go build - ./genconfig.sh - -$(docker_compose_yml): $(distro)_base.stamp | $(net_name) $(cache_dir) - @if [ "$(pki)" = "appchain" ]; then \ - $(docker_run_sh) 'cd /go/opt/docker ; make pki=appchain genconfig'; \ - else \ - $(MAKE) -e -C $(katzenpost_dir)/docker $@; \ - fi - -.PHONY: $(distro)_base.stamp -$(distro)_base.stamp: - $(MAKE) -e -C $(katzenpost_dir)/docker $@ - -$(net_dir)/pki.$(distro): $(katzenpost_dir)/docker/$(distro)_base.stamp $(docker_compose_yml) | $(net_name) $(cache_dir) - $(docker_run_sh) 'cd /go/opt/pki ; go build -trimpath -ldflags ${ldflags} && mv pki /$(net_name)/pki.$(distro)' - $(net_dir)/http_proxy.$(distro): $(katzenpost_dir)/docker/$(distro)_base.stamp | $(net_name) $(cache_dir) $(docker_run_sh) 'cd /go/opt/server_plugins/cbor_plugins/http_proxy/cmd/http_proxy ; go build -trimpath -ldflags ${ldflags} && mv http_proxy /$(net_name)/http_proxy.$(distro)' cp ../server_plugins/cbor_plugins/http_proxy/http_proxy_config.toml $(net_dir)/servicenode1/ diff --git a/docker/Makefile.appchain b/docker/Makefile.appchain new file mode 100644 index 0000000..b2e32e8 --- /dev/null +++ b/docker/Makefile.appchain @@ -0,0 +1,62 @@ +net ?= /tmp/appchain-mixnet +dir_base := /mixnet +dir_bin := /opt/zkn +docker_image ?= zkn/node:latest +docker_image_agent ?= zkn/agent:latest +docker := $(shell if which podman|grep -q .; then echo podman; else echo docker; fi) +docker_user ?= $(shell [ "$(docker)" = "podman" ] && echo 0:0 || echo $${SUDO_UID:-$$(id -u)}:$${SUDO_GID:-$$(id -g)}) +docker_compose ?= DOCKER_USER=$(docker_user) $(shell if which podman|grep -q .; then echo DOCKER_HOST="unix://$$XDG_RUNTIME_DIR/podman/podman.sock"; fi) docker compose +docker_args := --user ${docker_user} +docker_run := $(docker) run $(docker_args) --network=host --rm --volume $(shell readlink -f $(net)):$(dir_base) + +warped?=true +num_mixes=3 +num_gateways=1 +num_servicenodes=1 + +probe_count ?= 1 + +export + +$(net): + mkdir -vp $(net) + +image: $(net)/image.stamp +$(net)/image.stamp: $(net) + $(docker) build \ + --build-arg DIR_BIN=$(dir_bin) \ + --build-arg ENABLE_WARPED_EPOCH=$(warped) \ + --file ./node/Dockerfile \ + --tag $(docker_image) \ + ../ + touch $(net)/image.stamp + +config: $(net)/docker-compose.yml +$(net)/docker-compose.yml: genconfig.sh $(net) $(net)/image.stamp + ./genconfig.sh + +_start: $(net)/run.stamp +start: config image $(net)/run.stamp +$(net)/run.stamp: + cd $(net); $(docker_compose) up --remove-orphans -d; $(docker) compose top + touch $(net)/run.stamp + +wait: $(net)/run.stamp + $(docker_run) $(docker_image) $(dir_bin)/fetch -f $(dir_base)/client/client.toml + +probe: $(net)/run.stamp + $(docker_run) $(docker_image) $(dir_bin)/walletshield \ + -config $(dir_base)/client2/client.toml \ + -log_level DEBUG \ + -probe \ + -probe_count $(probe_count) + +.PHONY: stop +stop: + [ -e $(net) ] && cd $(net) && $(docker_compose) down --remove-orphans + rm -fv $(net)/run.stamp + +.PHONY: clean +clean: stop + $(docker) rmi $(docker_image) + rm -rfv $(net) diff --git a/docker/README.md b/docker/README.md index 620863b..9a3f9be 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,10 +1,67 @@ -# Docker Test Network +# Dockerized Test Networks -The Makefile and scripts here allow developers of 0KN mix network apps and server-side plugins to -locally run an offline Katzenpost test network with a podman-compatible docker-compose -configuration. It is meant for developing and testing client and server mix network components as -part of the core developer work flow. +This directory provides Makefiles and scripts to set up a local, offline test network for developing +and testing 0KN mix network applications and server-side plugins. The setup leverages a +Podman-compatible `docker-compose` configuration for simulating a Katzenpost network environment. -This Makefile covers 0KN-specifics and proxies other targets to Katzenpost's `docker/Makefile`. -Refer to [Katzenpost Docker test network](https://github.com/katzenpost/katzenpost/tree/main/docker) -for more info. +The goal is to support core development workflows by enabling local testing of both client and +server mix network components in isolated, controlled conditions. + +There are two Makefiles available, each corresponding to a different PKI. + +- **`Makefile`:** (Default) Manages a local test network using Katzenpost’s voting PKI. +- **`Makefile.appchain`:** Uses 0KN’s ZKAppChain PKI. + +## Voting PKI + +This setup, managed by the default `Makefile`, covers 0KN-specifics and proxies other targets to +Katzenpost's `docker/Makefile`. For additional details, refer to the [Katzenpost Docker Test +Network documentation](https://github.com/katzenpost/katzenpost/tree/main/docker). The voting PKI +functionality offers less complex local testing of 0KN mix plugins and client apps that do not +require the appchain. + +## Appchain PKI + +This Makefile builds and manages a network of dockerized nodes from +[`node/Dockerfile`](./node/Dockerfile). It uses the [genconfig](../genconfig/) utility to create +configurations for nodes from the network info in [network.yml](./network.yml) using the +appchain-powered [pki](../pki/). Node interactions with the appchain are managed through the +appchain-agent, utilizing UNIX domain sockets for communication. + +### Prerequisites + +To run the Appchain PKI network, ensure the following components are available: + +- [appchain-agent](https://github.com/0KnowledgeNetwork/appchain-agent) Docker image +- An operational 0KN ZKAppChain + +### Example Workflow + +```bash +# build the appchain-agent docker image +cd appchain-agent && make image + +# start local appchain instance, then: + +# build the docker image, configure, start the network, wait for the epoch, then probe +net=/tmp/appchain-mixnet make -f Makefile.appchain start wait probe + +# stop the network and clean up +net=/tmp/appchain-mixnet make -f Makefile.appchain clean + +# build the docker image and configure (without starting network) +# to inspect or manually edit the configuration files before continuing +net=/tmp/appchain-mixnet make -f Makefile.appchain config + +# start the network without rebuilding or reconfiguring, wait for the epoch +net=/tmp/appchain-mixnet make -f Makefile.appchain _start wait + +# test the network with a client sending 10 test probes +net=/tmp/appchain-mixnet probe_count=10 make -f Makefile.appchain probe + +# watch log files +tail -f /tmp/appchain-mixnet/*/*.log + +# stop the network (without cleaning up) +net=/tmp/appchain-mixnet make -f Makefile.appchain stop +``` diff --git a/docker/genconfig.sh b/docker/genconfig.sh index bb9d2d4..ee819a4 100755 --- a/docker/genconfig.sh +++ b/docker/genconfig.sh @@ -1,31 +1,33 @@ -#!/bin/bash +#!/bin/bash -e -# This script is invoked by ./Makefile to generate config files for a local -# test network using appchain pki. Variables set by the Makefile are read from -# the environment. This is intended to be run from within the katzenpost docker -# container. +# This script is invoked by ./Makefile to generate a docker-compose.yml file +# for a local test network using appchain pki. Variables set by the Makefile +# are read from the environment. port=30000 -dir_base="/${net_name}" -dir_out=${dir_base} -binary_suffix=".${distro}" - -rm -rf ${dir_out} && mkdir -p ${dir_out} +dir_out=${net} echo "Generating config files for local network:" -echo " num gateways: ${gateways}" -echo " num servicenodes: ${serviceNodes}" -echo " num mixes: ${mixes}" -echo " binary-suffix: ${binary_suffix}" -echo " distro: ${distro}" -echo " dir-base: ${dir_base}" -echo " dir-out: ${dir_out}" - -gencfg="../genconfig/cmd/genconfig/genconfig \ - -input ./network.yml \ - -binary-suffix ${binary_suffix} \ - -dir-base ${dir_base} \ - -dir-out ${dir_out}" +echo " dir_base: ${dir_base}" +echo " dir_bin: ${dir_bin}" +echo " dir_out: ${dir_out}" +echo " docker_image: ${docker_image}" +echo " docker_image_agent: ${docker_image_agent}" +echo " num_gateways: ${num_gateways}" +echo " num_servicenodes: ${num_servicenodes}" +echo " num_mixes: ${num_mixes}" + +gencfg="${docker} run ${docker_args} --rm \ + --volume $(readlink -f ./network.yml):/tmp/network.yml \ + --volume $(readlink -f ${dir_out}):${dir_base} \ + ${docker_image} \ + ${dir_bin}/genconfig \ + -input /tmp/network.yml \ + -binary-prefix ${dir_bin}/ \ + -dir-base ${dir_base} \ + -dir-out ${dir_base}" + +echo "genconfig: ${gencfg}" cat < ${dir_out}/prometheus.yml scrape_configs: @@ -38,9 +40,10 @@ EOF cat < ${dir_out}/docker-compose.yml x-common-service: &common-service restart: "no" - image: katzenpost-${distro}_base + image: ${docker_image} + user: ${docker_user} volumes: - - ./:${dir_base} + - ${dir_out}:${dir_base} network_mode: host services: @@ -49,7 +52,7 @@ services: restart: "no" image: docker.io/prom/prometheus volumes: - - ./:${dir_base} + - ${dir_out}:${dir_base} command: --config.file="${dir_base}/prometheus.yml" network_mode: host @@ -58,30 +61,53 @@ EOF function gencfg_node () { type=${1} id=${type}${2} + metrics="127.0.0.1:$((port+2))" + + ${gencfg} \ + -type ${type} \ + -identifier ${id} \ + -metrics ${metrics} \ + -port ${port} \ + || exit 1 - ${gencfg} -port ${port} -type ${type} -identifier ${id} || exit 1 + echo " - ${metrics}" >> ${dir_out}/prometheus.yml - echo " - 127.0.0.1:${port}" >> ${dir_out}/prometheus.yml - port=$((port+2)) + # increment port for the next node + port=$((port+10)) cat <> ${dir_out}/docker-compose.yml + ${id}-agent: + <<: *common-service + image: ${docker_image_agent} + command: > + pnpm run agent \ + --ipfs \ + --ipfs-data ${dir_base}/ipfs-data \ + --listen \ + --key ${dir_base}/${id}-auth/appchain.key \ + --socket ${dir_base}/${id}-auth/appchain.sock \ + --socket-format cbor \ + --tx-status-retries 20 \ + --debug + ${id}-auth: <<: *common-service - command: ${dir_base}/pki${binary_suffix} -f ${dir_base}/${id}-auth/authority.toml + command: ${dir_bin}/pki -f ${dir_base}/${id}-auth/authority.toml + depends_on: + - ${id}-agent ${id}: <<: *common-service - command: ${dir_base}/server${binary_suffix} -f ${dir_base}/${id}/katzenpost.toml + command: ${dir_bin}/server -f ${dir_base}/${id}/katzenpost.toml depends_on: - ${id}-auth EOF } -for i in $(seq 1 ${gateways}); do gencfg_node gateway ${i}; done -for i in $(seq 1 ${serviceNodes}); do gencfg_node servicenode ${i}; done -for i in $(seq 1 ${mixes}); do gencfg_node mix ${i}; done - -# FIXME: client*/config.toml generated with, to include, gateway('s auth) -# ${gc} -type client1 -# ${gc} -type client2 +for i in $(seq 1 ${num_mixes}); do gencfg_node mix ${i}; done +for i in $(seq 1 ${num_gateways}); do gencfg_node gateway ${i}; done +for i in $(seq 1 ${num_servicenodes}); do + gencfg_node servicenode ${i} + cp ../server_plugins/cbor_plugins/http_proxy/http_proxy_config.toml ${dir_out}/servicenode${i}/http_proxy_config.toml +done diff --git a/docker/network.yml b/docker/network.yml index 85e4bc9..124466e 100644 --- a/docker/network.yml +++ b/docker/network.yml @@ -1,6 +1,6 @@ # Example network configuration file; configures local test network using appchain pki build_datetime: '1728353595' -kp_client_debug_DisableDecoyTraffic: false +kp_client_debug_DisableDecoyTraffic: true kp_client_debug_EnableTimeSync: false kp_client_debug_InitialMaxPKIRetrievalDelay: 0 kp_client_debug_PollingInterval: 0 diff --git a/docker/node/Dockerfile b/docker/node/Dockerfile new file mode 100644 index 0000000..afa7b20 --- /dev/null +++ b/docker/node/Dockerfile @@ -0,0 +1,71 @@ +FROM ubuntu:latest AS builder + +ARG ENABLE_WARPED_EPOCH=false +ARG KATZENPOST_DIR=/src/katzenpost +ARG VERSION_GO=1.22.3 + +ENV GOROOT=/usr/local/go +ENV PATH=$GOROOT/bin:$PATH +ENV GOCACHE=/root/.cache/go-build +ENV GO_BUILD_OPTS="-trimpath -ldflags=-buildid= -ldflags=-X=github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${ENABLE_WARPED_EPOCH}" + +# Install build dependencies +RUN \ + --mount=type=cache,target=/var/cache/apt \ + --mount=type=cache,target=/var/lib/apt \ + apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + git \ + wget + +# Install Go +RUN f=go${VERSION_GO}.linux-amd64.tar.gz \ + && wget https://dl.google.com/go/${f} \ + && tar -C /usr/local -xzf ${f} \ + && rm ${f} + +# Copy the project source +COPY . /src + +# Build Katzenpost components +RUN --mount=type=cache,target="${GOCACHE}" \ + mkdir /dest \ + # clone 0KN opt-specific katzenpost branch, if one does not exist + && if [ ! -d "${KATZENPOST_DIR}" ]; then make katzenpost_dir=${KATZENPOST_DIR} -C /src/docker clone-katzenpost ; fi \ + # a function to build and move the binary + && build() { cd ${KATZENPOST_DIR}/$1 ; b=$(basename $1) ; go build ${GO_BUILD_OPTS} ; chmod u+x $b ; mv $b /dest/$2; } \ + # clients + && build authority/cmd/fetch fetch \ + && build ping ping \ + # server + && build server/cmd/server server \ + ## servicenode plugins + && build http/proxy/client proxy_client \ + && build http/proxy/server proxy_server \ + && build memspool/server/cmd/memspool memspool \ + && build pigeonhole/server/cmd/pigeonhole pigeonhole \ + && build panda/server/cmd/panda_server panda_server \ + && build server_plugins/cbor_plugins/echo-go echo_server + +# Build 0KN mix network components +RUN --mount=type=cache,target="${GOCACHE}" \ + # a function to build and move the binary + build() { cd /src/$1 ; b=$(basename $1) ; go build ${GO_BUILD_OPTS} ; chmod u+x $b ; mv $b /dest/$2; } \ + # pki + && build pki pki \ + # genconfig + && build genconfig/cmd/genconfig genconfig \ + # servicenode plugins + && build server_plugins/cbor_plugins/http_proxy/cmd/http_proxy http_proxy \ + # clients + && build apps/walletshield walletshield + + +FROM ubuntu:latest AS node + +ARG DIR_BIN=/opt/zkn + +COPY --from=builder /dest ${DIR_BIN} + +CMD ["/bin/bash"] diff --git a/genconfig/genconfig.go b/genconfig/genconfig.go index dfffc98..38e5efc 100644 --- a/genconfig/genconfig.go +++ b/genconfig/genconfig.go @@ -51,6 +51,7 @@ type GenconfigInput struct { addrBind string baseDir string basePort int + binPrefix string binSuffix string cfgType string identifier string @@ -64,6 +65,7 @@ type GenconfigInput struct { type katzenpost struct { baseDir string outDir string + binPrefix string binSuffix string logLevel string logWriter io.Writer @@ -313,7 +315,7 @@ func (s *katzenpost) genNodeConfig(identifier string, isGateway bool, isServiceN spoolCfg := &sConfig.CBORPluginKaetzchen{ Capability: "spool", Endpoint: "+spool", - Command: s.baseDir + "/memspool" + s.binSuffix, + Command: s.binPrefix + "memspool" + s.binSuffix, MaxConcurrency: 1, Config: map[string]interface{}{ "data_store": s.baseDir + "/" + cfg.Server.Identifier + "/memspool.storage", @@ -325,7 +327,7 @@ func (s *katzenpost) genNodeConfig(identifier string, isGateway bool, isServiceN mapCfg := &sConfig.CBORPluginKaetzchen{ Capability: "pigeonhole", Endpoint: "+pigeonhole", - Command: s.baseDir + "/pigeonhole" + s.binSuffix, + Command: s.binPrefix + "pigeonhole" + s.binSuffix, MaxConcurrency: 1, Config: map[string]interface{}{ "db": s.baseDir + "/" + cfg.Server.Identifier + "/map.storage", @@ -338,7 +340,7 @@ func (s *katzenpost) genNodeConfig(identifier string, isGateway bool, isServiceN pandaCfg := &sConfig.CBORPluginKaetzchen{ Capability: "panda", Endpoint: "+panda", - Command: s.baseDir + "/panda_server" + s.binSuffix, + Command: s.binPrefix + "panda_server" + s.binSuffix, MaxConcurrency: 1, Config: map[string]interface{}{ "fileStore": s.baseDir + "/" + cfg.Server.Identifier + "/panda.storage", @@ -355,7 +357,7 @@ func (s *katzenpost) genNodeConfig(identifier string, isGateway bool, isServiceN proxyCfg := &sConfig.CBORPluginKaetzchen{ Capability: "http", Endpoint: "+http", - Command: s.baseDir + "/proxy_server" + s.binSuffix, + Command: s.binPrefix + "proxy_server" + s.binSuffix, MaxConcurrency: 1, Config: map[string]interface{}{ // allow connections to localhost:4242 @@ -367,6 +369,24 @@ func (s *katzenpost) genNodeConfig(identifier string, isGateway bool, isServiceN cfg.ServiceNode.CBORPluginKaetzchen = append(cfg.ServiceNode.CBORPluginKaetzchen, proxyCfg) s.hasProxy = true } + + // 0KN JSON RPC - HTTP Proxy + httpProxyCfg := &sConfig.CBORPluginKaetzchen{ + Capability: "http_proxy", + Endpoint: "http_proxy", + Command: s.binPrefix + "http_proxy" + s.binSuffix, + MaxConcurrency: 1, + Disable: false, + Config: map[string]interface{}{ + "config": s.baseDir + "/" + cfg.Server.Identifier + "/http_proxy_config.toml", + "log_dir": s.baseDir + "/" + cfg.Server.Identifier, + }, + } + cfg.ServiceNode.CBORPluginKaetzchen = append(cfg.ServiceNode.CBORPluginKaetzchen, httpProxyCfg) + // create empty default http_proxy_config.toml file + httpProxyConfigFile := filepath.Join(s.outDir, cfg.Server.Identifier, "http_proxy_config.toml") + saveFileContents(httpProxyConfigFile, "[Networks]\n") + cfg.Debug.NumKaetzchenWorkers = 4 } @@ -454,7 +474,7 @@ func (s *katzenpost) genAuthorizedNodes() ([]*vConfig.Node, []*vConfig.Node, []* for _, nodeCfg := range s.nodeConfigs { node := &vConfig.Node{ Identifier: nodeCfg.Server.Identifier, - IdentityPublicKeyPem: filepath.Join("../", nodeCfg.Server.Identifier, "identity.public.pem"), + IdentityPublicKeyPem: filepath.Join(s.outDir, nodeCfg.Server.Identifier, "identity.public.pem"), } if nodeCfg.Server.IsGatewayNode { gateways = append(gateways, node) @@ -483,6 +503,7 @@ func ParseFlags() GenconfigInput { flag.StringVar(&gi.addr, "address", addr, "Address to publish (and bind to if -address-bind not set)") flag.StringVar(&gi.addrBind, "address-bind", "", "Address to bind to") flag.StringVar(&gi.baseDir, "dir-base", "", "Absolute path as installation directory in config files (default -dir-out)") + flag.StringVar(&gi.binPrefix, "binary-prefix", "", "Prefix for binaries") flag.StringVar(&gi.binSuffix, "binary-suffix", "", "Suffix for binaries") flag.StringVar(&gi.cfgType, "type", "", "Type of config to generate: mix, gateway, servicenode, client1, client2") flag.StringVar(&gi.identifier, "identifier", "", "Node identifier; lowercase alphanumeric with 4 to 20 characters (default -type)") @@ -515,6 +536,7 @@ func Genconfig(gi GenconfigInput) error { addrBind := &gi.addrBind baseDir := &gi.baseDir basePort := &gi.basePort + binPrefix := &gi.binPrefix binSuffix := &gi.binSuffix cfgType := &gi.cfgType identifier := &gi.identifier @@ -617,6 +639,7 @@ func Genconfig(gi GenconfigInput) error { s.baseDir = *baseDir s.outDir = *outDir + s.binPrefix = *binPrefix s.binSuffix = *binSuffix s.basePort = uint16(*basePort) s.lastPort = s.basePort + 1 @@ -696,7 +719,6 @@ func Genconfig(gi GenconfigInput) error { } os.Mkdir(s.outDir, 0700) - os.Mkdir(filepath.Join(s.outDir, s.baseDir), 0700) if *voting { // Generate the voting authority configurations @@ -828,6 +850,19 @@ func saveCfg(cfg interface{}, outDir string) error { return enc.Encode(cfg) } +func saveFileContents(filename string, contents string) error { + log.Printf("writing %s", filename) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("os.Create(%s) failed: %s", filename, err) + } + defer f.Close() + if _, err := f.WriteString(contents); err != nil { + return fmt.Errorf("f.WriteString() failed: %s", err) + } + return nil +} + func cfgIdKey(cfg interface{}, outDir string) sign.PublicKey { var priv, public string var pkiSignatureScheme string