From 8411a384f4af75d55db40660dd755ba06f4f058d Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Sat, 19 Apr 2025 14:12:14 -0700 Subject: [PATCH 01/10] refactor(pki): restructure pki directory --- .gitignore | 2 +- docker/node/Dockerfile | 2 +- pki/Makefile | 8 +++++--- pki/{ => cmd/pki}/main.go | 9 +++++---- pki/{ => server}/config/config.go | 0 pki/{ => server}/server.go | 4 ++-- pki/{ => server}/state.go | 4 ++-- pki/{ => server}/state_chain_comm.go | 4 ++-- pki/{ => server}/wire_handler.go | 2 +- 9 files changed, 19 insertions(+), 16 deletions(-) rename pki/{ => cmd/pki}/main.go (85%) rename pki/{ => server}/config/config.go (100%) rename pki/{ => server}/server.go (99%) rename pki/{ => server}/state.go (99%) rename pki/{ => server}/state_chain_comm.go (98%) rename pki/{ => server}/wire_handler.go (99%) diff --git a/.gitignore b/.gitignore index cee3c32..e70a6dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ apps/walletshield/walletshield docker/node/.env genconfig/cmd/genconfig/genconfig -pki/pki +pki/cmd/pki/pki server_plugins/cbor_plugins/http_proxy/cmd/http_proxy/http_proxy diff --git a/docker/node/Dockerfile b/docker/node/Dockerfile index 5238c3b..93d5ef1 100644 --- a/docker/node/Dockerfile +++ b/docker/node/Dockerfile @@ -63,7 +63,7 @@ RUN --mount=type=cache,target="${GOCACHE}" \ # a function to build and move the binary build() { cd /src/$1 ; b=$(basename $1) ; go build ${GO_BUILD_OPTS} ; chmod u+x $b ; mv $b /dest/$2; } \ # pki - && build pki pki \ + && build pki/cmd/pki pki \ # genconfig && build genconfig/cmd/genconfig genconfig \ # servicenode plugins diff --git a/pki/Makefile b/pki/Makefile index 7c210ce..0ec2401 100644 --- a/pki/Makefile +++ b/pki/Makefile @@ -2,9 +2,11 @@ warped?=false ldflags="-buildid= -X github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${warped}" .PHONY: all -all: - go build -trimpath -ldflags ${ldflags} +all: cmd/pki/pki + +cmd/pki/pki: server/*.go cmd/pki/*.go + cd cmd/pki && go build -trimpath -ldflags ${ldflags} .PHONY: clean clean: - rm -f ./pki + rm -f cmd/pki/pki diff --git a/pki/main.go b/pki/cmd/pki/main.go similarity index 85% rename from pki/main.go rename to pki/cmd/pki/main.go index c33a138..1bc364c 100644 --- a/pki/main.go +++ b/pki/cmd/pki/main.go @@ -1,4 +1,4 @@ -// related: katzenpost:/authority/cmd/voting/main.go +// related: katzenpost:/authority/cmd/dirauth/main.go package main import ( @@ -10,7 +10,8 @@ import ( "github.com/carlmjohnson/versioninfo" - "github.com/ZeroKnowledgeNetwork/opt/pki/config" + "github.com/ZeroKnowledgeNetwork/opt/pki/server" + "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" "github.com/katzenpost/katzenpost/core/compat" ) @@ -43,9 +44,9 @@ func main() { signal.Notify(rotateCh, syscall.SIGHUP) // Start up the authority. - svr, err := New(cfg) + svr, err := server.New(cfg) if err != nil { - if err == ErrGenerateOnly { + if err == server.ErrGenerateOnly { os.Exit(0) } fmt.Fprintf(os.Stderr, "Failed to spawn authority instance: %v\n", err) diff --git a/pki/config/config.go b/pki/server/config/config.go similarity index 100% rename from pki/config/config.go rename to pki/server/config/config.go diff --git a/pki/server.go b/pki/server/server.go similarity index 99% rename from pki/server.go rename to pki/server/server.go index a967749..cbba5a5 100644 --- a/pki/server.go +++ b/pki/server/server.go @@ -1,6 +1,6 @@ // related: katzenpost:authority/voting/server/server.go -package main +package server import ( "crypto/sha256" @@ -29,7 +29,7 @@ import ( "github.com/katzenpost/katzenpost/core/utils" "github.com/katzenpost/katzenpost/http/common" - "github.com/ZeroKnowledgeNetwork/opt/pki/config" + "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" ) // ErrGenerateOnly is the error returned when the server initialization diff --git a/pki/state.go b/pki/server/state.go similarity index 99% rename from pki/state.go rename to pki/server/state.go index 7911a60..cc42e09 100644 --- a/pki/state.go +++ b/pki/server/state.go @@ -1,6 +1,6 @@ // related katzenpost:authority/voting/server/state.go -package main +package server import ( "crypto/hmac" @@ -25,7 +25,7 @@ import ( "github.com/katzenpost/katzenpost/core/worker" "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" - "github.com/ZeroKnowledgeNetwork/opt/pki/config" + "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" ) const ( diff --git a/pki/state_chain_comm.go b/pki/server/state_chain_comm.go similarity index 98% rename from pki/state_chain_comm.go rename to pki/server/state_chain_comm.go index 189a779..6d4e754 100644 --- a/pki/state_chain_comm.go +++ b/pki/server/state_chain_comm.go @@ -1,6 +1,6 @@ // AppChain communication (chainbridge) functions -package main +package server import ( "fmt" @@ -10,7 +10,7 @@ import ( "github.com/katzenpost/katzenpost/core/pki" "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" - "github.com/ZeroKnowledgeNetwork/opt/pki/config" + "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" ) func (s *state) chNodesGet(name string) (*chainbridge.Node, error) { diff --git a/pki/wire_handler.go b/pki/server/wire_handler.go similarity index 99% rename from pki/wire_handler.go rename to pki/server/wire_handler.go index cf847cc..d2bffea 100644 --- a/pki/wire_handler.go +++ b/pki/server/wire_handler.go @@ -1,6 +1,6 @@ // related: katzenpost:authority/voting/server/wire_handler.go -package main +package server import ( "crypto/hmac" From 1a01ac57a2c9063af8b15672eccbcac99f7dfe0b Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Sat, 19 Apr 2025 14:22:59 -0700 Subject: [PATCH 02/10] chore: go get katzenpost v0.0.48 --- go.mod | 10 +++++----- go.sum | 7 +++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 1b9ef95..75171fb 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/carlmjohnson/versioninfo v0.22.5 github.com/charmbracelet/log v0.4.0 github.com/fxamacker/cbor/v2 v2.7.0 - github.com/katzenpost/hpqc v0.0.45 - github.com/katzenpost/katzenpost v0.0.43 + github.com/katzenpost/hpqc v0.0.55 + github.com/katzenpost/katzenpost v0.0.48 github.com/quic-go/quic-go v0.50.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 @@ -26,11 +26,11 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9 // indirect github.com/henrydcase/nobs v0.0.0-20230313231516-25b66236df73 // indirect - github.com/katzenpost/chacha20 v0.0.0-20190910113340-7ce890d6a556 // indirect + github.com/katzenpost/chacha20 v0.0.1 // indirect github.com/katzenpost/circl v1.3.9-0.20240222183521-1cd9a34e9a0c // indirect github.com/katzenpost/nyquist v0.0.10 // indirect github.com/katzenpost/sntrup4591761 v0.0.0-20231024131303-8755eb1986b8 // indirect - github.com/katzenpost/sphincsplus v0.0.2-0.20240114192234-1dc77b544e31 // indirect + github.com/katzenpost/sphincsplus v0.0.2 // indirect github.com/lesismal/nbio v1.5.11 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.18 // indirect @@ -57,5 +57,5 @@ require ( golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.25.0 // indirect ) diff --git a/go.sum b/go.sum index d2d987c..026012f 100644 --- a/go.sum +++ b/go.sum @@ -39,18 +39,23 @@ github.com/henrydcase/nobs v0.0.0-20230313231516-25b66236df73 h1:d3rq/Tz+RJ5h1xk github.com/henrydcase/nobs v0.0.0-20230313231516-25b66236df73/go.mod h1:ptK2MJqVLVEa/V/oK8n+MEyUDCSjSylW+jeNmCG1DJo= github.com/katzenpost/chacha20 v0.0.0-20190910113340-7ce890d6a556 h1:9gHByAWH1LydGefFGorN1ZBRZ/Oz9iozdzMvRTWpyRw= github.com/katzenpost/chacha20 v0.0.0-20190910113340-7ce890d6a556/go.mod h1:d9kxwmGOcutgP6bQwr2xaLInaW5yJsxsoPRyUIG0J/E= +github.com/katzenpost/chacha20 v0.0.1/go.mod h1:/LIJK/8cUXVJrCh5NypZ8So3gDfQCoQT8lRvy1rYQZA= github.com/katzenpost/circl v1.3.9-0.20240222183521-1cd9a34e9a0c h1:FYy03rLIjdyjklBOI6YSCb3q7OubTx0dVDWYOgDsvA8= github.com/katzenpost/circl v1.3.9-0.20240222183521-1cd9a34e9a0c/go.mod h1:+EBrwiGYs9S+qZqaqxujN1CReTNCMAG6p+31KkEDeeA= github.com/katzenpost/hpqc v0.0.45 h1:CiNTvwUe7CaGdIeA0tEtHY+O3CKk6lTgdAb4iQfSy4k= github.com/katzenpost/hpqc v0.0.45/go.mod h1:yMxuQLTjgzgHdvQlJIbWFiusyizyMW94fpH6wxTTur8= +github.com/katzenpost/hpqc v0.0.55/go.mod h1:yaVqoZyKeBmgiKnGBpgLNf+nfO4ll81f54RAnaL6OpI= github.com/katzenpost/katzenpost v0.0.43 h1:BAZxLxl3he+bNodTaXv6GW0BYA9Qj6jGQcsVHOjeiN0= github.com/katzenpost/katzenpost v0.0.43/go.mod h1:+aRwtsFwBT7GTU9Mj07MlQ3QoM4XQ6YLP/w0/j1gOHc= +github.com/katzenpost/katzenpost v0.0.48 h1:nyvWYvVu5r+0UgPRCGnPyC/Jn7Dst9UluWsJ4XB7OCs= +github.com/katzenpost/katzenpost v0.0.48/go.mod h1:FcNs9Kc6PiJQ7t8g6iNYGAtlempt627iLQvUlrV3HHc= github.com/katzenpost/nyquist v0.0.10 h1:rh9TCEXCsutsg+cvbV6ASVFnzSAYBisWQ3fnwQSPa34= github.com/katzenpost/nyquist v0.0.10/go.mod h1:tyK92JiCptgsaE0iUAMlt5W2v2Rdw6mnUpIdIidIGHo= github.com/katzenpost/sntrup4591761 v0.0.0-20231024131303-8755eb1986b8 h1:TsKxH0x2RUwf5rBw67k15bqVM3oVbexA9oaTZQLIy3Y= github.com/katzenpost/sntrup4591761 v0.0.0-20231024131303-8755eb1986b8/go.mod h1:Hmcrwom7jcEmGdo0CsyuJNnldPeyS+M07FuCbo7I8fw= github.com/katzenpost/sphincsplus v0.0.2-0.20240114192234-1dc77b544e31 h1:fKGa/too1Br31gmoYmV2kE61gydj47Ed5K/g/CE+3Bs= github.com/katzenpost/sphincsplus v0.0.2-0.20240114192234-1dc77b544e31/go.mod h1:VFrCPnmbxQLBi+qJfWHUqvpvTMZrYBMZEEy0AidY0nE= +github.com/katzenpost/sphincsplus v0.0.2/go.mod h1:ChO9+ojgCH1yEuplGgW4mSI1FwZWtyEmEkG1xL3w264= github.com/lesismal/llib v1.1.13/go.mod h1:70tFXXe7P1FZ02AU9l8LgSOK7d7sRrpnkUr3rd3gKSg= github.com/lesismal/nbio v1.5.11 h1:MVjrzcej4NSJQMRT+S0dPZvVaiFUHD1JWnvr+FHIHOo= github.com/lesismal/nbio v1.5.11/go.mod h1:QsxE0fKFe1PioyjuHVDn2y8ktYK7xv9MFbpkoRFj8vI= @@ -135,6 +140,8 @@ golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 6224c8ad6b2800069f0782cb3e65784036336e1c Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Mon, 31 Mar 2025 20:56:41 -0700 Subject: [PATCH 03/10] chore(docker/Makefile): add extldflags --- docker/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Makefile b/docker/Makefile index 93d8960..779658d 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -13,7 +13,7 @@ sh=$(shell if echo ${distro}|grep -q alpine; then echo sh; else echo bash; fi) cache_dir=cache log_level=DEBUG docker=$(shell if which podman|grep -q .; then echo podman; else echo docker; fi) -ldflags="-buildid= -X github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${warped}" +ldflags="-extldflags '-Wl,-z,stack-size=0x1F40000' -buildid= -X github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${warped}" uid?=$(shell [ "$$SUDO_UID" != "" ] && echo "$$SUDO_UID" || id -u) gid?=$(shell [ "$$SUDO_GID" != "" ] && echo "$$SUDO_GID" || id -g) docker_user?=$(shell if echo ${docker}|grep -q podman; then echo 0:0; else echo ${uid}:${gid}; fi) From f85a874557415263a851a533f20fb3468414bbc5 Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Mon, 31 Mar 2025 20:54:47 -0700 Subject: [PATCH 04/10] chore(docker): network.yml: pkiSignatureScheme: (E -> e)d25519 --- docker/network.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/network.yml b/docker/network.yml index 124466e..239fc71 100644 --- a/docker/network.yml +++ b/docker/network.yml @@ -22,7 +22,7 @@ kp_config_kem: '' kp_config_log_level: DEBUG kp_config_nike: x25519 kp_config_ratchetNike: CTIDH512-X25519 -kp_config_pkiSignatureScheme: Ed25519 +kp_config_pkiSignatureScheme: ed25519 kp_config_wirekem: xwing kp_debug_ConnectTimeout: 60000 kp_debug_DecoySlack: 15000 From 1c7aa8f8d8fda9604bee682a09455728a27f97a5 Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Sat, 19 Apr 2025 22:27:03 -0700 Subject: [PATCH 05/10] wip(pki): replace pki/ with upstream katzenpost wipe out custom changes for a fresh start with improved integration --- pki/cmd/pki/main.go | 7 +- pki/server/config/config.go | 439 ------- pki/server/server.go | 41 +- pki/server/state.go | 2166 +++++++++++++++++++++++++++----- pki/server/state_chain_comm.go | 176 --- pki/server/wire_handler.go | 164 ++- 6 files changed, 2056 insertions(+), 937 deletions(-) delete mode 100644 pki/server/config/config.go delete mode 100644 pki/server/state_chain_comm.go diff --git a/pki/cmd/pki/main.go b/pki/cmd/pki/main.go index 1bc364c..7f48e34 100644 --- a/pki/cmd/pki/main.go +++ b/pki/cmd/pki/main.go @@ -1,4 +1,7 @@ -// related: katzenpost:/authority/cmd/dirauth/main.go +// upstream: katzenpost:/authority/cmd/dirauth/main.go +// main.go - Katzenpost voting-authority binary. +// with modifications for ZKN ZK-PKI + package main import ( @@ -11,7 +14,7 @@ import ( "github.com/carlmjohnson/versioninfo" "github.com/ZeroKnowledgeNetwork/opt/pki/server" - "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" + "github.com/katzenpost/katzenpost/authority/voting/server/config" "github.com/katzenpost/katzenpost/core/compat" ) diff --git a/pki/server/config/config.go b/pki/server/config/config.go deleted file mode 100644 index 676188a..0000000 --- a/pki/server/config/config.go +++ /dev/null @@ -1,439 +0,0 @@ -// related: katzenpost:authority/voting/server/config/config.go -package config - -import ( - "errors" - "fmt" - "io" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/BurntSushi/toml" - - "github.com/katzenpost/hpqc/kem/schemes" - "github.com/katzenpost/hpqc/rand" - signpem "github.com/katzenpost/hpqc/sign/pem" - signSchemes "github.com/katzenpost/hpqc/sign/schemes" - "github.com/katzenpost/katzenpost/core/sphinx/geo" - "github.com/katzenpost/katzenpost/core/utils" -) - -const ( - defaultAddress = ":62472" - defaultLogLevel = "NOTICE" - defaultLayers = 3 - defaultMinNodesPerLayer = 2 - absoluteMaxDelay = 6 * 60 * 60 * 1000 // 6 hours. - - // rate limiting of client connections - defaultSendRatePerMinute = 100 - - // Note: These values are picked primarily for debugging and need to - // be changed to something more suitable for a production deployment - // at some point. - defaultMu = 0.00025 - defaultMuMaxPercentile = 0.99999 - defaultLambdaP = 0.00025 - defaultLambdaPMaxPercentile = 0.99999 - defaultLambdaL = 0.00025 - defaultLambdaLMaxPercentile = 0.99999 - defaultLambdaD = 0.00025 - defaultLambdaDMaxPercentile = 0.99999 - defaultLambdaM = 0.00025 - defaultLambdaMMaxPercentile = 0.99999 - - publicKeyHashSize = 32 -) - -var defaultLogging = Logging{ - Disable: false, - File: "", - Level: defaultLogLevel, -} - -// Logging is the authority logging configuration. -type Logging struct { - // Disable disables logging entirely. - Disable bool - - // File specifies the log file, if omitted stdout will be used. - File string - - // Level specifies the log level. - Level string -} - -func (lCfg *Logging) validate() error { - lvl := strings.ToUpper(lCfg.Level) - switch lvl { - case "ERROR", "WARNING", "NOTICE", "INFO", "DEBUG": - case "": - lCfg.Level = defaultLogLevel - default: - return fmt.Errorf("config: Logging: Level '%v' is invalid", lCfg.Level) - } - lCfg.Level = lvl // Force uppercase. - return nil -} - -// Parameters is the network parameters. -type Parameters struct { - // SendRatePerMinute is the rate per minute. - SendRatePerMinute uint64 - - // Mu is the inverse of the mean of the exponential distribution - // that is used to select the delay for each hop. - Mu float64 - - // MuMaxDelay sets the maximum delay for Mu. - MuMaxDelay uint64 - - // LambdaP is the inverse of the mean of the exponential distribution - // that is used to select the delay between clients sending from their egress - // FIFO queue or drop decoy message. - LambdaP float64 - - // LambdaPMaxDelay sets the maximum delay for LambdaP. - LambdaPMaxDelay uint64 - - // LambdaL is the inverse of the mean of the exponential distribution - // that is used to select the delay between clients sending loop decoys. - LambdaL float64 - - // LambdaLMaxDelay sets the maximum delay for LambdaP. - LambdaLMaxDelay uint64 - - // LambdaD is the inverse of the mean of the exponential distribution - // that is used to select the delay between clients sending deop decoys. - LambdaD float64 - - // LambdaDMaxDelay sets the maximum delay for LambdaP. - LambdaDMaxDelay uint64 - - // LambdaM is the inverse of the mean of the exponential distribution - // that is used to select the delay between sending mix node decoys. - LambdaM float64 - - // LambdaG is the inverse of the mean of the exponential distribution - // that is used to select the delay between sending gateway node decoys. - // - // WARNING: This is not used via the TOML config file; this field is only - // used internally by the dirauth server state machine. - LambdaG float64 - - // LambdaMMaxDelay sets the maximum delay for LambdaP. - LambdaMMaxDelay uint64 - - // LambdaGMaxDelay sets the maximum delay for LambdaG. - LambdaGMaxDelay uint64 -} - -func (pCfg *Parameters) validate() error { - if pCfg.Mu < 0 { - return fmt.Errorf("config: Parameters: Mu %v is invalid", pCfg.Mu) - } - if pCfg.MuMaxDelay > absoluteMaxDelay { - return fmt.Errorf("config: Parameters: MuMaxDelay %v is out of range", pCfg.MuMaxDelay) - } - if pCfg.LambdaP < 0 { - return fmt.Errorf("config: Parameters: LambdaP %v is invalid", pCfg.LambdaP) - } - if pCfg.LambdaPMaxDelay > absoluteMaxDelay { - return fmt.Errorf("config: Parameters: LambdaPMaxDelay %v is out of range", pCfg.LambdaPMaxDelay) - } - if pCfg.LambdaL < 0 { - return fmt.Errorf("config: Parameters: LambdaL %v is invalid", pCfg.LambdaP) - } - if pCfg.LambdaLMaxDelay > absoluteMaxDelay { - return fmt.Errorf("config: Parameters: LambdaLMaxDelay %v is out of range", pCfg.LambdaPMaxDelay) - } - if pCfg.LambdaD < 0 { - return fmt.Errorf("config: Parameters: LambdaD %v is invalid", pCfg.LambdaP) - } - if pCfg.LambdaDMaxDelay > absoluteMaxDelay { - return fmt.Errorf("config: Parameters: LambdaDMaxDelay %v is out of range", pCfg.LambdaPMaxDelay) - } - if pCfg.LambdaM < 0 { - return fmt.Errorf("config: Parameters: LambdaM %v is invalid", pCfg.LambdaP) - } - if pCfg.LambdaMMaxDelay > absoluteMaxDelay { - return fmt.Errorf("config: Parameters: LambdaMMaxDelay %v is out of range", pCfg.LambdaPMaxDelay) - } - if pCfg.LambdaGMaxDelay > absoluteMaxDelay { - return fmt.Errorf("config: Parameters: LambdaGMaxDelay %v is out of range", pCfg.LambdaPMaxDelay) - } - if pCfg.LambdaGMaxDelay == 0 { - return errors.New("LambdaGMaxDelay must be set") - } - - return nil -} - -func (pCfg *Parameters) applyDefaults() { - if pCfg.SendRatePerMinute == 0 { - pCfg.SendRatePerMinute = defaultSendRatePerMinute - } - if pCfg.Mu == 0 { - pCfg.Mu = defaultMu - } - if pCfg.MuMaxDelay == 0 { - pCfg.MuMaxDelay = uint64(rand.ExpQuantile(pCfg.Mu, defaultMuMaxPercentile)) - if pCfg.MuMaxDelay > absoluteMaxDelay { - pCfg.MuMaxDelay = absoluteMaxDelay - } - } - if pCfg.LambdaP == 0 { - pCfg.LambdaP = defaultLambdaP - } - if pCfg.LambdaPMaxDelay == 0 { - pCfg.LambdaPMaxDelay = uint64(rand.ExpQuantile(pCfg.LambdaP, defaultLambdaPMaxPercentile)) - } - if pCfg.LambdaL == 0 { - pCfg.LambdaL = defaultLambdaL - } - if pCfg.LambdaLMaxDelay == 0 { - pCfg.LambdaLMaxDelay = uint64(rand.ExpQuantile(pCfg.LambdaL, defaultLambdaLMaxPercentile)) - } - if pCfg.LambdaD == 0 { - pCfg.LambdaD = defaultLambdaD - } - if pCfg.LambdaDMaxDelay == 0 { - pCfg.LambdaDMaxDelay = uint64(rand.ExpQuantile(pCfg.LambdaD, defaultLambdaDMaxPercentile)) - } - if pCfg.LambdaM == 0 { - pCfg.LambdaM = defaultLambdaM - } - if pCfg.LambdaMMaxDelay == 0 { - pCfg.LambdaMMaxDelay = uint64(rand.ExpQuantile(pCfg.LambdaM, defaultLambdaMMaxPercentile)) - } -} - -// Debug is the authority debug configuration. -type Debug struct { - // Layers is the number of non-provider layers in the network topology. - Layers int - - // MinNodesPerLayer is the minimum number of nodes per layer required to - // form a valid Document. - MinNodesPerLayer int - - // GenerateOnly halts and cleans up the server right after long term - // key generation. - GenerateOnly bool -} - -func (dCfg *Debug) validate() error { - if dCfg.Layers > defaultLayers { - // This is a limitation of the Sphinx implementation. - return fmt.Errorf("config: Debug: Layers %v exceeds maximum", dCfg.Layers) - } - return nil -} - -func (dCfg *Debug) applyDefaults() { - if dCfg.Layers <= 0 { - dCfg.Layers = defaultLayers - } - if dCfg.MinNodesPerLayer <= 0 { - dCfg.MinNodesPerLayer = defaultMinNodesPerLayer - } -} - -// Node is an authority mix node or provider entry. -type Node struct { - // Identifier is the human readable node identifier, to be set iff - // the node is a Provider. - Identifier string - - // IdentityPublicKeyPem is the node's public signing key also known - // as the identity key. - IdentityPublicKeyPem string -} - -type Server struct { - // Identifier is the human readable identifier for the node (eg: FQDN). - Identifier string - - // WireKEMScheme is the wire protocol KEM scheme to use. - WireKEMScheme string - - // PKISignatureScheme specifies the cryptographic signature scheme - PKISignatureScheme string - - // Addresses are the IP address/port combinations that the server will bind - // to for incoming connections. - Addresses []string - - // DataDir is the absolute path to the server's state files. - DataDir string -} - -// Validate parses and checks the Server configuration. -func (sCfg *Server) validate() error { - if sCfg.WireKEMScheme == "" { - return errors.New("WireKEMScheme was not set") - } else { - s := schemes.ByName(sCfg.WireKEMScheme) - if s == nil { - return errors.New("KEM Scheme not found") - } - } - - if sCfg.PKISignatureScheme == "" { - return errors.New("PKISignatureScheme was not set") - } else { - s := signSchemes.ByName(sCfg.PKISignatureScheme) - if s == nil { - return errors.New("PKI Signature Scheme not found") - } - } - - if sCfg.Addresses != nil { - for _, v := range sCfg.Addresses { - if u, err := url.Parse(v); err != nil { - return fmt.Errorf("config: Authority: Address '%v' is invalid: %v", v, err) - } else if u.Port() == "" { - return fmt.Errorf("config: Authority: Address '%v' is invalid: Must contain Port", v) - } - } - } else { - // Try to guess a "suitable" external IPv4 address. If people want - // to do loopback testing, they can manually specify one. If people - // want to use IPng, they can manually specify that as well. - addr, err := utils.GetExternalIPv4Address() - if err != nil { - return err - } - sCfg.Addresses = []string{addr.String() + defaultAddress} - } - if !filepath.IsAbs(sCfg.DataDir) { - return fmt.Errorf("config: Authority: DataDir '%v' is not an absolute path", sCfg.DataDir) - } - return nil -} - -// Config is the top level authority configuration. -type Config struct { - Server *Server - Logging *Logging - Parameters *Parameters - Debug *Debug - - // Note: these are an iterative step; useful to register non-volunteer nodes within the appchain - Mixes []*Node - GatewayNodes []*Node - ServiceNodes []*Node - Topology *Topology - - SphinxGeometry *geo.Geometry -} - -// Layer holds a slice of Nodes -type Layer struct { - Nodes []Node -} - -// Topology contains a slice of Layers, each containing a slice of Nodes -type Topology struct { - Layers []Layer -} - -// FixupAndValidate applies defaults to config entries and validates the -// supplied configuration. Most people should call one of the Load variants -// instead. -func (cfg *Config) FixupAndValidate(forceGenOnly bool) error { - - if cfg.SphinxGeometry == nil { - return errors.New("config: No SphinxGeometry block was present") - } - - err := cfg.SphinxGeometry.Validate() - if err != nil { - return err - } - - // Handle missing sections if possible. - if cfg.Server == nil { - return errors.New("config: No Authority block was present") - } - // Handle missing sections if possible. - if cfg.Logging == nil { - cfg.Logging = &defaultLogging - } - if cfg.Parameters == nil { - cfg.Parameters = &Parameters{} - } - if cfg.Debug == nil { - cfg.Debug = &Debug{} - } - - // Validate and fixup the various sections. - if err := cfg.Server.validate(); err != nil { - return err - } - if err := cfg.Logging.validate(); err != nil { - return err - } - if err := cfg.Parameters.validate(); err != nil { - return err - } - if err := cfg.Debug.validate(); err != nil { - return err - } - cfg.Parameters.applyDefaults() - cfg.Debug.applyDefaults() - - pkiSignatureScheme := signSchemes.ByName(cfg.Server.PKISignatureScheme) - - if forceGenOnly { - return nil - } - - ourPubKeyFile := filepath.Join(cfg.Server.DataDir, "identity.public.pem") - f, err := os.Open(ourPubKeyFile) - if err != nil { - return err - } - pemData, err := io.ReadAll(f) - if err != nil { - return err - } - - _, err = signpem.FromPublicPEMBytes(pemData, pkiSignatureScheme) - if err != nil { - return err - } - - return nil -} - -// Load parses and validates the provided buffer b as a config file body and -// returns the Config. -func Load(b []byte, forceGenOnly bool) (*Config, error) { - cfg := new(Config) - err := toml.Unmarshal(b, cfg) - if err != nil { - return nil, err - } - if err := cfg.FixupAndValidate(forceGenOnly); err != nil { - return nil, err - } - - if forceGenOnly { - cfg.Debug.GenerateOnly = true - } - - return cfg, nil -} - -// LoadFile loads, parses and validates the provided file and returns the -// Config. -func LoadFile(f string, forceGenOnly bool) (*Config, error) { - b, err := os.ReadFile(f) - if err != nil { - return nil, err - } - return Load(b, forceGenOnly) -} diff --git a/pki/server/server.go b/pki/server/server.go index cbba5a5..4a9519c 100644 --- a/pki/server/server.go +++ b/pki/server/server.go @@ -1,4 +1,6 @@ -// related: katzenpost:authority/voting/server/server.go +// upstream: katzenpost:authority/voting/server/server.go +// server.go - Katzenpost voting authority server. +// with modifications for ZKN ZK-PKI package server @@ -24,12 +26,11 @@ import ( signpem "github.com/katzenpost/hpqc/sign/pem" signSchemes "github.com/katzenpost/hpqc/sign/schemes" + "github.com/katzenpost/katzenpost/authority/voting/server/config" "github.com/katzenpost/katzenpost/core/log" "github.com/katzenpost/katzenpost/core/sphinx/geo" "github.com/katzenpost/katzenpost/core/utils" "github.com/katzenpost/katzenpost/http/common" - - "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" ) // ErrGenerateOnly is the error returned when the server initialization @@ -58,15 +59,6 @@ type Server struct { haltOnce sync.Once } -// used for dynamic topology derived from appchain, and not from config file -func computeLambdaGFromNodesPerLayer(cfg *config.Config, npl int) float64 { - n := float64(npl) - if n == 1 { - return cfg.Parameters.LambdaP + cfg.Parameters.LambdaL + cfg.Parameters.LambdaD - } - return n * math.Log(n) -} - func computeLambdaG(cfg *config.Config) float64 { n := float64(len(cfg.Topology.Layers[0].Nodes)) if n == 1 { @@ -276,6 +268,19 @@ func New(cfg *config.Config) (*Server, error) { if err != nil { return nil, err } + + /* NOTE(david): enable this check after we get things working again? + linkpubkey, err := kempem.FromPublicPEMFile(linkPublicKeyFile, scheme) + if err != nil { + return nil, err + } + s.log.Warning("attempting to call validate our config's peers against our own link public key") + err = cfg.ValidateAuthorities(linkpubkey) + if err != nil { + s.log.Error("config's peers validation failure. must be your own peer!") + return nil, err + } + */ } else if utils.BothNotExists(linkPrivateKeyFile, linkPublicKeyFile) { linkPublicKey, linkPrivateKey, err := scheme.GenerateKeyPair() if err != nil { @@ -307,6 +312,18 @@ func New(cfg *config.Config) (*Server, error) { return nil, ErrGenerateOnly } + // Ensure that there are enough mixes and providers whitelisted to form + // a topology, assuming all of them post a descriptor. + if len(cfg.GatewayNodes) < 1 { + return nil, fmt.Errorf("server: No GatewayNodes specified in the config") + } + if len(cfg.ServiceNodes) < 1 { + return nil, fmt.Errorf("server: No ServiceNodes specified in the config") + } + if len(cfg.Mixes) < cfg.Debug.Layers*cfg.Debug.MinNodesPerLayer { + return nil, fmt.Errorf("server: Insufficient nodes whitelisted, got %v , need %v", len(cfg.Mixes), cfg.Debug.Layers*cfg.Debug.MinNodesPerLayer) + } + // Past this point, failures need to call s.Shutdown() to do cleanup. isOk := false defer func() { diff --git a/pki/server/state.go b/pki/server/state.go index cc42e09..bde0133 100644 --- a/pki/server/state.go +++ b/pki/server/state.go @@ -1,82 +1,142 @@ -// related katzenpost:authority/voting/server/state.go +// upstream katzenpost:authority/voting/server/state.go +// state.go - Katzenpost voting authority server state. +// with modifications for ZKN ZK-PKI package server import ( + "bytes" + "context" "crypto/hmac" + "encoding/base64" + "encoding/binary" + "encoding/gob" "errors" "fmt" + "net" + "net/url" "path/filepath" "sort" + "strings" "sync" "time" + signSchemes "github.com/katzenpost/hpqc/sign/schemes" + + bolt "go.etcd.io/bbolt" + "golang.org/x/crypto/blake2b" "gopkg.in/op/go-logging.v1" - "github.com/fxamacker/cbor/v2" "github.com/katzenpost/hpqc/hash" + "github.com/katzenpost/hpqc/kem" + "github.com/katzenpost/hpqc/kem/schemes" + signpem "github.com/katzenpost/hpqc/sign/pem" + "github.com/katzenpost/hpqc/rand" "github.com/katzenpost/hpqc/sign" - signpem "github.com/katzenpost/hpqc/sign/pem" - signSchemes "github.com/katzenpost/hpqc/sign/schemes" + "github.com/katzenpost/katzenpost/authority/voting/client" + "github.com/katzenpost/katzenpost/authority/voting/server/config" + "github.com/katzenpost/katzenpost/core/cert" "github.com/katzenpost/katzenpost/core/epochtime" "github.com/katzenpost/katzenpost/core/pki" "github.com/katzenpost/katzenpost/core/sphinx/constants" + "github.com/katzenpost/katzenpost/core/sphinx/geo" + "github.com/katzenpost/katzenpost/core/wire" + "github.com/katzenpost/katzenpost/core/wire/commands" "github.com/katzenpost/katzenpost/core/worker" - - "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" - "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" + "github.com/katzenpost/katzenpost/http/common" ) const ( - stateBootstrap = "bootstrap" - stateDescriptorSend = "descriptor_send" - stateAcceptDescriptor = "accept_desc" - stateAcceptVote = "accept_vote" - stateConfirmConsensus = "confirm_consensus" + descriptorsBucket = "descriptors" + replicaDescriptorsBucket = "replica_descriptors" + documentsBucket = "documents" + stateAcceptDescriptor = "accept_desc" + stateAcceptVote = "accept_vote" + stateAcceptReveal = "accept_reveal" + stateAcceptCert = "accept_cert" + stateAcceptSignature = "accept_signature" + stateBootstrap = "bootstrap" publicKeyHashSize = 32 ) -// NOTE: 2024-11-01: -// Parts of katzenpost use MixPublishDeadline and PublishConsensusDeadline defined in -// katzenpost:authority/voting/server/state.go -// So, we preserve that aspect of the epoch schedule. var ( - MixPublishDeadline = epochtime.Period * 1 / 8 // Do NOT change this - DescriptorBlockDeadline = epochtime.Period * 2 / 8 - AuthorityVoteDeadline = epochtime.Period * 3 / 8 - PublishConsensusDeadline = epochtime.Period * 5 / 8 // Do NOT change this - DocGenerationDeadline = epochtime.Period * 7 / 8 - RandomCourtessyDelay = epochtime.Period * 1 / 16 // duration to distribute load across synchronized nodes + MixPublishDeadline = epochtime.Period / 8 + AuthorityVoteDeadline = MixPublishDeadline + epochtime.Period/8 + AuthorityRevealDeadline = AuthorityVoteDeadline + epochtime.Period/8 + AuthorityCertDeadline = AuthorityRevealDeadline + epochtime.Period/8 + PublishConsensusDeadline = AuthorityCertDeadline + epochtime.Period/8 errGone = errors.New("authority: Requested epoch will never get a Document") errNotYet = errors.New("authority: Document is not ready yet") errInvalidTopology = errors.New("authority: Invalid Topology") ) +type descriptor struct { + desc *pki.MixDescriptor + raw []byte +} + +type document struct { + doc *pki.Document + raw []byte +} + type state struct { sync.RWMutex worker.Worker - s *Server - log *logging.Logger - chainBridge *chainbridge.ChainBridge - ccbor cbor.EncMode // a la katzenpost:core/pki/document.go - - // locally registered node, only one allowed - // mix descriptor uploads to this authority are restricted to this node - authorizedNode *chainbridge.Node - - documents map[uint64]*pki.Document - descriptors map[uint64]map[[publicKeyHashSize]byte]*pki.MixDescriptor + s *Server + geo *geo.Geometry + log *logging.Logger + + db *bolt.DB + + reverseHash map[[publicKeyHashSize]byte]sign.PublicKey + authorizedMixes map[[publicKeyHashSize]byte]bool + authorizedGatewayNodes map[[publicKeyHashSize]byte]string + authorizedServiceNodes map[[publicKeyHashSize]byte]string + authorizedReplicaNodes map[[publicKeyHashSize]byte]string + authorizedAuthorities map[[publicKeyHashSize]byte]bool + authorityLinkKeys map[[publicKeyHashSize]byte]kem.PublicKey + authorityNames map[[publicKeyHashSize]byte]string + + documents map[uint64]*pki.Document + myconsensus map[uint64]*pki.Document + descriptors map[uint64]map[[publicKeyHashSize]byte]*pki.MixDescriptor + replicaDescriptors map[uint64]map[[publicKeyHashSize]byte]*pki.ReplicaDescriptor + votes map[uint64]map[[publicKeyHashSize]byte]*pki.Document + certificates map[uint64]map[[publicKeyHashSize]byte]*pki.Document + signatures map[uint64]map[[publicKeyHashSize]byte]*cert.Signature + priorSRV [][]byte + reveals map[uint64]map[[publicKeyHashSize]byte][]byte + commits map[uint64]map[[publicKeyHashSize]byte][]byte + verifiers map[[publicKeyHashSize]byte]sign.PublicKey + + updateCh chan interface{} votingEpoch uint64 genesisEpoch uint64 + threshold int + dissenters int state string } func (s *state) Halt() { s.Worker.Halt() + + // Gracefully close the persistence store. + s.db.Sync() + s.db.Close() +} + +func (s *state) onUpdate() { + // Non-blocking write, multiple invocations are harmless, the channel is + // buffered, and there is a fallback timer. + select { + case s.updateCh <- true: + default: + } } func (s *state) worker() { @@ -91,20 +151,14 @@ func (s *state) worker() { } } -// Returns a random delay to distribute load across synchronized nodes -func (s *state) courtessyDelay() time.Duration { - return time.Duration(rand.NewMath().Float64() * float64(RandomCourtessyDelay)) -} - func (s *state) fsm() <-chan time.Time { s.Lock() var sleep time.Duration epoch, elapsed, nextEpoch := epochtime.Now() - s.log.Debugf("Current epoch %d, remaining time: %s, state: %s", epoch, nextEpoch, s.state) + s.log.Debugf("Current epoch %d, remaining time: %s", epoch, nextEpoch) switch s.state { case stateBootstrap: - // TODO: ensure network is ready and locally registered node is eligible for participation s.genesisEpoch = 0 s.backgroundFetchConsensus(epoch - 1) s.backgroundFetchConsensus(epoch) @@ -115,48 +169,90 @@ func (s *state) fsm() <-chan time.Time { s.state = stateBootstrap } else { s.votingEpoch = epoch + 1 - s.state = stateDescriptorSend - sleep = MixPublishDeadline - elapsed + s.courtessyDelay() + s.state = stateAcceptDescriptor + sleep = MixPublishDeadline - elapsed if sleep < 0 { sleep = 0 } s.log.Noticef("Bootstrapping for %d", s.votingEpoch) } - case stateDescriptorSend: - // Send mix descriptor to the appchain - pk := hash.Sum256(s.authorizedNode.IdentityKey) - desc, ok := s.descriptors[s.votingEpoch][pk] - if ok { - s.submitDescriptorToAppchain(desc, s.votingEpoch) - } else { - s.log.Errorf("❌ No descriptor for epoch %d", s.votingEpoch) - } - s.state = stateAcceptDescriptor - sleep = DescriptorBlockDeadline - elapsed + s.courtessyDelay() case stateAcceptDescriptor: - doc, err := s.getVote(s.votingEpoch) + signed, err := s.getVote(s.votingEpoch) if err == nil { - s.sendVoteToAppchain(doc, s.votingEpoch) + serialized, err := signed.MarshalCertificate() + if err == nil { + s.sendVoteToAuthorities(serialized, s.votingEpoch) + } else { + s.log.Errorf("Failed to serialize certificate for epoch %v: %s", s.votingEpoch, err) + } } else { - s.log.Errorf("❌ Failed to compute vote for epoch %v: %s", s.votingEpoch, err) + s.log.Errorf("Failed to compute vote for epoch %v: %s", s.votingEpoch, err) } s.state = stateAcceptVote _, nowelapsed, _ := epochtime.Now() sleep = AuthorityVoteDeadline - nowelapsed case stateAcceptVote: - s.backgroundFetchConsensus(s.votingEpoch) - s.state = stateConfirmConsensus + signed := s.reveal(s.votingEpoch) + s.sendRevealToAuthorities(signed, s.votingEpoch) + s.state = stateAcceptReveal + _, nowelapsed, _ := epochtime.Now() + sleep = AuthorityRevealDeadline - nowelapsed + case stateAcceptReveal: + signed, err := s.getCertificate(s.votingEpoch) + if err == nil { + serialized, err := signed.MarshalCertificate() + if err == nil { + s.sendCertToAuthorities(serialized, s.votingEpoch) + } else { + s.log.Errorf("Failed to serialize certificate for epoch %v", s.votingEpoch) + } + } else { + s.log.Errorf("Failed to compute certificate for epoch %v", s.votingEpoch) + } + s.state = stateAcceptCert + _, nowelapsed, _ := epochtime.Now() + sleep = AuthorityCertDeadline - nowelapsed + case stateAcceptCert: + doc, err := s.getMyConsensus(s.votingEpoch) + if err == nil { + s.log.Noticef("my view of consensus: %x\n%s", s.identityPubKeyHash(), doc) + // detach signature and send to authorities + sig, ok := doc.Signatures[s.identityPubKeyHash()] + if !ok { + s.log.Errorf("Failed to find our signature for epoch %v", s.votingEpoch) + s.s.fatalErrCh <- err + break + } + serialized, err := sig.Marshal() + if err != nil { + s.log.Errorf("Failed to serialize our signature for epoch %v", s.votingEpoch) + s.s.fatalErrCh <- err + break + } + signed, err := cert.Sign(s.s.identityPrivateKey, s.s.identityPublicKey, serialized, s.votingEpoch) + if err != nil { + s.log.Errorf("Failed to sign our signature for epoch %v: %s", s.votingEpoch, err) + s.s.fatalErrCh <- err + break + } + s.sendSigToAuthorities(signed, s.votingEpoch) + } else { + s.log.Errorf("Failed to compute our view of consensus for %v with %s", s.votingEpoch, err) + } + s.state = stateAcceptSignature _, nowelapsed, _ := epochtime.Now() sleep = PublishConsensusDeadline - nowelapsed - case stateConfirmConsensus: - // See if consensus doc was retrieved from the appchain - _, ok := s.documents[epoch+1] - if ok { - s.state = stateDescriptorSend - sleep = MixPublishDeadline + nextEpoch + s.courtessyDelay() + case stateAcceptSignature: + // combine signatures over a certificate and see if we make a threshold consensus + s.log.Noticef("Combining signatures for epoch %v", s.votingEpoch) + _, err := s.getThresholdConsensus(s.votingEpoch) + _, _, nextEpoch := epochtime.Now() + if err == nil { + s.state = stateAcceptDescriptor + sleep = MixPublishDeadline + nextEpoch s.votingEpoch++ } else { - s.log.Error("No document for epoch %v", epoch+1) + s.log.Error(err.Error()) s.state = stateBootstrap s.votingEpoch = epoch + 2 // vote on epoch+2 in epoch+1 sleep = nextEpoch @@ -169,46 +265,73 @@ func (s *state) fsm() <-chan time.Time { return time.After(sleep) } -// getVote produces a pki.Document using all MixDescriptors recorded with the appchain +func (s *state) persistDocument(epoch uint64, doc []byte) { + if err := s.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(documentsBucket)) + return bkt.Put(epochToBytes(epoch), doc) + }); err != nil { + // Persistence failures are FATAL. + s.s.fatalErrCh <- err + } +} + +// getVote produces a pki.Document using all MixDescriptors that we have seen func (s *state) getVote(epoch uint64) (*pki.Document, error) { - // Is there a prior consensus? If so, obtain the GenesisEpoch + // Is there a prior consensus? If so, obtain the GenesisEpoch and prior SRV values if d, ok := s.documents[s.votingEpoch-1]; ok { s.log.Debugf("Restoring genesisEpoch %d from document cache", d.GenesisEpoch) s.genesisEpoch = d.GenesisEpoch + s.priorSRV = d.PriorSharedRandom d.PKISignatureScheme = s.s.cfg.Server.PKISignatureScheme } else { s.log.Debugf("Setting genesisEpoch %d from votingEpoch", s.votingEpoch) s.genesisEpoch = s.votingEpoch } - descriptors, err := s.chPKIGetMixDescriptors(epoch) - if err != nil { - return nil, err + descriptors := []*pki.MixDescriptor{} + for _, desc := range s.descriptors[epoch] { + descriptors = append(descriptors, desc) + } + + replicaDescriptors := []*pki.ReplicaDescriptor{} + for _, desc := range s.replicaDescriptors[epoch] { + replicaDescriptors = append(replicaDescriptors, desc) } // vote topology is irrelevent. - // TODO: use an appchain block hash as srv var zeros [32]byte - doc := s.getDocument(descriptors, s.s.cfg.Parameters, zeros[:]) - - // Note: For appchain-pki, upload unsigned document and sign it upon local save. - // simulate SignDocument's setting of doc version, required by IsDocumentWellFormed - doc.Version = pki.DocumentVersion + vote := s.getDocument(descriptors, replicaDescriptors, s.s.cfg.Parameters, zeros[:]) - if err := pki.IsDocumentWellFormed(doc, nil); err != nil { - s.log.Errorf("pki: ❌ getVote: IsDocumentWellFormed: %s", err) + // create our SharedRandom Commit + signedCommit, err := s.doCommit(epoch) + if err != nil { return nil, err } + commits := make(map[[hash.HashSize]byte][]byte) + commits[s.identityPubKeyHash()] = signedCommit + vote.SharedRandomCommit = commits - return doc, nil -} + _, err = s.doSignDocument(s.s.identityPrivateKey, s.s.identityPublicKey, vote) + if err != nil { + return nil, err + } -func (s *state) sendVoteToAppchain(doc *pki.Document, epoch uint64) { - if err := s.chPKISetDocument(doc); err != nil { - s.log.Errorf("❌ sendVoteToAppchain: Error setting document for epoch %d: %v", epoch, err) + s.log.Debugf("Ready to send our vote:\n%s", vote) + // save our own vote + if _, ok := s.votes[epoch]; !ok { + s.votes[epoch] = make(map[[publicKeyHashSize]byte]*pki.Document) + } + if _, ok := s.votes[epoch][s.identityPubKeyHash()]; !ok { + s.votes[epoch][s.identityPubKeyHash()] = vote } else { - s.log.Noticef("✅ sendVoteToAppchain: Set document for epoch %d", epoch) + return nil, errors.New("failure: vote already present, this should never happen") } + return vote, nil +} + +func (s *state) doParseDocument(b []byte) (*pki.Document, error) { + doc, err := pki.ParseDocument(b) + return doc, err } func (s *state) doSignDocument(signer sign.PrivateKey, verifier sign.PublicKey, d *pki.Document) ([]byte, error) { @@ -218,7 +341,161 @@ func (s *state) doSignDocument(signer sign.PrivateKey, verifier sign.PublicKey, return sig, err } -func (s *state) getDocument(descriptors []*pki.MixDescriptor, params *config.Parameters, srv []byte) *pki.Document { +// getCertificate is the same as a vote but it contains all SharedRandomCommits and SharedRandomReveals seen +func (s *state) getCertificate(epoch uint64) (*pki.Document, error) { + if s.TryLock() { + panic("write lock not held in getCertificate(epoch)") + } + + mixes, replicas, params, err := s.tallyVotes(epoch) + if err != nil { + s.log.Warningf("No document for epoch %v, aborting!, %v", epoch, err) + return nil, err + } + s.log.Debug("Mixes tallied, now making a document") + var zeros [32]byte + srv := zeros[:] + certificate := s.getDocument(mixes, replicas, params, srv) + // add the SharedRandomCommit and SharedRandomReveal that we have seen + certificate.SharedRandomCommit = s.commits[epoch] + certificate.SharedRandomReveal = s.reveals[epoch] + // if there are no prior SRV values, copy the current srv twice + if len(s.priorSRV) == 0 { + s.priorSRV = [][]byte{srv, srv} + } else if epoch%epochtime.WeekOfEpochs == 0 { + // rotate the weekly epochs if it is time to do so. + s.priorSRV = [][]byte{srv, s.priorSRV[0]} + } + _, err = s.doSignDocument(s.s.identityPrivateKey, s.s.identityPublicKey, certificate) + if err != nil { + return nil, err + } + err = pki.IsDocumentWellFormed(certificate, s.getVerifiers()) + if err != nil { + return nil, err + } + // save our own certificate + if _, ok := s.certificates[epoch]; !ok { + s.certificates[epoch] = make(map[[publicKeyHashSize]byte]*pki.Document) + } + if _, ok := s.certificates[epoch][s.identityPubKeyHash()]; !ok { + s.certificates[epoch][s.identityPubKeyHash()] = certificate + } else { + return nil, errors.New("failure: vote already present, this should never happen") + } + return certificate, nil +} + +// getConsensus computes the final document using the computed SharedRandomValue +func (s *state) getMyConsensus(epoch uint64) (*pki.Document, error) { + if s.TryLock() { + panic("write lock not held in getMyConsensus(epoch)") + } + + certificates, ok := s.certificates[epoch] + if !ok { + return nil, fmt.Errorf("No certificates for epoch %d", epoch) + } + + // well this isn't going to work then is it? + if len(certificates) < s.threshold { + return nil, fmt.Errorf("No way to make consensus with too few votes!, only %d certificates", len(certificates)) + } + + // verify that all shared random commit and reveal are present for this epoch + commits, reveals := s.verifyCommits(epoch) + if len(commits) < s.threshold { + return nil, fmt.Errorf("No way to make consensus with too few SharedRandom commits!, only %d commits", len(commits)) + } + if len(commits) != len(reveals) { + panic("ShouldNotBePossible") + } + + // compute the shared random for the consensus + srv, err := s.computeSharedRandom(epoch, commits, reveals) + if err != nil { + return nil, err + } + // if there are no prior SRV values, copy the current srv twice + if epoch == s.genesisEpoch { + s.priorSRV = [][]byte{srv, srv} + } else if epoch%epochtime.WeekOfEpochs == 0 { + // rotate the weekly epochs if it is time to do so. + s.priorSRV = [][]byte{srv, s.priorSRV[0]} + } + mixes, replicas, params, err := s.tallyVotes(epoch) + if err != nil { + return nil, err + } + consensusOfOne := s.getDocument(mixes, replicas, params, srv) + _, err = s.doSignDocument(s.s.identityPrivateKey, s.s.identityPublicKey, consensusOfOne) + if err != nil { + return nil, err + } + + // save our view of the conseusus + s.myconsensus[epoch] = consensusOfOne + return consensusOfOne, nil +} + +// getThresholdConsensus returns a *pki.Document iff a threshold consensus is reached or error +func (s *state) getThresholdConsensus(epoch uint64) (*pki.Document, error) { + // range over the certificates we have collected and see if we can collect enough signatures to make a consensus + if s.TryLock() { + panic("write lock not held in getThresholdConsensus(epoch)") + } + + ourConsensus, ok := s.myconsensus[epoch] + if !ok { + return nil, fmt.Errorf("We have no view of consensus!") + } + for pk, signature := range s.signatures[epoch] { + s.log.Debugf("Checking signature from %x on our certificates", pk) + v := s.reverseHash[pk] + err := ourConsensus.AddSignature(v, *signature) + if err != nil { + s.log.Errorf("Failed to AddSignature from %x on our consensus: %s", pk, err) + } + } + // now see if we managed to get a threshold number of signatures + signedConsensus, err := ourConsensus.MarshalCertificate() + if err != nil { + return nil, err + } + _, good, bad, err := cert.VerifyThreshold(s.getVerifiers(), s.threshold, signedConsensus) + for _, b := range bad { + s.log.Errorf("Consensus NOT signed by %s", s.authorityNames[hash.Sum256From(b)]) + } + for _, g := range good { + s.log.Noticef("Consensus signed by %s", s.authorityNames[hash.Sum256From(g)]) + } + if err == nil { + s.log.Noticef("Consensus made for epoch %d with %d/%d signatures: %v", epoch, len(good), len(s.verifiers), ourConsensus) + // Persist the document to disk. + s.persistDocument(epoch, signedConsensus) + s.documents[epoch] = ourConsensus + return ourConsensus, nil + } else { + s.log.Errorf("VerifyThreshold failed!: %s", err) + } + return nil, fmt.Errorf("No consensus found for epoch %d", epoch) +} + +func (s *state) getVerifiers() []sign.PublicKey { + v := make([]sign.PublicKey, len(s.verifiers)) + i := 0 + for _, val := range s.verifiers { + v[i] = val + i++ + } + return v +} + +func (s *state) identityPubKeyHash() [publicKeyHashSize]byte { + return hash.Sum256From(s.s.identityPublicKey) +} + +func (s *state) getDocument(descriptors []*pki.MixDescriptor, replicaDescriptors []*pki.ReplicaDescriptor, params *config.Parameters, srv []byte) *pki.Document { // Carve out the descriptors between providers and nodes. gateways := []*pki.MixDescriptor{} serviceNodes := []*pki.MixDescriptor{} @@ -237,18 +514,23 @@ func (s *state) getDocument(descriptors []*pki.MixDescriptor, params *config.Par // Assign nodes to layers. var topology [][]*pki.MixDescriptor - // We prefer to not randomize the topology if there is an existing topology to avoid - // partitioning the client anonymity set when messages from an earlier epoch are - // differentiable as such because of topology violations in the present epoch. - if d, ok := s.documents[s.votingEpoch-1]; ok { - topology = s.generateTopology(nodes, d, srv) + // if a static topology is specified, generate a fixed topology + if s.s.cfg.Topology != nil { + topology = s.generateFixedTopology(nodes, srv) } else { - topology = s.generateRandomTopology(nodes, srv) + // We prefer to not randomize the topology if there is an existing topology to avoid + // partitioning the client anonymity set when messages from an earlier epoch are + // differentiable as such because of topology violations in the present epoch. + + if d, ok := s.documents[s.votingEpoch-1]; ok { + topology = s.generateTopology(nodes, d, srv) + } else { + topology = s.generateRandomTopology(nodes, srv) + } } - nodesPerLayer := len(nodes) / s.s.cfg.Debug.Layers - lambdaG := computeLambdaGFromNodesPerLayer(s.s.cfg, nodesPerLayer) - s.log.Debugf("computed lambdaG from %d nodes per layer is %f", nodesPerLayer, lambdaG) + lambdaG := computeLambdaG(s.s.cfg) + s.log.Debugf("computed lambdaG is %f", lambdaG) // Build the Document. doc := &pki.Document{ @@ -270,167 +552,1179 @@ func (s *state) getDocument(descriptors []*pki.MixDescriptor, params *config.Par Topology: topology, GatewayNodes: gateways, ServiceNodes: serviceNodes, + StorageReplicas: replicaDescriptors, SharedRandomValue: srv, - PriorSharedRandom: [][]byte{srv}, // this is made up, only to suffice IsDocumentWellFormed - SphinxGeometryHash: s.s.geo.Hash(), + PriorSharedRandom: s.priorSRV, + SphinxGeometryHash: s.geo.Hash(), PKISignatureScheme: s.s.cfg.Server.PKISignatureScheme, } return doc } -func (s *state) generateTopology(nodeList []*pki.MixDescriptor, doc *pki.Document, srv []byte) [][]*pki.MixDescriptor { - s.log.Debugf("Generating mix topology.") +func (s *state) hasEnoughDescriptors(m map[[publicKeyHashSize]byte]*pki.MixDescriptor) bool { + // A Document will be generated iff there are at least: + // + // * Debug.Layers * Debug.MinNodesPerLayer nodes. + // * One gateway. + // * One service node. + // + // Otherwise, it's pointless to generate a unusable document. + nrGateways := 0 + nrServiceNodes := 0 + for _, v := range m { + if v.IsGatewayNode { + nrGateways++ + } + if v.IsServiceNode { + nrServiceNodes++ + } - nodeMap := make(map[[constants.NodeIDLength]byte]*pki.MixDescriptor) - for _, v := range nodeList { - id := hash.Sum256(v.IdentityKey) - nodeMap[id] = v } + nrNodes := len(m) - nrGateways - nrServiceNodes - // TODO: consider strategies for balancing topology? Should this happen automatically? - // the current strategy will rebalance by limiting the number of nodes that are - // (re)inserted at each layer and placing these nodes into another layer. + minNodes := s.s.cfg.Debug.Layers * s.s.cfg.Debug.MinNodesPerLayer + return (nrGateways > 0) && (nrServiceNodes > 0) && (nrNodes >= minNodes) +} - // Since there is an existing network topology, use that as the basis for - // generating the mix topology such that the number of nodes per layer is - // approximately equal, and as many nodes as possible retain their existing - // layer assignment to minimise network churn. - // The srv is used, when available, to ensure the ordering of new nodes - // is deterministic between authorities - rng, err := rand.NewDeterministicRandReader(srv[:]) - if err != nil { - s.log.Errorf("DeterministicRandReader() failed to initialize: %v", err) - s.s.fatalErrCh <- err +func (s *state) verifyCommits(epoch uint64) (map[[publicKeyHashSize]byte][]byte, map[[publicKeyHashSize]byte][]byte) { + if s.TryLock() { + panic("write lock not held in verifyCommits(epoch)") } - targetNodesPerLayer := len(nodeList) / s.s.cfg.Debug.Layers - topology := make([][]*pki.MixDescriptor, s.s.cfg.Debug.Layers) - // Assign nodes that still exist up to the target size. - for layer, nodes := range doc.Topology { - nodeIndexes := rng.Perm(len(nodes)) + // check that each authority presented the same commit to every other authority + badnodes := make(map[[publicKeyHashSize]byte]bool) + comitted := make(map[[publicKeyHashSize]byte][]byte) + revealed := make(map[[publicKeyHashSize]byte][]byte) - for _, idx := range nodeIndexes { - if len(topology[layer]) >= targetNodesPerLayer { + // verify that each authority only submitted one commit value to all the authorities + for pk, certificate := range s.certificates[epoch] { + // skip badnodes + if _, ok := badnodes[pk]; ok { + continue + } + for pk2, signedCommit := range certificate.SharedRandomCommit { + // skip badnodes + if _, ok := badnodes[pk2]; ok { + continue + } + // verify that pk2 is authorized + v, ok := s.reverseHash[pk2] + if !ok { + s.log.Errorf("Commit from invaid peer %x in certificate from %s", pk2, s.authorityNames[pk]) + badnodes[pk] = true + break + } + // verify that the allged commit is signed by pk2 + commit, err := cert.Verify(v, signedCommit) + if err != nil { + // pk didn't validate commit in its certificate! + badnodes[pk] = true + s.log.Errorf("Invalid signature over commit from %s in certificate from %s, rejecting %s from consensus", s.authorityNames[pk2], s.authorityNames[pk], s.authorityNames[pk]) + // do not bother checking any more of pk's SharedRandomCommits + break + } + // verify that the commit is accompanied by a reaveal + signedReveal, ok := certificate.SharedRandomReveal[pk2] + if !ok { + s.log.Errorf("Certificate from %s has Commit for %s but not Reveal", s.authorityNames[pk], s.authorityNames[pk2]) + badnodes[pk] = true + // do not bother checking any more of pk's SharedRandomCommits + break + } + // verify that the alleged reveal is signed by pk2 + reveal, err := cert.Verify(v, signedReveal) + if err != nil { + s.log.Errorf("Reveal in certificate from %s has invalid signature on reveal from %s", s.authorityNames[pk], s.authorityNames[pk2]) + badnodes[pk] = true + // do not bother checking any more of pk's SharedRandomCommits + break + } + srv := new(pki.SharedRandom) + srv.SetCommit(commit) + // verify that the SharedRandom is for the correct epoch + if srv.GetEpoch() != epoch { + s.log.Errorf("SharedRandomCommit in certificate from %s contains bad Epoch from %s", s.authorityNames[pk], s.authorityNames[pk2]) + badnodes[pk] = true + badnodes[pk2] = true + // do not bother checking any more of pk's SharedRandomCommits break } - id := hash.Sum256(nodes[idx].IdentityKey) - if n, ok := nodeMap[id]; ok { - // There is a new descriptor with the same identity key, - // as an existing descriptor in the previous document, - // so preserve the layering. - topology[layer] = append(topology[layer], n) - delete(nodeMap, id) + // verify that the commit is validate by the revealed value + if !srv.Verify(reveal) { + s.log.Errorf("Reveal in certificate from %s has invalid reveal from %s", s.authorityNames[pk], s.authorityNames[pk2]) + // pk should have validated the Reveal, and pk2 signed an invalid Reveal + badnodes[pk] = true + badnodes[pk2] = true + break + } + // see if we saw a different commit from pk2 + signedCommit2, ok := comitted[pk2] + if ok { + // check that the commits were the same + if !bytes.Equal(signedCommit, signedCommit2) { + s.log.Errorf("%s submitted commit %x to %s and previously submitted %x", s.authorityNames[pk2], signedCommit[:32], s.authorityNames[pk], signedCommit2[:32]) + badnodes[pk2] = true + } + } else { + // first time we saw a commit from pk2 + comitted[pk2] = signedCommit + } + // see if we saw a different reveal from pk2 + signedReveal2, ok := revealed[pk2] + if ok { + if !bytes.Equal(signedReveal, signedReveal2) { + s.log.Errorf("%s submitted commit %x to %s and previously submitted %x", s.authorityNames[pk2], signedReveal, s.authorityNames[pk], signedReveal2) + badnodes[pk2] = true + } + } else { + revealed[pk2] = signedReveal } } } - - // Flatten the map containing the nodes pending assignment. - toAssign := make([]*pki.MixDescriptor, 0, len(nodeMap)) - for _, n := range nodeMap { - toAssign = append(toAssign, n) + // ensure we have enough commits to make a threshold consensus + for pk, _ := range badnodes { + s.log.Warningf("Found bad node %s", s.authorityNames[pk]) + delete(comitted, pk) + delete(revealed, pk) } - // must sort toAssign by ID! - sortNodesByPublicKey(toAssign) + return comitted, revealed +} - assignIndexes := rng.Perm(len(toAssign)) +// IsPeerValid authenticates the remote peer's credentials +// for our link layer wire protocol as specified by +// the PeerAuthenticator interface in core/wire/session.go +func (s *state) IsPeerValid(creds *wire.PeerCredentials) bool { + var ad [publicKeyHashSize]byte + copy(ad[:], creds.AdditionalData[:publicKeyHashSize]) + _, ok := s.authorizedAuthorities[ad] + if ok { + return true + } + return false +} - // Fill out any layers that are under the target size, by - // randomly assigning from the pending list. - idx := 0 - for layer := range doc.Topology { - for len(topology[layer]) < targetNodesPerLayer { - n := toAssign[assignIndexes[idx]] - topology[layer] = append(topology[layer], n) - idx++ +func (s *state) sendCommandToPeer(peer *config.Authority, cmd commands.Command) (commands.Command, error) { + var conn net.Conn + var err error + for i, a := range peer.Addresses { + u, err := url.Parse(a) + if err != nil { + continue + } + defaultDialer := &net.Dialer{} + ctx, cancelFn := context.WithCancel(context.Background()) + conn, err = common.DialURL(u, ctx, defaultDialer.DialContext) + cancelFn() + if err == nil { + defer conn.Close() + break + } else { + s.log.Errorf("Got err from Peer: %v", err) + } + if i == len(peer.Addresses)-1 { + return nil, err } } + s.s.Add(1) + defer s.s.Done() + identityHash := hash.Sum256From(s.s.identityPublicKey) - // Assign the remaining nodes. - for layer := 0; idx < len(assignIndexes); idx++ { - n := toAssign[assignIndexes[idx]] - topology[layer] = append(topology[layer], n) - layer++ - layer = layer % len(topology) + kemscheme := schemes.ByName(s.s.cfg.Server.WireKEMScheme) + if kemscheme == nil { + panic("kem scheme not found in registry") } - return topology -} - -func (s *state) generateRandomTopology(nodes []*pki.MixDescriptor, srv []byte) [][]*pki.MixDescriptor { - s.log.Debugf("Generating random mix topology.") - - // If there is no node history in the form of a previous consensus, - // then the simplest thing to do is to randomly assign nodes to the - // various layers. - - if len(srv) != 32 { - err := errors.New("SharedRandomValue too short") - s.log.Errorf("srv: %s", srv) - s.s.fatalErrCh <- err + cfg := &wire.SessionConfig{ + KEMScheme: kemscheme, + Geometry: s.geo, + Authenticator: s, + AdditionalData: identityHash[:], + AuthenticationKey: s.s.linkKey, + RandomReader: rand.Reader, } - rng, err := rand.NewDeterministicRandReader(srv[:]) + session, err := wire.NewPKISession(cfg, true) if err != nil { - s.log.Errorf("DeterministicRandReader() failed to initialize: %v", err) - s.s.fatalErrCh <- err + return nil, err } + defer session.Close() - nodeIndexes := rng.Perm(len(nodes)) - topology := make([][]*pki.MixDescriptor, s.s.cfg.Debug.Layers) - for idx, layer := 0, 0; idx < len(nodes); idx++ { - n := nodes[nodeIndexes[idx]] - topology[layer] = append(topology[layer], n) - layer++ - layer = layer % len(topology) + if err = session.Initialize(conn); err != nil { + return nil, err } - - return topology + err = session.SendCommand(cmd) + if err != nil { + return nil, err + } + resp, err := session.RecvCommand() + if err != nil { + return nil, err + } + return resp, nil } -func (s *state) pruneDocuments() { - // Looking a bit into the past is probably ok, if more past documents - // need to be accessible, then methods that query the DB could always - // be added. - const preserveForPastEpochs = 3 +// sendCommitToAuthorities sends our cert to all Directory Authorities +func (s *state) sendCertToAuthorities(cert []byte, epoch uint64) { + if s.TryLock() { + panic("write lock not held in sendCertToAuthorities(cert, epoch)") + } - now, _, _ := epochtime.Now() - cmpEpoch := now - preserveForPastEpochs + s.log.Noticef("Sending Certificate for epoch %v, to all Directory Authorities.", epoch) + cmd := &commands.Cert{ + Epoch: epoch, + PublicKey: s.s.IdentityKey(), + Payload: cert, + } + + for _, peer := range s.s.cfg.Authorities { + peer := peer + if peer.IdentityPublicKey.Equal(s.s.identityPublicKey) { + continue // skip self + } + s.Go(func() { + s.log.Noticef("Sending cert to %s", peer.Identifier) + resp, err := s.sendCommandToPeer(peer, cmd) + if err != nil { + s.log.Error("Failed to send cert to %s", peer.Identifier) + return + } + r, ok := resp.(*commands.CertStatus) + if !ok { + s.log.Warningf("Cert response resulted in unexpected reply: %T", resp) + return + } + switch r.ErrorCode { + case commands.CertOk: + s.log.Notice("Cert submitted to %s", peer.Identifier) + case commands.CertTooLate: + s.log.Warningf("Cert rejected with CertTooLate by %s", peer.Identifier) + case commands.CertTooEarly: + s.log.Warningf("Cert rejected with CertTooEarly by %s", peer.Identifier) + case commands.CertAlreadyReceived: + s.log.Warningf("Cert rejected with CertAlreadyReceived by %s", peer.Identifier) + case commands.CertNotAuthorized: + s.log.Warningf("Cert rejected with CertNotAuthoritzed by %s", peer.Identifier) + case commands.CertNotSigned: + s.log.Warningf("Cert rejected with CertNotSigned by %s", peer.Identifier) + default: + s.log.Warningf("Cert rejected with unknown error code received by %s", peer.Identifier) + } + }) + } +} + +// sendVoteToAuthorities sends s.descriptors[epoch] to all Directory Authorities +func (s *state) sendVoteToAuthorities(vote []byte, epoch uint64) { + if s.TryLock() { + panic("write lock not held in sendVoteToAuthorities(vote, epoch)") + } + + s.log.Noticef("Sending Vote for epoch %v, to all Directory Authorities.", epoch) + + cmd := &commands.Vote{ + Epoch: epoch, + PublicKey: s.s.IdentityKey(), + Payload: vote, + } + + for _, peer := range s.s.cfg.Authorities { + peer := peer + if peer.IdentityPublicKey.Equal(s.s.identityPublicKey) { + continue // skip self + } + s.Go(func() { + s.log.Noticef("Sending Vote to %s", peer.Identifier) + resp, err := s.sendCommandToPeer(peer, cmd) + if err != nil { + s.log.Error("Failed to send vote to %s: %s", peer.Identifier, err) + return + } + r, ok := resp.(*commands.VoteStatus) + if !ok { + s.log.Warningf("Vote response resulted in unexpected reply: %T", resp) + return + } + switch r.ErrorCode { + case commands.VoteOk: + s.log.Notice("Vote submitted to %s", peer.Identifier) + case commands.VoteTooLate: + s.log.Warningf("Vote rejected with VoteTooLate by %s", peer.Identifier) + case commands.VoteTooEarly: + s.log.Warningf("Vote rejected with VoteTooEarly by %s", peer.Identifier) + default: + s.log.Warningf("Vote rejected with unknown error code received by %s", peer.Identifier) + } + }) + } +} + +// sendRevealToAuthorities sends a Shared Random Reveal command to +// all Directory Authorities +func (s *state) sendRevealToAuthorities(reveal []byte, epoch uint64) { + s.log.Noticef("Sending Shared Random Reveal for epoch %v, to all Directory Authorities.", epoch) + + cmd := &commands.Reveal{ + Epoch: epoch, + PublicKey: s.s.IdentityKey(), + Payload: reveal, + } + for _, peer := range s.s.cfg.Authorities { + peer := peer + if peer.IdentityPublicKey.Equal(s.s.identityPublicKey) { + continue // skip self + } + s.Go(func() { + s.log.Noticef("Sending Reveal to %s", peer.Identifier) + resp, err := s.sendCommandToPeer(peer, cmd) + if err != nil { + s.log.Error("Failed to send reveal to %s: %s", peer.Identifier, err) + return + } + r, ok := resp.(*commands.RevealStatus) + if !ok { + s.log.Error("Reveal response resulted in unexpected reply: %T", resp) + return + } + switch r.ErrorCode { + case commands.RevealOk: + s.log.Notice("Reveal submitted to %s", peer.Identifier) + case commands.RevealTooLate: + s.log.Warningf("Reveal rejected with RevealTooLate by %s", peer.Identifier) + case commands.RevealTooEarly: + s.log.Warningf("Reveal rejected with RevealTooEarly by %s", peer.Identifier) + case commands.RevealAlreadyReceived: + s.log.Warningf("Reveal rejected with RevealAlreadyReceived by %s", peer.Identifier) + case commands.RevealNotAuthorized: + s.log.Warningf("Reveal rejected with RevealNotAuthoritzed by %s", peer.Identifier) + case commands.RevealNotSigned: + s.log.Warningf("Reveal rejected with RevealNotSigned by %s", peer.Identifier) + default: + s.log.Warningf("reveal rejected with unknown error code received by %s", peer.Identifier) + } + }) + } +} + +func (s *state) sendSigToAuthorities(sig []byte, epoch uint64) { + if s.TryLock() { + panic("write lock not held in sendSigToAuthorities(sig, epoch)") + } + + s.log.Noticef("Sending Signature for epoch %v, to all Directory Authorities.", epoch) + + cmd := &commands.Sig{ + Epoch: epoch, + PublicKey: s.s.IdentityKey(), + Payload: sig, + } + + for _, peer := range s.s.cfg.Authorities { + peer := peer + if peer.IdentityPublicKey.Equal(s.s.identityPublicKey) { + continue // skip self + } + s.Go(func() { + s.log.Noticef("Sending Signature to %s", peer.Identifier) + resp, err := s.sendCommandToPeer(peer, cmd) + if err != nil { + s.log.Error("Failed to send Signature to %s", peer.Identifier) + return + } + r, ok := resp.(*commands.SigStatus) + if !ok { + s.log.Warningf("Signature resulted in unexpected reply: %T", resp) + return + } + switch r.ErrorCode { + case commands.SigOk: + s.log.Notice("Signature submitted to %s", peer.Identifier) + case commands.SigTooLate: + s.log.Warningf("Signature rejected with SigTooLate by %s", peer.Identifier) + case commands.SigTooEarly: + s.log.Warningf("Signature rejected with SigTooEarly by %s", peer.Identifier) + default: + s.log.Warningf("Signature rejected with unknown error code received by %s", peer.Identifier) + } + }) + } +} + +func (s *state) tallyVotes(epoch uint64) ([]*pki.MixDescriptor, []*pki.ReplicaDescriptor, *config.Parameters, error) { + if s.TryLock() { + panic("write lock not held in tallyVotes(epoch)") + } + + _, ok := s.votes[epoch] + if !ok { + return nil, nil, nil, fmt.Errorf("no votes for epoch %v", epoch) + } + if len(s.votes[epoch]) < s.threshold { + return nil, nil, nil, fmt.Errorf("not enough votes for epoch %v", epoch) + } + + nodes := make([]*pki.MixDescriptor, 0) + mixTally := make(map[string][]*pki.Document) + mixParams := make(map[string][]*pki.Document) + replicaTally := make(map[string][]*pki.Document) + replicaNodes := make([]*pki.ReplicaDescriptor, 0) + for id, vote := range s.votes[epoch] { + // serialize the vote parameters and tally these as well. + params := &config.Parameters{ + SendRatePerMinute: vote.SendRatePerMinute, + Mu: vote.Mu, + MuMaxDelay: vote.MuMaxDelay, + LambdaP: vote.LambdaP, + LambdaPMaxDelay: vote.LambdaPMaxDelay, + LambdaL: vote.LambdaL, + LambdaLMaxDelay: vote.LambdaLMaxDelay, + LambdaD: vote.LambdaD, + LambdaDMaxDelay: vote.LambdaDMaxDelay, + LambdaM: vote.LambdaM, + LambdaMMaxDelay: vote.LambdaMMaxDelay, + LambdaG: computeLambdaG(s.s.cfg), + LambdaGMaxDelay: vote.LambdaGMaxDelay, + } + b := bytes.Buffer{} + e := gob.NewEncoder(&b) + err := e.Encode(params) + if err != nil { + s.log.Errorf("Skipping vote from Authority %s whose MixParameters failed to encode?! %v", s.authorityNames[id], err) + continue + } + bs := b.String() + if _, ok := mixParams[bs]; !ok { + mixParams[bs] = make([]*pki.Document, 0) + } + mixParams[bs] = append(mixParams[bs], vote) + + // include edge nodes in the tally. + for _, desc := range vote.GatewayNodes { + rawDesc, err := desc.MarshalBinary() + if err != nil { + s.log.Errorf("Skipping vote from Authority %s whose MixDescriptor failed to encode?! %v", s.authorityNames[id], err) + continue + } + k := string(rawDesc) + if _, ok := mixTally[k]; !ok { + mixTally[k] = make([]*pki.Document, 0) + } + mixTally[k] = append(mixTally[k], vote) + } + for _, desc := range vote.ServiceNodes { + rawDesc, err := desc.MarshalBinary() + if err != nil { + s.log.Errorf("Skipping vote from Authority %s whose MixDescriptor failed to encode?! %v", s.authorityNames[id], err) + continue + } + k := string(rawDesc) + if _, ok := mixTally[k]; !ok { + mixTally[k] = make([]*pki.Document, 0) + } + mixTally[k] = append(mixTally[k], vote) + } + // include the rest of the mixes in the tally. + for _, l := range vote.Topology { + for _, desc := range l { + rawDesc, err := desc.MarshalBinary() + if err != nil { + s.log.Errorf("Skipping vote from Authority %s whose MixDescriptor failed to encode?! %v", s.authorityNames[id], err) + continue + } + + k := string(rawDesc) + if _, ok := mixTally[k]; !ok { + mixTally[k] = make([]*pki.Document, 0) + } + mixTally[k] = append(mixTally[k], vote) + } + } + for _, desc := range vote.StorageReplicas { + rawDesc, err := desc.Marshal() + if err != nil { + s.log.Errorf("Skipping vote from Authority %s whose ReplicaDescriptor failed to encode?! %v", s.authorityNames[id], err) + continue + } + k := string(rawDesc) + if _, ok := replicaTally[k]; !ok { + replicaTally[k] = make([]*pki.Document, 0) + } + replicaTally[k] = append(replicaTally[k], vote) + } + } + // include mixes that have a threshold of votes + for rawDesc, votes := range mixTally { + if len(votes) >= s.threshold { + // this shouldn't fail as the descriptors have already been verified + desc := new(pki.MixDescriptor) + err := desc.UnmarshalBinary([]byte(rawDesc)) + if err != nil { + return nil, nil, nil, err + } + // only add nodes we have authorized + if s.isDescriptorAuthorized(desc) { + nodes = append(nodes, desc) + } + } + } + for rawDesc, votes := range replicaTally { + if len(votes) >= s.threshold { + // this shouldn't fail as the descriptors have already been verified + desc := new(pki.ReplicaDescriptor) + err := desc.Unmarshal([]byte(rawDesc)) + if err != nil { + return nil, nil, nil, err + } + // only add nodes we have authorized + if s.isReplicaDescriptorAuthorized(desc) { + replicaNodes = append(replicaNodes, desc) + } + } + } + // include parameters that have a threshold of votes + for bs, votes := range mixParams { + params := &config.Parameters{} + d := gob.NewDecoder(strings.NewReader(bs)) + if err := d.Decode(params); err != nil { + s.log.Errorf("tallyVotes: failed to decode params: err=%v: bs=%v", err, bs) + continue + } + + if len(votes) >= s.threshold { + sortNodesByPublicKey(nodes) + // successful tally + return nodes, replicaNodes, params, nil + } else if len(votes) >= s.dissenters { + s.log.Errorf("tallyVotes: failed threshold with params: %v", params) + continue + } + + } + return nil, nil, nil, errors.New("consensus failure (mixParams empty)") +} + +func (s *state) computeSharedRandom(epoch uint64, commits map[[publicKeyHashSize]byte][]byte, reveals map[[publicKeyHashSize]byte][]byte) ([]byte, error) { + if len(commits) < s.threshold { + s.log.Errorf("Insufficient commits for epoch %d to make consensus", epoch) + for id, _ := range commits { + s.log.Errorf("Have commits for epoch %d from %x", epoch, id) + } + return nil, errors.New("Insuffiient commits to make threshold vote") + } + type Reveal struct { + PublicKey [publicKeyHashSize]byte + Digest []byte + } + sortedreveals := make([]Reveal, 0, len(reveals)) + for pk, srr := range reveals { + digest, err := cert.GetCertified(srr) + if err != nil { + return nil, err + } + sortedreveals = append(sortedreveals, Reveal{PublicKey: pk, Digest: digest}) + } + srv, err := blake2b.New256(nil) + if err != nil { + panic(err) + } + + srv.Write([]byte("shared-random")) + srv.Write(epochToBytes(epoch)) + + sort.Slice(sortedreveals, func(i, j int) bool { + return string(sortedreveals[i].Digest) > string(sortedreveals[j].Digest) + }) + + for _, reveal := range sortedreveals { + srv.Write(reveal.PublicKey[:]) + srv.Write(reveal.Digest) + } + // XXX: Tor also hashes in the previous srv or 32 bytes of 0x00 + // How do we bootstrap a new authority? + zeros := make([]byte, 32) + if vot, ok := s.documents[s.votingEpoch-1]; ok { + srv.Write(vot.SharedRandomValue) + } else { + srv.Write(zeros) + } + return srv.Sum(nil), nil +} + +func (s *state) generateTopology(nodeList []*pki.MixDescriptor, doc *pki.Document, srv []byte) [][]*pki.MixDescriptor { + s.log.Debugf("Generating mix topology.") + + nodeMap := make(map[[constants.NodeIDLength]byte]*pki.MixDescriptor) + for _, v := range nodeList { + id := hash.Sum256(v.IdentityKey) + nodeMap[id] = v + } + + // TODO: consider strategies for balancing topology? Should this happen automatically? + // the current strategy will rebalance by limiting the number of nodes that are + // (re)inserted at each layer and placing these nodes into another layer. + + // Since there is an existing network topology, use that as the basis for + // generating the mix topology such that the number of nodes per layer is + // approximately equal, and as many nodes as possible retain their existing + // layer assignment to minimise network churn. + // The srv is used, when available, to ensure the ordering of new nodes + // is deterministic between authorities + rng, err := rand.NewDeterministicRandReader(srv[:]) + if err != nil { + s.log.Errorf("DeterministicRandReader() failed to initialize: %v", err) + s.s.fatalErrCh <- err + } + targetNodesPerLayer := len(nodeList) / s.s.cfg.Debug.Layers + topology := make([][]*pki.MixDescriptor, s.s.cfg.Debug.Layers) + + // Assign nodes that still exist up to the target size. + for layer, nodes := range doc.Topology { + nodeIndexes := rng.Perm(len(nodes)) + + for _, idx := range nodeIndexes { + if len(topology[layer]) >= targetNodesPerLayer { + break + } + + id := hash.Sum256(nodes[idx].IdentityKey) + if n, ok := nodeMap[id]; ok { + // There is a new descriptor with the same identity key, + // as an existing descriptor in the previous document, + // so preserve the layering. + topology[layer] = append(topology[layer], n) + delete(nodeMap, id) + } + } + } + + // Flatten the map containing the nodes pending assignment. + toAssign := make([]*pki.MixDescriptor, 0, len(nodeMap)) + for _, n := range nodeMap { + toAssign = append(toAssign, n) + } + // must sort toAssign by ID! + sortNodesByPublicKey(toAssign) + + assignIndexes := rng.Perm(len(toAssign)) + + // Fill out any layers that are under the target size, by + // randomly assigning from the pending list. + idx := 0 + for layer := range doc.Topology { + for len(topology[layer]) < targetNodesPerLayer { + n := toAssign[assignIndexes[idx]] + topology[layer] = append(topology[layer], n) + idx++ + } + } + + // Assign the remaining nodes. + for layer := 0; idx < len(assignIndexes); idx++ { + n := toAssign[assignIndexes[idx]] + topology[layer] = append(topology[layer], n) + layer++ + layer = layer % len(topology) + } + + return topology +} + +// generateFixedTopology returns an array of layers, which are an array of raw descriptors +// topology is represented as an array of arrays where the contents are the raw descriptors +// because a mix that does not submit a descriptor must not be in the consensus, the topology section must be populated at runtime and checked for sanity before a consensus is made +func (s *state) generateFixedTopology(nodes []*pki.MixDescriptor, srv []byte) [][]*pki.MixDescriptor { + nodeMap := make(map[[constants.NodeIDLength]byte]*pki.MixDescriptor) + // collect all of the identity keys from the current set of descriptors + for _, v := range nodes { + id := hash.Sum256(v.IdentityKey) + nodeMap[id] = v + } + + pkiSignatureScheme := signSchemes.ByName(s.s.cfg.Server.PKISignatureScheme) + + // range over the keys in the configuration file and collect the descriptors for each layer + topology := make([][]*pki.MixDescriptor, len(s.s.cfg.Topology.Layers)) + for strata, layer := range s.s.cfg.Topology.Layers { + for _, node := range layer.Nodes { + + var identityPublicKey sign.PublicKey + var err error + if filepath.IsAbs(node.IdentityPublicKeyPem) { + identityPublicKey, err = signpem.FromPublicPEMFile(node.IdentityPublicKeyPem, pkiSignatureScheme) + if err != nil { + panic(err) + } + } else { + pemFilePath := filepath.Join(s.s.cfg.Server.DataDir, node.IdentityPublicKeyPem) + identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) + if err != nil { + panic(err) + } + } + + id := hash.Sum256From(identityPublicKey) + + // if the listed node is in the current descriptor set, place it in the layer + if n, ok := nodeMap[id]; ok { + topology[strata] = append(topology[strata], n) + } + } + } + return topology +} + +func (s *state) generateRandomTopology(nodes []*pki.MixDescriptor, srv []byte) [][]*pki.MixDescriptor { + s.log.Debugf("Generating random mix topology.") + + // If there is no node history in the form of a previous consensus, + // then the simplest thing to do is to randomly assign nodes to the + // various layers. + + if len(srv) != 32 { + err := errors.New("SharedRandomValue too short") + s.log.Errorf("srv: %s", srv) + s.s.fatalErrCh <- err + } + rng, err := rand.NewDeterministicRandReader(srv[:]) + if err != nil { + s.log.Errorf("DeterministicRandReader() failed to initialize: %v", err) + s.s.fatalErrCh <- err + } + + nodeIndexes := rng.Perm(len(nodes)) + topology := make([][]*pki.MixDescriptor, s.s.cfg.Debug.Layers) + for idx, layer := 0, 0; idx < len(nodes); idx++ { + n := nodes[nodeIndexes[idx]] + topology[layer] = append(topology[layer], n) + layer++ + layer = layer % len(topology) + } + + return topology +} + +func (s *state) pruneDocuments() { + if s.TryLock() { + panic("write lock not held in pruneDocuments()") + } + + // Looking a bit into the past is probably ok, if more past documents + // need to be accessible, then methods that query the DB could always + // be added. + const preserveForPastEpochs = 3 + + now, _, _ := epochtime.Now() + cmpEpoch := now - preserveForPastEpochs + + for e := range s.documents { + if e < cmpEpoch { + delete(s.documents, e) + } + } + for e := range s.descriptors { + if e < cmpEpoch { + delete(s.descriptors, e) + } + } + for e := range s.votes { + if e < cmpEpoch { + delete(s.votes, e) + } + } + for e := range s.certificates { + if e < cmpEpoch { + delete(s.certificates, e) + } + } + for e := range s.myconsensus { + if e < cmpEpoch { + delete(s.myconsensus, e) + } + } +} + +func (s *state) isReplicaDescriptorAuthorized(desc *pki.ReplicaDescriptor) bool { + pk := hash.Sum256(desc.IdentityKey) + name, ok := s.authorizedReplicaNodes[pk] + if !ok { + return false + } + return name == desc.Name +} + +func (s *state) isDescriptorAuthorized(desc *pki.MixDescriptor) bool { + pk := hash.Sum256(desc.IdentityKey) + if !desc.IsGatewayNode && !desc.IsServiceNode { + return s.authorizedMixes[pk] + } + if desc.IsGatewayNode { + name, ok := s.authorizedGatewayNodes[pk] + if !ok { + return false + } + return name == desc.Name + } + if desc.IsServiceNode { + name, ok := s.authorizedServiceNodes[pk] + if !ok { + return false + } + return name == desc.Name + } + panic("impossible") +} + +func (s *state) dupSig(sig commands.Sig) bool { + if _, ok := s.signatures[s.votingEpoch][hash.Sum256From(sig.PublicKey)]; ok { + return true + } + return false +} + +func (s *state) dupVote(vote commands.Vote) bool { + if _, ok := s.votes[s.votingEpoch][hash.Sum256From(vote.PublicKey)]; ok { + return true + } + return false +} + +// a certificate is a vote that has a full set of sharedrandom commit and reveals as seen by the peer +func (s *state) onCertUpload(certificate *commands.Cert) commands.Command { + s.Lock() + defer s.Unlock() + resp := commands.CertStatus{} + pk := hash.Sum256From(certificate.PublicKey) - for e := range s.documents { - if e < cmpEpoch { - delete(s.documents, e) - } + // if not authorized + _, ok := s.authorizedAuthorities[pk] + if !ok { + s.log.Error("Voter not authorized.") + resp.ErrorCode = commands.CertNotAuthorized + return &resp } - for e := range s.descriptors { - if e < cmpEpoch { - delete(s.descriptors, e) + + // XXX: this ought to use state, to prevent out-of-order protocol events, in case + // we have any bugs in our implmementation + if certificate.Epoch < s.votingEpoch { + s.log.Errorf("Certificate from %s received too early: %d < %d", s.authorityNames[pk], certificate.Epoch, s.votingEpoch) + resp.ErrorCode = commands.CertTooEarly + return &resp + } + if certificate.Epoch > s.votingEpoch { + s.log.Errorf("Certificate from %s too late: %d > %d", s.authorityNames[pk], certificate.Epoch, s.votingEpoch) + resp.ErrorCode = commands.CertTooLate + return &resp + } + + // ensure certificate.PublicKey verifies the payload (ie Vote has a signature from this peer) + _, err := cert.Verify(certificate.PublicKey, certificate.Payload) + if err != nil { + s.log.Error("Certificate from %s failed to verify.", s.authorityNames[pk]) + resp.ErrorCode = commands.CertNotSigned + return &resp + } + + // verify the structure of the certificate + doc, err := s.doParseDocument(certificate.Payload) + if err != nil { + s.log.Error("Certficate from %s failed to verify: %s", s.authorityNames[pk], certificate.PublicKey, err) + resp.ErrorCode = commands.CertNotSigned + return &resp + } + + // haven't received a vote from this peer yet for this epoch + if _, ok := s.votes[s.votingEpoch][pk]; !ok { + s.log.Errorf("Certficate from %s received before peer's vote?.", s.authorityNames[pk]) + resp.ErrorCode = commands.CertTooEarly + return &resp + } + + // the first certificate received this round + if _, ok := s.certificates[s.votingEpoch]; !ok { + s.certificates[s.votingEpoch] = make(map[[publicKeyHashSize]byte]*pki.Document) + } + + // already received a certificate for this round + if _, ok := s.certificates[s.votingEpoch][pk]; ok { + s.log.Error("Another Cert received from peer %s", s.authorityNames[pk]) + resp.ErrorCode = commands.CertAlreadyReceived + return &resp + } + s.log.Noticef("Cert OK from: %s\n%s", s.authorityNames[pk], doc) + s.certificates[s.votingEpoch][pk] = doc + resp.ErrorCode = commands.CertOk + return &resp +} + +func (s *state) onRevealUpload(reveal *commands.Reveal) commands.Command { + s.Lock() + defer s.Unlock() + resp := commands.RevealStatus{} + pk := hash.Sum256From(reveal.PublicKey) + + // if not authorized + _, ok := s.authorizedAuthorities[pk] + if !ok { + s.log.Error("Voter not authorized.") + resp.ErrorCode = commands.RevealNotAuthorized + return &resp + } + + // verify the signature on the payload + certified, err := cert.Verify(reveal.PublicKey, reveal.Payload) + if err != nil { + s.log.Error("Reveal from %s failed to verify.", s.authorityNames[pk]) + resp.ErrorCode = commands.RevealNotSigned + return &resp + } + + e := epochFromBytes(certified[:8]) + // received too late + if e < s.votingEpoch { + s.log.Errorf("Reveal from %s received too late: %d < %d", s.authorityNames[pk], e, s.votingEpoch) + resp.ErrorCode = commands.RevealTooLate + return &resp + } + + // received too early + if e > s.votingEpoch { + s.log.Errorf("Reveal from %s received too early: %d > %d", s.authorityNames[pk], e, s.votingEpoch) + resp.ErrorCode = commands.RevealTooEarly + return &resp + } + + // haven't received a commit yet for this epoch + if _, ok := s.commits[s.votingEpoch]; !ok { + s.log.Errorf("Reveal from %s received before any commit.", s.authorityNames[pk]) + resp.ErrorCode = commands.RevealTooEarly + return &resp + } + + // haven't received a commit from this peer yet for this epoch + if _, ok := s.commits[s.votingEpoch][pk]; !ok { + s.log.Errorf("Reveal from %s received before peer's vote.", s.authorityNames[pk]) + resp.ErrorCode = commands.RevealTooEarly + return &resp + } + + // the first reveal received this round + if _, ok := s.reveals[s.votingEpoch]; !ok { + s.reveals[s.votingEpoch] = make(map[[publicKeyHashSize]byte][]byte) + } + + // already received a reveal for this round + if _, ok := s.reveals[s.votingEpoch][pk]; ok { + s.log.Errorf("Reveal from %s already received", s.authorityNames[pk]) + resp.ErrorCode = commands.RevealAlreadyReceived + return &resp + } + s.log.Noticef("Reveal OK from: %s\n%x", s.authorityNames[pk], certified) + s.reveals[s.votingEpoch][pk] = reveal.Payload + resp.ErrorCode = commands.RevealOk + return &resp +} + +func (s *state) onVoteUpload(vote *commands.Vote) commands.Command { + s.Lock() + defer s.Unlock() + resp := commands.VoteStatus{} + pk := hash.Sum256From(vote.PublicKey) + + // if not authorized + _, ok := s.authorizedAuthorities[pk] + if !ok { + s.log.Error("Voter not authorized.") + resp.ErrorCode = commands.VoteNotAuthorized + return &resp + } + + // XXX: this ought to use state, to prevent out-of-order protocol events, in case + // we have any bugs in our implmementation + if vote.Epoch < s.votingEpoch { + s.log.Errorf("Vote from %s received too early: %d < %d", s.authorityNames[pk], vote.Epoch, s.votingEpoch) + resp.ErrorCode = commands.VoteTooEarly + return &resp + } + if vote.Epoch > s.votingEpoch { + s.log.Errorf("Vote from %s received too late: %d > %d", s.authorityNames[pk], vote.Epoch, s.votingEpoch) + resp.ErrorCode = commands.VoteTooLate + return &resp + } + + // haven't received a vote yet for this epoch + if _, ok := s.votes[s.votingEpoch]; !ok { + s.votes[s.votingEpoch] = make(map[[publicKeyHashSize]byte]*pki.Document) + } + + // haven't received a commit yet for this epoch + if _, ok = s.commits[s.votingEpoch]; !ok { + s.commits[s.votingEpoch] = make(map[[publicKeyHashSize]byte][]byte) + } + + // peer has already voted for this epoch + _, ok = s.votes[s.votingEpoch][pk] + if ok { + s.log.Errorf("Vote from %s already received", s.authorityNames[pk]) + resp.ErrorCode = commands.VoteAlreadyReceived + return &resp + } + + // ensure vote.PublicKey verifies the payload (ie Vote has a signature from this peer) + _, err := cert.Verify(vote.PublicKey, vote.Payload) + if err != nil { + s.log.Errorf("Vote from %s failed to verify.", s.authorityNames[pk]) + resp.ErrorCode = commands.VoteNotSigned + return &resp + } + + doc, err := s.doParseDocument(vote.Payload) + if err != nil { + s.log.Errorf("Vote from %s failed signature verification.", s.authorityNames[pk]) + resp.ErrorCode = commands.VoteNotSigned + return &resp + } + + // Check that the deserialiezd payload was signed for the correct Epoch + if doc.Epoch != s.votingEpoch { + s.log.Errorf("Vote from %s contains wrong Epoch %d", s.authorityNames[pk], doc.Epoch) + resp.ErrorCode = commands.VoteMalformed + return &resp + } + + // extract commit from document and verify that it was signed by this peer + // IsDocumentWellFormed has already verified that any commit is for + // this Epoch and is signed by a known verifier + commit, ok := doc.SharedRandomCommit[pk] + if !ok { + // It's possible that an authority submitted another authoritys vote on its behalf, + // but we are not going to allow that behavior as it is not specified. + s.log.Error("Vote from %s did not contain SharedRandom Commit.", s.authorityNames[pk]) + resp.ErrorCode = commands.VoteMalformed + return &resp + } + // save the vote + s.votes[s.votingEpoch][pk] = doc + // save the commit + s.commits[s.votingEpoch][pk] = commit + s.log.Noticef("Vote OK from: %s\n%s", s.authorityNames[pk], doc) + resp.ErrorCode = commands.VoteOk + return &resp +} + +func (s *state) onSigUpload(sig *commands.Sig) commands.Command { + s.Lock() + defer s.Unlock() + resp := commands.SigStatus{} + pk := hash.Sum256From(sig.PublicKey) + + _, ok := s.authorizedAuthorities[pk] + if !ok { + s.log.Error("Signature not authorized.") + resp.ErrorCode = commands.SigNotAuthorized + return &resp + } + if sig.Epoch < s.votingEpoch { + s.log.Errorf("Signature from %s received too early: %d < %d", s.authorityNames[pk], sig.Epoch, s.votingEpoch) + resp.ErrorCode = commands.SigTooEarly + return &resp + } + if sig.Epoch > s.votingEpoch { + s.log.Errorf("Signature from %s received too late: %d > %d", s.authorityNames[pk], sig.Epoch, s.votingEpoch) + resp.ErrorCode = commands.SigTooLate + return &resp + } + verified, err := cert.Verify(sig.PublicKey, sig.Payload) + if err != nil { + s.log.Error("Sig failed signature verification.") + resp.ErrorCode = commands.SigNotSigned + return &resp + } + + // haven't received a sig yet for this epoch + if _, ok := s.signatures[s.votingEpoch]; !ok { + s.signatures[s.votingEpoch] = make(map[[publicKeyHashSize]byte]*cert.Signature) + } + + // peer has not yet submitted a signature + if !s.dupSig(*sig) { + csig := new(cert.Signature) + err := csig.Unmarshal(verified) + if err != nil { + resp.ErrorCode = commands.SigInvalid + s.log.Errorf("Signature failed to deserialize from: %s", s.authorityNames[pk]) + return &resp } + s.log.Noticef("Signature OK from: %s", s.authorityNames[pk]) + s.signatures[s.votingEpoch][hash.Sum256From(sig.PublicKey)] = csig + resp.ErrorCode = commands.SigOk + return &resp + } else { + // peer is behaving strangely + // error; two sigs from same peer + s.log.Error("Sig command invalid: more than one sig from same peer is not allowed.") + resp.ErrorCode = commands.SigAlreadyReceived + return &resp } } -// Ensure that the descriptor is from the local registered node -func (s *state) isDescriptorAuthorized(desc *pki.MixDescriptor) bool { - node := s.authorizedNode +func (s *state) onReplicaDescriptorUpload(rawDesc []byte, desc *pki.ReplicaDescriptor, epoch uint64) error { + s.Lock() + defer s.Unlock() + // Note: Caller ensures that the epoch is the current epoch +- 1. pk := hash.Sum256(desc.IdentityKey) - if pk != hash.Sum256(node.IdentityKey) { - s.log.Debugf("pki: ❌ isDescriptorAuthorized: IdentityKey mismatch for node %s", desc.Name) - return false + + // Get the public key -> descriptor map for the epoch. + _, ok := s.replicaDescriptors[epoch] + if !ok { + s.replicaDescriptors[epoch] = make(map[[publicKeyHashSize]byte]*pki.ReplicaDescriptor) } - if desc.IsGatewayNode != node.IsGatewayNode { - return false + // Check for redundant uploads. + d, ok := s.replicaDescriptors[epoch][pk] + if ok { + // If the descriptor changes, then it will be rejected to prevent + // nodes from reneging on uploads. + serialized, err := d.Marshal() + if err != nil { + return err + } + if !hmac.Equal(serialized, rawDesc) { + return fmt.Errorf("state: node %s (%x): Conflicting descriptor for epoch %v", desc.Name, hash.Sum256(desc.IdentityKey), epoch) + } + + // Redundant uploads that don't change are harmless. + return nil } - if desc.IsServiceNode != node.IsServiceNode { - return false + // Ok, this is a new descriptor. + if s.documents[epoch] != nil { + // If there is a document already, the descriptor is late, and will + // never appear in a document, so reject it. + return fmt.Errorf("state: Node %v: Late descriptor upload for for epoch %v", desc.IdentityKey, epoch) + } + + // Persist the raw descriptor to disk. + if err := s.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(replicaDescriptorsBucket)) + eBkt, err := bkt.CreateBucketIfNotExists(epochToBytes(epoch)) + if err != nil { + return err + } + return eBkt.Put(pk[:], rawDesc) + }); err != nil { + // Persistence failures are FATAL. + s.s.fatalErrCh <- err } - return true + // Store the parsed descriptor + s.replicaDescriptors[epoch][pk] = desc + + s.log.Noticef("Node %x: Successfully submitted replica descriptor for epoch %v.", pk, epoch) + s.onUpdate() + return nil } func (s *state) onDescriptorUpload(rawDesc []byte, desc *pki.MixDescriptor, epoch uint64) error { - s.log.Noticef("pki: ⭐ onDescriptorUpload; Node name=%v, epoch=%v", desc.Name, epoch) s.Lock() defer s.Unlock() @@ -467,26 +1761,29 @@ func (s *state) onDescriptorUpload(rawDesc []byte, desc *pki.MixDescriptor, epoc return fmt.Errorf("state: Node %v: Late descriptor upload for for epoch %v", desc.IdentityKey, epoch) } + // Persist the raw descriptor to disk. + if err := s.db.Update(func(tx *bolt.Tx) error { + bkt := tx.Bucket([]byte(descriptorsBucket)) + eBkt, err := bkt.CreateBucketIfNotExists(epochToBytes(epoch)) + if err != nil { + return err + } + return eBkt.Put(pk[:], rawDesc) + }); err != nil { + // Persistence failures are FATAL. + s.s.fatalErrCh <- err + } + // Store the parsed descriptor s.descriptors[epoch][pk] = desc s.log.Noticef("Node %x: Successfully submitted descriptor for epoch %v.", pk, epoch) + s.onUpdate() return nil } -func (s *state) submitDescriptorToAppchain(desc *pki.MixDescriptor, epoch uint64) { - // Register the mix descriptor with the appchain, which will: - // - reject redundant descriptors (even those that didn't change) - // - reject descriptors if document for the epoch exists - if err := s.chPKISetMixDescriptor(desc, epoch); err != nil { - s.log.Errorf("❌ submitDescriptorToAppchain: Failed to set mix descriptor for node %v, epoch=%v: %v", desc.Name, epoch, err) - } - epochCurrent, _, _ := epochtime.Now() - s.log.Noticef("✅ submitDescriptorToAppchain: Submitted descriptor to appchain for node %v, epoch=%v (in epoch=%v)", desc.Name, epoch, epochCurrent) -} - func (s *state) documentForEpoch(epoch uint64) ([]byte, error) { - s.log.Debugf("pki: documentForEpoch(%v)", epoch) + var generationDeadline = 7 * (epochtime.Period / 8) s.RLock() defer s.RUnlock() @@ -508,8 +1805,8 @@ func (s *state) documentForEpoch(epoch uint64) ([]byte, error) { case now + 1: // If it's past the time by which we should have generated a document // then we will never be able to service this. - if elapsed > DocGenerationDeadline { - s.log.Errorf("No document for next epoch %v and it's already past DocGenerationDeadline of previous epoch", now+1) + if elapsed > generationDeadline { + s.log.Errorf("No document for next epoch %v and it's already past 7/8 of previous epoch", now+1) return nil, errGone } return nil, errNotYet @@ -526,103 +1823,295 @@ func (s *state) documentForEpoch(epoch uint64) ([]byte, error) { // NOTREACHED } +func (s *state) restorePersistence() error { + const ( + metadataBucket = "metadata" + versionKey = "version" + ) + + return s.db.Update(func(tx *bolt.Tx) error { + // Ensure that all the buckets exist. + bkt, err := tx.CreateBucketIfNotExists([]byte(metadataBucket)) + if err != nil { + return err + } + descsBkt, err := tx.CreateBucketIfNotExists([]byte(descriptorsBucket)) + if err != nil { + return err + } + replicaDescsBkt, err := tx.CreateBucketIfNotExists([]byte(replicaDescriptorsBucket)) + if err != nil { + return err + } + docsBkt, err := tx.CreateBucketIfNotExists([]byte(documentsBucket)) + if err != nil { + return err + } + + if b := bkt.Get([]byte(versionKey)); b != nil { + // Well it looks like we loaded as opposed to created. + if len(b) != 1 || b[0] != 0 { + return fmt.Errorf("state: incompatible version: %d", uint(b[0])) + } + + // Figure out which epochs to restore for. + now, _, _ := epochtime.Now() + epochs := []uint64{now - 1, now, now + 1} + + // Restore the replica descriptors. + for _, epoch := range epochs { + epochBytes := epochToBytes(epoch) + eDescsBkt := replicaDescsBkt.Bucket(epochBytes) + if eDescsBkt == nil { + s.log.Debugf("No persisted Descriptors for epoch: %v.", epoch) + continue + } + c := eDescsBkt.Cursor() + for wantHash, rawDesc := c.First(); wantHash != nil; wantHash, rawDesc = c.Next() { + if len(wantHash) != publicKeyHashSize { + panic("stored hash should be 32 bytes") + } + desc := new(pki.ReplicaDescriptor) + err := desc.Unmarshal(rawDesc) + if err != nil { + s.log.Errorf("Failed to validate persisted descriptor: %v", err) + continue + } + idHash := hash.Sum256(desc.IdentityKey) + if !hmac.Equal(wantHash, idHash[:]) { + s.log.Errorf("Discarding persisted descriptor: key mismatch") + continue + } + + if !s.isReplicaDescriptorAuthorized(desc) { + s.log.Warningf("Discarding persisted descriptor: %v", desc) + continue + } + + _, ok := s.replicaDescriptors[epoch] + if !ok { + s.replicaDescriptors[epoch] = make(map[[publicKeyHashSize]byte]*pki.ReplicaDescriptor) + } + + s.replicaDescriptors[epoch][hash.Sum256(desc.IdentityKey)] = desc + s.log.Debugf("Restored replica descriptor for epoch %v: %+v", epoch, desc) + } + } + + // Restore the documents and descriptors. + for _, epoch := range epochs { + epochBytes := epochToBytes(epoch) + if rawDoc := docsBkt.Get(epochBytes); rawDoc != nil { + _, _, _, err := cert.VerifyThreshold(s.getVerifiers(), s.threshold, rawDoc) + if err != nil { + s.log.Errorf("Failed to verify threshold on restored document") + break // or continue? + } + doc, err := s.doParseDocument(rawDoc) + if err != nil { + s.log.Errorf("Failed to validate persisted document: %v", err) + } else if doc.Epoch != epoch { + // The document for the wrong epoch was persisted? + s.log.Errorf("Persisted document has unexpected epoch: %v", doc.Epoch) + } else { + s.log.Debugf("Restored Document for epoch %v: %v.", epoch, doc) + s.documents[epoch] = doc + } + } + + eDescsBkt := descsBkt.Bucket(epochBytes) + if eDescsBkt == nil { + s.log.Debugf("No persisted Descriptors for epoch: %v.", epoch) + continue + } + + c := eDescsBkt.Cursor() + for wantHash, rawDesc := c.First(); wantHash != nil; wantHash, rawDesc = c.Next() { + if len(wantHash) != publicKeyHashSize { + panic("stored hash should be 32 bytes") + } + desc := new(pki.MixDescriptor) + err := desc.UnmarshalBinary(rawDesc) + if err != nil { + s.log.Errorf("Failed to validate persisted descriptor: %v", err) + continue + } + idHash := hash.Sum256(desc.IdentityKey) + if !hmac.Equal(wantHash, idHash[:]) { + s.log.Errorf("Discarding persisted descriptor: key mismatch") + continue + } + + if !s.isDescriptorAuthorized(desc) { + s.log.Warningf("Discarding persisted descriptor: %v", desc) + continue + } + + _, ok := s.descriptors[epoch] + if !ok { + s.descriptors[epoch] = make(map[[publicKeyHashSize]byte]*pki.MixDescriptor) + } + + s.descriptors[epoch][hash.Sum256(desc.IdentityKey)] = desc + s.log.Debugf("Restored descriptor for epoch %v: %+v", epoch, desc) + } + } + return nil + } + + // We created a new database, so populate the new `metadata` bucket. + return bkt.Put([]byte(versionKey), []byte{0}) + }) +} + func newState(s *Server) (*state, error) { + const dbFile = "persistence.db" + st := new(state) st.s = s + st.geo = s.geo st.log = s.logBackend.GetLogger("state") // set voting schedule at runtime st.log.Debugf("State initialized with epoch Period: %s", epochtime.Period) - st.log.Debugf("State initialized with RandomCourtessyDelay: %s", RandomCourtessyDelay) st.log.Debugf("State initialized with MixPublishDeadline: %s", MixPublishDeadline) - st.log.Debugf("State initialized with DescriptorBlockDeadline: %s", DescriptorBlockDeadline) st.log.Debugf("State initialized with AuthorityVoteDeadline: %s", AuthorityVoteDeadline) + st.log.Debugf("State initialized with AuthorityRevealDeadline: %s", AuthorityRevealDeadline) st.log.Debugf("State initialized with PublishConsensusDeadline: %s", PublishConsensusDeadline) - st.log.Debugf("State initialized with DocGenerationDeadline: %s", DocGenerationDeadline) - - ccbor, err := cbor.CanonicalEncOptions().EncMode() - if err != nil { - panic(err) + st.verifiers = make(map[[publicKeyHashSize]byte]sign.PublicKey) + for _, auth := range s.cfg.Authorities { + st.verifiers[hash.Sum256From(auth.IdentityPublicKey)] = auth.IdentityPublicKey } - st.ccbor = ccbor + st.verifiers[hash.Sum256From(s.IdentityKey())] = sign.PublicKey(s.IdentityKey()) + st.threshold = len(st.verifiers)/2 + 1 + st.dissenters = len(s.cfg.Authorities)/2 - 1 + + st.s.cfg.Server.PKISignatureScheme = s.cfg.Server.PKISignatureScheme + pkiSignatureScheme := signSchemes.ByName(s.cfg.Server.PKISignatureScheme) + + // Initialize the authorized peer tables. + st.reverseHash = make(map[[publicKeyHashSize]byte]sign.PublicKey) + st.authorizedMixes = make(map[[publicKeyHashSize]byte]bool) + for _, v := range st.s.cfg.Mixes { + var identityPublicKey sign.PublicKey + var err error + if filepath.IsAbs(v.IdentityPublicKeyPem) { + identityPublicKey, err = signpem.FromPublicPEMFile(v.IdentityPublicKeyPem, pkiSignatureScheme) + if err != nil { + panic(err) + } + } else { + pemFilePath := filepath.Join(s.cfg.Server.DataDir, v.IdentityPublicKeyPem) + identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) + if err != nil { + panic(err) + } + } - // Init AppChain communications (chainbridge) - chainBridgeLogger := s.logBackend.GetLogger("state:chain") - st.chainBridge = chainbridge.NewChainBridge(filepath.Join(s.cfg.Server.DataDir, "appchain.sock")) - st.chainBridge.SetErrorHandler(func(err error) { - chainBridgeLogger.Errorf("Error: %v", err) - }) - st.chainBridge.SetLogHandler(func(msg string) { - chainBridgeLogger.Infof(msg) - }) - if err := st.chainBridge.Start(); err != nil { - chainBridgeLogger.Fatalf("Error: %v", err) + pk := hash.Sum256From(identityPublicKey) + st.authorizedMixes[pk] = true + st.reverseHash[pk] = identityPublicKey } + st.authorizedGatewayNodes = make(map[[publicKeyHashSize]byte]string) + for _, v := range st.s.cfg.GatewayNodes { + var identityPublicKey sign.PublicKey + var err error - // Load the authorized local node from configuration - - // return a single node configuration and its node type - extractNodeFromCfg := func() (*config.Node, bool, bool) { - if len(st.s.cfg.GatewayNodes) == 1 { - return st.s.cfg.GatewayNodes[0], true, false - } - if len(st.s.cfg.ServiceNodes) == 1 { - return st.s.cfg.ServiceNodes[0], false, true - } - if len(st.s.cfg.Mixes) == 1 { - return st.s.cfg.Mixes[0], false, false + if filepath.IsAbs(v.IdentityPublicKeyPem) { + identityPublicKey, err = signpem.FromPublicPEMFile(v.IdentityPublicKeyPem, pkiSignatureScheme) + if err != nil { + panic(err) + } + } else { + pemFilePath := filepath.Join(s.cfg.Server.DataDir, v.IdentityPublicKeyPem) + identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) + if err != nil { + panic(err) + } } - return nil, false, false - } - v, isGatewayNode, isServiceNode := extractNodeFromCfg() - if v == nil { - st.log.Fatalf("❌ Error: Invalid configuration for a single local node") + + pk := hash.Sum256From(identityPublicKey) + st.authorizedGatewayNodes[pk] = v.Identifier + st.reverseHash[pk] = identityPublicKey } + st.authorizedServiceNodes = make(map[[publicKeyHashSize]byte]string) + for _, v := range st.s.cfg.ServiceNodes { + var identityPublicKey sign.PublicKey + var err error - pkiSignatureScheme := signSchemes.ByName(st.s.cfg.Server.PKISignatureScheme) - var identityPublicKey sign.PublicKey - if filepath.IsAbs(v.IdentityPublicKeyPem) { - identityPublicKey, err = signpem.FromPublicPEMFile(v.IdentityPublicKeyPem, pkiSignatureScheme) - if err != nil { - panic(err) - } - } else { - pemFilePath := filepath.Join(st.s.cfg.Server.DataDir, v.IdentityPublicKeyPem) - identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) - if err != nil { - panic(err) + if filepath.IsAbs(v.IdentityPublicKeyPem) { + identityPublicKey, err = signpem.FromPublicPEMFile(v.IdentityPublicKeyPem, pkiSignatureScheme) + if err != nil { + panic(err) + } + } else { + pemFilePath := filepath.Join(s.cfg.Server.DataDir, v.IdentityPublicKeyPem) + identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) + if err != nil { + panic(err) + } } + + pk := hash.Sum256From(identityPublicKey) + st.authorizedServiceNodes[pk] = v.Identifier + st.reverseHash[pk] = identityPublicKey } + st.authorizedReplicaNodes = make(map[[publicKeyHashSize]byte]string) + for _, v := range st.s.cfg.StorageReplicas { + var identityPublicKey sign.PublicKey + var err error - // Node Registration: check if node is already registered before registering and rechecking - st.authorizedNode, err = st.chNodesGet(v.Identifier) - if err != nil { - if err := st.chNodesRegister(v, identityPublicKey, isGatewayNode, isServiceNode); err != nil { - st.log.Fatalf("❌ Error: node registration failed:", err) - } - time.Sleep(time.Duration(1) * time.Second) - st.authorizedNode, err = st.chNodesGet(v.Identifier) - if err != nil { - s.log.Fatalf("❌ Error: Failed to get node=%s from appchain: %v", v.Identifier, err) + if filepath.IsAbs(v.IdentityPublicKeyPem) { + identityPublicKey, err = signpem.FromPublicPEMFile(v.IdentityPublicKeyPem, pkiSignatureScheme) + if err != nil { + panic(err) + } + } else { + pemFilePath := filepath.Join(s.cfg.Server.DataDir, v.IdentityPublicKeyPem) + identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) + if err != nil { + panic(err) + } } - } - // Ensure node appchain registration matches the local node configuration - pk := hash.Sum256From(identityPublicKey) - if pk != hash.Sum256(st.authorizedNode.IdentityKey) { - s.log.Fatalf("❌ Error: IdentityKey mismatch between node registration and configuration") + pk := hash.Sum256From(identityPublicKey) + st.authorizedReplicaNodes[pk] = v.Identifier } - st.log.Noticef("✅ Node registered with Identifier '%s', Identity key hash '%x'", v.Identifier, pk) - - st.log.Debugf("State initialized with epoch Period: %s", epochtime.Period) + st.authorizedAuthorities = make(map[[publicKeyHashSize]byte]bool) + st.authorityLinkKeys = make(map[[publicKeyHashSize]byte]kem.PublicKey) + st.authorityNames = make(map[[publicKeyHashSize]byte]string) + for _, v := range st.s.cfg.Authorities { + pk := hash.Sum256From(v.IdentityPublicKey) + st.authorizedAuthorities[pk] = true + st.authorityLinkKeys[pk] = v.LinkPublicKey + st.reverseHash[pk] = v.IdentityPublicKey + st.authorityNames[pk] = v.Identifier + } + st.reverseHash[hash.Sum256From(st.s.identityPublicKey)] = st.s.identityPublicKey st.documents = make(map[uint64]*pki.Document) + st.myconsensus = make(map[uint64]*pki.Document) st.descriptors = make(map[uint64]map[[publicKeyHashSize]byte]*pki.MixDescriptor) - - epoch, elapsed, nextEpoch := epochtime.Now() - st.log.Debugf("Epoch: %d, elapsed: %s, remaining time: %s", epoch, elapsed, nextEpoch) + st.replicaDescriptors = make(map[uint64]map[[publicKeyHashSize]byte]*pki.ReplicaDescriptor) + st.votes = make(map[uint64]map[[publicKeyHashSize]byte]*pki.Document) + st.certificates = make(map[uint64]map[[publicKeyHashSize]byte]*pki.Document) + st.reveals = make(map[uint64]map[[publicKeyHashSize]byte][]byte) + st.signatures = make(map[uint64]map[[publicKeyHashSize]byte]*cert.Signature) + st.commits = make(map[uint64]map[[publicKeyHashSize]byte][]byte) + st.priorSRV = make([][]byte, 0) + + // Initialize the persistence store and restore state. + dbPath := filepath.Join(s.cfg.Server.DataDir, dbFile) + var err error + if st.db, err = bolt.Open(dbPath, 0600, nil); err != nil { + return nil, err + } + if err = st.restorePersistence(); err != nil { + st.db.Close() + return nil, err + } // Set the initial state to bootstrap st.state = stateBootstrap @@ -634,13 +2123,36 @@ func (s *state) backgroundFetchConsensus(epoch uint64) { panic("write lock not held in backgroundFetchConsensus(epoch)") } - // If there isn't a consensus for the previous epoch, ask the appchain for a consensus. + // If there isn't a consensus for the previous epoch, ask the other + // authorities for a consensus. _, ok := s.documents[epoch] if !ok { + kemscheme := schemes.ByName(s.s.cfg.Server.WireKEMScheme) + if kemscheme == nil { + panic("kem scheme not found in registry") + } + pkiSignatureScheme := signSchemes.ByName(s.s.cfg.Server.PKISignatureScheme) + if pkiSignatureScheme == nil { + panic("pki signature scheme not found in registry") + } s.Go(func() { - doc, err := s.chPKIGetDocument(epoch) + cfg := &client.Config{ + KEMScheme: kemscheme, + PKISignatureScheme: pkiSignatureScheme, + LinkKey: s.s.linkKey, + LogBackend: s.s.logBackend, + Authorities: s.s.cfg.Authorities, + DialContextFn: nil, + Geo: s.geo, + } + c, err := client.New(cfg) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) + defer cancel() + doc, _, err := c.Get(ctx, epoch) if err != nil { - s.log.Debugf("pki: FetchConsensus: Failed to fetch document for epoch %v: %v", epoch, err) return } s.Lock() @@ -650,19 +2162,22 @@ func (s *state) backgroundFetchConsensus(epoch uint64) { // if backgroundFetchConsensus was called // multiple times during bootstrapping if _, ok := s.documents[epoch]; !ok { - // sign the locally-stored document - _, err := s.doSignDocument(s.s.identityPrivateKey, s.s.identityPublicKey, doc) - if err != nil { - s.log.Errorf("pki: FetchConsensus: Error signing document for epoch %v: %v", epoch, err) - return - } s.documents[epoch] = doc - s.log.Debugf("pki: FetchConsensus: ✅ Set doc for epoch %v: %s", epoch, doc.String()) } }) } } +func epochToBytes(e uint64) []byte { + ret := make([]byte, 8) + binary.BigEndian.PutUint64(ret, e) + return ret +} + +func epochFromBytes(b []byte) uint64 { + return binary.BigEndian.Uint64(b[0:8]) +} + func sortNodesByPublicKey(nodes []*pki.MixDescriptor) { dTos := func(d *pki.MixDescriptor) string { pk := hash.Sum256(d.IdentityKey) @@ -670,3 +2185,62 @@ func sortNodesByPublicKey(nodes []*pki.MixDescriptor) { } sort.Slice(nodes, func(i, j int) bool { return dTos(nodes[i]) < dTos(nodes[j]) }) } + +func sha256b64(raw []byte) string { + hash := blake2b.Sum256(raw) + return base64.StdEncoding.EncodeToString(hash[:]) +} + +// validate the topology +func (s *state) verifyTopology(topology [][]*pki.MixDescriptor) error { + if len(topology) < s.s.cfg.Debug.Layers { + return errInvalidTopology + } + + for strata, _ := range topology { + if len(topology[strata]) < s.s.cfg.Debug.MinNodesPerLayer { + return errInvalidTopology + } + } + return nil +} + +// generate commit and reveal values and save them +func (s *state) doCommit(epoch uint64) ([]byte, error) { + s.log.Debugf("Generating SharedRandom Commit for %d", epoch) + srv := new(pki.SharedRandom) + commit, err := srv.Commit(epoch) + if err != nil { + return nil, err + } + // sign the serialized commit + signedCommit, err := cert.Sign(s.s.identityPrivateKey, s.s.identityPublicKey, commit, epoch) + if err != nil { + return nil, err + } + // sign the reveal + signedReveal, err := cert.Sign(s.s.identityPrivateKey, s.s.identityPublicKey, srv.Reveal(), epoch) + if err != nil { + return nil, err + } + // save our commit + if _, ok := s.commits[epoch]; !ok { + s.commits[epoch] = make(map[[pki.PublicKeyHashSize]byte][]byte) + } + s.commits[epoch][s.identityPubKeyHash()] = signedCommit + + // save our reveal + if _, ok := s.reveals[epoch]; !ok { + s.reveals[epoch] = make(map[[pki.PublicKeyHashSize]byte][]byte) + } + s.reveals[epoch][s.identityPubKeyHash()] = signedReveal + return signedCommit, nil +} + +func (s *state) reveal(epoch uint64) []byte { + signed, ok := s.reveals[epoch][s.identityPubKeyHash()] + if !ok { + s.s.fatalErrCh <- errors.New("reveal() called without commit") + } + return signed +} diff --git a/pki/server/state_chain_comm.go b/pki/server/state_chain_comm.go deleted file mode 100644 index 6d4e754..0000000 --- a/pki/server/state_chain_comm.go +++ /dev/null @@ -1,176 +0,0 @@ -// AppChain communication (chainbridge) functions - -package server - -import ( - "fmt" - - "github.com/fxamacker/cbor/v2" - "github.com/katzenpost/hpqc/sign" - "github.com/katzenpost/katzenpost/core/pki" - - "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" - "github.com/ZeroKnowledgeNetwork/opt/pki/server/config" -) - -func (s *state) chNodesGet(name string) (*chainbridge.Node, error) { - chCommand := fmt.Sprintf(chainbridge.Cmd_nodes_getNode, name) - chResponse, err := s.chainBridge.Command(chCommand, nil) - if err != nil { - return nil, fmt.Errorf("state: ChainBridge command error: %v", err) - } - - var node chainbridge.Node - if err = s.chainBridge.DataUnmarshal(chResponse, &node); err != nil { - return nil, err - } - - return &node, nil -} - -func (st *state) chNodesRegister(v *config.Node, identityPublicKey sign.PublicKey, isGatewayNode bool, isServiceNode bool) error { - payload, err := identityPublicKey.MarshalBinary() - if err != nil { - return fmt.Errorf("failed to marshal identityPublicKey: %v", err) - } - chCommand := fmt.Sprintf( - chainbridge.Cmd_nodes_register, - v.Identifier, - chainbridge.Bool2int(isGatewayNode), - chainbridge.Bool2int(isServiceNode)) - chResponse, err := st.chainBridge.Command(chCommand, payload) - st.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) - if err != nil { - return fmt.Errorf("ChainBridge command error: %v", err) - } - if chResponse.Error != "" && chResponse.Error != chainbridge.Err_nodes_alreadyRegistered { - return fmt.Errorf("ChainBridge response error: %v", chResponse.Error) - } - - return nil -} - -func (s *state) chPKIGetGenesisEpoch() (uint64, error) { - chResponse, err := s.chainBridge.Command(chainbridge.Cmd_pki_getGenesisEpoch, nil) - if err != nil { - return 0, fmt.Errorf("state: ChainBridge command error: %v", err) - } - genesisEpoch, err := s.chainBridge.GetDataUInt(chResponse) - if err != nil { - return 0, fmt.Errorf("state: ChainBridge data error: %v", err) - } - return genesisEpoch, nil -} - -func (s *state) chPKIGetDocument(epoch uint64) (*pki.Document, error) { - chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getDocucment, epoch) - chResponse, err := s.chainBridge.Command(chCommand, nil) - if err != nil { - return nil, fmt.Errorf("state: ChainBridge command error: %v", err) - } - - chDoc, err := s.chainBridge.GetDataBytes(chResponse) - if err != nil { - return nil, err - } - - var doc pki.Document - // X: if err = doc.UnmarshalCertificate(chDoc); err != nil { - if err = cbor.Unmarshal(chDoc, (*pki.Document)(&doc)); err != nil { - return nil, fmt.Errorf("state: failed to unmarshal PKI document: %v", err) - } - - return &doc, nil -} - -// register the PKI doc with the appchain -func (s *state) chPKISetDocument(doc *pki.Document) error { - // register with the appchain an unsigned certificate-less doc, - // so authorities submit the same doc hash as their vote - // X: payload, err := doc.MarshalCertificate() - payload, err := s.ccbor.Marshal((*pki.Document)(doc)) - if err != nil { - return err - } - - if err != nil { - return fmt.Errorf("state: failed to marshal PKI document: %v", err) - } - chCommand := fmt.Sprintf(chainbridge.Cmd_pki_setDocument, doc.Epoch) - chResponse, err := s.chainBridge.Command(chCommand, payload) - s.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) - if err != nil { - return fmt.Errorf("state: ChainBridge command error: %v", err) - } - - // ignore the most likely chResponse.Error: "Document already exists for the epoch" - // if chResponse.Error != "" { - // return fmt.Errorf("state: ChainBridge response error: %v", chResponse.Error) - // } - - return nil -} - -// get number of descriptors for the given epoch from appchain -func (s *state) chPKIGetMixDescriptorCounter(epoch uint64) (uint64, error) { - chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getMixDescriptorCounter, epoch) - chResponse, err := s.chainBridge.Command(chCommand, nil) - if err != nil { - return 0, fmt.Errorf("state: ChainBridge command error: %v", err) - } - numDescriptors, err := s.chainBridge.GetDataUInt(chResponse) - if err != nil && err != chainbridge.ErrNoData { - return 0, fmt.Errorf("state: ChainBridge data error: %v", err) - } - return numDescriptors, nil -} - -func (s *state) chPKIGetMixDescriptors(epoch uint64) ([]*pki.MixDescriptor, error) { - numDescriptors, err := s.chPKIGetMixDescriptorCounter(epoch) - if err != nil { - return nil, err - } - - descriptors := []*pki.MixDescriptor{} - for i := 0; i < int(numDescriptors); i++ { - chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getMixDescriptorByIndex, epoch, i) - chResponse, err := s.chainBridge.Command(chCommand, nil) - if err != nil { - s.log.Error("ChainBridge command error: %v", err) - continue - } - dataAsBytes, err := s.chainBridge.GetDataBytes(chResponse) - if err != nil { - s.log.Error("ChainBridge data error: %v", err) - continue - } - var desc pki.MixDescriptor - if err = desc.UnmarshalBinary(dataAsBytes); err != nil { - s.log.Error("Failed to unmarshal descriptor: %v", err) - continue - } - descriptors = append(descriptors, &desc) - } - - return descriptors, nil -} - -// Register the mix descriptor with the appchain, which will: -// - reject redundant descriptors (even those that didn't change) -// - reject descriptors if document for the epoch exists -func (s *state) chPKISetMixDescriptor(desc *pki.MixDescriptor, epoch uint64) error { - payload, err := desc.MarshalBinary() - if err != nil { - return fmt.Errorf("state: failed to marshal descriptor: %v", err) - } - chCommand := fmt.Sprintf(chainbridge.Cmd_pki_setMixDescriptor, epoch, desc.Name) - chResponse, err := s.chainBridge.Command(chCommand, payload) - s.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) - if err != nil { - return fmt.Errorf("state: ChainBridge command error: %v", err) - } - if chResponse.Error != "" { - return fmt.Errorf("state: ChainBridge response error: %v", chResponse.Error) - } - return nil -} diff --git a/pki/server/wire_handler.go b/pki/server/wire_handler.go index d2bffea..017ab0e 100644 --- a/pki/server/wire_handler.go +++ b/pki/server/wire_handler.go @@ -1,4 +1,6 @@ -// related: katzenpost:authority/voting/server/wire_handler.go +// upstream: katzenpost:authority/voting/server/wire_handler.go +// wire_handler.go - Katzenpost non-voting authority connection handler. +// with modifications for ZKN ZK-PKI package server @@ -81,6 +83,10 @@ func (s *Server) onConn(conn net.Conn) { resp = s.onClient(rAddr, cmd) } else if auth.isMix { resp = s.onMix(rAddr, cmd, auth.peerIdentityKeyHash) + } else if auth.isReplica { + resp = s.onReplica(rAddr, cmd, auth.peerIdentityKeyHash) + } else if auth.isAuthority { + resp = s.onAuthority(rAddr, cmd) } else { panic("wtf") // should only happen if there is a bug in wireAuthenticator } @@ -122,8 +128,40 @@ func (s *Server) onMix(rAddr net.Addr, cmd commands.Command, peerIdentityKeyHash return resp } +func (s *Server) onReplica(rAddr net.Addr, cmd commands.Command, peerIdentityKeyHash []byte) commands.Command { + s.log.Debug("onReplica") + var resp commands.Command + switch c := cmd.(type) { + case *commands.PostReplicaDescriptor: + resp = s.onPostReplicaDescriptor(rAddr, c, peerIdentityKeyHash) + default: + s.log.Debugf("Peer %v: Invalid request: %T", rAddr, c) + return nil + } + return resp +} + +func (s *Server) onAuthority(rAddr net.Addr, cmd commands.Command) commands.Command { + var resp commands.Command + switch c := cmd.(type) { + case *commands.GetConsensus: + resp = s.onGetConsensus(rAddr, c) + case *commands.Vote: + resp = s.state.onVoteUpload(c) + case *commands.Cert: + resp = s.state.onCertUpload(c) + case *commands.Reveal: + resp = s.state.onRevealUpload(c) + case *commands.Sig: + resp = s.state.onSigUpload(c) + default: + s.log.Debugf("Peer %v: Invalid request: %T", rAddr, c) + return nil + } + return resp +} + func (s *Server) onGetConsensus(rAddr net.Addr, cmd *commands.GetConsensus) commands.Command { - s.log.Debugf("onGetConsensus: rAddr: %v, cmd: %+v", rAddr, cmd) resp := &commands.Consensus{} doc, err := s.state.documentForEpoch(cmd.Epoch) if err != nil { @@ -141,8 +179,82 @@ func (s *Server) onGetConsensus(rAddr net.Addr, cmd *commands.GetConsensus) comm return resp } +func (s *Server) onPostReplicaDescriptor(rAddr net.Addr, cmd *commands.PostReplicaDescriptor, pubKeyHash []byte) commands.Command { + resp := &commands.PostReplicaDescriptorStatus{ + ErrorCode: commands.DescriptorInvalid, + } + + // Ensure the epoch is somewhat sane. + now, _, _ := epochtime.Now() + switch cmd.Epoch { + case now - 1, now, now + 1: + // Nodes will always publish the descriptor for the current epoch on + // launch, which may be off by one period, depending on how skewed + // the node's clock is and the current time. + default: + // The peer is publishing for an epoch that's invalid. + s.log.Errorf("Peer %v: Invalid descriptor epoch '%v'", rAddr, cmd.Epoch) + return resp + } + + // Validate and deserialize the SignedReplicaUpload. + signedUpload := new(pki.SignedReplicaUpload) + err := signedUpload.Unmarshal(cmd.Payload) + if err != nil { + s.log.Errorf("Peer %v: Invalid descriptor: %v", rAddr, err) + return resp + } + + desc := signedUpload.ReplicaDescriptor + + // Ensure that the descriptor is signed by the peer that is posting. + identityKeyHash := hash.Sum256(desc.IdentityKey) + if !hmac.Equal(identityKeyHash[:], pubKeyHash) { + s.log.Errorf("Peer %v: Identity key hash '%x' is not link key '%v'.", rAddr, hash.Sum256(desc.IdentityKey), pubKeyHash) + resp.ErrorCode = commands.DescriptorForbidden + return resp + } + pkiSignatureScheme := signSchemes.ByName(s.cfg.Server.PKISignatureScheme) + + descIdPubKey, err := pkiSignatureScheme.UnmarshalBinaryPublicKey(desc.IdentityKey) + if err != nil { + s.log.Error("failed to unmarshal descriptor IdentityKey") + resp.ErrorCode = commands.DescriptorForbidden + return resp + } + + if !signedUpload.Verify(descIdPubKey) { + s.log.Error("PostDescriptorStatus contained a SignedUpload with an invalid signature") + resp.ErrorCode = commands.DescriptorForbidden + return resp + } + + // Ensure that the descriptor is from an allowed peer. + if !s.state.isReplicaDescriptorAuthorized(desc) { + s.log.Errorf("Peer %v: Identity key hash '%x' not authorized", rAddr, hash.Sum256(desc.IdentityKey)) + resp.ErrorCode = commands.DescriptorForbidden + return resp + } + + // Hand the replica descriptor off to the state worker. As long as this returns + // a nil, the authority "accepts" the replica descriptor. + err = s.state.onReplicaDescriptorUpload(cmd.Payload, desc, cmd.Epoch) + if err != nil { + // This is either a internal server error or the peer is trying to + // retroactively modify their descriptor. This should disambituate + // the condition, but the latter is more likely. + s.log.Errorf("Peer %v: Rejected probably a conflict: %v", rAddr, err) + resp.ErrorCode = commands.DescriptorConflict + return resp + } + + // Return a successful response. + s.log.Debugf("Peer %v: Accepted replica descriptor for epoch %v", rAddr, cmd.Epoch) + resp.ErrorCode = commands.DescriptorOk + return resp +} + func (s *Server) onPostDescriptor(rAddr net.Addr, cmd *commands.PostDescriptor, pubKeyHash []byte) commands.Command { - s.log.Debugf("onPostDescriptor: from rAddr: %v, for epoch: %d", rAddr, cmd.Epoch) resp := &commands.PostDescriptorStatus{ ErrorCode: commands.DescriptorInvalid, } @@ -199,13 +311,16 @@ func (s *Server) onPostDescriptor(rAddr net.Addr, cmd *commands.PostDescriptor, return resp } - // TODO: Use the packet loss statistics to make decisions about how to generate the consensus document. + // TODO(david): Use the packet loss statistics to make decisions about how to generate the consensus document. - // Hand the descriptor off to the state. As long as this returns + // Hand the descriptor off to the state worker. As long as this returns // a nil, the authority "accepts" the descriptor. err = s.state.onDescriptorUpload(cmd.Payload, desc, cmd.Epoch) if err != nil { - s.log.Errorf("Peer %v: Rejected descriptor for epoch %v: %v", rAddr, cmd.Epoch, err) + // This is either a internal server error or the peer is trying to + // retroactively modify their descriptor. This should disambituate + // the condition, but the latter is more likely. + s.log.Errorf("Peer %v: Rejected probably a conflict: %v", rAddr, err) resp.ErrorCode = commands.DescriptorConflict return resp } @@ -222,6 +337,8 @@ type wireAuthenticator struct { peerIdentityKeyHash []byte isClient bool isMix bool + isReplica bool + isAuthority bool } func (a *wireAuthenticator) IsPeerValid(creds *wire.PeerCredentials) bool { @@ -240,15 +357,38 @@ func (a *wireAuthenticator) IsPeerValid(creds *wire.PeerCredentials) bool { pk := [hash.HashSize]byte{} copy(pk[:], creds.AdditionalData[:hash.HashSize]) - isRegistered := (pk == hash.Sum256(a.s.state.authorizedNode.IdentityKey)) - if isRegistered { - a.s.log.Debugf("Accepting authority authentication from locally registered node with public key '%x'", pk) + _, isMix := a.s.state.authorizedMixes[pk] + _, isGatewayNode := a.s.state.authorizedGatewayNodes[pk] + _, isServiceNode := a.s.state.authorizedServiceNodes[pk] + _, isReplicaNode := a.s.state.authorizedReplicaNodes[pk] + _, isAuthority := a.s.state.authorizedAuthorities[pk] + + switch { + case isMix || isGatewayNode || isServiceNode: a.isMix = true // Gateways and service nodes and mixes are all mixes. return true - } else { + case isAuthority: + linkKey, ok := a.s.state.authorityLinkKeys[pk] + if !ok { + a.s.log.Warning("Rejecting authority authentication, no link key entry.") + return false + } + if creds.PublicKey == nil { + a.s.log.Warning("Rejecting authority authentication, public key is nil.") + return false + } + if !linkKey.Equal(creds.PublicKey) { + a.s.log.Warning("Rejecting authority authentication, public key mismatch.") + return false + } + a.isAuthority = true + return true + case isReplicaNode: + a.isReplica = true + return true + default: a.s.log.Warning("Rejecting authority authentication, public key mismatch.") return false } - - return false // Not reached. + // not reached } From 2b001020ff0eb83a24ab1cd6019675d67d60ff6a Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Sun, 20 Apr 2025 21:11:09 -0700 Subject: [PATCH 06/10] fix(genconfig): use relative path for IdentityPublicKeyPem --- genconfig/genconfig.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/genconfig/genconfig.go b/genconfig/genconfig.go index d876808..7d5dc7d 100644 --- a/genconfig/genconfig.go +++ b/genconfig/genconfig.go @@ -474,7 +474,9 @@ func (s *katzenpost) genAuthorizedNodes() ([]*vConfig.Node, []*vConfig.Node, []* for _, nodeCfg := range s.nodeConfigs { node := &vConfig.Node{ Identifier: nodeCfg.Server.Identifier, - IdentityPublicKeyPem: filepath.Join(s.outDir, nodeCfg.Server.Identifier, "identity.public.pem"), + IdentityPublicKeyPem: filepath.Join("..", nodeCfg.Server.Identifier, "identity.public.pem"), + // Note: 20250421 KP config.Load does not support Abs path for this key, so use relative + // IdentityPublicKeyPem: filepath.Join(s.outDir, nodeCfg.Server.Identifier, "identity.public.pem"), } if nodeCfg.Server.IsGatewayNode { gateways = append(gateways, node) From ff9439f1879d6b38ac47fe6b93a108bb444b6319 Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Sun, 20 Apr 2025 21:13:49 -0700 Subject: [PATCH 07/10] feat(pki): add ZK-PKI support with custom state machine --- pki/server/aux.go | 302 +++++++++++++++++++++++++++++++++ pki/server/server.go | 2 + pki/server/state.go | 41 ++++- pki/server/state_chain_comm.go | 181 ++++++++++++++++++++ 4 files changed, 525 insertions(+), 1 deletion(-) create mode 100644 pki/server/aux.go create mode 100644 pki/server/state_chain_comm.go diff --git a/pki/server/aux.go b/pki/server/aux.go new file mode 100644 index 0000000..97baef5 --- /dev/null +++ b/pki/server/aux.go @@ -0,0 +1,302 @@ +/****** ZK-PKI ******/ +// ZK-PKI specifics for minimal changes to the upstream code + +package server + +import ( + "math" + "path/filepath" + "time" + + "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" + "github.com/katzenpost/hpqc/hash" + "github.com/katzenpost/hpqc/rand" + "github.com/katzenpost/hpqc/sign" + signpem "github.com/katzenpost/hpqc/sign/pem" + signSchemes "github.com/katzenpost/hpqc/sign/schemes" + "github.com/katzenpost/katzenpost/authority/voting/server/config" + "github.com/katzenpost/katzenpost/core/epochtime" + "github.com/katzenpost/katzenpost/core/pki" +) + +const ( + // stateBootstrap = "bootstrap" + stateDescriptorSend = "descriptor_send" + // stateAcceptDescriptor = "accept_desc" + // stateAcceptVote = "accept_vote" + stateConfirmConsensus = "confirm_consensus" +) + +// NOTE: 2024-11-01: +// Parts of katzenpost use MixPublishDeadline and PublishConsensusDeadline defined in +// katzenpost:authority/voting/server/state.go +// So, we preserve that aspect of the epoch schedule. +var ( + // MixPublishDeadline = epochtime.Period * 1 / 8 // Do NOT change this + DescriptorBlockDeadline = epochtime.Period * 2 / 8 + // AuthorityVoteDeadline = epochtime.Period * 3 / 8 + // PublishConsensusDeadline = epochtime.Period * 5 / 8 // Do NOT change this + DocGenerationDeadline = epochtime.Period * 7 / 8 // Do NOT change this (see katzenpost:authority/voting/server/state.go:documentForEpoch) +) + +var ( + JitterMax = epochtime.Period * 1 / 16 // max duration to distribute network load across synchronized nodes +) + +func (s *state) fsm() <-chan time.Time { + s.Lock() + var sleep time.Duration + epoch, elapsed, nextEpoch := epochtime.Now() + s.log.Debugf("Current epoch %d, remaining time: %s, state: %s", epoch, nextEpoch, s.state) + + switch s.state { + case stateBootstrap: + // TODO: ensure network is ready and locally registered node is eligible for participation + s.genesisEpoch = 0 + s.backgroundFetchConsensus(epoch - 1) + s.backgroundFetchConsensus(epoch) + if elapsed > MixPublishDeadline { + s.log.Errorf("Too late to vote this round, sleeping until %s", nextEpoch) + sleep = nextEpoch + s.votingEpoch = epoch + 2 + s.state = stateBootstrap + } else { + s.votingEpoch = epoch + 1 + s.state = stateDescriptorSend + sleep = MixPublishDeadline - elapsed + s.zkpki_jitter() + if sleep < 0 { + sleep = 0 + } + s.log.Noticef("Bootstrapping for %d", s.votingEpoch) + } + + case stateDescriptorSend: + // Send mix descriptor to the appchain + pk := hash.Sum256(s.authorizedNode.IdentityKey) + desc, ok := s.descriptors[s.votingEpoch][pk] + if ok { + s.zkpki_submitDescriptor(desc, s.votingEpoch) + } else { + s.log.Errorf("❌ No descriptor for epoch %d", s.votingEpoch) + } + s.state = stateAcceptDescriptor + sleep = DescriptorBlockDeadline - elapsed + s.zkpki_jitter() + + case stateAcceptDescriptor: + doc, err := s.getVote(s.votingEpoch) + if err == nil { + s.zkpki_sendVote(doc, s.votingEpoch) + } else { + s.log.Errorf("❌ Failed to compute vote for epoch %v: %s", s.votingEpoch, err) + } + s.state = stateAcceptVote + _, nowelapsed, _ := epochtime.Now() + sleep = AuthorityVoteDeadline - nowelapsed + + case stateAcceptVote: + s.backgroundFetchConsensus(s.votingEpoch) + s.state = stateConfirmConsensus + _, nowelapsed, _ := epochtime.Now() + sleep = PublishConsensusDeadline - nowelapsed + + case stateConfirmConsensus: + // See if consensus doc was retrieved from the appchain + _, ok := s.documents[epoch+1] + if ok { + s.state = stateDescriptorSend + sleep = MixPublishDeadline + nextEpoch + s.zkpki_jitter() + s.votingEpoch++ + } else { + s.log.Error("No document for epoch %v", epoch+1) + s.state = stateBootstrap + s.votingEpoch = epoch + 2 // vote on epoch+2 in epoch+1 + sleep = nextEpoch + } + + default: + } + + s.pruneDocuments() + s.log.Debugf("authority: FSM in state %v until %s", s.state, sleep) + s.Unlock() + return time.After(sleep) +} + +// related: pki/server/server.go:computeLambdaG +// compute lambdaG value for dynamic topology from number of nodes per layer, and not from config file +func zkpki_computeLambdaGFromNodesPerLayer(cfg *config.Config, npl int) float64 { + n := float64(npl) + if n == 1 { + return cfg.Parameters.LambdaP + cfg.Parameters.LambdaL + cfg.Parameters.LambdaD + } + return n * math.Log(n) +} + +// Returns a random delay to distribute load across synchronized nodes +func (s *state) zkpki_jitter() time.Duration { + return time.Duration(rand.NewMath().Float64() * float64(JitterMax)) +} + +func (s *state) zkpki_submitDescriptor(desc *pki.MixDescriptor, epoch uint64) { + // Register the mix descriptor with the appchain, which will: + // - reject redundant descriptors (even those that didn't change) + // - reject descriptors if document for the epoch exists + if err := s.chPKISetMixDescriptor(desc, epoch); err != nil { + s.log.Errorf("❌ submitDescriptorToAppchain: Failed to set mix descriptor for node %v, epoch=%v: %v", desc.Name, epoch, err) + } + epochCurrent, _, _ := epochtime.Now() + s.log.Noticef("✅ submitDescriptorToAppchain: Submitted descriptor to appchain for node %v, epoch=%v (in epoch=%v)", desc.Name, epoch, epochCurrent) +} + +func (s *state) zkpki_sendVote(doc *pki.Document, epoch uint64) { + if err := s.chPKISetDocument(doc); err != nil { + s.log.Errorf("❌ sendVoteToAppchain: Error setting document for epoch %d: %v", epoch, err) + } else { + s.log.Noticef("✅ sendVoteToAppchain: Set document for epoch %d", epoch) + } +} + +func (s *state) zkpki_backgroundFetchConsensus(epoch uint64) { + // If there isn't a consensus for the previous epoch, ask the appchain for a consensus. + _, ok := s.documents[epoch] + if !ok { + s.Go(func() { + doc, err := s.chPKIGetDocument(epoch) + if err != nil { + s.log.Debugf("pki: FetchConsensus: Failed to fetch document for epoch %v: %v", epoch, err) + return + } + s.Lock() + defer s.Unlock() + + // It's possible that the state has changed + // if backgroundFetchConsensus was called + // multiple times during bootstrapping + if _, ok := s.documents[epoch]; !ok { + // sign the locally-stored document + _, err := s.doSignDocument(s.s.identityPrivateKey, s.s.identityPublicKey, doc) + if err != nil { + s.log.Errorf("pki: FetchConsensus: Error signing document for epoch %v: %v", epoch, err) + return + } + s.documents[epoch] = doc + s.log.Debugf("pki: FetchConsensus: ✅ Set doc for epoch %v: %s", epoch, doc.String()) + } + }) + } +} + +// produces a pki.Document using all MixDescriptors from the appchain +func (s *state) zkpki_getVote(epoch uint64) (*pki.Document, error) { + descriptors, err := s.chPKIGetMixDescriptors(epoch) + if err != nil { + return nil, err + } + + // TODO replicaDescriptors not yet recorded to appchain + replicaDescriptors := []*pki.ReplicaDescriptor{} + + // vote topology is irrelevent. + // TODO: idea: use an appchain block hash as srv + var zeros [32]byte + doc := s.getDocument(descriptors, replicaDescriptors, s.s.cfg.Parameters, zeros[:]) + + // Note: upload unsigned document and sign it upon local save to use the doc CID as the vote. + + // simulate SignDocument's setting of doc version, required by IsDocumentWellFormed + doc.Version = pki.DocumentVersion + + if err := pki.IsDocumentWellFormed(doc, nil); err != nil { + s.log.Errorf("pki: ❌ getVote: IsDocumentWellFormed: %s", err) + return nil, err + } + + return doc, nil +} + +func zkpki_newState(st *state) error { + + st.log.Debugf("[ZK-PKI] State initialized with epoch Period: %s", epochtime.Period) + st.log.Debugf("[ZK-PKI] State initialized with JitterMax: %s", JitterMax) + st.log.Debugf("[ZK-PKI] State initialized with MixPublishDeadline: %s", MixPublishDeadline) + st.log.Debugf("[ZK-PKI] State initialized with DescriptorBlockDeadline: %s", DescriptorBlockDeadline) + st.log.Debugf("[ZK-PKI] State initialized with AuthorityVoteDeadline: %s", AuthorityVoteDeadline) + st.log.Debugf("[ZK-PKI] State initialized with PublishConsensusDeadline: %s", PublishConsensusDeadline) + st.log.Debugf("[ZK-PKI] State initialized with DocGenerationDeadline: %s", DocGenerationDeadline) + + // Init AppChain communications (chainbridge) + chlog := st.s.logBackend.GetLogger("state:chain") + st.chainBridge = chainbridge.NewChainBridge(filepath.Join(st.s.cfg.Server.DataDir, "appchain.sock")) + st.chainBridge.SetErrorHandler(func(err error) { + chlog.Errorf("Error: %v", err) + }) + st.chainBridge.SetLogHandler(func(msg string) { + chlog.Infof(msg) + }) + if err := st.chainBridge.Start(); err != nil { + chlog.Fatalf("Error: %v", err) + } + + // Load the authorized local node from configuration + + // return a single node configuration and its node type + extractNodeFromCfg := func() (*config.Node, bool, bool, bool) { + if len(st.s.cfg.GatewayNodes) == 1 { + return st.s.cfg.GatewayNodes[0], true, false, false + } + if len(st.s.cfg.ServiceNodes) == 1 { + return st.s.cfg.ServiceNodes[0], false, true, false + } + if len(st.s.cfg.Mixes) == 1 { + return st.s.cfg.Mixes[0], false, false, false + } + if len(st.s.cfg.StorageReplicas) == 1 { + return st.s.cfg.StorageReplicas[0], false, false, true + } + return nil, false, false, false + } + v, isGatewayNode, isServiceNode, _ /*isStorageReplica*/ := extractNodeFromCfg() + if v == nil { + st.log.Fatalf("❌ Error: Invalid configuration for a single local node") + } + + // load the authorized node's identity public key + pkiSignatureScheme := signSchemes.ByName(st.s.cfg.Server.PKISignatureScheme) + var err error + var identityPublicKey sign.PublicKey + if filepath.IsAbs(v.IdentityPublicKeyPem) { + identityPublicKey, err = signpem.FromPublicPEMFile(v.IdentityPublicKeyPem, pkiSignatureScheme) + if err != nil { + panic(err) + } + } else { + pemFilePath := filepath.Join(st.s.cfg.Server.DataDir, v.IdentityPublicKeyPem) + identityPublicKey, err = signpem.FromPublicPEMFile(pemFilePath, pkiSignatureScheme) + if err != nil { + panic(err) + } + } + + // Node Registration: check if node is already registered before registering and rechecking + st.authorizedNode, err = st.chNodesGet(v.Identifier) + if err != nil { + if err := st.chNodesRegister(v, identityPublicKey, isGatewayNode, isServiceNode); err != nil { + st.log.Fatalf("❌ Error: node registration failed:", err) + } + time.Sleep(time.Duration(1) * time.Second) + st.authorizedNode, err = st.chNodesGet(v.Identifier) + if err != nil { + st.log.Fatalf("❌ Error: Failed to get node=%s from appchain: %v", v.Identifier, err) + } + } + + // Ensure node appchain registration matches the local node configuration + pk := hash.Sum256From(identityPublicKey) + if pk != hash.Sum256(st.authorizedNode.IdentityKey) { + st.log.Fatalf("❌ Error: IdentityKey mismatch between node registration and configuration") + } + + st.log.Noticef("✅ Node registered with Identifier '%s', Identity key hash '%x'", v.Identifier, pk) + + return nil +} diff --git a/pki/server/server.go b/pki/server/server.go index 4a9519c..81903bf 100644 --- a/pki/server/server.go +++ b/pki/server/server.go @@ -312,6 +312,7 @@ func New(cfg *config.Config) (*Server, error) { return nil, ErrGenerateOnly } + /****** ZK-PKI ****** Nodes do not depend on other nodes // Ensure that there are enough mixes and providers whitelisted to form // a topology, assuming all of them post a descriptor. if len(cfg.GatewayNodes) < 1 { @@ -323,6 +324,7 @@ func New(cfg *config.Config) (*Server, error) { if len(cfg.Mixes) < cfg.Debug.Layers*cfg.Debug.MinNodesPerLayer { return nil, fmt.Errorf("server: Insufficient nodes whitelisted, got %v , need %v", len(cfg.Mixes), cfg.Debug.Layers*cfg.Debug.MinNodesPerLayer) } + ****** ZK-PKI ******/ // Past this point, failures need to call s.Shutdown() to do cleanup. isOk := false diff --git a/pki/server/state.go b/pki/server/state.go index bde0133..c910f13 100644 --- a/pki/server/state.go +++ b/pki/server/state.go @@ -1,6 +1,6 @@ // upstream katzenpost:authority/voting/server/state.go // state.go - Katzenpost voting authority server state. -// with modifications for ZKN ZK-PKI +// with modifications for ZK-PKI package server @@ -21,6 +21,7 @@ import ( "sync" "time" + "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" signSchemes "github.com/katzenpost/hpqc/sign/schemes" bolt "go.etcd.io/bbolt" @@ -120,6 +121,13 @@ type state struct { threshold int dissenters int state string + + /****** ZK-PKI ******/ + // locally registered node, only one allowed + // mix descriptor uploads to this authority are restricted to this node + authorizedNode *chainbridge.Node + chainBridge *chainbridge.ChainBridge + /****** ZK-PKI ******/ } func (s *state) Halt() { @@ -151,6 +159,7 @@ func (s *state) worker() { } } +/****** ZK-PKI ****** Use custom state machine and epoch schedule func (s *state) fsm() <-chan time.Time { s.Lock() var sleep time.Duration @@ -264,6 +273,7 @@ func (s *state) fsm() <-chan time.Time { s.Unlock() return time.After(sleep) } +****** ZK-PKI ******/ func (s *state) persistDocument(epoch uint64, doc []byte) { if err := s.db.Update(func(tx *bolt.Tx) error { @@ -288,6 +298,11 @@ func (s *state) getVote(epoch uint64) (*pki.Document, error) { s.genesisEpoch = s.votingEpoch } + /****** ZK-PKI ******/ + vote_, err := s.zkpki_getVote(epoch) + return vote_, err + /****** ZK-PKI ******/ + descriptors := []*pki.MixDescriptor{} for _, desc := range s.descriptors[epoch] { descriptors = append(descriptors, desc) @@ -529,8 +544,14 @@ func (s *state) getDocument(descriptors []*pki.MixDescriptor, replicaDescriptors } } + /****** ZK-PKI ****** Compute lambdaG from nodes per layer, not config-file topology lambdaG := computeLambdaG(s.s.cfg) s.log.Debugf("computed lambdaG is %f", lambdaG) + ****** ZK-PKI ******/ + nodesPerLayer := len(nodes) / s.s.cfg.Debug.Layers + lambdaG := zkpki_computeLambdaGFromNodesPerLayer(s.s.cfg, nodesPerLayer) + s.log.Debugf("computed lambdaG from %d nodes per layer is %f", nodesPerLayer, lambdaG) + /****** ZK-PKI ******/ // Build the Document. doc := &pki.Document{ @@ -558,6 +579,11 @@ func (s *state) getDocument(descriptors []*pki.MixDescriptor, replicaDescriptors SphinxGeometryHash: s.geo.Hash(), PKISignatureScheme: s.s.cfg.Server.PKISignatureScheme, } + + /****** ZK-PKI ******/ + doc.PriorSharedRandom = [][]byte{srv} // this is made up, to suffice IsDocumentWellFormed + /****** ZK-PKI ******/ + return doc } @@ -1974,11 +2000,13 @@ func newState(s *Server) (*state, error) { // set voting schedule at runtime + /****** ZK-PKI ****** Use custom epoch schedule st.log.Debugf("State initialized with epoch Period: %s", epochtime.Period) st.log.Debugf("State initialized with MixPublishDeadline: %s", MixPublishDeadline) st.log.Debugf("State initialized with AuthorityVoteDeadline: %s", AuthorityVoteDeadline) st.log.Debugf("State initialized with AuthorityRevealDeadline: %s", AuthorityRevealDeadline) st.log.Debugf("State initialized with PublishConsensusDeadline: %s", PublishConsensusDeadline) + ****** ZK-PKI ******/ st.verifiers = make(map[[publicKeyHashSize]byte]sign.PublicKey) for _, auth := range s.cfg.Authorities { st.verifiers[hash.Sum256From(auth.IdentityPublicKey)] = auth.IdentityPublicKey @@ -2102,6 +2130,12 @@ func newState(s *Server) (*state, error) { st.commits = make(map[uint64]map[[publicKeyHashSize]byte][]byte) st.priorSRV = make([][]byte, 0) + /****** ZK-PKI ******/ + if err := zkpki_newState(st); err != nil { + return nil, err + } + /****** ZK-PKI ******/ + // Initialize the persistence store and restore state. dbPath := filepath.Join(s.cfg.Server.DataDir, dbFile) var err error @@ -2123,6 +2157,11 @@ func (s *state) backgroundFetchConsensus(epoch uint64) { panic("write lock not held in backgroundFetchConsensus(epoch)") } + /****** ZK-PKI ******/ + s.zkpki_backgroundFetchConsensus(epoch) + return + /****** ZK-PKI ******/ + // If there isn't a consensus for the previous epoch, ask the other // authorities for a consensus. _, ok := s.documents[epoch] diff --git a/pki/server/state_chain_comm.go b/pki/server/state_chain_comm.go new file mode 100644 index 0000000..98d6da7 --- /dev/null +++ b/pki/server/state_chain_comm.go @@ -0,0 +1,181 @@ +// AppChain communication (chainbridge) functions + +package server + +import ( + "fmt" + + "github.com/ZeroKnowledgeNetwork/appchain-agent/clients/go/chainbridge" + "github.com/fxamacker/cbor/v2" + "github.com/katzenpost/hpqc/sign" + "github.com/katzenpost/katzenpost/authority/voting/server/config" + "github.com/katzenpost/katzenpost/core/pki" +) + +func (s *state) chNodesGet(name string) (*chainbridge.Node, error) { + chCommand := fmt.Sprintf(chainbridge.Cmd_nodes_getNode, name) + chResponse, err := s.chainBridge.Command(chCommand, nil) + if err != nil { + return nil, fmt.Errorf("state: ChainBridge command error: %v", err) + } + + var node chainbridge.Node + if err = s.chainBridge.DataUnmarshal(chResponse, &node); err != nil { + return nil, err + } + + return &node, nil +} + +func (st *state) chNodesRegister(v *config.Node, identityPublicKey sign.PublicKey, isGatewayNode bool, isServiceNode bool) error { + payload, err := identityPublicKey.MarshalBinary() + if err != nil { + return fmt.Errorf("failed to marshal identityPublicKey: %v", err) + } + chCommand := fmt.Sprintf( + chainbridge.Cmd_nodes_register, + v.Identifier, + chainbridge.Bool2int(isGatewayNode), + chainbridge.Bool2int(isServiceNode)) + chResponse, err := st.chainBridge.Command(chCommand, payload) + st.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) + if err != nil { + return fmt.Errorf("ChainBridge command error: %v", err) + } + if chResponse.Error != "" && chResponse.Error != chainbridge.Err_nodes_alreadyRegistered { + return fmt.Errorf("ChainBridge response error: %v", chResponse.Error) + } + + return nil +} + +func (s *state) chPKIGetGenesisEpoch() (uint64, error) { + chResponse, err := s.chainBridge.Command(chainbridge.Cmd_pki_getGenesisEpoch, nil) + if err != nil { + return 0, fmt.Errorf("state: ChainBridge command error: %v", err) + } + genesisEpoch, err := s.chainBridge.GetDataUInt(chResponse) + if err != nil { + return 0, fmt.Errorf("state: ChainBridge data error: %v", err) + } + return genesisEpoch, nil +} + +func (s *state) chPKIGetDocument(epoch uint64) (*pki.Document, error) { + chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getDocucment, epoch) + chResponse, err := s.chainBridge.Command(chCommand, nil) + if err != nil { + return nil, fmt.Errorf("state: ChainBridge command error: %v", err) + } + + chDoc, err := s.chainBridge.GetDataBytes(chResponse) + if err != nil { + return nil, err + } + + var doc pki.Document + // X: if err = doc.UnmarshalCertificate(chDoc); err != nil { + if err = cbor.Unmarshal(chDoc, (*pki.Document)(&doc)); err != nil { + return nil, fmt.Errorf("state: failed to unmarshal PKI document: %v", err) + } + + return &doc, nil +} + +// register the PKI doc with the appchain +func (s *state) chPKISetDocument(doc *pki.Document) error { + //cbor.EncMode a la katzenpost:core/pki/document.go + ccbor, err := cbor.CanonicalEncOptions().EncMode() + if err != nil { + panic(err) + } + + // register with the appchain an unsigned certificate-less doc, + // so authorities submit the same doc hash as their vote + // X: payload, err := doc.MarshalCertificate() + payload, err := ccbor.Marshal((*pki.Document)(doc)) + if err != nil { + return err + } + + if err != nil { + return fmt.Errorf("state: failed to marshal PKI document: %v", err) + } + chCommand := fmt.Sprintf(chainbridge.Cmd_pki_setDocument, doc.Epoch) + chResponse, err := s.chainBridge.Command(chCommand, payload) + s.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) + if err != nil { + return fmt.Errorf("state: ChainBridge command error: %v", err) + } + + // ignore the most likely chResponse.Error: "Document already exists for the epoch" + // if chResponse.Error != "" { + // return fmt.Errorf("state: ChainBridge response error: %v", chResponse.Error) + // } + + return nil +} + +// get number of descriptors for the given epoch from appchain +func (s *state) chPKIGetMixDescriptorCounter(epoch uint64) (uint64, error) { + chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getMixDescriptorCounter, epoch) + chResponse, err := s.chainBridge.Command(chCommand, nil) + if err != nil { + return 0, fmt.Errorf("state: ChainBridge command error: %v", err) + } + numDescriptors, err := s.chainBridge.GetDataUInt(chResponse) + if err != nil && err != chainbridge.ErrNoData { + return 0, fmt.Errorf("state: ChainBridge data error: %v", err) + } + return numDescriptors, nil +} + +func (s *state) chPKIGetMixDescriptors(epoch uint64) ([]*pki.MixDescriptor, error) { + numDescriptors, err := s.chPKIGetMixDescriptorCounter(epoch) + if err != nil { + return nil, err + } + + descriptors := []*pki.MixDescriptor{} + for i := 0; i < int(numDescriptors); i++ { + chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getMixDescriptorByIndex, epoch, i) + chResponse, err := s.chainBridge.Command(chCommand, nil) + if err != nil { + s.log.Error("ChainBridge command error: %v", err) + continue + } + dataAsBytes, err := s.chainBridge.GetDataBytes(chResponse) + if err != nil { + s.log.Error("ChainBridge data error: %v", err) + continue + } + var desc pki.MixDescriptor + if err = desc.UnmarshalBinary(dataAsBytes); err != nil { + s.log.Error("Failed to unmarshal descriptor: %v", err) + continue + } + descriptors = append(descriptors, &desc) + } + + return descriptors, nil +} + +// Register the mix descriptor with the appchain, which will: +// - reject redundant descriptors (even those that didn't change) +// - reject descriptors if document for the epoch exists +func (s *state) chPKISetMixDescriptor(desc *pki.MixDescriptor, epoch uint64) error { + payload, err := desc.MarshalBinary() + if err != nil { + return fmt.Errorf("state: failed to marshal descriptor: %v", err) + } + chCommand := fmt.Sprintf(chainbridge.Cmd_pki_setMixDescriptor, epoch, desc.Name) + chResponse, err := s.chainBridge.Command(chCommand, payload) + s.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) + if err != nil { + return fmt.Errorf("state: ChainBridge command error: %v", err) + } + if chResponse.Error != "" { + return fmt.Errorf("state: ChainBridge response error: %v", chResponse.Error) + } + return nil +} From 812ddfe8b7809572cc6e8538c9ad445f0fd0ce12 Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Mon, 21 Apr 2025 14:32:56 -0700 Subject: [PATCH 08/10] chore: go mod tidy --- go.mod | 4 ++-- go.sum | 18 +++++------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 75171fb..2b80c70 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/katzenpost/hpqc v0.0.55 github.com/katzenpost/katzenpost v0.0.48 github.com/quic-go/quic-go v0.50.0 + go.etcd.io/bbolt v1.3.10 + golang.org/x/crypto v0.36.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 ) @@ -48,9 +50,7 @@ require ( gitlab.com/yawning/aez.git v0.0.0-20211027044916-e49e68abd344 // indirect gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect gitlab.com/yawning/x448.git v0.0.0-20221003101044-617eb9b7d9b7 // indirect - go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/mock v0.5.0 // indirect - golang.org/x/crypto v0.36.0 // indirect golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.37.0 // indirect diff --git a/go.sum b/go.sum index 026012f..9982315 100644 --- a/go.sum +++ b/go.sum @@ -37,24 +37,19 @@ github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9 h1:q5g0N9eal4bmJwXHC5 github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/henrydcase/nobs v0.0.0-20230313231516-25b66236df73 h1:d3rq/Tz+RJ5h1xk6Lt3jbObJN3WhvZm7rV41OCIzUyI= github.com/henrydcase/nobs v0.0.0-20230313231516-25b66236df73/go.mod h1:ptK2MJqVLVEa/V/oK8n+MEyUDCSjSylW+jeNmCG1DJo= -github.com/katzenpost/chacha20 v0.0.0-20190910113340-7ce890d6a556 h1:9gHByAWH1LydGefFGorN1ZBRZ/Oz9iozdzMvRTWpyRw= -github.com/katzenpost/chacha20 v0.0.0-20190910113340-7ce890d6a556/go.mod h1:d9kxwmGOcutgP6bQwr2xaLInaW5yJsxsoPRyUIG0J/E= +github.com/katzenpost/chacha20 v0.0.1 h1:Scu6Pqyginw083FhypKMIBmCI2gQTC1RFD6vqpWwI2Y= github.com/katzenpost/chacha20 v0.0.1/go.mod h1:/LIJK/8cUXVJrCh5NypZ8So3gDfQCoQT8lRvy1rYQZA= github.com/katzenpost/circl v1.3.9-0.20240222183521-1cd9a34e9a0c h1:FYy03rLIjdyjklBOI6YSCb3q7OubTx0dVDWYOgDsvA8= github.com/katzenpost/circl v1.3.9-0.20240222183521-1cd9a34e9a0c/go.mod h1:+EBrwiGYs9S+qZqaqxujN1CReTNCMAG6p+31KkEDeeA= -github.com/katzenpost/hpqc v0.0.45 h1:CiNTvwUe7CaGdIeA0tEtHY+O3CKk6lTgdAb4iQfSy4k= -github.com/katzenpost/hpqc v0.0.45/go.mod h1:yMxuQLTjgzgHdvQlJIbWFiusyizyMW94fpH6wxTTur8= +github.com/katzenpost/hpqc v0.0.55 h1:dRdk3k+sJsvRb5Z2CLVUmxOhY/WhgQ6JkAGWYFUNdLg= github.com/katzenpost/hpqc v0.0.55/go.mod h1:yaVqoZyKeBmgiKnGBpgLNf+nfO4ll81f54RAnaL6OpI= -github.com/katzenpost/katzenpost v0.0.43 h1:BAZxLxl3he+bNodTaXv6GW0BYA9Qj6jGQcsVHOjeiN0= -github.com/katzenpost/katzenpost v0.0.43/go.mod h1:+aRwtsFwBT7GTU9Mj07MlQ3QoM4XQ6YLP/w0/j1gOHc= github.com/katzenpost/katzenpost v0.0.48 h1:nyvWYvVu5r+0UgPRCGnPyC/Jn7Dst9UluWsJ4XB7OCs= github.com/katzenpost/katzenpost v0.0.48/go.mod h1:FcNs9Kc6PiJQ7t8g6iNYGAtlempt627iLQvUlrV3HHc= github.com/katzenpost/nyquist v0.0.10 h1:rh9TCEXCsutsg+cvbV6ASVFnzSAYBisWQ3fnwQSPa34= github.com/katzenpost/nyquist v0.0.10/go.mod h1:tyK92JiCptgsaE0iUAMlt5W2v2Rdw6mnUpIdIidIGHo= github.com/katzenpost/sntrup4591761 v0.0.0-20231024131303-8755eb1986b8 h1:TsKxH0x2RUwf5rBw67k15bqVM3oVbexA9oaTZQLIy3Y= github.com/katzenpost/sntrup4591761 v0.0.0-20231024131303-8755eb1986b8/go.mod h1:Hmcrwom7jcEmGdo0CsyuJNnldPeyS+M07FuCbo7I8fw= -github.com/katzenpost/sphincsplus v0.0.2-0.20240114192234-1dc77b544e31 h1:fKGa/too1Br31gmoYmV2kE61gydj47Ed5K/g/CE+3Bs= -github.com/katzenpost/sphincsplus v0.0.2-0.20240114192234-1dc77b544e31/go.mod h1:VFrCPnmbxQLBi+qJfWHUqvpvTMZrYBMZEEy0AidY0nE= +github.com/katzenpost/sphincsplus v0.0.2 h1:W1UWejLK62Lk0uK2R08H/sWEaQrRHWCaMEKO181SoOE= github.com/katzenpost/sphincsplus v0.0.2/go.mod h1:ChO9+ojgCH1yEuplGgW4mSI1FwZWtyEmEkG1xL3w264= github.com/lesismal/llib v1.1.13/go.mod h1:70tFXXe7P1FZ02AU9l8LgSOK7d7sRrpnkUr3rd3gKSg= github.com/lesismal/nbio v1.5.11 h1:MVjrzcej4NSJQMRT+S0dPZvVaiFUHD1JWnvr+FHIHOo= @@ -92,9 +87,10 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schwarmco/go-cartesian-product v0.0.0-20230921023625-e02d1c150053 h1:h7EwPM2KjupG0zVAG+EYxbR2cHnbiP1d4DTAZ+G09LY= +github.com/schwarmco/go-cartesian-product v0.0.0-20230921023625-e02d1c150053/go.mod h1:/TRiIlxvQQAtfnBXEqqbnYBYPmE6XT5iZxSx+hJ9zGw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -123,7 +119,6 @@ golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -138,8 +133,6 @@ golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -148,6 +141,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 4591de60e35aa47108d70ac95dac8cd3e2a35b3f Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Mon, 21 Apr 2025 14:31:28 -0700 Subject: [PATCH 09/10] refactor(pki): use custom logger for ZK-PKI logs --- pki/server/aux.go | 57 +++++++++++++++++----------------- pki/server/state.go | 1 + pki/server/state_chain_comm.go | 38 +++++++++++------------ 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/pki/server/aux.go b/pki/server/aux.go index 97baef5..a7b28b8 100644 --- a/pki/server/aux.go +++ b/pki/server/aux.go @@ -47,7 +47,7 @@ func (s *state) fsm() <-chan time.Time { s.Lock() var sleep time.Duration epoch, elapsed, nextEpoch := epochtime.Now() - s.log.Debugf("Current epoch %d, remaining time: %s, state: %s", epoch, nextEpoch, s.state) + s.zlog.Debugf("Current epoch %d, remaining time: %s, state: %s", epoch, nextEpoch, s.state) switch s.state { case stateBootstrap: @@ -56,7 +56,7 @@ func (s *state) fsm() <-chan time.Time { s.backgroundFetchConsensus(epoch - 1) s.backgroundFetchConsensus(epoch) if elapsed > MixPublishDeadline { - s.log.Errorf("Too late to vote this round, sleeping until %s", nextEpoch) + s.zlog.Errorf("Too late to vote this round, sleeping until %s", nextEpoch) sleep = nextEpoch s.votingEpoch = epoch + 2 s.state = stateBootstrap @@ -67,7 +67,7 @@ func (s *state) fsm() <-chan time.Time { if sleep < 0 { sleep = 0 } - s.log.Noticef("Bootstrapping for %d", s.votingEpoch) + s.zlog.Noticef("Bootstrapping for %d", s.votingEpoch) } case stateDescriptorSend: @@ -77,7 +77,7 @@ func (s *state) fsm() <-chan time.Time { if ok { s.zkpki_submitDescriptor(desc, s.votingEpoch) } else { - s.log.Errorf("❌ No descriptor for epoch %d", s.votingEpoch) + s.zlog.Errorf("❌ No descriptor for epoch %d", s.votingEpoch) } s.state = stateAcceptDescriptor sleep = DescriptorBlockDeadline - elapsed + s.zkpki_jitter() @@ -87,7 +87,7 @@ func (s *state) fsm() <-chan time.Time { if err == nil { s.zkpki_sendVote(doc, s.votingEpoch) } else { - s.log.Errorf("❌ Failed to compute vote for epoch %v: %s", s.votingEpoch, err) + s.zlog.Errorf("❌ Failed to compute vote for epoch %v: %s", s.votingEpoch, err) } s.state = stateAcceptVote _, nowelapsed, _ := epochtime.Now() @@ -107,7 +107,7 @@ func (s *state) fsm() <-chan time.Time { sleep = MixPublishDeadline + nextEpoch + s.zkpki_jitter() s.votingEpoch++ } else { - s.log.Error("No document for epoch %v", epoch+1) + s.zlog.Error("No document for epoch %v", epoch+1) s.state = stateBootstrap s.votingEpoch = epoch + 2 // vote on epoch+2 in epoch+1 sleep = nextEpoch @@ -117,7 +117,7 @@ func (s *state) fsm() <-chan time.Time { } s.pruneDocuments() - s.log.Debugf("authority: FSM in state %v until %s", s.state, sleep) + s.zlog.Debugf("authority: FSM in state %v until %s", s.state, sleep) s.Unlock() return time.After(sleep) } @@ -142,17 +142,17 @@ func (s *state) zkpki_submitDescriptor(desc *pki.MixDescriptor, epoch uint64) { // - reject redundant descriptors (even those that didn't change) // - reject descriptors if document for the epoch exists if err := s.chPKISetMixDescriptor(desc, epoch); err != nil { - s.log.Errorf("❌ submitDescriptorToAppchain: Failed to set mix descriptor for node %v, epoch=%v: %v", desc.Name, epoch, err) + s.zlog.Errorf("❌ submitDescriptor: Failed to set mix descriptor for node %v, epoch=%v: %v", desc.Name, epoch, err) } epochCurrent, _, _ := epochtime.Now() - s.log.Noticef("✅ submitDescriptorToAppchain: Submitted descriptor to appchain for node %v, epoch=%v (in epoch=%v)", desc.Name, epoch, epochCurrent) + s.zlog.Noticef("✅ submitDescriptor: Set mix descriptor for node %v, epoch=%v (in epoch=%v)", desc.Name, epoch, epochCurrent) } func (s *state) zkpki_sendVote(doc *pki.Document, epoch uint64) { if err := s.chPKISetDocument(doc); err != nil { - s.log.Errorf("❌ sendVoteToAppchain: Error setting document for epoch %d: %v", epoch, err) + s.zlog.Errorf("❌ sendVote: Error setting document for epoch %d: %v", epoch, err) } else { - s.log.Noticef("✅ sendVoteToAppchain: Set document for epoch %d", epoch) + s.zlog.Noticef("✅ sendVote: Set document for epoch %d", epoch) } } @@ -163,7 +163,7 @@ func (s *state) zkpki_backgroundFetchConsensus(epoch uint64) { s.Go(func() { doc, err := s.chPKIGetDocument(epoch) if err != nil { - s.log.Debugf("pki: FetchConsensus: Failed to fetch document for epoch %v: %v", epoch, err) + s.zlog.Debugf("FetchConsensus: Failed to fetch document for epoch %v: %v", epoch, err) return } s.Lock() @@ -176,11 +176,11 @@ func (s *state) zkpki_backgroundFetchConsensus(epoch uint64) { // sign the locally-stored document _, err := s.doSignDocument(s.s.identityPrivateKey, s.s.identityPublicKey, doc) if err != nil { - s.log.Errorf("pki: FetchConsensus: Error signing document for epoch %v: %v", epoch, err) + s.zlog.Errorf("FetchConsensus: Error signing document for epoch %v: %v", epoch, err) return } s.documents[epoch] = doc - s.log.Debugf("pki: FetchConsensus: ✅ Set doc for epoch %v: %s", epoch, doc.String()) + s.zlog.Debugf("FetchConsensus: ✅ Set local doc for epoch %v: %s", epoch, doc.String()) } }) } @@ -207,7 +207,7 @@ func (s *state) zkpki_getVote(epoch uint64) (*pki.Document, error) { doc.Version = pki.DocumentVersion if err := pki.IsDocumentWellFormed(doc, nil); err != nil { - s.log.Errorf("pki: ❌ getVote: IsDocumentWellFormed: %s", err) + s.zlog.Errorf("❌ getVote: IsDocumentWellFormed: %s", err) return nil, err } @@ -215,17 +215,18 @@ func (s *state) zkpki_getVote(epoch uint64) (*pki.Document, error) { } func zkpki_newState(st *state) error { + st.zlog = st.s.logBackend.GetLogger("state/zkpki") - st.log.Debugf("[ZK-PKI] State initialized with epoch Period: %s", epochtime.Period) - st.log.Debugf("[ZK-PKI] State initialized with JitterMax: %s", JitterMax) - st.log.Debugf("[ZK-PKI] State initialized with MixPublishDeadline: %s", MixPublishDeadline) - st.log.Debugf("[ZK-PKI] State initialized with DescriptorBlockDeadline: %s", DescriptorBlockDeadline) - st.log.Debugf("[ZK-PKI] State initialized with AuthorityVoteDeadline: %s", AuthorityVoteDeadline) - st.log.Debugf("[ZK-PKI] State initialized with PublishConsensusDeadline: %s", PublishConsensusDeadline) - st.log.Debugf("[ZK-PKI] State initialized with DocGenerationDeadline: %s", DocGenerationDeadline) + st.zlog.Debugf("State initialized with epoch Period: %s", epochtime.Period) + st.zlog.Debugf("State initialized with JitterMax: %s", JitterMax) + st.zlog.Debugf("State initialized with MixPublishDeadline: %s", MixPublishDeadline) + st.zlog.Debugf("State initialized with DescriptorBlockDeadline: %s", DescriptorBlockDeadline) + st.zlog.Debugf("State initialized with AuthorityVoteDeadline: %s", AuthorityVoteDeadline) + st.zlog.Debugf("State initialized with PublishConsensusDeadline: %s", PublishConsensusDeadline) + st.zlog.Debugf("State initialized with DocGenerationDeadline: %s", DocGenerationDeadline) // Init AppChain communications (chainbridge) - chlog := st.s.logBackend.GetLogger("state:chain") + chlog := st.s.logBackend.GetLogger("state/zkpki/chain") st.chainBridge = chainbridge.NewChainBridge(filepath.Join(st.s.cfg.Server.DataDir, "appchain.sock")) st.chainBridge.SetErrorHandler(func(err error) { chlog.Errorf("Error: %v", err) @@ -257,7 +258,7 @@ func zkpki_newState(st *state) error { } v, isGatewayNode, isServiceNode, _ /*isStorageReplica*/ := extractNodeFromCfg() if v == nil { - st.log.Fatalf("❌ Error: Invalid configuration for a single local node") + st.zlog.Fatalf("❌ Error: Invalid configuration for a single local node") } // load the authorized node's identity public key @@ -281,22 +282,22 @@ func zkpki_newState(st *state) error { st.authorizedNode, err = st.chNodesGet(v.Identifier) if err != nil { if err := st.chNodesRegister(v, identityPublicKey, isGatewayNode, isServiceNode); err != nil { - st.log.Fatalf("❌ Error: node registration failed:", err) + st.zlog.Fatalf("❌ Error: node registration failed:", err) } time.Sleep(time.Duration(1) * time.Second) st.authorizedNode, err = st.chNodesGet(v.Identifier) if err != nil { - st.log.Fatalf("❌ Error: Failed to get node=%s from appchain: %v", v.Identifier, err) + st.zlog.Fatalf("❌ Error: Failed to get node=%s from appchain: %v", v.Identifier, err) } } // Ensure node appchain registration matches the local node configuration pk := hash.Sum256From(identityPublicKey) if pk != hash.Sum256(st.authorizedNode.IdentityKey) { - st.log.Fatalf("❌ Error: IdentityKey mismatch between node registration and configuration") + st.zlog.Fatalf("❌ Error: IdentityKey mismatch between node registration and configuration") } - st.log.Noticef("✅ Node registered with Identifier '%s', Identity key hash '%x'", v.Identifier, pk) + st.zlog.Noticef("✅ Node registered with Identifier '%s', Identity key hash '%x'", v.Identifier, pk) return nil } diff --git a/pki/server/state.go b/pki/server/state.go index c910f13..a9fa807 100644 --- a/pki/server/state.go +++ b/pki/server/state.go @@ -127,6 +127,7 @@ type state struct { // mix descriptor uploads to this authority are restricted to this node authorizedNode *chainbridge.Node chainBridge *chainbridge.ChainBridge + zlog *logging.Logger /****** ZK-PKI ******/ } diff --git a/pki/server/state_chain_comm.go b/pki/server/state_chain_comm.go index 98d6da7..08ff2a9 100644 --- a/pki/server/state_chain_comm.go +++ b/pki/server/state_chain_comm.go @@ -16,7 +16,7 @@ func (s *state) chNodesGet(name string) (*chainbridge.Node, error) { chCommand := fmt.Sprintf(chainbridge.Cmd_nodes_getNode, name) chResponse, err := s.chainBridge.Command(chCommand, nil) if err != nil { - return nil, fmt.Errorf("state: ChainBridge command error: %v", err) + return nil, fmt.Errorf("ChainBridge command error: %v", err) } var node chainbridge.Node @@ -30,7 +30,7 @@ func (s *state) chNodesGet(name string) (*chainbridge.Node, error) { func (st *state) chNodesRegister(v *config.Node, identityPublicKey sign.PublicKey, isGatewayNode bool, isServiceNode bool) error { payload, err := identityPublicKey.MarshalBinary() if err != nil { - return fmt.Errorf("failed to marshal identityPublicKey: %v", err) + return fmt.Errorf("Failed to marshal identityPublicKey: %v", err) } chCommand := fmt.Sprintf( chainbridge.Cmd_nodes_register, @@ -38,7 +38,7 @@ func (st *state) chNodesRegister(v *config.Node, identityPublicKey sign.PublicKe chainbridge.Bool2int(isGatewayNode), chainbridge.Bool2int(isServiceNode)) chResponse, err := st.chainBridge.Command(chCommand, payload) - st.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) + st.zlog.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) if err != nil { return fmt.Errorf("ChainBridge command error: %v", err) } @@ -52,11 +52,11 @@ func (st *state) chNodesRegister(v *config.Node, identityPublicKey sign.PublicKe func (s *state) chPKIGetGenesisEpoch() (uint64, error) { chResponse, err := s.chainBridge.Command(chainbridge.Cmd_pki_getGenesisEpoch, nil) if err != nil { - return 0, fmt.Errorf("state: ChainBridge command error: %v", err) + return 0, fmt.Errorf("ChainBridge command error: %v", err) } genesisEpoch, err := s.chainBridge.GetDataUInt(chResponse) if err != nil { - return 0, fmt.Errorf("state: ChainBridge data error: %v", err) + return 0, fmt.Errorf("ChainBridge data error: %v", err) } return genesisEpoch, nil } @@ -65,7 +65,7 @@ func (s *state) chPKIGetDocument(epoch uint64) (*pki.Document, error) { chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getDocucment, epoch) chResponse, err := s.chainBridge.Command(chCommand, nil) if err != nil { - return nil, fmt.Errorf("state: ChainBridge command error: %v", err) + return nil, fmt.Errorf("ChainBridge command error: %v", err) } chDoc, err := s.chainBridge.GetDataBytes(chResponse) @@ -76,7 +76,7 @@ func (s *state) chPKIGetDocument(epoch uint64) (*pki.Document, error) { var doc pki.Document // X: if err = doc.UnmarshalCertificate(chDoc); err != nil { if err = cbor.Unmarshal(chDoc, (*pki.Document)(&doc)); err != nil { - return nil, fmt.Errorf("state: failed to unmarshal PKI document: %v", err) + return nil, fmt.Errorf("Failed to unmarshal PKI document: %v", err) } return &doc, nil @@ -99,13 +99,13 @@ func (s *state) chPKISetDocument(doc *pki.Document) error { } if err != nil { - return fmt.Errorf("state: failed to marshal PKI document: %v", err) + return fmt.Errorf("Failed to marshal PKI document: %v", err) } chCommand := fmt.Sprintf(chainbridge.Cmd_pki_setDocument, doc.Epoch) chResponse, err := s.chainBridge.Command(chCommand, payload) - s.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) + s.zlog.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) if err != nil { - return fmt.Errorf("state: ChainBridge command error: %v", err) + return fmt.Errorf("ChainBridge command error: %v", err) } // ignore the most likely chResponse.Error: "Document already exists for the epoch" @@ -121,11 +121,11 @@ func (s *state) chPKIGetMixDescriptorCounter(epoch uint64) (uint64, error) { chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getMixDescriptorCounter, epoch) chResponse, err := s.chainBridge.Command(chCommand, nil) if err != nil { - return 0, fmt.Errorf("state: ChainBridge command error: %v", err) + return 0, fmt.Errorf("ChainBridge command error: %v", err) } numDescriptors, err := s.chainBridge.GetDataUInt(chResponse) if err != nil && err != chainbridge.ErrNoData { - return 0, fmt.Errorf("state: ChainBridge data error: %v", err) + return 0, fmt.Errorf("ChainBridge data error: %v", err) } return numDescriptors, nil } @@ -141,17 +141,17 @@ func (s *state) chPKIGetMixDescriptors(epoch uint64) ([]*pki.MixDescriptor, erro chCommand := fmt.Sprintf(chainbridge.Cmd_pki_getMixDescriptorByIndex, epoch, i) chResponse, err := s.chainBridge.Command(chCommand, nil) if err != nil { - s.log.Error("ChainBridge command error: %v", err) + s.zlog.Error("ChainBridge command error: %v", err) continue } dataAsBytes, err := s.chainBridge.GetDataBytes(chResponse) if err != nil { - s.log.Error("ChainBridge data error: %v", err) + s.zlog.Error("ChainBridge data error: %v", err) continue } var desc pki.MixDescriptor if err = desc.UnmarshalBinary(dataAsBytes); err != nil { - s.log.Error("Failed to unmarshal descriptor: %v", err) + s.zlog.Error("Failed to unmarshal descriptor: %v", err) continue } descriptors = append(descriptors, &desc) @@ -166,16 +166,16 @@ func (s *state) chPKIGetMixDescriptors(epoch uint64) ([]*pki.MixDescriptor, erro func (s *state) chPKISetMixDescriptor(desc *pki.MixDescriptor, epoch uint64) error { payload, err := desc.MarshalBinary() if err != nil { - return fmt.Errorf("state: failed to marshal descriptor: %v", err) + return fmt.Errorf("Failed to marshal descriptor: %v", err) } chCommand := fmt.Sprintf(chainbridge.Cmd_pki_setMixDescriptor, epoch, desc.Name) chResponse, err := s.chainBridge.Command(chCommand, payload) - s.log.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) + s.zlog.Debugf("ChainBridge response (%s): %+v", chCommand, chResponse) if err != nil { - return fmt.Errorf("state: ChainBridge command error: %v", err) + return fmt.Errorf("ChainBridge command error: %v", err) } if chResponse.Error != "" { - return fmt.Errorf("state: ChainBridge response error: %v", chResponse.Error) + return fmt.Errorf("ChainBridge response error: %v", chResponse.Error) } return nil } From 8d093b4a20b17a75d863bbead6f2702dcf913eb9 Mon Sep 17 00:00:00 2001 From: Xendarboh <1435589+xendarboh@users.noreply.github.com> Date: Mon, 21 Apr 2025 14:31:52 -0700 Subject: [PATCH 10/10] feat(docker): Add ping command to Makefile.appchain --- docker/Makefile.appchain | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/Makefile.appchain b/docker/Makefile.appchain index f22313b..5519ad6 100644 --- a/docker/Makefile.appchain +++ b/docker/Makefile.appchain @@ -61,6 +61,13 @@ $(net)/run.stamp: wait: $(net)/run.stamp $(docker_run) $(docker_image) $(dir_bin)/fetch -f $(dir_base)/client/client.toml +ping: $(net)/run.stamp + $(docker_run) $(docker_image) $(dir_bin)/ping \ + -c $(dir_base)/client2/client.toml \ + -s echo \ + -printDiff \ + -n 10 + probe: $(net)/run.stamp $(docker_run) $(docker_image) $(dir_bin)/walletshield \ -config $(dir_base)/client2/client.toml \