feat: sync katzenpost v0.0.63

Makefile

@@ -1,22 +1,26 @@
+compression?=true
+warped?=true
+ldflags="-X github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${warped} -X github.com/ZeroKnowledgeNetwork/opt/common.CompressionEnabled=${compression}"
+
 .PHONY: all app-walletshield genconfig http_proxy pki clean
 
 all: app-walletshield http_proxy genconfig pki
 
 app-walletshield:
-	make -C apps/walletshield
+	cd apps/walletshield && go build -trimpath -ldflags ${ldflags}
 
 genconfig:
 	cd genconfig/cmd/genconfig && go build
 
 http_proxy:
-	cd server_plugins/cbor_plugins/http_proxy/cmd/http_proxy && go build
+	cd server_plugins/cbor_plugins/http_proxy/cmd/http_proxy && go build -trimpath -ldflags ${ldflags}
 
 pki:
 	make -C pki
 
 clean:
 	make -C pki clean
-	make -C apps/walletshield clean
 	rm -f \
+		apps/walletshield/walletshield \
 		genconfig/cmd/genconfig/genconfig \
 		server_plugins/cbor_plugins/http_proxy/cmd/http_proxy/http_proxy

apps/walletshield/main.go

@@ -38,9 +38,31 @@ var (
 	UserForwardPayloadLength = 30000
 )
 
-func sendRequest(thin *thin.ThinClient, payload []byte) ([]byte, error) {
+func sendRequest(thin *thin.ThinClient, httpRequestBytes []byte) ([]byte, error) {
+	// Compress the HTTP request
+	compressedPayload, err := common.CompressData(httpRequestBytes)
+	if err != nil {
+		return nil, fmt.Errorf("common.CompressData failed: %w", err)
+	}
+
+	// Create the request wrapper
+	request := &http_proxy.Request{
+		Payload: compressedPayload,
+	}
+
+	// Marshal to CBOR
+	blob, err := cbor.Marshal(request)
+	if err != nil {
+		return nil, fmt.Errorf("cbor.Marshal failed: %w", err)
+	}
+
+	// Validate payload size
+	if len(blob) > UserForwardPayloadLength {
+		return nil, fmt.Errorf("payload size %d exceeds maximum %d bytes", len(blob), UserForwardPayloadLength)
+	}
+
 	surbID := &[sConstants.SURBIDLength]byte{}
-	_, err := rand.Reader.Read(surbID[:])
+	_, err = rand.Reader.Read(surbID[:])
 	if err != nil {
 		panic(err)
 	}
@@ -53,7 +75,7 @@ func sendRequest(thin *thin.ThinClient, payload []byte) ([]byte, error) {
 	nodeId := hash.Sum256(target.MixDescriptor.IdentityKey)
 
 	timeoutCtx, _ := context.WithTimeout(context.TODO(), time.Duration(timeout)*time.Second)
-	return thin.BlockingSendMessage(timeoutCtx, payload, &nodeId, target.RecipientQueueID)
+	return thin.BlockingSendMessage(timeoutCtx, blob, &nodeId, target.RecipientQueueID)
 }
 
 type Server struct {
@@ -72,6 +94,7 @@ func main() {
 	var testProbeCount int
 	var testProbeResponseDelay int
 	var testProbeSendDelay int
+	var thinClientOnly bool
 
 	flag.StringVar(&configPath, "config", "", "file path of the client configuration TOML file")
 	flag.IntVar(&delayStart, "delay_start", 0, "max random seconds to delay start")
@@ -83,6 +106,7 @@ func main() {
 	flag.IntVar(&testProbeResponseDelay, "probe_response_delay", 0, "test probe response deplay")
 	flag.IntVar(&testProbeSendDelay, "probe_send_delay", 10, "test probe delay between probes")
 	flag.IntVar(&timeout, "timeout", timeout, "seconds to wait for a request")
+	flag.BoolVar(&thinClientOnly, "thin", false, "use thin client mode (connect to existing daemon)")
 	flag.Parse()
 
 	if listenAddr == "" && !testProbe {
@@ -109,27 +133,48 @@ func main() {
 	}
 
 	// start client2 daemon
-	cfg, err := config.LoadFile(configPath)
-	if err != nil {
-		panic(err)
-	}
+	var d *client2.Daemon
+	var cfgThin *thin.Config
+	if !thinClientOnly {
+		cfg, err := config.LoadFile(configPath)
+		if err != nil {
+			panic(err)
+		}
 
-	if listenAddrClient != "" {
-		cfg.ListenAddress = listenAddrClient
-	}
+		if listenAddrClient != "" {
+			cfg.ListenAddress = listenAddrClient
+		}
 
-	d, err := client2.NewDaemon(cfg)
-	if err != nil {
-		panic(err)
-	}
-	err = d.Start()
-	if err != nil {
-		panic(err)
+		d, err := client2.NewDaemon(cfg)
+		if err != nil {
+			panic(err)
+		}
+		err = d.Start()
+		if err != nil {
+			panic(err)
+		}
+
+		cfgThin = thin.FromConfig(cfg)
+
+		fmt.Println("Sleeping for 3 seconds to let the client daemon startup...")
+		time.Sleep(time.Second * 3) // XXX ugly hack but works: FIXME
+	} else {
+		cfgThin, err = thin.LoadFile(configPath)
+
+		if listenAddrClient != "" {
+			cfgThin.Address = listenAddrClient
+		}
+		if err != nil {
+			panic(fmt.Errorf("failed to open thin client config: %s", err))
+		}
 	}
 
-	time.Sleep(time.Second * 3) // XXX ugly hack but works: FIXME
+	logging := &config.Logging{
+		Disable: false,
+		Level:   level.String(),
+	}
 
-	thin := thin.NewThinClient(cfg)
+	thin := thin.NewThinClient(cfgThin, logging)
 	err = thin.Dial()
 	if err != nil {
 		panic(err)
@@ -144,6 +189,7 @@ func main() {
 
 	if testProbe {
 		server.SendTestProbes(testProbeSendDelay, testProbeCount, testProbeResponseDelay)
+		d.Shutdown()
 	} else {
 		http.HandleFunc("/", server.Handler)
 		err := http.ListenAndServe(listenAddr, nil)
@@ -172,33 +218,17 @@ func (s *Server) Handler(w http.ResponseWriter, req *http.Request) {
 	buf := new(bytes.Buffer)
 	req.Write(buf)
 
-	request := new(http_proxy.Request)
-	request.Payload, err = common.CompressData(buf.Bytes())
-	if err != nil {
-		s.log.Errorf("common.CompressData failed: %s", err)
-		return
-	}
-
 	s.log.Debugf("RAW HTTP REQUEST:\n%s", string(buf.Bytes()))
 
-	blob, err := cbor.Marshal(request)
-	if err != nil {
-		panic(err)
-	}
-
-	// FIXME: resolve with multi-packet transimssion to transcend payload size limitation
-	// While better solution is developing, pre-emptively reject oversized payloads
-	size := len(blob)
-	if size > UserForwardPayloadLength {
-		s.log.Errorf("(WIP) Rejecting message with oversized payload: %d > %d bytes", size, UserForwardPayloadLength)
-		http.Error(w, "custom 500", http.StatusInternalServerError)
-		return
-	}
-
-	rawReply, err := sendRequest(s.thin, blob)
+	rawReply, err := sendRequest(s.thin, buf.Bytes())
 	if err != nil {
 		s.log.Errorf("Failed to send message: %s", err)
-		http.Error(w, "custom 404", http.StatusNotFound)
+		// Check if it's a payload size error
+		if strings.Contains(err.Error(), "exceeds maximum") {
+			http.Error(w, "custom 500", http.StatusInternalServerError)
+		} else {
+			http.Error(w, "custom 404", http.StatusNotFound)
+		}
 		return
 	}
 
@@ -237,18 +267,13 @@ func (s *Server) Handler(w http.ResponseWriter, req *http.Request) {
 func (s *Server) SendTestProbes(testProbeSendDelay int, testProbeCount int, testProbeResponseDelay int) {
 	url := fmt.Sprintf("http://nowhere/_/probe/%d", testProbeResponseDelay)
 	req, err := http.NewRequest("GET", url, nil)
-	buf := new(bytes.Buffer)
-	req.Write(buf)
-	request := new(http_proxy.Request)
-	request.Payload, err = common.CompressData(buf.Bytes())
 	if err != nil {
-		s.log.Errorf("common.CompressData failed: %s", err)
+		s.log.Errorf("http.NewRequest failed: %s", err)
 		return
 	}
-	blob, err := cbor.Marshal(request)
-	if err != nil {
-		panic(err)
-	}
+	buf := new(bytes.Buffer)
+	req.Write(buf)
+	httpRequestBytes := buf.Bytes()
 
 	var packetsTransmitted, packetsReceived int
 	var rttMin, rttMax, rttTotal float64
@@ -258,7 +283,7 @@ func (s *Server) SendTestProbes(testProbeSendDelay int, testProbeCount int, test
 		packetsTransmitted++
 		t := time.Now()
 
-		_, err = sendRequest(s.thin, blob)
+		_, err = sendRequest(s.thin, httpRequestBytes)
 		elapsed := time.Since(t).Seconds()
 		if err != nil {
 			s.log.Errorf("Probe failed after %.2fs: %s", elapsed, err)

common/compression.go

@@ -6,9 +6,16 @@ import (
 	"io"
 )
 
+// CompressionEnabled is a flag that can be passed at build time to disable compression
+var CompressionEnabled string = "true"
+
 func CompressData(data []byte) ([]byte, error) {
+	if CompressionEnabled != "true" {
+		return data, nil
+	}
+
 	var buf bytes.Buffer
-	writer, err := flate.NewWriter(&buf, flate.BestCompression)
+	writer, err := flate.NewWriter(&buf, flate.BestSpeed)
 	if err != nil {
 		return nil, err
 	}
@@ -24,6 +31,10 @@ func CompressData(data []byte) ([]byte, error) {
 }
 
 func DecompressData(data []byte) ([]byte, error) {
+	if CompressionEnabled != "true" {
+		return data, nil
+	}
+
 	reader := flate.NewReader(bytes.NewReader(data))
 	defer reader.Close()
 	decompressedData, err := io.ReadAll(reader)

docker/Makefile

@@ -1,4 +1,6 @@
 warped?=true
+compression?=true
+
 mixes=3
 auths=3
 gateways=1
@@ -13,7 +15,7 @@ sh=$(shell if echo ${distro}|grep -q alpine; then echo sh; else echo bash; fi)
 cache_dir=cache
 log_level=DEBUG
 docker=$(shell if which podman|grep -q .; then echo podman; else echo docker; fi)
-ldflags="-extldflags '-Wl,-z,stack-size=0x1F40000' -buildid= -X github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${warped}"
+ldflags="-extldflags '-Wl,-z,stack-size=0x1F40000' -buildid= -X github.com/katzenpost/katzenpost/core/epochtime.WarpedEpoch=${warped} -X github.com/ZeroKnowledgeNetwork/opt/common.CompressionEnabled=${compression}"
 uid?=$(shell [ "$$SUDO_UID" != "" ] && echo "$$SUDO_UID" || id -u)
 gid?=$(shell [ "$$SUDO_GID" != "" ] && echo "$$SUDO_GID" || id -g)
 docker_user?=$(shell if echo ${docker}|grep -q podman; then echo 0:0; else echo ${uid}:${gid}; fi)
@@ -79,12 +81,17 @@ $(net_dir)/walletshield.$(distro): $(katzenpost_dir)/docker/$(distro)_base.stamp
 .PHONY: walletshield-probe
 walletshield-probe: $(net_dir)/walletshield.$(distro) $(net_dir)/running.stamp | $(cache_dir)
 	$(docker) run --network=host $(docker_args) $(mount_net_name) --rm katzenpost-$(distro)_base \
-		/$(net_name)/walletshield.$(distro) -config /$(net_name)/client2/client.toml -log_level DEBUG -probe --probe_count 3
+		/$(net_name)/walletshield.$(distro) -config /$(net_name)/client2/thinclient.toml -thin -log_level DEBUG -probe --probe_count 3
+
+.PHONY: walletshield-probe-full
+walletshield-probe-full: $(net_dir)/walletshield.$(distro) $(net_dir)/running.stamp | $(cache_dir)
+	$(docker) run --network=host $(docker_args) $(mount_net_name) --rm katzenpost-$(distro)_base \
+		/$(net_name)/walletshield.$(distro) -config /$(net_name)/client2/client.toml --listen_client :64333 -log_level DEBUG -probe --probe_count 3
 
 .PHONY: walletshield-start
 walletshield-start: $(net_dir)/walletshield.$(distro) $(net_dir)/running.stamp | $(cache_dir)
 	$(docker) run -d --network=host $(docker_args) $(mount_net_name) --name walletshield katzenpost-$(distro)_base \
-		/$(net_name)/walletshield.$(distro) -config /$(net_name)/client2/client.toml -listen :7070 -log_level DEBUG
+		/$(net_name)/walletshield.$(distro) -config /$(net_name)/client2/thinclient.toml -thin -listen :7070 -log_level DEBUG
 
 .PHONY: walletshield-logs
 walletshield-logs: