Add compression functions into /common

apps/walletshield/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/katzenpost/katzenpost/client2/thin"
 	sConstants "github.com/katzenpost/katzenpost/core/sphinx/constants"
 
+	"github.com/ZeroKnowledgeNetwork/opt/common"
 	"github.com/ZeroKnowledgeNetwork/opt/server_plugins/cbor_plugins/http_proxy"
 )
 
@@ -172,7 +173,11 @@ func (s *Server) Handler(w http.ResponseWriter, req *http.Request) {
 	req.Write(buf)
 
 	request := new(http_proxy.Request)
-	request.Payload = buf.Bytes()
+	request.Payload, err = common.CompressData(buf.Bytes())
+	if err != nil {
+		s.log.Errorf("common.CompressData failed: %s", err)
+		return
+	}
 
 	s.log.Debugf("RAW HTTP REQUEST:\n%s", string(buf.Bytes()))
 
@@ -210,14 +215,23 @@ func (s *Server) Handler(w http.ResponseWriter, req *http.Request) {
 
 	if response.Error != "" {
 		s.log.Errorf("Response Error: %s", response.Error)
-	} else {
-		s.log.Infof("Response: %s", response.Payload)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
 	}
 
+	responsePayload, err := common.DecompressData(response.Payload)
+	if err != nil {
+		s.log.Errorf("common.DecompressData failed: %s", err)
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	s.log.Infof("Response: %s", responsePayload)
+
 	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(response.Payload)))
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(responsePayload)))
 	w.WriteHeader(http.StatusOK)
-	fmt.Fprintf(w, string(response.Payload))
+	fmt.Fprintf(w, string(responsePayload))
 }
 
 func (s *Server) SendTestProbes(testProbeSendDelay int, testProbeCount int, testProbeResponseDelay int) {
@@ -226,7 +240,11 @@ func (s *Server) SendTestProbes(testProbeSendDelay int, testProbeCount int, test
 	buf := new(bytes.Buffer)
 	req.Write(buf)
 	request := new(http_proxy.Request)
-	request.Payload = buf.Bytes()
+	request.Payload, err = common.CompressData(buf.Bytes())
+	if err != nil {
+		s.log.Errorf("common.CompressData failed: %s", err)
+		return
+	}
 	blob, err := cbor.Marshal(request)
 	if err != nil {
 		panic(err)

common/compression.go

@@ -0,0 +1,34 @@
+package common
+
+import (
+	"bytes"
+	"compress/flate"
+	"io"
+)
+
+func CompressData(data []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	writer, err := flate.NewWriter(&buf, flate.BestCompression)
+	if err != nil {
+		return nil, err
+	}
+	_, err = writer.Write(data)
+	if err != nil {
+		return nil, err
+	}
+	err = writer.Close()
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func DecompressData(data []byte) ([]byte, error) {
+	reader := flate.NewReader(bytes.NewReader(data))
+	defer reader.Close()
+	decompressedData, err := io.ReadAll(reader)
+	if err != nil && err != io.EOF {
+		return nil, err
+	}
+	return decompressedData, nil
+}

common/compression_test.go

@@ -0,0 +1,18 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompressData(t *testing.T) {
+	data := []byte("test data")
+	compressedData, err := CompressData(data)
+	require.NoError(t, err)
+	require.NotEqual(t, data, compressedData)
+
+	decompressedData, err := DecompressData(compressedData)
+	require.NoError(t, err)
+	require.Equal(t, data, decompressedData)
+}

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/katzenpost/hpqc v0.0.55
 	github.com/katzenpost/katzenpost v0.0.48
 	github.com/quic-go/quic-go v0.50.0
+	github.com/stretchr/testify v1.9.0
 	go.etcd.io/bbolt v1.3.10
 	golang.org/x/crypto v0.36.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
@@ -23,6 +24,7 @@ require (
 	filippo.io/mlkem768 v0.0.0-20240221181710-5ce91625fdc1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/lipgloss v0.10.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-faster/xor v1.0.0 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
@@ -43,6 +45,7 @@ require (
 	github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect
 	github.com/oasisprotocol/deoxysii v0.0.0-20220228165953-2091330c22b7 // indirect
 	github.com/onsi/ginkgo/v2 v2.20.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect

server_plugins/cbor_plugins/http_proxy/cmd/http_proxy/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/katzenpost/katzenpost/core/log"
 	"github.com/katzenpost/katzenpost/server/cborplugin"
 
+	"github.com/ZeroKnowledgeNetwork/opt/common"
 	"github.com/ZeroKnowledgeNetwork/opt/server_plugins/cbor_plugins/http_proxy"
 )
 
@@ -141,13 +142,19 @@ func (s *proxyRequestHandler) OnCommand(cmd cborplugin.Command) error {
 
 		s.log.Debugf("Raw request payload: %s", req.Payload)
 
-		request, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(req.Payload)))
+		requestPayload, err := common.DecompressData(req.Payload)
+		if err != nil {
+			s.log.Errorf("common.DecompressData failed: %s", err)
+			return s.sendError(r.ID, r.SURB, "Failed to decompress request")
+		}
+		request, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(requestPayload)))
 		if err != nil {
 			s.log.Errorf("http.ReadRequest failed: %s", err)
 			return s.sendError(r.ID, r.SURB, "Failed to read HTTP request")
 		}
 
-		s.log.Debugf("REQUEST URL PATH: %s", request.URL.Path)
+		s.log.Debugf("Request URL Path: %s", request.URL.Path)
+		s.log.Debugf("Request payload: %s", requestPayload)
 
 		err = validateRequestURL(request.URL.Path)
 		if err != nil {
@@ -203,12 +210,17 @@ func (s *proxyRequestHandler) OnCommand(cmd cborplugin.Command) error {
 		s.log.Debugf("Reply payload: %s", body)
 
 		var response *http_proxy.Response
-		if len(body) > MaxPayloadSize {
+		responsePayload, err := common.CompressData(body)
+		if err != nil {
+			s.log.Errorf("common.CompressData failed: %s", err)
+			return s.sendError(r.ID, r.SURB, "Failed to compress response")
+		}
+		if len(responsePayload) > MaxPayloadSize {
 			s.log.Error("HTTP response body exceeds max Sphinx payload")
 			return s.sendError(r.ID, r.SURB, "HTTP response is too big")
 		} else {
 			response = &http_proxy.Response{
-				Payload: body,
+				Payload: responsePayload,
 			}
 		}
 

tests/e2e/walletshield/base/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x2105","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x2105"}

tests/e2e/walletshield/base/sepolia/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x14a34","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x14a34"}

tests/e2e/walletshield/bitcoin/out.json

@@ -1 +1 @@
-{"result":"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09","error":null,"id":"test"}
+{"error":null,"id":"test","result":"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"}

tests/e2e/walletshield/bitcoin/testnet/out.json

@@ -1 +1 @@
-{"result":"00000000373403049c5fff2cd653590e8cbe6f7ac639db270e7d1a7503d698df","error":null,"id":"test"}
+{"error":null,"id":"test","result":"00000000373403049c5fff2cd653590e8cbe6f7ac639db270e7d1a7503d698df"}

tests/e2e/walletshield/bsc/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x38","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x38"}

tests/e2e/walletshield/bsc/testnet/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x61","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x61"}

tests/e2e/walletshield/ethereum/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x1","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x1"}

tests/e2e/walletshield/ethereum/sepolia/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0xaa36a7","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0xaa36a7"}

tests/e2e/walletshield/ethereum:large-response/in.json

@@ -0,0 +1,15 @@
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "eth_getLogs",
+  "params": [
+    {
+      "fromBlock": "0x12C0000",
+      "toBlock": "0x12C000D",
+      "address": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48",
+      "topics": [
+        "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
+      ]
+    }
+  ]
+}

tests/e2e/walletshield/mina/devnet/out.json

@@ -1 +1 @@
-{"data":{"daemonStatus":{"chainId":"29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6"}}}
+{"data":{"daemonStatus":{"chainId":"29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6"}}}

tests/e2e/walletshield/mina/out.json

@@ -1 +1 @@
-{"data":{"daemonStatus":{"chainId":"a7351abc7ddf2ea92d1b38cc8e636c271c1dfd2c081c637f62ebc2af34eb7cc1"}}}
+{"data":{"daemonStatus":{"chainId":"a7351abc7ddf2ea92d1b38cc8e636c271c1dfd2c081c637f62ebc2af34eb7cc1"}}}

tests/e2e/walletshield/polygon/amoy/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x13882","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x13882"}

tests/e2e/walletshield/polygon/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"0x89","id":"1729115977487"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x89"}

tests/e2e/walletshield/solana/devnet/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"EtWTRABZaYq6iMfeYKouRu166VU2xqa1wcaWoxPkrZBG","id":1}
+{"id":1,"jsonrpc":"2.0","result":"EtWTRABZaYq6iMfeYKouRu166VU2xqa1wcaWoxPkrZBG"}

tests/e2e/walletshield/solana/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","result":"5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d","id":1}
+{"id":1,"jsonrpc":"2.0","result":"5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d"}

tests/e2e/walletshield/test.sh

@@ -7,20 +7,22 @@ status=0
 run_test() {
   test_dir="$1"
   test_case="${test_dir#${dir}}"
+  network="${test_case%%:*}"
   output=$(mktemp)
 
   if [ -f "${test_dir}/in.json" ]; then
     curl \
       -X POST \
       -H 'Content-Type: application/json' \
       -d "$(cat ${test_dir}/in.json)" \
-      --output ${output} \
       --silent \
-      "${uri}${test_case}"
+      "${uri}${network}" |
+      jq -cS . 2>/dev/null >${output}
 
     if ! diff "${test_dir}/out.json" ${output} > /dev/null; then
       echo "${test_case}: ❌ failed"
       status=1
+      [[ "${WRITE:-0}" == 1 ]] && cp -v ${output} "${test_dir}/out.json"
     else
       echo "${test_case}: ✅ passed"
     fi
@@ -32,6 +34,8 @@ run_test() {
   sleep 1s
 }
 
+[[ "${WRITE:-0}" == 1 ]] && echo "Writing output."
+
 if [ -z "${1}" ]; then
   for test_dir in $(find "${dir}" -mindepth 1 -type d)
   do

tests/e2e/walletshield/xdc/out.json

@@ -1 +1 @@
-{"jsonrpc":"2.0","id":"1729115977487","result":"0x32"}
+{"id":"1729115977487","jsonrpc":"2.0","result":"0x32"}