feat: support conditional compress

encoding/codecv1/codecv1.go

@@ -18,9 +18,6 @@ import (
 	"github.com/scroll-tech/da-codec/encoding/codecv0"
 )
 
-// BLSModulus is the BLS modulus defined in EIP-4844.
-var BLSModulus = new(big.Int).SetBytes(common.FromHex("0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001"))
-
 // MaxNumChunks is the maximum number of chunks that a batch can contain.
 const MaxNumChunks = 15
 
@@ -261,7 +258,7 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
 	copy(challengePreimage[0:], hash[:])
 
 	// convert raw data to BLSFieldElements
-	blob, err := MakeBlobCanonical(blobBytes)
+	blob, err := encoding.MakeBlobCanonical(blobBytes)
 	if err != nil {
 		return nil, common.Hash{}, nil, err
 	}
@@ -278,7 +275,7 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
 
 	// compute z = challenge_digest % BLS_MODULUS
 	challengeDigest := crypto.Keccak256Hash(challengePreimage)
-	pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), BLSModulus)
+	pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), encoding.BLSModulus)
 	pointBytes := pointBigInt.Bytes()
 
 	// the challenge point z
@@ -289,31 +286,6 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
 	return blob, blobVersionedHash, &z, nil
 }
 
-// MakeBlobCanonical converts the raw blob data into the canonical blob representation of 4096 BLSFieldElements.
-func MakeBlobCanonical(blobBytes []byte) (*kzg4844.Blob, error) {
-	// blob contains 131072 bytes but we can only utilize 31/32 of these
-	if len(blobBytes) > 126976 {
-		return nil, fmt.Errorf("oversized batch payload, blob bytes length: %v, max length: %v", len(blobBytes), 126976)
-	}
-
-	// the canonical (padded) blob payload
-	var blob kzg4844.Blob
-
-	// encode blob payload by prepending every 31 bytes with 1 zero byte
-	index := 0
-
-	for from := 0; from < len(blobBytes); from += 31 {
-		to := from + 31
-		if to > len(blobBytes) {
-			to = len(blobBytes)
-		}
-		copy(blob[index+1:], blobBytes[from:to])
-		index += 32
-	}
-
-	return &blob, nil
-}
-
 // NewDABatchFromBytes decodes the given byte slice into a DABatch.
 // Note: This function only populates the batch header, it leaves the blob-related fields empty.
 func NewDABatchFromBytes(data []byte) (*DABatch, error) {
@@ -374,24 +346,7 @@ func (b *DABatch) BlobDataProof() ([]byte, error) {
 		return nil, fmt.Errorf("failed to create KZG proof at point, err: %w, z: %v", err, hex.EncodeToString(b.z[:]))
 	}
 
-	return BlobDataProofFromValues(*b.z, y, commitment, proof), nil
-}
-
-// BlobDataProofFromValues creates the blob data proof from the given values.
-// Memory layout of ``_blobDataProof``:
-// | z       | y       | kzg_commitment | kzg_proof |
-// |---------|---------|----------------|-----------|
-// | bytes32 | bytes32 | bytes48        | bytes48   |
-
-func BlobDataProofFromValues(z kzg4844.Point, y kzg4844.Claim, commitment kzg4844.Commitment, proof kzg4844.Proof) []byte {
-	result := make([]byte, 32+32+48+48)
-
-	copy(result[0:32], z[:])
-	copy(result[32:64], y[:])
-	copy(result[64:112], commitment[:])
-	copy(result[112:160], proof[:])
-
-	return result
+	return encoding.BlobDataProofFromValues(*b.z, y, commitment, proof), nil
 }
 
 // Blob returns the blob of the batch.
@@ -406,7 +361,7 @@ func EstimateChunkL1CommitBlobSize(c *encoding.Chunk) (uint64, error) {
 	if err != nil {
 		return 0, err
 	}
-	return CalculatePaddedBlobSize(metadataSize + chunkDataSize), nil
+	return encoding.CalculatePaddedBlobSize(metadataSize + chunkDataSize), nil
 }
 
 // EstimateBatchL1CommitBlobSize estimates the total size of the L1 commit blob for a batch.
@@ -420,7 +375,7 @@ func EstimateBatchL1CommitBlobSize(b *encoding.Batch) (uint64, error) {
 		}
 		batchDataSize += chunkDataSize
 	}
-	return CalculatePaddedBlobSize(metadataSize + batchDataSize), nil
+	return encoding.CalculatePaddedBlobSize(metadataSize + batchDataSize), nil
 }
 
 func chunkL1CommitBlobDataSize(c *encoding.Chunk) (uint64, error) {
@@ -558,15 +513,3 @@ func EstimateBatchL1CommitCalldataSize(b *encoding.Batch) uint64 {
 	}
 	return totalL1CommitCalldataSize
 }
-
-// CalculatePaddedBlobSize calculates the required size on blob storage
-// where every 32 bytes can store only 31 bytes of actual data, with the first byte being zero.
-func CalculatePaddedBlobSize(dataSize uint64) uint64 {
-	paddedSize := (dataSize / 31) * 32
-
-	if dataSize%31 != 0 {
-		paddedSize += 1 + dataSize%31 // Add 1 byte for the first empty byte plus the remainder bytes
-	}
-
-	return paddedSize
-}

encoding/codecv2/codecv2.go

@@ -1,19 +1,12 @@
 package codecv2
 
-/*
-#include <stdint.h>
-char* compress_scroll_batch_bytes(uint8_t* src, uint64_t src_size, uint8_t* output_buf, uint64_t *output_buf_size);
-*/
-import "C"
-
 import (
 	"crypto/sha256"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
 	"math/big"
-	"unsafe"
 
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/core/types"
@@ -23,11 +16,9 @@ import (
 
 	"github.com/scroll-tech/da-codec/encoding"
 	"github.com/scroll-tech/da-codec/encoding/codecv1"
+	"github.com/scroll-tech/da-codec/encoding/zstd"
 )
 
-// BLSModulus is the BLS modulus defined in EIP-4844.
-var BLSModulus = codecv1.BLSModulus
-
 // MaxNumChunks is the maximum number of chunks that a batch can contain.
 const MaxNumChunks = 45
 
@@ -88,7 +79,7 @@ func NewDABatch(batch *encoding.Batch) (*DABatch, error) {
 	}
 
 	// blob payload
-	blob, blobVersionedHash, z, err := ConstructBlobPayload(batch.Chunks, false /* no mock */)
+	blob, blobVersionedHash, z, _, err := ConstructBlobPayload(batch.Chunks, false /* no mock */)
 	if err != nil {
 		return nil, err
 	}
@@ -118,7 +109,7 @@ func ComputeBatchDataHash(chunks []*encoding.Chunk, totalL1MessagePoppedBefore u
 }
 
 // ConstructBlobPayload constructs the 4844 blob payload.
-func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg4844.Blob, common.Hash, *kzg4844.Point, error) {
+func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg4844.Blob, common.Hash, *kzg4844.Point, []byte, error) {
 	// metadata consists of num_chunks (2 bytes) and chunki_size (4 bytes per chunk)
 	metadataLength := 2 + MaxNumChunks*4
 
@@ -149,7 +140,7 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
 				// encode L2 txs into blob payload
 				rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(tx, useMockTxData)
 				if err != nil {
-					return nil, common.Hash{}, nil, err
+					return nil, common.Hash{}, nil, nil, err
 				}
 				batchBytes = append(batchBytes, rlpTxData...)
 			}
@@ -178,30 +169,35 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
 	copy(challengePreimage[0:], hash[:])
 
 	// blobBytes represents the compressed blob payload (batchBytes)
-	blobBytes, err := compressScrollBatchBytes(batchBytes)
+	blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
 	if err != nil {
-		return nil, common.Hash{}, nil, err
+		return nil, common.Hash{}, nil, nil, err
 	}
 
 	// Only apply this check when the uncompressed batch data has exceeded 128 KiB.
 	if !useMockTxData && len(batchBytes) > 131072 {
 		// Check compressed data compatibility.
 		if err = encoding.CheckCompressedDataCompatibility(blobBytes); err != nil {
 			log.Error("ConstructBlobPayload: compressed data compatibility check failed", "err", err, "batchBytes", hex.EncodeToString(batchBytes), "blobBytes", hex.EncodeToString(blobBytes))
-			return nil, common.Hash{}, nil, err
+			return nil, common.Hash{}, nil, nil, err
 		}
 	}
 
+	if len(blobBytes) > 126976 {
+		log.Error("ConstructBlobPayload: Blob payload exceeds maximum size", "size", len(blobBytes), "blobBytes", hex.EncodeToString(blobBytes))
+		return nil, common.Hash{}, nil, nil, errors.New("Blob payload exceeds maximum size")
+	}
+
 	// convert raw data to BLSFieldElements
-	blob, err := MakeBlobCanonical(blobBytes)
+	blob, err := encoding.MakeBlobCanonical(blobBytes)
 	if err != nil {
-		return nil, common.Hash{}, nil, err
+		return nil, common.Hash{}, nil, nil, err
 	}
 
 	// compute blob versioned hash
 	c, err := kzg4844.BlobToCommitment(blob)
 	if err != nil {
-		return nil, common.Hash{}, nil, errors.New("failed to create blob commitment")
+		return nil, common.Hash{}, nil, nil, errors.New("failed to create blob commitment")
 	}
 	blobVersionedHash := kzg4844.CalcBlobHashV1(sha256.New(), &c)
 
@@ -210,20 +206,15 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
 
 	// compute z = challenge_digest % BLS_MODULUS
 	challengeDigest := crypto.Keccak256Hash(challengePreimage)
-	pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), BLSModulus)
+	pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), encoding.BLSModulus)
 	pointBytes := pointBigInt.Bytes()
 
 	// the challenge point z
 	var z kzg4844.Point
 	start := 32 - len(pointBytes)
 	copy(z[start:], pointBytes)
 
-	return blob, blobVersionedHash, &z, nil
-}
-
-// MakeBlobCanonical converts the raw blob data into the canonical blob representation of 4096 BLSFieldElements.
-func MakeBlobCanonical(blobBytes []byte) (*kzg4844.Blob, error) {
-	return codecv1.MakeBlobCanonical(blobBytes)
+	return blob, blobVersionedHash, &z, blobBytes, nil
 }
 
 // NewDABatchFromBytes decodes the given byte slice into a DABatch.
@@ -286,7 +277,7 @@ func (b *DABatch) BlobDataProof() ([]byte, error) {
 		return nil, fmt.Errorf("failed to create KZG proof at point, err: %w, z: %v", err, hex.EncodeToString(b.z[:]))
 	}
 
-	return codecv1.BlobDataProofFromValues(*b.z, y, commitment, proof), nil
+	return encoding.BlobDataProofFromValues(*b.z, y, commitment, proof), nil
 }
 
 // Blob returns the blob of the batch.
@@ -296,38 +287,38 @@ func (b *DABatch) Blob() *kzg4844.Blob {
 
 // EstimateChunkL1CommitBatchSizeAndBlobSize estimates the L1 commit uncompressed batch size and compressed blob size for a single chunk.
 func EstimateChunkL1CommitBatchSizeAndBlobSize(c *encoding.Chunk) (uint64, uint64, error) {
-	batchBytes, err := constructBatchPayload([]*encoding.Chunk{c})
+	batchBytes, err := encoding.ConstructBatchPayloadInBlob([]*encoding.Chunk{c}, MaxNumChunks)
 	if err != nil {
 		return 0, 0, err
 	}
-	blobBytes, err := compressScrollBatchBytes(batchBytes)
+	blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
 	if err != nil {
 		return 0, 0, err
 	}
-	return uint64(len(batchBytes)), CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
+	return uint64(len(batchBytes)), encoding.CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
 }
 
 // EstimateBatchL1CommitBatchSizeAndBlobSize estimates the L1 commit uncompressed batch size and compressed blob size for a batch.
 func EstimateBatchL1CommitBatchSizeAndBlobSize(b *encoding.Batch) (uint64, uint64, error) {
-	batchBytes, err := constructBatchPayload(b.Chunks)
+	batchBytes, err := encoding.ConstructBatchPayloadInBlob(b.Chunks, MaxNumChunks)
 	if err != nil {
 		return 0, 0, err
 	}
-	blobBytes, err := compressScrollBatchBytes(batchBytes)
+	blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
 	if err != nil {
 		return 0, 0, err
 	}
-	return uint64(len(batchBytes)), CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
+	return uint64(len(batchBytes)), encoding.CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
 }
 
 // CheckChunkCompressedDataCompatibility checks the compressed data compatibility for a batch built from a single chunk.
 // It constructs a batch payload, compresses the data, and checks the compressed data compatibility if the uncompressed data exceeds 128 KiB.
 func CheckChunkCompressedDataCompatibility(c *encoding.Chunk) (bool, error) {
-	batchBytes, err := constructBatchPayload([]*encoding.Chunk{c})
+	batchBytes, err := encoding.ConstructBatchPayloadInBlob([]*encoding.Chunk{c}, MaxNumChunks)
 	if err != nil {
 		return false, err
 	}
-	blobBytes, err := compressScrollBatchBytes(batchBytes)
+	blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
 	if err != nil {
 		return false, err
 	}
@@ -345,11 +336,11 @@ func CheckChunkCompressedDataCompatibility(c *encoding.Chunk) (bool, error) {
 // CheckBatchCompressedDataCompatibility checks the compressed data compatibility for a batch.
 // It constructs a batch payload, compresses the data, and checks the compressed data compatibility if the uncompressed data exceeds 128 KiB.
 func CheckBatchCompressedDataCompatibility(b *encoding.Batch) (bool, error) {
-	batchBytes, err := constructBatchPayload(b.Chunks)
+	batchBytes, err := encoding.ConstructBatchPayloadInBlob(b.Chunks, MaxNumChunks)
 	if err != nil {
 		return false, err
 	}
-	blobBytes, err := compressScrollBatchBytes(batchBytes)
+	blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
 	if err != nil {
 		return false, err
 	}
@@ -388,63 +379,3 @@ func EstimateChunkL1CommitGas(c *encoding.Chunk) uint64 {
 func EstimateBatchL1CommitGas(b *encoding.Batch) uint64 {
 	return codecv1.EstimateBatchL1CommitGas(b)
 }
-
-// constructBatchPayload constructs the batch payload.
-// This function is only used in compressed batch payload length estimation.
-func constructBatchPayload(chunks []*encoding.Chunk) ([]byte, error) {
-	// metadata consists of num_chunks (2 bytes) and chunki_size (4 bytes per chunk)
-	metadataLength := 2 + MaxNumChunks*4
-
-	// batchBytes represents the raw (un-compressed and un-padded) blob payload
-	batchBytes := make([]byte, metadataLength)
-
-	// batch metadata: num_chunks
-	binary.BigEndian.PutUint16(batchBytes[0:], uint16(len(chunks)))
-
-	// encode batch metadata and L2 transactions,
-	for chunkID, chunk := range chunks {
-		currentChunkStartIndex := len(batchBytes)
-
-		for _, block := range chunk.Blocks {
-			for _, tx := range block.Transactions {
-				if tx.Type == types.L1MessageTxType {
-					continue
-				}
-
-				// encode L2 txs into batch payload
-				rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(tx, false /* no mock */)
-				if err != nil {
-					return nil, err
-				}
-				batchBytes = append(batchBytes, rlpTxData...)
-			}
-		}
-
-		// batch metadata: chunki_size
-		if chunkSize := len(batchBytes) - currentChunkStartIndex; chunkSize != 0 {
-			binary.BigEndian.PutUint32(batchBytes[2+4*chunkID:], uint32(chunkSize))
-		}
-	}
-	return batchBytes, nil
-}
-
-// compressScrollBatchBytes compresses the given batch of bytes.
-// The output buffer is allocated with an extra 128 bytes to accommodate metadata overhead or error message.
-func compressScrollBatchBytes(batchBytes []byte) ([]byte, error) {
-	srcSize := C.uint64_t(len(batchBytes))
-	outbufSize := C.uint64_t(len(batchBytes) + 128) // Allocate output buffer with extra 128 bytes
-	outbuf := make([]byte, outbufSize)
-
-	if err := C.compress_scroll_batch_bytes((*C.uchar)(unsafe.Pointer(&batchBytes[0])), srcSize,
-		(*C.uchar)(unsafe.Pointer(&outbuf[0])), &outbufSize); err != nil {
-		return nil, fmt.Errorf("failed to compress scroll batch bytes: %s", C.GoString(err))
-	}
-
-	return outbuf[:int(outbufSize)], nil
-}
-
-// CalculatePaddedBlobSize calculates the required size on blob storage
-// where every 32 bytes can store only 31 bytes of actual data, with the first byte being zero.
-func CalculatePaddedBlobSize(dataSize uint64) uint64 {
-	return codecv1.CalculatePaddedBlobSize(dataSize)
-}