diff --git a/integration/map.go b/integration/map.go index 941a2176b9..b06394c2a4 100644 --- a/integration/map.go +++ b/integration/map.go @@ -30,6 +30,8 @@ import ( "github.com/google/trillian/testonly" ) +const treeID = int64(0) + // RunMapIntegration runs a map integration test using the given map ID and client. func RunMapIntegration(ctx context.Context, mapID int64, pubKey crypto.PublicKey, client trillian.TrillianMapClient) error { { @@ -127,8 +129,8 @@ func RunMapIntegration(ctx context.Context, mapID int64, pubKey crypto.PublicKey if got, want := leaf.LeafValue, ev.LeafValue; !bytes.Equal(got, want) { return fmt.Errorf("got value %s, want %s", got, want) } - leafHash := h.HashLeaf(leaf.LeafValue) - if err := merkle.VerifyMapInclusionProof(leaf.Index, leafHash, r.GetMapRoot().GetRootHash(), incl.Inclusion, h); err != nil { + leafHash := h.HashLeaf(treeID, leaf.Index, h.BitLen(), leaf.LeafValue) + if err := merkle.VerifyMapInclusionProof(treeID, leaf.Index, leafHash, r.GetMapRoot().GetRootHash(), incl.Inclusion, h); err != nil { return fmt.Errorf("verifyMapInclusionProof(%x): %v", leaf.Index, err) } } @@ -157,8 +159,8 @@ func testForNonExistentLeaf(ctx context.Context, mapID int64, if got, want := len(leaf.LeafValue), 0; got != want { return fmt.Errorf("len(GetLeaves(%s).LeafValue): %v, want, %v", index1, got, want) } - leafHash := h.HashLeaf(leaf.LeafValue) - if err := merkle.VerifyMapInclusionProof(leaf.Index, leafHash, latestRoot.RootHash, incl.Inclusion, h); err != nil { + leafHash := h.HashLeaf(treeID, leaf.Index, h.BitLen(), leaf.LeafValue) + if err := merkle.VerifyMapInclusionProof(treeID, leaf.Index, leafHash, latestRoot.RootHash, incl.Inclusion, h); err != nil { return fmt.Errorf("VerifyMapInclusionProof(%x): %v", leaf.Index, err) } } diff --git a/merkle/coniks/coniks.go b/merkle/coniks/coniks.go new file mode 100644 index 0000000000..043ab31fbb --- /dev/null +++ b/merkle/coniks/coniks.go @@ -0,0 +1,123 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package coniks provides hashing for maps. +package coniks + +import ( + "crypto" + "encoding/binary" + "fmt" + + "github.com/google/trillian" + "github.com/google/trillian/merkle/hashers" +) + +func init() { + hashers.RegisterMapHasher(trillian.HashStrategy_CONIKS_SHA512_256, Default) +} + +// Domain separation prefixes +var ( + leafIdentifier = []byte("L") + emptyIdentifier = []byte("E") +) + +// Default is the standard CONIKS hasher. +var Default = New(crypto.SHA512_256) + +// hasher implements the sparse merkle tree hashing algorithm specified in the CONIKS paper. +type hasher struct { + crypto.Hash +} + +// New creates a new hashers.TreeHasher using the passed in hash function. +func New(h crypto.Hash) hashers.MapHasher { + return &hasher{Hash: h} +} + +// EmptyRoot returns the root of an empty tree. +func (m *hasher) EmptyRoot() []byte { + panic("EmptyRoot() not defined for coniks.Hasher") +} + +// HashEmpty returns the hash of an empty branch at a given height. +// A height of 0 indicates the hash of an empty leaf. +// Empty branches within the tree are plain interior nodes e1 = H(e0, e0) etc. +func (m *hasher) HashEmpty(treeID int64, index []byte, height int) []byte { + depth := m.BitLen() - height + + h := m.New() + h.Write(emptyIdentifier) + binary.Write(h, binary.BigEndian, uint64(treeID)) + h.Write(m.maskIndex(index, depth)) + binary.Write(h, binary.BigEndian, uint32(depth)) + return h.Sum(nil) +} + +// HashLeaf calculate the merkle tree leaf value: +// H(Identifier || treeID || depth || index || dataHash) +func (m *hasher) HashLeaf(treeID int64, index []byte, height int, leaf []byte) []byte { + depth := m.BitLen() - height + + h := m.New() + h.Write(leafIdentifier) + binary.Write(h, binary.BigEndian, uint64(treeID)) + h.Write(m.maskIndex(index, depth)) + binary.Write(h, binary.BigEndian, uint32(depth)) + h.Write(leaf) + return h.Sum(nil) +} + +// HashChildren returns the internal Merkle tree node hash of the the two child nodes l and r. +// The hashed structure is H(l || r). +func (m *hasher) HashChildren(l, r []byte) []byte { + h := m.New() + h.Write(l) + h.Write(r) + return h.Sum(nil) +} + +// BitLen returns the number of bits in the hash function. +func (m *hasher) BitLen() int { + return m.Size() * 8 +} + +// leftmask contains bitmasks indexed such that the left x bits are set. It is +// indexed by byte position from 0-7 0 is special cased to 0xFF since 8 mod 8 +// is 0. leftmask is only used to mask the last byte. +var leftmask = [8]byte{0xFF, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE} + +// maskIndex returns index with only the left depth bits set. +// index must be of size m.Size() and 0 <= depth <= m.BitLen(). +// e.g. +func (m *hasher) maskIndex(index []byte, depth int) []byte { + if got, want := len(index), m.Size(); got != want { + panic(fmt.Sprintf("index len: %d, want %d", got, want)) + } + if got, want := depth, m.BitLen(); got < 0 || got > want { + panic(fmt.Sprintf("depth: %d, want <= %d && > 0", got, want)) + } + + // Create an empty index Size() bytes long. + ret := make([]byte, m.Size()) + if depth > 0 { + // Copy the first depthBytes. + depthBytes := (depth + 7) >> 3 + copy(ret, index[:depthBytes]) + // Mask off unwanted bits in the last byte. + ret[depthBytes-1] = ret[depthBytes-1] & leftmask[depth%8] + } + return ret +} diff --git a/merkle/coniks/coniks_test.go b/merkle/coniks/coniks_test.go new file mode 100644 index 0000000000..4924291e93 --- /dev/null +++ b/merkle/coniks/coniks_test.go @@ -0,0 +1,85 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package coniks + +import ( + "bytes" + "crypto" + "encoding/hex" + "testing" +) + +// h2b converts a hex string into a bytes string +func h2b(h string) []byte { + b, err := hex.DecodeString(h) + if err != nil { + panic("invalid hex string") + } + return b +} + +func TestVectors(t *testing.T) { + for _, tc := range []struct { + treeID int64 + index []byte + depth int + leaf []byte + want []byte + }{ + {0, h2b("0000000000000000000000000000000000000000000000000000000000000000"), 128, []byte(""), h2b("5f4bf72f8175e8db7b58c96354d870b60fb98ce7e4fdde7d601a4d3e5b5d1f20")}, + {1, h2b("0000000000000000000000000000000000000000000000000000000000000000"), 128, []byte(""), h2b("a5f5d0c1e86a15c1ab9c8b88f7e8b7ef17b246350c141c6f21ab81e51d5a6ef2")}, + {0, h2b("1111111111111111111111111111111111111111111111111111111111111111"), 128, []byte(""), h2b("f7ab5ae11bdea50c293a59c0399f5704fd3401ab4144b3ce6230a6866efe2304")}, + {0, h2b("0000000000000000000000000000000000000000000000000000000000000000"), 127, []byte(""), h2b("8a8170ff167d7dcdf1b580c89b2f6a6cc3a085c957d1d637d6314e38b83732a0")}, + {0, h2b("0000000000000000000000000000000000000000000000000000000000000000"), 128, []byte("foo"), h2b("0d394ddaca7acbf2ad6f9bede5f652be966e3c9e94eaccc472c9b2ca139d06ec")}, + // Test vector from Key Transparency + {0, h2b("1111111111111111111111111111111111111111111111111111111111111111"), 128, []byte("leaf"), h2b("d77b4bb8e8fdd941976d285a8a0cd8db27b6f7e889e51134e1428224306b6f52")}, + } { + height := Default.BitLen() - tc.depth + if got, want := Default.HashLeaf(tc.treeID, tc.index, height, tc.leaf), tc.want; !bytes.Equal(got, want) { + t.Errorf("HashLeaf(%v, %s, %v, %s): %x, want %x", tc.treeID, tc.index, tc.depth, tc.leaf, got, want) + } + } +} + +func TestMaskIndex(t *testing.T) { + h := &hasher{crypto.SHA1} // Use a shorter hash for shorter test vectors. + for _, tc := range []struct { + index []byte + depth int + want []byte + }{ + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 0, want: h2b("0000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 1, want: h2b("8000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 2, want: h2b("C000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 3, want: h2b("E000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 4, want: h2b("F000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 5, want: h2b("F800000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 6, want: h2b("FC00000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 7, want: h2b("FE00000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 8, want: h2b("FF00000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 9, want: h2b("FF80000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 10, want: h2b("FFC0000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 159, want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 160, want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 1, want: h2b("0000000000000000000000000000000000000000")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 17, want: h2b("0001000000000000000000000000000000000000")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 159, want: h2b("000102030405060708090A0B0C0D0E0F10111212")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 160, want: h2b("000102030405060708090A0B0C0D0E0F10111213")}, + } { + if got, want := h.maskIndex(tc.index, tc.depth), tc.want; !bytes.Equal(got, want) { + t.Errorf("maskIndex(%x, %v): %x, want %x", tc.index, tc.depth, got, want) + } + } +} diff --git a/merkle/hashers/tree_hasher.go b/merkle/hashers/tree_hasher.go index 69a24c0cf5..efb209f938 100644 --- a/merkle/hashers/tree_hasher.go +++ b/merkle/hashers/tree_hasher.go @@ -38,15 +38,18 @@ type LogHasher interface { type MapHasher interface { // HashEmpty returns the hash of an empty branch at a given depth. // A height of 0 indicates an empty leaf. The maximum height is Size*8. - HashEmpty(height int) []byte + // TODO(gbelvin) fully define index. + HashEmpty(treeID int64, index []byte, height int) []byte // HashLeaf computes the hash of a leaf that exists. - HashLeaf(leaf []byte) []byte + HashLeaf(treeID int64, index []byte, height int, leaf []byte) []byte // HashChildren computes interior nodes. HashChildren(l, r []byte) []byte // Size is the number of bits in the underlying hash function. // It is also the height of the merkle tree. // TODO(gbelvin): Replace Size() with BitLength(). Size() int + // BitLen returns the number of bits in the underlying hash function. + BitLen() int } var ( diff --git a/merkle/hstar2.go b/merkle/hstar2.go index 4a444b5bfb..216bc3b2c0 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -39,12 +39,14 @@ type HStar2LeafHash struct { // HStar2 is a recursive implementation for calculating the root hash of a sparse // Merkle tree. type HStar2 struct { + treeID int64 hasher hashers.MapHasher } // NewHStar2 creates a new HStar2 tree calculator based on the passed in MapHasher. -func NewHStar2(hasher hashers.MapHasher) HStar2 { +func NewHStar2(treeID int64, hasher hashers.MapHasher) HStar2 { return HStar2{ + treeID: treeID, hasher: hasher, } } @@ -56,7 +58,7 @@ func (s *HStar2) HStar2Root(n int, values []HStar2LeafHash) ([]byte, error) { offset := big.NewInt(0) return s.hStar2b(n, values, offset, func(depth int, index *big.Int) ([]byte, error) { - return s.hasher.HashEmpty(depth), nil + return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth), nil }, func(int, *big.Int, []byte) error { return nil }) } @@ -98,7 +100,7 @@ func (s *HStar2) HStar2Nodes(treeDepth, treeLevelOffset int, values []HStar2Leaf return h, nil } // otherwise just return the null hash for this level - return s.hasher.HashEmpty(depth + treeLevelOffset), nil + return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth+treeLevelOffset), nil }, func(depth int, index *big.Int, hash []byte) error { return set(treeDepth-depth, index, hash) @@ -121,13 +123,12 @@ func (s *HStar2) hStar2b(n int, values []HStar2LeafHash, offset *big.Int, get Sp } return values[0].LeafHash, nil } - - split := new(big.Int).Lsh(smtOne, uint(n-1)) - split.Add(split, offset) if len(values) == 0 { return get(n, offset) } + split := new(big.Int).Lsh(smtOne, uint(n-1)) + split.Add(split, offset) i := sort.Search(len(values), func(i int) bool { return values[i].Index.Cmp(split) >= 0 }) lhs, err := s.hStar2b(n-1, values[:i], offset, get, set) if err != nil { @@ -176,3 +177,13 @@ func (s *valueSorter) Less(i, j int) bool { func indexLess(a, b *HStar2LeafHash) bool { return a.Index.Cmp(b.Index) < 0 } + +// PaddedBytes takes a big.Int and returns it's value, left padded with zeros. +// e.g. 1 -> 0000000000000000000000000000000000000001 +func PaddedBytes(i *big.Int, size int) []byte { + b := i.Bytes() + ret := make([]byte, size) + padBytes := len(ret) - len(b) + copy(ret[padBytes:], b) + return ret +} diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index 0e3d4c879b..876a5794f2 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -16,7 +16,7 @@ package merkle import ( "bytes" - "crypto/sha256" + "encoding/hex" "fmt" "math/big" "testing" @@ -26,6 +26,8 @@ import ( "github.com/google/trillian/testonly" ) +const treeID = int64(0) + // This root was calculated with the C++/Python sparse Merkle tree code in the // github.com/google/certificate-transparency repo. // TODO(alcutter): replace with hash-dependent computation. How is this computed? @@ -35,13 +37,13 @@ var sparseEmptyRootHashB64 = testonly.MustDecodeBase64("xmifEIEqCYCXbZUz2Dh1KCFm // passing into a the HStar2 sparse Merkle tree implementation. // The map keys will be SHA256 hashed before being added to the returned // structs. -func createHStar2Leaves(hasher hashers.MapHasher, values map[string]string) []HStar2LeafHash { +func createHStar2Leaves(treeID int64, hasher hashers.MapHasher, values map[string]string) []HStar2LeafHash { r := []HStar2LeafHash{} for k := range values { - khash := sha256.Sum256([]byte(k)) - vhash := hasher.HashLeaf([]byte(values[k])) + khash := testonly.HashKey(k) + vhash := hasher.HashLeaf(treeID, khash, hasher.BitLen(), []byte(values[k])) r = append(r, HStar2LeafHash{ - Index: new(big.Int).SetBytes(khash[:]), + Index: new(big.Int).SetBytes(khash), LeafHash: vhash[:], }) } @@ -49,7 +51,7 @@ func createHStar2Leaves(hasher hashers.MapHasher, values map[string]string) []HS } func TestHStar2EmptyRootKAT(t *testing.T) { - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, maphasher.Default) root, err := s.HStar2Root(s.hasher.Size()*8, []HStar2LeafHash{}) if err != nil { t.Fatalf("Failed to calculate root: %v", err) @@ -72,12 +74,12 @@ var simpleTestVector = []struct { } func TestHStar2SimpleDataSetKAT(t *testing.T) { - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, maphasher.Default) m := make(map[string]string) for i, x := range simpleTestVector { m[x.k] = x.v - values := createHStar2Leaves(maphasher.Default, m) + values := createHStar2Leaves(treeID, maphasher.Default, m) root, err := s.HStar2Root(s.hasher.Size()*8, values) if err != nil { t.Errorf("Failed to calculate root at iteration %d: %v", i, err) @@ -95,17 +97,18 @@ func TestHStar2GetSet(t *testing.T) { // Node cache is shared between tree builds and in effect plays the role of // the TreeStorage layer. cache := make(map[string][]byte) + hasher := maphasher.Default for i, x := range simpleTestVector { - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, hasher) m := make(map[string]string) m[x.k] = x.v - values := createHStar2Leaves(maphasher.Default, m) + values := createHStar2Leaves(treeID, hasher, m) // ensure we're going incrementally, one leaf at a time. if len(values) != 1 { t.Fatalf("Should only have 1 leaf per run, got %d", len(values)) } - root, err := s.HStar2Nodes(s.hasher.Size()*8, 0, values, + root, err := s.HStar2Nodes(s.hasher.BitLen(), 0, values, func(depth int, index *big.Int) ([]byte, error) { return cache[fmt.Sprintf("%x/%d", index, depth)], nil }, @@ -118,7 +121,7 @@ func TestHStar2GetSet(t *testing.T) { continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root:\n%x\n, want:\n%x", got, want) + t.Errorf("Root:\n%x, want:\n%x", got, want) } } } @@ -126,7 +129,7 @@ func TestHStar2GetSet(t *testing.T) { // Checks that we calculate the same empty root hash as a 256-level tree has // when calculating top subtrees using an appropriate offset. func TestHStar2OffsetEmptyRootKAT(t *testing.T) { - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, maphasher.Default) for size := 1; size < 255; size++ { root, err := s.HStar2Nodes(size, s.hasher.Size()*8-size, []HStar2LeafHash{}, @@ -146,7 +149,7 @@ func TestHStar2OffsetEmptyRootKAT(t *testing.T) { // 256-prefixSize, and can be passed in as leaves to top-subtree calculation. func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HStar2LeafHash { var ret []HStar2LeafHash - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, maphasher.Default) for i := range lh { prefix := new(big.Int).Rsh(lh[i].Index, uint(s.hasher.Size()*8-prefixSize)) b := lh[i].Index.Bytes() @@ -168,7 +171,7 @@ func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HS // (single top subtree of size n, and multipl bottom subtrees of size 256-n) // still arrives at the same Known Answers for root hash. func TestHStar2OffsetRootKAT(t *testing.T) { - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, maphasher.Default) m := make(map[string]string) @@ -178,7 +181,7 @@ func TestHStar2OffsetRootKAT(t *testing.T) { // requirement. for size := 24; size < 256; size += 8 { m[x.k] = x.v - intermediates := rootsForTrimmedKeys(t, size, createHStar2Leaves(maphasher.Default, m)) + intermediates := rootsForTrimmedKeys(t, size, createHStar2Leaves(treeID, maphasher.Default, m)) root, err := s.HStar2Nodes(size, s.hasher.Size()*8-size, intermediates, func(int, *big.Int) ([]byte, error) { return nil, nil }, @@ -195,7 +198,7 @@ func TestHStar2OffsetRootKAT(t *testing.T) { } func TestHStar2NegativeTreeLevelOffset(t *testing.T) { - s := NewHStar2(maphasher.Default) + s := NewHStar2(treeID, maphasher.Default) _, err := s.HStar2Nodes(32, -1, []HStar2LeafHash{}, func(int, *big.Int) ([]byte, error) { return nil, nil }, @@ -204,3 +207,27 @@ func TestHStar2NegativeTreeLevelOffset(t *testing.T) { t.Fatalf("Hstar2Nodes(): %v, want %v", got, want) } } + +func TestPaddedBytes(t *testing.T) { + size := 160 / 8 + for _, tc := range []struct { + i *big.Int + want []byte + }{ + {i: big.NewInt(0), want: h2b("0000000000000000000000000000000000000000")}, + {i: big.NewInt(1), want: h2b("0000000000000000000000000000000000000001")}, + {i: new(big.Int).SetBytes(h2b("00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")), want: h2b("00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")}, + {i: new(big.Int).SetBytes(h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")), want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")}, + } { + if got, want := PaddedBytes(tc.i, size), tc.want; !bytes.Equal(got, want) { + t.Errorf("PaddedBytes(%d): %x, want %x", tc.i, got, want) + } + } +} +func h2b(h string) []byte { + b, err := hex.DecodeString(h) + if err != nil { + panic("invalid hex string") + } + return b +} diff --git a/merkle/map_verifier.go b/merkle/map_verifier.go index 1a116420ab..64e380e63c 100644 --- a/merkle/map_verifier.go +++ b/merkle/map_verifier.go @@ -29,17 +29,17 @@ import ( // append-only logs, but adds support for nil/"default" proof nodes. // // Returns nil on a successful verification, and an error otherwise. -func VerifyMapInclusionProof(index, leafHash, expectedRoot []byte, proof [][]byte, h hashers.MapHasher) error { +func VerifyMapInclusionProof(treeID int64, index, leafHash, expectedRoot []byte, proof [][]byte, h hashers.MapHasher) error { hBits := h.Size() * 8 if got, want := len(proof), hBits; got != want { - return fmt.Errorf("invalid proof length %d, want %d", got, want) + return fmt.Errorf("proof len: %d, want %d", got, want) } if got, want := len(index)*8, hBits; got != want { - return fmt.Errorf("invalid index length %d, want %d", got, want) + return fmt.Errorf("index len: %d, want %d", got, want) } if got, want := len(leafHash)*8, hBits; got != want { - return fmt.Errorf("invalid leafHash length %d, want %d", got, want) + return fmt.Errorf("leafHash len: %d, want %d", got, want) } // TODO(al): Remove this dep on storage, since clients will want to use this code. @@ -52,7 +52,7 @@ func VerifyMapInclusionProof(index, leafHash, expectedRoot []byte, proof [][]byt proofIsRightHandElement := nID.Bit(bit) == 0 pElement := proof[bit] if len(pElement) == 0 { - pElement = h.HashEmpty(bit) + pElement = h.HashEmpty(treeID, index, bit) } if got, want := len(pElement)*8, hBits; got != want { return fmt.Errorf("invalid proof: element has length %d, want %d", got, want) @@ -65,7 +65,7 @@ func VerifyMapInclusionProof(index, leafHash, expectedRoot []byte, proof [][]byt } if got, want := runningHash, expectedRoot; !bytes.Equal(got, want) { - return fmt.Errorf("invalid proof: calculated roothash \n%x, want \n%x", got, want) + return fmt.Errorf("calculated root: %x, want \n%x", got, want) } return nil } diff --git a/merkle/map_verifier_test.go b/merkle/map_verifier_test.go index 3a9ff37eca..57580143c7 100644 --- a/merkle/map_verifier_test.go +++ b/merkle/map_verifier_test.go @@ -50,8 +50,8 @@ func TestVerifyMap(t *testing.T) { {"excess proof", tv.Key, tv.Value, tv.ExpectedRoot, make([][]byte, h.Size()*8+1), false}, } { index := testonly.HashKey(test.key) - leafHash := h.HashLeaf(test.leaf) - err := VerifyMapInclusionProof(index, leafHash, test.root, test.proof, h) + leafHash := h.HashLeaf(treeID, index, h.BitLen(), test.leaf) + err := VerifyMapInclusionProof(treeID, index, leafHash, test.root, test.proof, h) if got := err == nil; got != test.want { t.Errorf("%v: VerifyMapInclusionProof(): %v, want %v", test.desc, err, test.want) } diff --git a/merkle/maphasher/maphasher.go b/merkle/maphasher/maphasher.go index 3783c0acf9..ddfefb4372 100644 --- a/merkle/maphasher/maphasher.go +++ b/merkle/maphasher/maphasher.go @@ -59,16 +59,16 @@ func (m *MapHasher) String() string { // HashEmpty returns the hash of an empty branch at a given depth. // A depth of 0 indicates the hash of an empty leaf. // Empty branches within the tree are plain interior nodes e1 = H(e0, e0) etc. -func (m *MapHasher) HashEmpty(depth int) []byte { - if depth < 0 || depth >= len(m.nullHashes) { - panic(fmt.Sprintf("HashEmpty(%v) out of bounds", depth)) +func (m *MapHasher) HashEmpty(treeID int64, index []byte, height int) []byte { + if height < 0 || height >= len(m.nullHashes) { + panic(fmt.Sprintf("HashEmpty(%v) out of bounds", height)) } - return m.nullHashes[depth] + return m.nullHashes[height] } // HashLeaf returns the Merkle tree leaf hash of the data passed in through leaf. // The hashed structure is leafHashPrefix||leaf. -func (m *MapHasher) HashLeaf(leaf []byte) []byte { +func (m *MapHasher) HashLeaf(treeID int64, index []byte, height int, leaf []byte) []byte { h := m.New() h.Write([]byte{leafHashPrefix}) h.Write(leaf) @@ -85,6 +85,11 @@ func (m *MapHasher) HashChildren(l, r []byte) []byte { return h.Sum(nil) } +// BitLen returns the number of bits in the hash function. +func (m *MapHasher) BitLen() int { + return m.Size() * 8 +} + // initNullHashes sets the cache of empty hashes, one for each level in the sparse tree, // starting with the hash of an empty leaf, all the way up to the root hash of an empty tree. // These empty branches are not stored on disk in a sparse tree. They are computed since their @@ -94,7 +99,7 @@ func (m *MapHasher) initNullHashes() { // There are Size()*8 edges, and Size()*8 + 1 nodes in the tree. nodes := m.Size()*8 + 1 r := make([][]byte, nodes, nodes) - r[0] = m.HashLeaf(nil) + r[0] = m.HashLeaf(0, nil, m.Size()*8, nil) for i := 1; i < nodes; i++ { r[i] = m.HashChildren(r[i-1], r[i-1]) } diff --git a/merkle/maphasher/maphasher_test.go b/merkle/maphasher/maphasher_test.go index 8f62e63e06..181dbdd326 100644 --- a/merkle/maphasher/maphasher_test.go +++ b/merkle/maphasher/maphasher_test.go @@ -27,6 +27,7 @@ const ( // This was taken from the C++ SparseMerkleTree tests in // github.com/google/certificate-transparency. emptyMapRootB64 = "xmifEIEqCYCXbZUz2Dh1KCFmFZVn7DUVVxbBQTr1PWo=" + treeID = int64(0) ) func TestEmptyRoot(t *testing.T) { @@ -35,8 +36,7 @@ func TestEmptyRoot(t *testing.T) { t.Fatalf("couldn't decode empty root base64 constant.") } mh := New(crypto.SHA256) - rootLevel := mh.Size() * 8 - if got, want := mh.HashEmpty(rootLevel), emptyRoot; !bytes.Equal(got, want) { + if got, want := mh.HashEmpty(treeID, nil, mh.BitLen()), emptyRoot; !bytes.Equal(got, want) { t.Fatalf("HashEmpty(0): %x, want %x", got, want) } } @@ -46,11 +46,11 @@ func TestHStar2Equivalence(t *testing.T) { m := New(crypto.SHA256) star := hstar{ hasher: m, - hStarEmptyCache: [][]byte{m.HashLeaf([]byte(""))}, + hStarEmptyCache: [][]byte{m.HashLeaf(treeID, nil, m.BitLen(), []byte(""))}, } fullDepth := m.Size() * 8 for i := 0; i < fullDepth; i++ { - if got, want := m.HashEmpty(i), star.hStarEmpty(i); !bytes.Equal(got, want) { + if got, want := m.HashEmpty(treeID, nil, i), star.hStarEmpty(i); !bytes.Equal(got, want) { t.Errorf("HashEmpty(%v): \n%x, want: \n%x", i, got, want) } } diff --git a/merkle/objhasher/objhasher.go b/merkle/objhasher/objhasher.go index e6c25dc060..4e51bc987d 100644 --- a/merkle/objhasher/objhasher.go +++ b/merkle/objhasher/objhasher.go @@ -57,7 +57,7 @@ func (o *objloghasher) HashLeaf(leaf []byte) []byte { } // HashLeaf returns the object hash of leaf, which must be a JSON object. -func (o *objmaphasher) HashLeaf(leaf []byte) []byte { +func (o *objmaphasher) HashLeaf(treeID int64, index []byte, height int, leaf []byte) []byte { hash := objecthash.CommonJSONHash(string(leaf)) return hash[:] } diff --git a/merkle/objhasher/objhasher_test.go b/merkle/objhasher/objhasher_test.go index 792fd6d922..69c5a4a438 100644 --- a/merkle/objhasher/objhasher_test.go +++ b/merkle/objhasher/objhasher_test.go @@ -24,6 +24,8 @@ import ( "github.com/google/trillian/merkle/rfc6962" ) +const treeID = int64(0) + func TestLeafHash(t *testing.T) { h := NewLogHasher(rfc6962.New(crypto.SHA256)) @@ -53,7 +55,7 @@ func TestHashEmpty(t *testing.T) { h := NewMapHasher(maphasher.New(crypto.SHA256)) rfc := maphasher.New(crypto.SHA256) - if got, want := h.HashEmpty(0), rfc.HashEmpty(0); !bytes.Equal(got, want) { + if got, want := h.HashEmpty(treeID, nil, 0), rfc.HashEmpty(treeID, nil, 0); !bytes.Equal(got, want) { t.Errorf("HashEmpty():\n%x, want\n%x", got, want) } } diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index ecf0aae8ee..5127ace578 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -83,6 +83,7 @@ type getSubtreeFunc func(ctx context.Context, prefix []byte) (Subtree, error) // subtreeWriter knows how to calculate and store nodes for a subtree. type subtreeWriter struct { + treeID int64 // prefix is the path to the root of this subtree in the full tree. // i.e. all paths/indices under this tree share the same prefix. prefix []byte @@ -235,7 +236,7 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { } // calculate new root, and intermediate nodes: - hs2 := NewHStar2(s.treeHasher) + hs2 := NewHStar2(s.treeID, s.treeHasher) treeDepthOffset := (s.treeHasher.Size()-len(s.prefix))*8 - s.subtreeDepth addressSize := len(s.prefix) + s.subtreeDepth/8 root, err := hs2.HStar2Nodes(s.subtreeDepth, treeDepthOffset, leaves, @@ -317,12 +318,13 @@ func leafQueueSize(depths []int) int { } // newLocalSubtreeWriter creates a new local go-routine based subtree worker. -func newLocalSubtreeWriter(ctx context.Context, rev int64, prefix []byte, depths []int, newTX newTXFunc, h hashers.MapHasher) (Subtree, error) { +func newLocalSubtreeWriter(ctx context.Context, treeID, rev int64, prefix []byte, depths []int, newTX newTXFunc, h hashers.MapHasher) (Subtree, error) { tx, err := newTX() if err != nil { return nil, err } tree := subtreeWriter{ + treeID: treeID, treeRevision: rev, // TODO(al): figure out if we actually need these copies and remove it not. prefix: append(make([]byte, 0, len(prefix)), prefix...), @@ -334,7 +336,7 @@ func newLocalSubtreeWriter(ctx context.Context, rev int64, prefix []byte, depths treeHasher: h, getSubtree: func(ctx context.Context, p []byte) (Subtree, error) { myPrefix := bytes.Join([][]byte{prefix, p}, []byte{}) - return newLocalSubtreeWriter(ctx, rev, myPrefix, depths[1:], newTX, h) + return newLocalSubtreeWriter(ctx, treeID, rev, myPrefix, depths[1:], newTX, h) }, } @@ -347,10 +349,10 @@ func newLocalSubtreeWriter(ctx context.Context, rev int64, prefix []byte, depths // NewSparseMerkleTreeWriter returns a new SparseMerkleTreeWriter, which will // write data back into the tree at the specified revision, using the passed // in MapHasher to calculate/verify tree hashes, storing via tx. -func NewSparseMerkleTreeWriter(ctx context.Context, rev int64, h hashers.MapHasher, newTX newTXFunc) (*SparseMerkleTreeWriter, error) { +func NewSparseMerkleTreeWriter(ctx context.Context, treeID, rev int64, h hashers.MapHasher, newTX newTXFunc) (*SparseMerkleTreeWriter, error) { // TODO(al): allow the tree layering sizes to be customisable somehow. const topSubtreeSize = 8 // must be a multiple of 8 for now. - tree, err := newLocalSubtreeWriter(ctx, rev, []byte{}, []int{topSubtreeSize, h.Size()*8 - topSubtreeSize}, newTX, h) + tree, err := newLocalSubtreeWriter(ctx, treeID, rev, []byte{}, []int{topSubtreeSize, h.Size()*8 - topSubtreeSize}, newTX, h) if err != nil { return nil, err } diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index 6849b243e8..e41f540b6e 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -86,10 +86,10 @@ func getSparseMerkleTreeReaderWithMockTX(ctrl *gomock.Controller, rev int64) (*S return NewSparseMerkleTreeReader(rev, maphasher.Default, tx), tx } -func getSparseMerkleTreeWriterWithMockTX(ctx context.Context, ctrl *gomock.Controller, rev int64) (*SparseMerkleTreeWriter, *storage.MockMapTreeTX) { +func getSparseMerkleTreeWriterWithMockTX(ctx context.Context, ctrl *gomock.Controller, treeID, rev int64) (*SparseMerkleTreeWriter, *storage.MockMapTreeTX) { tx := storage.NewMockMapTreeTX(ctrl) tx.EXPECT().WriteRevision().AnyTimes().Return(rev) - tree, err := NewSparseMerkleTreeWriter(ctx, rev, maphasher.Default, newTX(tx)) + tree, err := NewSparseMerkleTreeWriter(ctx, treeID, rev, maphasher.Default, newTX(tx)) if err != nil { panic(err) } @@ -343,7 +343,7 @@ func testSparseTreeCalculatedRoot(ctx context.Context, t *testing.T, vec sparseT defer mockCtrl.Finish() const rev = 100 - w, tx := getSparseMerkleTreeWriterWithMockTX(ctx, mockCtrl, rev) + w, tx := getSparseMerkleTreeWriterWithMockTX(ctx, mockCtrl, treeID, rev) tx.EXPECT().Commit().AnyTimes().Return(nil) tx.EXPECT().Close().AnyTimes().Return(nil) @@ -356,7 +356,11 @@ func testSparseTreeCalculatedRoot(ctx context.Context, t *testing.T, vec sparseT func testSparseTreeCalculatedRootWithWriter(ctx context.Context, t *testing.T, rev int64, vec sparseTestVector, w *SparseMerkleTreeWriter) { var leaves []HashKeyValue for _, kv := range vec.kv { - leaves = append(leaves, HashKeyValue{testonly.HashKey(kv.k), w.hasher.HashLeaf([]byte(kv.v))}) + index := testonly.HashKey(kv.k) + leaves = append(leaves, HashKeyValue{ + HashedKey: index, + HashedValue: w.hasher.HashLeaf(treeID, index, w.hasher.BitLen(), []byte(kv.v)), + }) } if err := w.SetLeaves(ctx, leaves); err != nil { @@ -410,7 +414,7 @@ func testSparseTreeFetches(ctx context.Context, t *testing.T, vec sparseTestVect defer mockCtrl.Finish() const rev = 100 - w, tx := getSparseMerkleTreeWriterWithMockTX(ctx, mockCtrl, rev) + w, tx := getSparseMerkleTreeWriterWithMockTX(ctx, mockCtrl, treeID, rev) tx.EXPECT().Commit().AnyTimes().Return(nil) tx.EXPECT().Close().AnyTimes().Return(nil) @@ -573,7 +577,8 @@ func TestSparseMerkleTreeWriterFetchesMultipleLeaves(t *testing.T) { testSparseTreeFetches(context.Background(), t, vec) } -func DISABLEDTestSparseMerkleTreeWriterBigBatch(t *testing.T) { +func TestSparseMerkleTreeWriterBigBatch(t *testing.T) { + t.Skip("Disabled: BigBatch takes too long") ctx := context.Background() mockCtrl := gomock.NewController(t) @@ -581,7 +586,7 @@ func DISABLEDTestSparseMerkleTreeWriterBigBatch(t *testing.T) { defer maybeProfileCPU(t)() const rev = 100 - w, tx := getSparseMerkleTreeWriterWithMockTX(ctx, mockCtrl, rev) + w, tx := getSparseMerkleTreeWriterWithMockTX(ctx, mockCtrl, treeID, rev) tx.EXPECT().Close().AnyTimes().Return(nil) tx.EXPECT().Commit().AnyTimes().Return(nil) @@ -593,8 +598,9 @@ func DISABLEDTestSparseMerkleTreeWriterBigBatch(t *testing.T) { for x := 0; x < numBatches; x++ { h := make([]HashKeyValue, batchSize) for y := 0; y < batchSize; y++ { - h[y].HashedKey = testonly.HashKey(fmt.Sprintf("key-%d-%d", x, y)) - h[y].HashedValue = w.hasher.HashLeaf([]byte(fmt.Sprintf("value-%d-%d", x, y))) + index := testonly.HashKey(fmt.Sprintf("key-%d-%d", x, y)) + h[y].HashedKey = index + h[y].HashedValue = w.hasher.HashLeaf(treeID, index, w.hasher.BitLen(), []byte(fmt.Sprintf("value-%d-%d", x, y))) } if err := w.SetLeaves(ctx, h); err != nil { t.Fatalf("Failed to batch %d: %v", x, err) diff --git a/server/trillian_log_server/main.go b/server/trillian_log_server/main.go index 014ac34ba2..1de1982db6 100644 --- a/server/trillian_log_server/main.go +++ b/server/trillian_log_server/main.go @@ -29,6 +29,7 @@ import ( "github.com/google/trillian/crypto/keys" "github.com/google/trillian/extension" _ "github.com/google/trillian/merkle/objhasher" // Load hashers + _ "github.com/google/trillian/merkle/rfc6962" // Load hashers "github.com/google/trillian/monitoring" "github.com/google/trillian/monitoring/prometheus" mysqlq "github.com/google/trillian/quota/mysql" diff --git a/server/trillian_log_signer/main.go b/server/trillian_log_signer/main.go index 189f880ae8..30b225f26f 100644 --- a/server/trillian_log_signer/main.go +++ b/server/trillian_log_signer/main.go @@ -27,6 +27,8 @@ import ( "github.com/google/trillian/cmd" "github.com/google/trillian/crypto/keys" "github.com/google/trillian/extension" + _ "github.com/google/trillian/merkle/objhasher" // Load hashers + _ "github.com/google/trillian/merkle/rfc6962" // Load hashers "github.com/google/trillian/monitoring/prometheus" "github.com/google/trillian/quota" "github.com/google/trillian/server" diff --git a/server/trillian_map_server.go b/server/trillian_map_server.go index 61bb63a527..254ecd1467 100644 --- a/server/trillian_map_server.go +++ b/server/trillian_map_server.go @@ -158,6 +158,7 @@ func (t *TrillianMapServer) SetLeaves(ctx context.Context, req *trillian.SetMapL glog.V(2).Infof("%v: Writing at revision %v", mapID, tx.WriteRevision()) smtWriter, err := merkle.NewSparseMerkleTreeWriter( ctx, + req.MapId, tx.WriteRevision(), hasher, func() (storage.TreeTX, error) { return t.registry.MapStorage.BeginForTree(ctx, req.MapId) @@ -171,8 +172,8 @@ func (t *TrillianMapServer) SetLeaves(ctx context.Context, req *trillian.SetMapL return nil, status.Errorf(codes.InvalidArgument, "len(%x): %v, want %v", l.Index, got, want) } - // TODO(gbelvin) use LeafHash rather than computing here. - l.LeafHash = hasher.HashLeaf(l.LeafValue) + // TODO(gbelvin) use LeafHash rather than computing here. #423 + l.LeafHash = hasher.HashLeaf(mapID, l.Index, hasher.BitLen(), l.LeafValue) if err = tx.Set(ctx, l.Index, *l); err != nil { return nil, err diff --git a/server/trillian_map_server/main.go b/server/trillian_map_server/main.go index 457c061b03..8a0c0f18d2 100644 --- a/server/trillian_map_server/main.go +++ b/server/trillian_map_server/main.go @@ -20,6 +20,7 @@ import ( _ "net/http/pprof" _ "github.com/go-sql-driver/mysql" // Load MySQL driver + _ "github.com/google/trillian/merkle/coniks" // Make hashers available _ "github.com/google/trillian/merkle/maphasher" // Make hashers available "github.com/golang/glog" diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index 02ce4097fc..b05162e76f 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -416,7 +416,7 @@ func makeSuffixKey(depth int, index int64) (string, error) { // subtree Leaves map. // // This uses HStar2 to repopulate internal nodes. -func PopulateMapSubtreeNodes(hasher hashers.MapHasher) storage.PopulateSubtreeFunc { +func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.PopulateSubtreeFunc { return func(st *storagepb.SubtreeProto) error { st.InternalNodes = make(map[string][]byte) rootID := storage.NewNodeIDFromHash(st.Prefix) @@ -434,9 +434,8 @@ func PopulateMapSubtreeNodes(hasher hashers.MapHasher) storage.PopulateSubtreeFu Index: big.NewInt(int64(k[1])), }) } - hs2 := merkle.NewHStar2(hasher) - fullTreeDepth := hasher.Size() * 8 - offset := fullTreeDepth - rootID.PrefixLenBits - int(st.Depth) + hs2 := merkle.NewHStar2(treeID, hasher) + offset := hasher.BitLen() - rootID.PrefixLenBits - int(st.Depth) root, err := hs2.HStar2Nodes(int(st.Depth), offset, leaves, func(depth int, index *big.Int) ([]byte, error) { return nil, nil diff --git a/storage/cache/subtree_cache_test.go b/storage/cache/subtree_cache_test.go index ffdc608dcc..ab66b9c3a1 100644 --- a/storage/cache/subtree_cache_test.go +++ b/storage/cache/subtree_cache_test.go @@ -56,8 +56,10 @@ var splitTestVector = []struct { var defaultLogStrata = []int{8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8} var defaultMapStrata = []int{8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 176} +const treeID = int64(0) + func TestSplitNodeID(t *testing.T) { - c := NewSubtreeCache(defaultMapStrata, PopulateMapSubtreeNodes(maphasher.Default), PrepareMapSubtreeWrite()) + c := NewSubtreeCache(defaultMapStrata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) for i, v := range splitTestVector { n := storage.NewNodeIDFromHash(v.inPath) n.PrefixLenBits = v.inPathLenBits @@ -82,7 +84,7 @@ func TestCacheFillOnlyReadsSubtrees(t *testing.T) { defer mockCtrl.Finish() m := NewMockNodeStorage(mockCtrl) - c := NewSubtreeCache(defaultLogStrata, PopulateMapSubtreeNodes(maphasher.Default), PrepareMapSubtreeWrite()) + c := NewSubtreeCache(defaultLogStrata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) nodeID := storage.NewNodeIDFromHash([]byte("1234")) // When we loop around asking for all 0..32 bit prefix lengths of the above @@ -111,7 +113,7 @@ func TestCacheGetNodesReadsSubtrees(t *testing.T) { defer mockCtrl.Finish() m := NewMockNodeStorage(mockCtrl) - c := NewSubtreeCache(defaultLogStrata, PopulateMapSubtreeNodes(maphasher.Default), PrepareMapSubtreeWrite()) + c := NewSubtreeCache(defaultLogStrata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) nodeIDs := []storage.NodeID{ storage.NewNodeIDFromHash([]byte("1234")), @@ -164,7 +166,7 @@ func TestCacheFlush(t *testing.T) { defer mockCtrl.Finish() m := NewMockNodeStorage(mockCtrl) - c := NewSubtreeCache(defaultMapStrata, PopulateMapSubtreeNodes(maphasher.Default), PrepareMapSubtreeWrite()) + c := NewSubtreeCache(defaultMapStrata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) h := "0123456789abcdef0123456789abcdef" nodeID := storage.NewNodeIDFromHash([]byte(h)) @@ -318,7 +320,7 @@ func TestPrefixLengths(t *testing.T) { strata := []int{8, 8, 16, 32, 64, 128} stratumInfo := []stratumInfo{{0, 8}, {1, 8}, {2, 16}, {2, 16}, {4, 32}, {4, 32}, {4, 32}, {4, 32}, {8, 64}, {8, 64}, {8, 64}, {8, 64}, {8, 64}, {8, 64}, {8, 64}, {8, 64}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}, {16, 128}} - c := NewSubtreeCache(strata, PopulateMapSubtreeNodes(maphasher.Default), PrepareMapSubtreeWrite()) + c := NewSubtreeCache(strata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) if diff := pretty.Compare(c.stratumInfo, stratumInfo); diff != "" { t.Fatalf("prefixLengths diff:\n%v", diff) @@ -326,7 +328,7 @@ func TestPrefixLengths(t *testing.T) { } func TestGetStratumInfo(t *testing.T) { - c := NewSubtreeCache(defaultMapStrata, PopulateMapSubtreeNodes(maphasher.Default), PrepareMapSubtreeWrite()) + c := NewSubtreeCache(defaultMapStrata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) testVec := []struct { depth int info stratumInfo diff --git a/storage/mysql/map_storage.go b/storage/mysql/map_storage.go index 597a6f300c..4e24af4130 100644 --- a/storage/mysql/map_storage.go +++ b/storage/mysql/map_storage.go @@ -107,7 +107,7 @@ func (m *mySQLMapStorage) begin(ctx context.Context, treeID int64, readonly bool return nil, err } - stCache := cache.NewSubtreeCache(defaultMapStrata, cache.PopulateMapSubtreeNodes(hasher), cache.PrepareMapSubtreeWrite()) + stCache := cache.NewSubtreeCache(defaultMapStrata, cache.PopulateMapSubtreeNodes(treeID, hasher), cache.PrepareMapSubtreeWrite()) ttx, err := m.beginTreeTx(ctx, treeID, hasher.Size(), stCache) if err != nil { return nil, err diff --git a/storage/mysql/storage.sql b/storage/mysql/storage.sql index 5b26629889..1fb3de94ed 100644 --- a/storage/mysql/storage.sql +++ b/storage/mysql/storage.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS Trees( TreeId BIGINT NOT NULL, TreeState ENUM('ACTIVE', 'FROZEN', 'SOFT_DELETED', 'HARD_DELETED') NOT NULL, TreeType ENUM('LOG', 'MAP') NOT NULL, - HashStrategy ENUM('RFC6962_SHA256', 'TEST_MAP_HASHER') NOT NULL, + HashStrategy ENUM('RFC6962_SHA256', 'TEST_MAP_HASHER', 'OBJECT_RFC6962_SHA256', 'CONIKS_SHA512_256') NOT NULL, HashAlgorithm ENUM('SHA256') NOT NULL, SignatureAlgorithm ENUM('ECDSA', 'RSA') NOT NULL, DisplayName VARCHAR(20), diff --git a/trillian.pb.go b/trillian.pb.go index 5f8ec29e2f..ee7667caab 100644 --- a/trillian.pb.go +++ b/trillian.pb.go @@ -35,6 +35,8 @@ const ( // Append-only log strategy where leaf nodes are defined as the ObjectHash. // All other properties are equal to RFC6962_SHA256. HashStrategy_OBJECT_RFC6962_SHA256 HashStrategy = 3 + // The CONIKS sparse tree hasher with SHA512_256 as the hash algorithm. + HashStrategy_CONIKS_SHA512_256 HashStrategy = 4 ) var HashStrategy_name = map[int32]string{ @@ -42,12 +44,14 @@ var HashStrategy_name = map[int32]string{ 1: "RFC6962_SHA256", 2: "TEST_MAP_HASHER", 3: "OBJECT_RFC6962_SHA256", + 4: "CONIKS_SHA512_256", } var HashStrategy_value = map[string]int32{ "UNKNOWN_HASH_STRATEGY": 0, "RFC6962_SHA256": 1, "TEST_MAP_HASHER": 2, "OBJECT_RFC6962_SHA256": 3, + "CONIKS_SHA512_256": 4, } func (x HashStrategy) String() string { @@ -494,70 +498,71 @@ func init() { func init() { proto.RegisterFile("trillian.proto", fileDescriptor3) } var fileDescriptor3 = []byte{ - // 1038 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xed, 0x6e, 0xdb, 0x36, - 0x17, 0xae, 0x62, 0xd7, 0xb1, 0x8f, 0x3f, 0xa2, 0x30, 0x1f, 0xaf, 0x92, 0xbe, 0x58, 0x33, 0x6f, - 0xc0, 0xb2, 0x6c, 0xb0, 0x07, 0xa7, 0x09, 0x30, 0x14, 0xc3, 0xe0, 0x24, 0x4a, 0xf3, 0xe9, 0x18, - 0x92, 0xb6, 0xa1, 0xfd, 0x43, 0xd0, 0x36, 0x2b, 0x13, 0x95, 0x2c, 0x56, 0xa2, 0x8b, 0xaa, 0xd7, - 0x30, 0x60, 0xf7, 0xb3, 0xeb, 0xd9, 0x5d, 0xec, 0xcf, 0x40, 0x8a, 0xb2, 0x9d, 0xa4, 0x5b, 0x8a, - 0x61, 0x7f, 0x12, 0xf2, 0x39, 0xcf, 0xf3, 0xf0, 0xe3, 0x9c, 0x43, 0x19, 0x1a, 0x22, 0x66, 0x41, - 0xc0, 0xc8, 0xa4, 0xc5, 0xe3, 0x48, 0x44, 0xa8, 0x9c, 0xcf, 0xb7, 0x0f, 0x7c, 0x26, 0xc6, 0xd3, - 0x41, 0x6b, 0x18, 0x85, 0x6d, 0x3f, 0x8a, 0xfc, 0x80, 0xb6, 0xf3, 0x58, 0x7b, 0x18, 0xa7, 0x5c, - 0x44, 0xed, 0x37, 0x34, 0x4d, 0xf8, 0x40, 0xff, 0xcb, 0x0c, 0xb6, 0xf7, 0x1f, 0x96, 0x25, 0xcc, - 0xe7, 0x83, 0xec, 0xaf, 0x16, 0x6d, 0x69, 0xa6, 0x9a, 0x0d, 0xa6, 0xaf, 0xdb, 0x64, 0x92, 0xea, - 0xd0, 0x67, 0x77, 0x43, 0xa3, 0x69, 0x4c, 0x04, 0x8b, 0xf4, 0x86, 0xb7, 0x9f, 0xde, 0x8d, 0x0b, - 0x16, 0xd2, 0x44, 0x90, 0x90, 0x67, 0x84, 0xe6, 0x6f, 0xcb, 0x50, 0xf4, 0x62, 0x4a, 0xd1, 0xff, - 0x60, 0x59, 0xc4, 0x94, 0x62, 0x36, 0xb2, 0x8c, 0x1d, 0x63, 0xb7, 0xe0, 0x94, 0xe4, 0xf4, 0x7c, - 0x84, 0x3a, 0x00, 0x2a, 0x90, 0x08, 0x22, 0xa8, 0xb5, 0xb4, 0x63, 0xec, 0x36, 0x3a, 0x6b, 0xad, - 0xd9, 0xc5, 0x48, 0xb1, 0x2b, 0x43, 0x4e, 0x45, 0xe4, 0x43, 0xd4, 0x06, 0x35, 0xc1, 0x22, 0xe5, - 0xd4, 0x2a, 0x28, 0x09, 0xba, 0x2d, 0xf1, 0x52, 0x4e, 0x9d, 0xb2, 0xd0, 0x23, 0xf4, 0x1c, 0xea, - 0x63, 0x92, 0x8c, 0x71, 0x22, 0x62, 0x22, 0xa8, 0x9f, 0x5a, 0x45, 0x25, 0xda, 0x9c, 0x8b, 0xce, - 0x48, 0x32, 0x76, 0x75, 0xd4, 0xa9, 0x8d, 0x17, 0x66, 0xe8, 0x12, 0x1a, 0x4a, 0x4c, 0x02, 0x3f, - 0x8a, 0x99, 0x18, 0x87, 0xd6, 0x63, 0xa5, 0xfe, 0xb2, 0x95, 0xdd, 0xe2, 0x09, 0xf3, 0x99, 0x20, - 0x41, 0x90, 0xba, 0xcc, 0x9f, 0xd0, 0x91, 0xb2, 0xea, 0xe6, 0x5c, 0x47, 0x2d, 0x3c, 0x9b, 0xa2, - 0x57, 0xb0, 0x96, 0x30, 0x7f, 0x42, 0xc4, 0x34, 0xa6, 0x0b, 0x8e, 0x25, 0xe5, 0xf8, 0xf5, 0xdf, - 0x38, 0xba, 0xb9, 0x62, 0x6e, 0x8b, 0x92, 0x7b, 0x18, 0x22, 0xb0, 0x39, 0xf7, 0x1e, 0x32, 0x3e, - 0xa6, 0x31, 0x4e, 0xa6, 0x4c, 0x50, 0x0b, 0x29, 0xfb, 0x6f, 0x1e, 0xb2, 0x3f, 0x56, 0x1a, 0x57, - 0x4a, 0x9c, 0xf5, 0xe4, 0x23, 0x28, 0xfa, 0x1c, 0x6a, 0x23, 0x96, 0xf0, 0x80, 0xa4, 0x78, 0x42, - 0x42, 0x6a, 0x95, 0x77, 0x8c, 0xdd, 0x8a, 0x53, 0xd5, 0x58, 0x8f, 0x84, 0x14, 0xed, 0x40, 0x75, - 0x44, 0x93, 0x61, 0xcc, 0xb8, 0x2c, 0x14, 0xab, 0xa2, 0x19, 0x73, 0x08, 0x1d, 0x40, 0x95, 0xc7, - 0xec, 0x1d, 0x11, 0x14, 0xbf, 0xa1, 0xa9, 0x55, 0xdb, 0x31, 0x76, 0xab, 0x9d, 0xf5, 0x56, 0x56, - 0x4b, 0xad, 0xbc, 0x96, 0x5a, 0xdd, 0x49, 0xea, 0x80, 0x26, 0x5e, 0xd2, 0x14, 0xfd, 0x08, 0x66, - 0x22, 0xa2, 0x98, 0xf8, 0x14, 0x27, 0x54, 0x08, 0x36, 0xf1, 0x13, 0xab, 0xfe, 0x0f, 0xda, 0x15, - 0xcd, 0x76, 0x35, 0x19, 0x7d, 0x07, 0xc0, 0xa7, 0x83, 0x80, 0x0d, 0xd5, 0xb2, 0x0d, 0x25, 0x5d, - 0x6d, 0xe9, 0x06, 0xea, 0xab, 0xc8, 0x25, 0x4d, 0x9d, 0x0a, 0xcf, 0x87, 0xc8, 0x86, 0xd5, 0x90, - 0xbc, 0xc7, 0x71, 0x14, 0x09, 0x9c, 0x97, 0xbe, 0xb5, 0xa2, 0x84, 0x5b, 0xf7, 0xd6, 0x3c, 0xd1, - 0x04, 0x67, 0x25, 0x24, 0xef, 0x9d, 0x28, 0x12, 0x39, 0x80, 0x9e, 0x43, 0x75, 0x18, 0x53, 0x79, - 0x5e, 0xd9, 0x1f, 0x96, 0xa9, 0x0c, 0xb6, 0xef, 0x19, 0x78, 0x79, 0xf3, 0x38, 0x90, 0xd1, 0x25, - 0x20, 0xc5, 0x53, 0x3e, 0x9a, 0x89, 0x57, 0x1f, 0x16, 0x67, 0x74, 0x09, 0x5c, 0x14, 0xcb, 0xcb, - 0x66, 0xf9, 0xa2, 0x58, 0x06, 0xb3, 0x7a, 0x51, 0x2c, 0x57, 0xcd, 0x5a, 0xf3, 0x57, 0x03, 0xd6, - 0xb3, 0xbc, 0xdb, 0x13, 0x11, 0xa7, 0x33, 0x19, 0xfa, 0x0a, 0x56, 0x66, 0xdd, 0x8b, 0x27, 0x64, - 0x12, 0x25, 0xba, 0x53, 0x1b, 0x33, 0xb8, 0x27, 0x51, 0xb4, 0x01, 0xa5, 0x20, 0xf2, 0x65, 0x27, - 0x2f, 0xa9, 0xf8, 0xe3, 0x20, 0xf2, 0xcf, 0x47, 0xe8, 0x19, 0x54, 0x66, 0x25, 0xa3, 0x9a, 0xb2, - 0xda, 0xd9, 0xfc, 0x78, 0xc1, 0x39, 0x73, 0x62, 0xf3, 0x0f, 0x03, 0xea, 0x19, 0x7a, 0x15, 0xf9, - 0xf2, 0xd2, 0x3e, 0x7d, 0x1f, 0x4f, 0xa0, 0xa2, 0x12, 0x23, 0x1b, 0x4c, 0x6d, 0xa5, 0xe6, 0x94, - 0x25, 0x20, 0xfb, 0x4f, 0x06, 0xb3, 0x67, 0x85, 0x7d, 0xc8, 0x76, 0x53, 0xc8, 0x9e, 0x03, 0x97, - 0x7d, 0xa0, 0xb7, 0xb7, 0x5a, 0xfc, 0xc4, 0xad, 0x2e, 0x9c, 0xfb, 0xf1, 0xe2, 0xb9, 0xbf, 0x80, - 0xba, 0x5a, 0x29, 0xa6, 0xef, 0x58, 0x22, 0xeb, 0xa3, 0xa4, 0xa2, 0x35, 0x09, 0x3a, 0x1a, 0x6b, - 0xfe, 0x6e, 0x40, 0xe3, 0x9a, 0x70, 0x4e, 0xe3, 0x6b, 0x2a, 0xc8, 0x88, 0x08, 0x82, 0x9a, 0x50, - 0x4f, 0xa2, 0x69, 0x3c, 0xa4, 0x58, 0xbb, 0x1a, 0xea, 0x08, 0xd5, 0x0c, 0xbc, 0x52, 0xde, 0x3f, - 0xc0, 0x93, 0x31, 0xf3, 0xc7, 0x34, 0x11, 0xf8, 0xf5, 0x34, 0x08, 0x52, 0x3c, 0x8c, 0x42, 0x1e, - 0x50, 0x41, 0x47, 0x38, 0xa1, 0x6f, 0xf5, 0xfd, 0x5b, 0x9a, 0x72, 0x2a, 0x19, 0xc7, 0x39, 0xc1, - 0xa5, 0x6f, 0x91, 0x0d, 0x4f, 0x73, 0x39, 0x27, 0xb1, 0x60, 0xe4, 0xbe, 0x45, 0x76, 0x35, 0xff, - 0xd7, 0xb4, 0x7e, 0xce, 0x5a, 0xb4, 0x69, 0xfe, 0x39, 0xcb, 0xd1, 0x35, 0xe1, 0xff, 0x61, 0x8e, - 0x9e, 0x41, 0x39, 0xd4, 0xb7, 0xa1, 0x0b, 0xc6, 0x9a, 0x3f, 0xc8, 0xb7, 0x6f, 0xcb, 0x99, 0x31, - 0xff, 0x7d, 0xf2, 0x42, 0xc2, 0x17, 0x92, 0x17, 0x12, 0x7e, 0x3e, 0x92, 0xef, 0x99, 0x84, 0xef, - 0xe4, 0xae, 0x1a, 0x12, 0x9e, 0xa7, 0x6e, 0x2f, 0x84, 0xda, 0xe2, 0xc7, 0x01, 0x6d, 0xc1, 0xc6, - 0x4f, 0xbd, 0xcb, 0xde, 0xcd, 0x2f, 0x3d, 0x7c, 0xd6, 0x75, 0xcf, 0xb0, 0xeb, 0x39, 0x5d, 0xcf, - 0x7e, 0xf1, 0xd2, 0x7c, 0x84, 0x10, 0x34, 0x9c, 0xd3, 0xe3, 0xc3, 0xef, 0x0f, 0x3b, 0xd8, 0x3d, - 0xeb, 0x76, 0x0e, 0x0e, 0x4d, 0x03, 0xad, 0xc1, 0x8a, 0x67, 0xbb, 0x1e, 0xbe, 0xee, 0xf6, 0x15, - 0xdf, 0x76, 0xcc, 0x25, 0xe9, 0x71, 0x73, 0x74, 0x61, 0x1f, 0x7b, 0xf8, 0x0e, 0xbf, 0xb0, 0x87, - 0xa1, 0x32, 0xfb, 0xe6, 0xa1, 0x4d, 0x40, 0xf9, 0x5a, 0x9e, 0x63, 0xdb, 0xd8, 0xf5, 0xba, 0x9e, - 0x6d, 0x3e, 0x42, 0x00, 0xa5, 0xee, 0xb1, 0x77, 0xfe, 0xb3, 0x6d, 0x1a, 0x72, 0x7c, 0xea, 0xdc, - 0xbc, 0xb2, 0x7b, 0xe6, 0x12, 0x32, 0xa1, 0xe6, 0xde, 0x9c, 0x7a, 0xf8, 0xc4, 0xbe, 0xb2, 0x3d, - 0xfb, 0xc4, 0x2c, 0x48, 0xe4, 0xac, 0xeb, 0x9c, 0xcc, 0x90, 0xe2, 0xde, 0x3e, 0x94, 0xf3, 0x2f, - 0x24, 0xda, 0x80, 0xd5, 0x5b, 0xfe, 0xde, 0xcb, 0xbe, 0xb4, 0x5f, 0x86, 0xc2, 0xd5, 0xcd, 0x0b, - 0xd3, 0x90, 0x83, 0xeb, 0x6e, 0xdf, 0x5c, 0x3a, 0xfa, 0x16, 0xb6, 0x86, 0x51, 0x98, 0x3f, 0x3a, - 0xb7, 0x7f, 0xb6, 0x1c, 0xd5, 0x3d, 0x3d, 0xef, 0xcb, 0x69, 0xdf, 0x18, 0x94, 0x14, 0xbe, 0xff, - 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1c, 0x65, 0x2f, 0x7d, 0xe0, 0x08, 0x00, 0x00, + // 1056 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5f, 0x6f, 0xdb, 0xb6, + 0x17, 0xad, 0x62, 0xd7, 0xb1, 0xaf, 0xff, 0x44, 0x61, 0x93, 0xfc, 0x94, 0xf4, 0x87, 0x35, 0xf3, + 0x06, 0x2c, 0xcb, 0x06, 0x7b, 0x73, 0x9a, 0x00, 0x43, 0x31, 0x0c, 0x8e, 0xa3, 0x34, 0x7f, 0x6d, + 0x43, 0xd2, 0x36, 0xb4, 0x2f, 0x04, 0x6d, 0xb3, 0x32, 0x51, 0xc9, 0x52, 0x25, 0xba, 0xa8, 0xfa, + 0xbc, 0xc7, 0x01, 0xfb, 0x3e, 0xfb, 0x3c, 0xfb, 0x16, 0x7b, 0x19, 0x48, 0x51, 0xb2, 0x93, 0x74, + 0x4b, 0x31, 0xec, 0x25, 0x21, 0xcf, 0x3d, 0xe7, 0xe8, 0x8a, 0xf7, 0x5e, 0xca, 0xd0, 0xe0, 0x11, + 0xf3, 0x3c, 0x46, 0x66, 0xad, 0x30, 0x0a, 0x78, 0x80, 0xca, 0xd9, 0x7e, 0xe7, 0xd0, 0x65, 0x7c, + 0x3a, 0x1f, 0xb5, 0xc6, 0x81, 0xdf, 0x76, 0x83, 0xc0, 0xf5, 0x68, 0x3b, 0x8b, 0xb5, 0xc7, 0x51, + 0x12, 0xf2, 0xa0, 0xfd, 0x9a, 0x26, 0x71, 0x38, 0x52, 0xff, 0x52, 0x83, 0x9d, 0x83, 0xfb, 0x65, + 0x31, 0x73, 0xc3, 0x51, 0xfa, 0x57, 0x89, 0xb6, 0x15, 0x53, 0xee, 0x46, 0xf3, 0x57, 0x6d, 0x32, + 0x4b, 0x54, 0xe8, 0x93, 0xdb, 0xa1, 0xc9, 0x3c, 0x22, 0x9c, 0x05, 0x2a, 0xe1, 0x9d, 0x27, 0xb7, + 0xe3, 0x9c, 0xf9, 0x34, 0xe6, 0xc4, 0x0f, 0x53, 0x42, 0xf3, 0xb7, 0x55, 0x28, 0x3a, 0x11, 0xa5, + 0xe8, 0x7f, 0xb0, 0xca, 0x23, 0x4a, 0x31, 0x9b, 0x18, 0xda, 0xae, 0xb6, 0x57, 0xb0, 0x4a, 0x62, + 0x7b, 0x3e, 0x41, 0x1d, 0x00, 0x19, 0x88, 0x39, 0xe1, 0xd4, 0x58, 0xd9, 0xd5, 0xf6, 0x1a, 0x9d, + 0x47, 0xad, 0xfc, 0x60, 0x84, 0xd8, 0x16, 0x21, 0xab, 0xc2, 0xb3, 0x25, 0x6a, 0x83, 0xdc, 0x60, + 0x9e, 0x84, 0xd4, 0x28, 0x48, 0x09, 0xba, 0x29, 0x71, 0x92, 0x90, 0x5a, 0x65, 0xae, 0x56, 0xe8, + 0x19, 0xd4, 0xa7, 0x24, 0x9e, 0xe2, 0x98, 0x47, 0x84, 0x53, 0x37, 0x31, 0x8a, 0x52, 0xb4, 0xb5, + 0x10, 0x9d, 0x91, 0x78, 0x6a, 0xab, 0xa8, 0x55, 0x9b, 0x2e, 0xed, 0xd0, 0x25, 0x34, 0xa4, 0x98, + 0x78, 0x6e, 0x10, 0x31, 0x3e, 0xf5, 0x8d, 0x87, 0x52, 0xfd, 0x79, 0x2b, 0x3d, 0xc5, 0x13, 0xe6, + 0x32, 0x4e, 0x3c, 0x2f, 0xb1, 0x99, 0x3b, 0xa3, 0x13, 0x69, 0xd5, 0xcd, 0xb8, 0x96, 0x7c, 0x70, + 0xbe, 0x45, 0x2f, 0xe1, 0x51, 0xcc, 0xdc, 0x19, 0xe1, 0xf3, 0x88, 0x2e, 0x39, 0x96, 0xa4, 0xe3, + 0x97, 0x7f, 0xe3, 0x68, 0x67, 0x8a, 0x85, 0x2d, 0x8a, 0xef, 0x60, 0x88, 0xc0, 0xd6, 0xc2, 0x7b, + 0xcc, 0xc2, 0x29, 0x8d, 0x70, 0x3c, 0x67, 0x9c, 0x1a, 0x48, 0xda, 0x7f, 0x75, 0x9f, 0x7d, 0x4f, + 0x6a, 0x6c, 0x21, 0xb1, 0x36, 0xe2, 0x0f, 0xa0, 0xe8, 0x53, 0xa8, 0x4d, 0x58, 0x1c, 0x7a, 0x24, + 0xc1, 0x33, 0xe2, 0x53, 0xa3, 0xbc, 0xab, 0xed, 0x55, 0xac, 0xaa, 0xc2, 0xfa, 0xc4, 0xa7, 0x68, + 0x17, 0xaa, 0x13, 0x1a, 0x8f, 0x23, 0x16, 0x8a, 0x46, 0x31, 0x2a, 0x8a, 0xb1, 0x80, 0xd0, 0x21, + 0x54, 0xc3, 0x88, 0xbd, 0x25, 0x9c, 0xe2, 0xd7, 0x34, 0x31, 0x6a, 0xbb, 0xda, 0x5e, 0xb5, 0xb3, + 0xd1, 0x4a, 0x7b, 0xa9, 0x95, 0xf5, 0x52, 0xab, 0x3b, 0x4b, 0x2c, 0x50, 0xc4, 0x4b, 0x9a, 0xa0, + 0x1f, 0x40, 0x8f, 0x79, 0x10, 0x11, 0x97, 0xe2, 0x98, 0x72, 0xce, 0x66, 0x6e, 0x6c, 0xd4, 0xff, + 0x41, 0xbb, 0xa6, 0xd8, 0xb6, 0x22, 0xa3, 0x6f, 0x00, 0xc2, 0xf9, 0xc8, 0x63, 0x63, 0xf9, 0xd8, + 0x86, 0x94, 0xae, 0xb7, 0xd4, 0x00, 0x0d, 0x65, 0xe4, 0x92, 0x26, 0x56, 0x25, 0xcc, 0x96, 0xc8, + 0x84, 0x75, 0x9f, 0xbc, 0xc3, 0x51, 0x10, 0x70, 0x9c, 0xb5, 0xbe, 0xb1, 0x26, 0x85, 0xdb, 0x77, + 0x9e, 0x79, 0xa2, 0x08, 0xd6, 0x9a, 0x4f, 0xde, 0x59, 0x41, 0xc0, 0x33, 0x00, 0x3d, 0x83, 0xea, + 0x38, 0xa2, 0xe2, 0x7d, 0xc5, 0x7c, 0x18, 0xba, 0x34, 0xd8, 0xb9, 0x63, 0xe0, 0x64, 0xc3, 0x63, + 0x41, 0x4a, 0x17, 0x80, 0x10, 0xcf, 0xc3, 0x49, 0x2e, 0x5e, 0xbf, 0x5f, 0x9c, 0xd2, 0x05, 0x70, + 0x51, 0x2c, 0xaf, 0xea, 0xe5, 0x8b, 0x62, 0x19, 0xf4, 0xea, 0x45, 0xb1, 0x5c, 0xd5, 0x6b, 0xcd, + 0x5f, 0x35, 0xd8, 0x48, 0xeb, 0x6e, 0xce, 0x78, 0x94, 0xe4, 0x32, 0xf4, 0x05, 0xac, 0xe5, 0xd3, + 0x8b, 0x67, 0x64, 0x16, 0xc4, 0x6a, 0x52, 0x1b, 0x39, 0xdc, 0x17, 0x28, 0xda, 0x84, 0x92, 0x17, + 0xb8, 0x62, 0x92, 0x57, 0x64, 0xfc, 0xa1, 0x17, 0xb8, 0xe7, 0x13, 0xf4, 0x14, 0x2a, 0x79, 0xcb, + 0xc8, 0xa1, 0xac, 0x76, 0xb6, 0x3e, 0xdc, 0x70, 0xd6, 0x82, 0xd8, 0xfc, 0x43, 0x83, 0x7a, 0x8a, + 0x5e, 0x05, 0xae, 0x38, 0xb4, 0x8f, 0xcf, 0xe3, 0x31, 0x54, 0x64, 0x61, 0xc4, 0x80, 0xc9, 0x54, + 0x6a, 0x56, 0x59, 0x00, 0x62, 0xfe, 0x44, 0x30, 0xbd, 0x56, 0xd8, 0xfb, 0x34, 0x9b, 0x42, 0x7a, + 0x1d, 0xd8, 0xec, 0x3d, 0xbd, 0x99, 0x6a, 0xf1, 0x23, 0x53, 0x5d, 0x7a, 0xef, 0x87, 0xcb, 0xef, + 0xfd, 0x19, 0xd4, 0xe5, 0x93, 0x22, 0xfa, 0x96, 0xc5, 0xa2, 0x3f, 0x4a, 0x32, 0x5a, 0x13, 0xa0, + 0xa5, 0xb0, 0xe6, 0xef, 0x1a, 0x34, 0xae, 0x49, 0x18, 0xd2, 0xe8, 0x9a, 0x72, 0x32, 0x21, 0x9c, + 0xa0, 0x26, 0xd4, 0xe3, 0x60, 0x1e, 0x8d, 0x29, 0x56, 0xae, 0x9a, 0x7c, 0x85, 0x6a, 0x0a, 0x5e, + 0x49, 0xef, 0xef, 0xe1, 0xf1, 0x94, 0xb9, 0x53, 0x1a, 0x73, 0xfc, 0x6a, 0xee, 0x79, 0x09, 0x1e, + 0x07, 0x7e, 0xe8, 0x51, 0x4e, 0x27, 0x38, 0xa6, 0x6f, 0xd4, 0xf9, 0x1b, 0x8a, 0x72, 0x2a, 0x18, + 0xbd, 0x8c, 0x60, 0xd3, 0x37, 0xc8, 0x84, 0x27, 0x99, 0x3c, 0x24, 0x11, 0x67, 0xe4, 0xae, 0x45, + 0x7a, 0x34, 0xff, 0x57, 0xb4, 0x61, 0xc6, 0x5a, 0xb6, 0x69, 0xfe, 0x99, 0xd7, 0xe8, 0x9a, 0x84, + 0xff, 0x61, 0x8d, 0x9e, 0x42, 0xd9, 0x57, 0xa7, 0xa1, 0x1a, 0xc6, 0x58, 0x5c, 0xc8, 0x37, 0x4f, + 0xcb, 0xca, 0x99, 0xff, 0xbe, 0x78, 0x3e, 0x09, 0x97, 0x8a, 0xe7, 0x93, 0xf0, 0x7c, 0x22, 0xee, + 0x33, 0x01, 0xdf, 0xaa, 0x5d, 0xd5, 0x27, 0x61, 0x56, 0xba, 0xfd, 0x5f, 0x34, 0xa8, 0x2d, 0x7f, + 0x1d, 0xd0, 0x36, 0x6c, 0xfe, 0xd8, 0xbf, 0xec, 0x0f, 0x7e, 0xee, 0xe3, 0xb3, 0xae, 0x7d, 0x86, + 0x6d, 0xc7, 0xea, 0x3a, 0xe6, 0xf3, 0x17, 0xfa, 0x03, 0x84, 0xa0, 0x61, 0x9d, 0xf6, 0x8e, 0xbe, + 0x3b, 0xea, 0x60, 0xfb, 0xac, 0xdb, 0x39, 0x3c, 0xd2, 0x35, 0xf4, 0x08, 0xd6, 0x1c, 0xd3, 0x76, + 0xf0, 0x75, 0x77, 0x28, 0xf9, 0xa6, 0xa5, 0xaf, 0x08, 0x8f, 0xc1, 0xf1, 0x85, 0xd9, 0x73, 0xf0, + 0x2d, 0x7e, 0x01, 0x6d, 0xc2, 0x7a, 0x6f, 0xd0, 0x3f, 0xbf, 0xb4, 0x05, 0x74, 0xf8, 0x6d, 0x07, + 0x0b, 0xb8, 0xb8, 0x8f, 0xa1, 0x92, 0x7f, 0x0b, 0xd1, 0x16, 0xa0, 0x2c, 0x05, 0xc7, 0x32, 0x4d, + 0x6c, 0x3b, 0x5d, 0xc7, 0xd4, 0x1f, 0x20, 0x80, 0x52, 0xb7, 0xe7, 0x9c, 0xff, 0x64, 0xea, 0x9a, + 0x58, 0x9f, 0x5a, 0x83, 0x97, 0x66, 0x5f, 0x5f, 0x41, 0x3a, 0xd4, 0xec, 0xc1, 0xa9, 0x83, 0x4f, + 0xcc, 0x2b, 0xd3, 0x31, 0x4f, 0xf4, 0x82, 0x40, 0xce, 0xba, 0xd6, 0x49, 0x8e, 0x14, 0xf7, 0x0f, + 0xa0, 0x9c, 0x7d, 0x39, 0x45, 0x0e, 0x37, 0xfc, 0x9d, 0x17, 0x43, 0x61, 0xbf, 0x0a, 0x85, 0xab, + 0xc1, 0x73, 0x5d, 0x13, 0x8b, 0xeb, 0xee, 0x50, 0x5f, 0x39, 0xfe, 0x1a, 0xb6, 0xc7, 0x81, 0x9f, + 0x5d, 0x46, 0x37, 0x7f, 0xce, 0x1c, 0xd7, 0x1d, 0xb5, 0x1f, 0x8a, 0xed, 0x50, 0x1b, 0x95, 0x24, + 0x7e, 0xf0, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x17, 0x75, 0x85, 0xca, 0xf8, 0x08, 0x00, 0x00, } diff --git a/trillian.proto b/trillian.proto index 7e5e37a18e..d30b58f678 100644 --- a/trillian.proto +++ b/trillian.proto @@ -48,6 +48,9 @@ enum HashStrategy { // Append-only log strategy where leaf nodes are defined as the ObjectHash. // All other properties are equal to RFC6962_SHA256. OBJECT_RFC6962_SHA256 = 3; + + // The CONIKS sparse tree hasher with SHA512_256 as the hash algorithm. + CONIKS_SHA512_256 = 4; } // State of the tree. diff --git a/vmap/toy/vmap_toy.go b/vmap/toy/vmap_toy.go index 7437d442ba..33e8936e77 100644 --- a/vmap/toy/vmap_toy.go +++ b/vmap/toy/vmap_toy.go @@ -86,6 +86,7 @@ func main() { defer tx.Close() w, err := merkle.NewSparseMerkleTreeWriter( ctx, + mapID, tx.WriteRevision(), hasher, func() (storage.TreeTX, error) { @@ -98,8 +99,9 @@ func main() { glog.Infof("Starting batch %d...", x) h := make([]merkle.HashKeyValue, batchSize) for y := 0; y < batchSize; y++ { - h[y].HashedKey = testonly.HashKey(fmt.Sprintf("key-%d-%d", x, y)) - h[y].HashedValue = hasher.HashLeaf([]byte(fmt.Sprintf("value-%d-%d", x, y))) + index := testonly.HashKey(fmt.Sprintf("key-%d-%d", x, y)) + h[y].HashedKey = index + h[y].HashedValue = hasher.HashLeaf(mapID, index, hasher.BitLen(), []byte(fmt.Sprintf("value-%d-%d", x, y))) } glog.Infof("Created %d k/v pairs...", len(h))