From de0e59fad42402929762882482748cbf47e0ed16 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Tue, 4 Jul 2017 07:45:00 +0100 Subject: [PATCH 01/15] map_verifier fix: use neighbor index in HashEmpty --- merkle/common.go | 10 ++++++++++ merkle/common_test.go | 25 +++++++++++++++++++++++++ merkle/map_verifier.go | 3 ++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/merkle/common.go b/merkle/common.go index e2dce5ada1..9f3db0bdd4 100644 --- a/merkle/common.go +++ b/merkle/common.go @@ -32,3 +32,13 @@ func bit(index []byte, i int) uint { bIndex := (IndexBits - i - 1) / 8 return uint((index[bIndex] >> uint(i%8)) & 0x01) } + +// flipBit returns index with the i'th bit from the right flipped. +func flipBit(index []byte, i int) []byte { + r := make([]byte, len(index)) + copy(r, index) + IndexBits := len(index) * 8 + bIndex := (IndexBits - i - 1) / 8 + r[bIndex] ^= 1 << uint(i%8) + return r +} diff --git a/merkle/common_test.go b/merkle/common_test.go index 40dbd21773..3522938020 100644 --- a/merkle/common_test.go +++ b/merkle/common_test.go @@ -15,6 +15,7 @@ package merkle import ( + "bytes" "testing" ) @@ -41,3 +42,27 @@ func TestBit(t *testing.T) { } } } + +func TestFlipBit(t *testing.T) { + for _, tc := range []struct { + index []byte + i int + want []byte + }{ + {index: h2b("00"), i: 0, want: h2b("01")}, + {index: h2b("00"), i: 7, want: h2b("80")}, + {index: h2b("000b"), i: 0, want: h2b("000a")}, + {index: h2b("000b"), i: 1, want: h2b("0009")}, + {index: h2b("000b"), i: 2, want: h2b("000f")}, + {index: h2b("000b"), i: 3, want: h2b("0003")}, + {index: h2b("0001"), i: 0, want: h2b("0000")}, + {index: h2b("8000"), i: 15, want: h2b("0000")}, + {index: h2b("0000000000000001"), i: 0, want: h2b("0000000000000000")}, + {index: h2b("0000000000010000"), i: 16, want: h2b("0000000000000000")}, + {index: h2b("8000000000000000"), i: 63, want: h2b("0000000000000000")}, + } { + if got, want := flipBit(tc.index, tc.i), tc.want; !bytes.Equal(got, want) { + t.Errorf("flipBit(%x, %d): %x, want %x", tc.index, tc.i, got, want) + } + } +} diff --git a/merkle/map_verifier.go b/merkle/map_verifier.go index f249cf9571..15b0460206 100644 --- a/merkle/map_verifier.go +++ b/merkle/map_verifier.go @@ -48,7 +48,8 @@ func VerifyMapInclusionProof(treeID int64, index, leafHash, expectedRoot []byte, proofIsRightHandElement := bit(index, level) == 0 pElement := proof[level] if len(pElement) == 0 { - pElement = h.HashEmpty(treeID, index, level) + neighborIndex := flipBit(index, level) + pElement = h.HashEmpty(treeID, neighborIndex, level) } if proofIsRightHandElement { runningHash = h.HashChildren(runningHash, pElement) From 5be3b2a63a489027e762813f7d0d673347aea8f8 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Wed, 5 Jul 2017 13:39:04 +0100 Subject: [PATCH 02/15] Add index to HStar2Nodes. Tests pass --- merkle/hstar2.go | 115 ++++++++++++++++-------------- merkle/hstar2_test.go | 30 ++++---- merkle/sparse_merkle_tree.go | 18 +++-- merkle/sparse_merkle_tree_test.go | 4 +- storage/cache/subtree_cache.go | 3 +- storage/types.go | 5 ++ 6 files changed, 95 insertions(+), 80 deletions(-) diff --git a/merkle/hstar2.go b/merkle/hstar2.go index 1cbfee32ed..a241ff03ff 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -17,6 +17,7 @@ package merkle import ( "errors" "fmt" + "log" "math/big" "sort" @@ -54,15 +55,11 @@ func NewHStar2(treeID int64, hasher hashers.MapHasher) HStar2 { } } -// HStar2Root calculates the root of a sparse Merkle tree of depth n which contains -// the given set of non-null leaves. -func (s *HStar2) HStar2Root(n int, values []HStar2LeafHash) ([]byte, error) { +// HStar2Root calculates the root of a sparse Merkle tree of a given depth +// which contains the given set of non-null leaves. +func (s *HStar2) HStar2Root(depth int, values []HStar2LeafHash) ([]byte, error) { sort.Sort(ByIndex{values}) - return s.hStar2b(n, values, smtZero, - func(depth int, index *big.Int) ([]byte, error) { - return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth), nil - }, - func(int, *big.Int, []byte) error { return nil }) + return s.hStar2b(0, depth, values, smtZero, nil, nil) } // SparseGetNodeFunc should return any pre-existing node hash for the node address. @@ -71,83 +68,93 @@ type SparseGetNodeFunc func(depth int, index *big.Int) ([]byte, error) // SparseSetNodeFunc should store the passed node hash, associating it with the address. type SparseSetNodeFunc func(depth int, index *big.Int, hash []byte) error -// HStar2Nodes calculates the root hash of a pre-existing sparse Merkle tree -// plus the extra values passed in. Get and set are used to fetch and store -// internal node values. Values must not contain multiple leaves for the same -// index. +// HStar2Nodes calculates the root hash of a pre-existing sparse Merkle tree (SMT). +// HStar2Nodes can also calculate the root nodes of subtrees inside a SMT. +// Get and set are used to fetch and store internal node values. +// Values must not contain multiple leaves for the same index. // -// The treeLevelOffset argument is used when the tree to be calculated is part -// of a larger tree. It identifes the level in the larger tree at which the -// root of the subtree being calculated is found. -// e.g. Imagine a tree 256 levels deep, and that you already (somehow) happen -// to have the intermediate hash values for the non-null nodes 8 levels below -// the root already calculated (i.e. you just need to calculate the top 8 -// levels of a 256-level tree). To do this, you'd set treeDepth=8, and -// treeLevelOffset=248 (256-8). -func (s *HStar2) HStar2Nodes(treeDepth, treeLevelOffset int, values []HStar2LeafHash, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { +// prefix is the location of this subtree within the larger tree. Root is at nil. +// subtreeDepth is the number of levels in this subtree. +// The height of the whole tree is assumed to be hasher.BitLen() +func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2LeafHash, + get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { if glog.V(3) { - glog.Infof("HStar2Nodes(%v, %v, %v)", treeDepth, treeLevelOffset, len(values)) + glog.Infof("HStar2Nodes(%x, %v, %v, %v)", prefix, depth, subtreeDepth, len(values)) for _, v := range values { glog.Infof(" %x: %x", v.Index.Bytes(), v.LeafHash) } } - if treeLevelOffset < 0 { + depth := len(prefix) * 8 + totalDepth := depth + subtreeDepth + if totalDepth > s.hasher.BitLen() { return nil, ErrNegativeTreeLevelOffset } sort.Sort(ByIndex{values}) - return s.hStar2b(treeDepth, values, smtZero, - func(depth int, index *big.Int) ([]byte, error) { - // if we've got a function for getting existing node values, try it: - h, err := get(treeDepth-depth, index) - if err != nil { - return nil, err - } - // if we got a value then we'll use that - if h != nil { - return h, nil - } - // otherwise just return the null hash for this level - return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth+treeLevelOffset), nil - }, - func(depth int, index *big.Int, hash []byte) error { - return set(treeDepth-depth, index, hash) - }) + offset := new(big.Int).SetBytes(prefix) + offset = offset.Lsh(offset, uint(s.hasher.BitLen()-depth)) // shift prefix into place. + return s.hStar2b(depth, totalDepth, values, offset, get, set) } -// hStar2b is the recursive implementation for calculating a sparse Merkle tree -// root value. -func (s *HStar2) hStar2b(n int, values []HStar2LeafHash, offset *big.Int, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { - if n == 0 { +// hStar2b computes a sparse Merkle tree root value recursively. +func (s *HStar2) hStar2b(depth, maxDepth int, values []HStar2LeafHash, offset *big.Int, + get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { + log.Printf("hStar2b(%3v, %3v, %2d values, %x)", depth, maxDepth, len(values), offset.Bytes()) + if depth == maxDepth { switch { case len(values) == 0: - return get(n, offset) - case len(values) != 1: + return s.get(offset, depth, get) + case len(values) == 1: + return values[0].LeafHash, nil + default: return nil, fmt.Errorf("hStar2b base case: len(values): %d, want 1", len(values)) } - return values[0].LeafHash, nil } if len(values) == 0 { - return get(n, offset) + return s.get(offset, depth, get) } - split := new(big.Int).Lsh(smtOne, uint(n-1)) + bitsLeft := maxDepth - depth + split := new(big.Int).Lsh(smtOne, uint(bitsLeft-1)) split.Add(split, offset) i := sort.Search(len(values), func(i int) bool { return values[i].Index.Cmp(split) >= 0 }) - lhs, err := s.hStar2b(n-1, values[:i], offset, get, set) + lhs, err := s.hStar2b(depth+1, maxDepth, values[:i], offset, get, set) if err != nil { return nil, err } - rhs, err := s.hStar2b(n-1, values[i:], split, get, set) + rhs, err := s.hStar2b(depth+1, maxDepth, values[i:], split, get, set) if err != nil { return nil, err } h := s.hasher.HashChildren(lhs, rhs) - if set != nil { - set(n, offset, h) - } + s.set(offset, depth, h, set) return h, nil } +// get attempts to use getter. If getter fails, returns the HashEmpty value. +func (s *HStar2) get(index *big.Int, depth int, getter SparseGetNodeFunc) ([]byte, error) { + // if we've got a function for getting existing node values, try it: + if getter != nil { + h, err := getter(depth, index) + if err != nil { + return nil, err + } + // if we got a value then we'll use that + if h != nil { + return h, nil + } + } + height := s.hasher.BitLen() - depth + return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), height), nil +} + +// set attempts to use setter if it not nil. +func (s *HStar2) set(index *big.Int, depth int, hash []byte, setter SparseSetNodeFunc) error { + if setter != nil { + return setter(depth, index, hash) + } + return nil +} + // HStar2LeafHash sorting boilerplate below. // Leaves is a slice of HStar2LeafHash diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index 43e1277443..ea65e339fc 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -87,7 +87,7 @@ func TestHStar2SimpleDataSetKAT(t *testing.T) { continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root: \n%x, want:\n%x", got, want) + t.Errorf("Root: %x, want: %x", got, want) } } } @@ -107,7 +107,7 @@ func TestHStar2GetSet(t *testing.T) { if len(values) != 1 { t.Fatalf("Should only have 1 leaf per run, got %d", len(values)) } - root, err := s.HStar2Nodes(s.hasher.BitLen(), 0, values, + root, err := s.HStar2Nodes(nil, s.hasher.BitLen(), values, func(depth int, index *big.Int) ([]byte, error) { return cache[fmt.Sprintf("%x/%d", index, depth)], nil }, @@ -120,7 +120,7 @@ func TestHStar2GetSet(t *testing.T) { continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root:\n%x, want:\n%x", got, want) + t.Errorf("Root: %x, want: %x", got, want) } } } @@ -132,18 +132,18 @@ func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HS var ret []HStar2LeafHash s := NewHStar2(treeID, maphasher.Default) for i := range lh { - prefix := new(big.Int).Rsh(lh[i].Index, uint(s.hasher.BitLen()-prefixSize)) - b := lh[i].Index.Bytes() - // ensure we've got any chopped of leading zero bytes - for len(b) < 32 { - b = append([]byte{0}, b...) + subtreeDepth := s.hasher.BitLen() - prefixSize + prefix := lh[i].Index.Bytes() + // ensure we've got any chopped off leading zero bytes + for len(prefix) < 32 { + prefix = append([]byte{0}, prefix...) } - lh[i].Index.SetBytes(b[prefixSize/8:]) - root, err := s.HStar2Root(s.hasher.BitLen()-prefixSize, []HStar2LeafHash{lh[i]}) + prefix = prefix[:prefixSize/8] // We only want the first prefixSize bytes. + root, err := s.HStar2Nodes(prefix, subtreeDepth, []HStar2LeafHash{lh[i]}, nil, nil) if err != nil { t.Fatalf("Failed to calculate root %v", err) } - ret = append(ret, HStar2LeafHash{prefix, root}) + ret = append(ret, HStar2LeafHash{new(big.Int).SetBytes(prefix), root}) } return ret } @@ -163,15 +163,13 @@ func TestHStar2OffsetRootKAT(t *testing.T) { leaves := createHStar2Leaves(treeID, maphasher.Default, iv...) intermediates := rootsForTrimmedKeys(t, size, leaves) - root, err := s.HStar2Nodes(size, s.hasher.BitLen()-size, intermediates, - func(int, *big.Int) ([]byte, error) { return nil, nil }, - func(int, *big.Int, []byte) error { return nil }) + root, err := s.HStar2Nodes(nil, size, intermediates, nil, nil) if err != nil { t.Errorf("Failed to calculate root at iteration %d: %v", i, err) continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root: %x, want: %x", got, want) + t.Errorf("HStar2Nodes(i: %v, size:%v): %x, want: %x", i, size, got, want) } } } @@ -180,7 +178,7 @@ func TestHStar2OffsetRootKAT(t *testing.T) { func TestHStar2NegativeTreeLevelOffset(t *testing.T) { s := NewHStar2(treeID, maphasher.Default) - _, err := s.HStar2Nodes(32, -1, []HStar2LeafHash{}, + _, err := s.HStar2Nodes(make([]byte, 31), 9, []HStar2LeafHash{}, func(int, *big.Int) ([]byte, error) { return nil, nil }, func(int, *big.Int, []byte) error { return nil }) if got, want := err, ErrNegativeTreeLevelOffset; got != want { diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index a38903b04d..61da906998 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -56,6 +56,7 @@ type indexAndHash struct { // rootHashOrError represents a (sub-)tree root hash, or an error which // prevented the calculation from completing. +// TODO(gdbelvin): represent an empty subtree with a nil hash? type rootHashOrError struct { hash []byte err error @@ -170,7 +171,9 @@ func (s *subtreeWriter) SetLeaf(ctx context.Context, index []byte, hash []byte) return subtree.SetLeaf(ctx, index[s.subtreeDepth/8:], hash) case indexLen == s.subtreeDepth: - s.leafQueue <- func() (*indexAndHash, error) { return &indexAndHash{index: index, hash: hash}, nil } + s.leafQueue <- func() (*indexAndHash, error) { + return &indexAndHash{index: index, hash: hash}, nil + } return nil } @@ -209,21 +212,24 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { s.root <- rootHashOrError{hash: nil, err: err} return } - leaves = append(leaves, HStar2LeafHash{Index: new(big.Int).SetBytes(ih.index), LeafHash: ih.hash}) + leaves = append(leaves, HStar2LeafHash{ + Index: new(big.Int).SetBytes(ih.index), + LeafHash: ih.hash, + }) nodesToStore = append(nodesToStore, storage.Node{ - NodeID: storage.NewNodeIDFromHash(bytes.Join([][]byte{s.prefix, ih.index}, []byte{})), + NodeID: storage.NewNodeIDFromHash( + bytes.Join([][]byte{s.prefix, ih.index}, []byte{})), Hash: ih.hash, NodeRevision: s.treeRevision, }) - } // calculate new root, and intermediate nodes: hs2 := NewHStar2(s.treeID, s.treeHasher) - treeDepthOffset := (s.treeHasher.Size()-len(s.prefix))*8 - s.subtreeDepth totalDepth := len(s.prefix)*8 + s.subtreeDepth - root, err := hs2.HStar2Nodes(s.subtreeDepth, treeDepthOffset, leaves, + prefixDepth := len(s.prefix) * 8 + root, err := hs2.HStar2Nodes(nil, prefixDepth, s.subtreeDepth, leaves, func(height int, index *big.Int) ([]byte, error) { nodeID := storage.NewNodeIDFromRelativeBigInt(s.prefix, s.subtreeDepth, height, index, totalDepth) glog.V(4).Infof("buildSubtree.get(%x, %d) nid: %x, %v", diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index 91f36b44a5..8ac9418b6c 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -369,8 +369,8 @@ func testSparseTreeCalculatedRootWithWriter(ctx context.Context, t *testing.T, r if err != nil { t.Fatalf("Failed to commit map changes: %v", err) } - if expected, got := vec.expectedRoot, root; !bytes.Equal(expected, got) { - t.Errorf("Expected root:\n%s, but got root:\n%s", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got)) + if got, want := root, vec.expectedRoot; !bytes.Equal(got, want) { + t.Errorf("got root: %x, want %x", got, want) } } diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index 784d7fb7de..d8301c87eb 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -416,8 +416,7 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop }) } hs2 := merkle.NewHStar2(treeID, hasher) - offset := hasher.BitLen() - rootID.PrefixLenBits - int(st.Depth) - root, err := hs2.HStar2Nodes(int(st.Depth), offset, leaves, + root, err := hs2.HStar2Nodes(st.Prefix, rootID.PrefixLenBits, int(st.Depth), leaves, func(depth int, index *big.Int) ([]byte, error) { return nil, nil }, diff --git a/storage/types.go b/storage/types.go index c60f2ea39c..73ba1bbd9c 100644 --- a/storage/types.go +++ b/storage/types.go @@ -330,6 +330,11 @@ func (n *NodeID) Equivalent(other NodeID) bool { return n.String() == other.String() } +// Equal returns true iff a and b have the same string representation. +func Equal(a, b *NodeID) bool { + return a.String() == b.String() +} + // PopulateSubtreeFunc is a function which knows how to re-populate a subtree // from just its leaf nodes. type PopulateSubtreeFunc func(*storagepb.SubtreeProto) error From fabb822ce712752effa91f494149d136cdaed97f Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Wed, 5 Jul 2017 19:52:44 +0100 Subject: [PATCH 03/15] Use absolute indexes --- integration/maptest/map_test.go | 7 ++++ merkle/hstar2.go | 18 ++++----- merkle/hstar2_test.go | 11 ++++- merkle/map_verifier.go | 2 +- merkle/sparse_merkle_tree.go | 67 +++++++++++++++++-------------- merkle/sparse_merkle_tree_test.go | 12 ++++-- server/map_rpc_server.go | 2 +- storage/cache/subtree_cache.go | 12 +++--- storage/types.go | 12 +++--- 9 files changed, 80 insertions(+), 63 deletions(-) diff --git a/integration/maptest/map_test.go b/integration/maptest/map_test.go index 7927602a11..f0b260d0ac 100644 --- a/integration/maptest/map_test.go +++ b/integration/maptest/map_test.go @@ -140,6 +140,13 @@ func TestInclusion(t *testing.T) { {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, }, }, + { + desc: "CONIKS", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + leaves: []*trillian.MapLeaf{ + {Index: h2b("4100000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + }, + }, } { tree, hasher, err := newTreeWithHasher(ctx, env, tc.HashStrategy) if err != nil { diff --git a/merkle/hstar2.go b/merkle/hstar2.go index a241ff03ff..3d882611e9 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -17,7 +17,6 @@ package merkle import ( "errors" "fmt" - "log" "math/big" "sort" @@ -68,18 +67,17 @@ type SparseGetNodeFunc func(depth int, index *big.Int) ([]byte, error) // SparseSetNodeFunc should store the passed node hash, associating it with the address. type SparseSetNodeFunc func(depth int, index *big.Int, hash []byte) error -// HStar2Nodes calculates the root hash of a pre-existing sparse Merkle tree (SMT). -// HStar2Nodes can also calculate the root nodes of subtrees inside a SMT. -// Get and set are used to fetch and store internal node values. -// Values must not contain multiple leaves for the same index. +// HStar2Nodes calculates the root hash of a pre-existing sparse Merkle tree +// plus the extra values passed in. Get and set are used to fetch and store +// internal node values. Values must not contain multiple leaves for the same +// index. // // prefix is the location of this subtree within the larger tree. Root is at nil. // subtreeDepth is the number of levels in this subtree. -// The height of the whole tree is assumed to be hasher.BitLen() func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2LeafHash, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { if glog.V(3) { - glog.Infof("HStar2Nodes(%x, %v, %v, %v)", prefix, depth, subtreeDepth, len(values)) + glog.Infof("HStar2Nodes(%x, %v, %v)", prefix, subtreeDepth, len(values)) for _, v := range values { glog.Infof(" %x: %x", v.Index.Bytes(), v.LeafHash) } @@ -98,7 +96,6 @@ func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2Lea // hStar2b computes a sparse Merkle tree root value recursively. func (s *HStar2) hStar2b(depth, maxDepth int, values []HStar2LeafHash, offset *big.Int, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { - log.Printf("hStar2b(%3v, %3v, %2d values, %x)", depth, maxDepth, len(values), offset.Bytes()) if depth == maxDepth { switch { case len(values) == 0: @@ -113,7 +110,7 @@ func (s *HStar2) hStar2b(depth, maxDepth int, values []HStar2LeafHash, offset *b return s.get(offset, depth, get) } - bitsLeft := maxDepth - depth + bitsLeft := s.hasher.BitLen() - depth split := new(big.Int).Lsh(smtOne, uint(bitsLeft-1)) split.Add(split, offset) i := sort.Search(len(values), func(i int) bool { return values[i].Index.Cmp(split) >= 0 }) @@ -144,7 +141,8 @@ func (s *HStar2) get(index *big.Int, depth int, getter SparseGetNodeFunc) ([]byt } } height := s.hasher.BitLen() - depth - return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), height), nil + indexBytes := PaddedBytes(index, s.hasher.Size()) + return s.hasher.HashEmpty(s.treeID, indexBytes, height), nil } // set attempts to use setter if it not nil. diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index ea65e339fc..8e8452b29f 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -130,7 +130,8 @@ func TestHStar2GetSet(t *testing.T) { // 256-prefixSize, and can be passed in as leaves to top-subtree calculation. func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HStar2LeafHash { var ret []HStar2LeafHash - s := NewHStar2(treeID, maphasher.Default) + hasher := maphasher.Default + s := NewHStar2(treeID, hasher) for i := range lh { subtreeDepth := s.hasher.BitLen() - prefixSize prefix := lh[i].Index.Bytes() @@ -143,7 +144,13 @@ func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HS if err != nil { t.Fatalf("Failed to calculate root %v", err) } - ret = append(ret, HStar2LeafHash{new(big.Int).SetBytes(prefix), root}) + + index := new(big.Int).SetBytes(prefix) + index = index.Lsh(index, uint(hasher.BitLen()-prefixSize)) + ret = append(ret, HStar2LeafHash{ + Index: index, + LeafHash: root, + }) } return ret } diff --git a/merkle/map_verifier.go b/merkle/map_verifier.go index 15b0460206..7392e421d9 100644 --- a/merkle/map_verifier.go +++ b/merkle/map_verifier.go @@ -48,7 +48,7 @@ func VerifyMapInclusionProof(treeID int64, index, leafHash, expectedRoot []byte, proofIsRightHandElement := bit(index, level) == 0 pElement := proof[level] if len(pElement) == 0 { - neighborIndex := flipBit(index, level) + neighborIndex := Neighbor(index, level) pElement = h.HashEmpty(treeID, neighborIndex, level) } if proofIsRightHandElement { diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index 61da906998..2e8383928d 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "log" "math/big" "sync" @@ -110,7 +111,7 @@ type subtreeWriter struct { tx storage.TreeTX treeRevision int64 - treeHasher hashers.MapHasher + hasher hashers.MapHasher getSubtree getSubtreeFunc } @@ -152,32 +153,31 @@ func (s *subtreeWriter) getOrCreateChildSubtree(ctx context.Context, childPrefix return subtree, nil } -// SetLeaf sets a single leaf hash for incorporation into the sparse Merkle -// tree. +// SetLeaf sets a single leaf hash for incorporation into the sparse Merkle tree. +// index is the full path of the leaf, starting from the root (not the subtree's root). func (s *subtreeWriter) SetLeaf(ctx context.Context, index []byte, hash []byte) error { - indexLen := len(index) * 8 + depth := len(index) * 8 + absSubtreeDepth := len(s.prefix)*8 + s.subtreeDepth switch { - case indexLen < s.subtreeDepth: - return fmt.Errorf("index length %d is < our depth %d", indexLen, s.subtreeDepth) + case depth < absSubtreeDepth: + return fmt.Errorf("depth: %d, want >= %d", depth, absSubtreeDepth) - case indexLen > s.subtreeDepth: - childPrefix := index[:s.subtreeDepth/8] + case depth > absSubtreeDepth: + childPrefix := index[:absSubtreeDepth/8] subtree, err := s.getOrCreateChildSubtree(ctx, childPrefix) if err != nil { return err } - return subtree.SetLeaf(ctx, index[s.subtreeDepth/8:], hash) + return subtree.SetLeaf(ctx, index, hash) - case indexLen == s.subtreeDepth: + default: // depth == absSubtreeDepth: s.leafQueue <- func() (*indexAndHash, error) { return &indexAndHash{index: index, hash: hash}, nil } return nil } - - return fmt.Errorf("internal logic error in SetLeaf. index length: %d, subtreeDepth: %d", indexLen, s.subtreeDepth) } // CalculateRoot initiates the process of calculating the subtree root. @@ -212,28 +212,27 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { s.root <- rootHashOrError{hash: nil, err: err} return } + index := new(big.Int).SetBytes(ih.index) + index = index.Lsh(index, uint(s.hasher.BitLen()-len(ih.index)*8)) leaves = append(leaves, HStar2LeafHash{ - Index: new(big.Int).SetBytes(ih.index), + Index: index, LeafHash: ih.hash, }) nodesToStore = append(nodesToStore, storage.Node{ - NodeID: storage.NewNodeIDFromHash( - bytes.Join([][]byte{s.prefix, ih.index}, []byte{})), + NodeID: storage.NewNodeIDFromHash(ih.index), Hash: ih.hash, NodeRevision: s.treeRevision, }) } // calculate new root, and intermediate nodes: - hs2 := NewHStar2(s.treeID, s.treeHasher) - totalDepth := len(s.prefix)*8 + s.subtreeDepth - prefixDepth := len(s.prefix) * 8 - root, err := hs2.HStar2Nodes(nil, prefixDepth, s.subtreeDepth, leaves, - func(height int, index *big.Int) ([]byte, error) { - nodeID := storage.NewNodeIDFromRelativeBigInt(s.prefix, s.subtreeDepth, height, index, totalDepth) + hs2 := NewHStar2(s.treeID, s.hasher) + root, err := hs2.HStar2Nodes(s.prefix, s.subtreeDepth, leaves, + func(depth int, index *big.Int) ([]byte, error) { + nodeID := storage.NewNodeIDFromBigInt(depth, index, s.hasher.BitLen()) glog.V(4).Infof("buildSubtree.get(%x, %d) nid: %x, %v", - index.Bytes(), height, nodeID.Path, nodeID.PrefixLenBits) + index.Bytes(), depth, nodeID.Path, nodeID.PrefixLenBits) nodes, err := s.tx.GetMerkleNodes(ctx, s.treeRevision, []storage.NodeID{nodeID}) if err != nil { return nil, err @@ -241,23 +240,23 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { if len(nodes) == 0 { return nil, nil } - if expected, got := nodeID, nodes[0].NodeID; !expected.Equivalent(got) { - return nil, fmt.Errorf("expected node ID %s from storage, but got %s", expected.String(), got.String()) + if got, want := nodes[0].NodeID, nodeID; !got.Equivalent(want) { + return nil, fmt.Errorf("got node %s from storage, want %s", got, want) } - if expected, got := s.treeRevision, nodes[0].NodeRevision; got > expected { - return nil, fmt.Errorf("expected node revision <= %d, but got %d", expected, got) + if got, want := nodes[0].NodeRevision, s.treeRevision; got > want { + return nil, fmt.Errorf("got node revision %d, want <= %d", got, want) } return nodes[0].Hash, nil }, - func(height int, index *big.Int, h []byte) error { + func(depth int, index *big.Int, h []byte) error { // Don't store the root node of the subtree - that's part of the parent // tree. - if height == 0 && len(s.prefix) > 0 { + if depth == len(s.prefix)*8 && len(s.prefix) > 0 { return nil } - nodeID := storage.NewNodeIDFromRelativeBigInt(s.prefix, s.subtreeDepth, height, index, totalDepth) + nodeID := storage.NewNodeIDFromBigInt(depth, index, s.hasher.BitLen()) glog.V(4).Infof("buildSubtree.set(%x, %v) nid: %x, %v : %x", - index.Bytes(), height, nodeID.Path, nodeID.PrefixLenBits, h) + index.Bytes(), depth, nodeID.Path, nodeID.PrefixLenBits, h) nodesToStore = append(nodesToStore, storage.Node{ NodeID: nodeID, @@ -327,7 +326,7 @@ func newLocalSubtreeWriter(ctx context.Context, treeID, rev int64, prefix []byte root: make(chan rootHashOrError, 1), children: make(map[string]Subtree), tx: tx, - treeHasher: h, + hasher: h, getSubtree: func(ctx context.Context, p []byte) (Subtree, error) { myPrefix := bytes.Join([][]byte{prefix, p}, []byte{}) return newLocalSubtreeWriter(ctx, treeID, rev, myPrefix, depths[1:], newTX, h) @@ -388,14 +387,20 @@ func (s SparseMerkleTreeReader) RootAtRevision(ctx context.Context, rev int64) ( func (s SparseMerkleTreeReader) InclusionProof(ctx context.Context, rev int64, index []byte) ([][]byte, error) { nid := storage.NewNodeIDFromHash(index) sibs := nid.Siblings() + log.Printf("Siblings: ") + for _, s := range sibs { + log.Printf(" %x", s.Path) + } nodes, err := s.tx.GetMerkleNodes(ctx, rev, sibs) if err != nil { return nil, err } nodeMap := make(map[string]*storage.Node) + log.Printf("Got Nodes: ") for _, n := range nodes { n := n // need this or we'll end up with the same node hash repeated in the map + log.Printf(" %x, %d: %x", n.NodeID.Path, len(n.NodeID.String()), n.Hash) nodeMap[n.NodeID.String()] = &n } diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index 8ac9418b6c..fa4801ab56 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -437,10 +437,14 @@ func testSparseTreeFetches(ctx context.Context, t *testing.T, vec sparseTestVect id := sibs[j].String() pathNode := nodeID.String()[:len(id)] if _, ok := reads[pathNode]; ok { - // we're modifying both children of a node because two keys are - // intersecting, since both will be recalculated neither will be read - // from storage so we remove the previously set expectation for this - // node's sibling, and skip adding one for this node: + // we're modifying both children of a + // node because two keys are + // intersecting, since both will be + // recalculated neither will be read + // from storage so we remove the + // previously set expectation for this + // node's sibling, and skip adding one + // for this node: delete(reads, pathNode) continue } diff --git a/server/map_rpc_server.go b/server/map_rpc_server.go index 2222d19c5d..eb881695d6 100644 --- a/server/map_rpc_server.go +++ b/server/map_rpc_server.go @@ -167,7 +167,7 @@ func (t *TrillianMapServer) SetLeaves(ctx context.Context, req *trillian.SetMapL "len(%x): %v, want %v", l.Index, got, want) } // TODO(gbelvin) use LeafHash rather than computing here. #423 - l.LeafHash = hasher.HashLeaf(mapID, l.Index, hasher.BitLen(), l.LeafValue) + l.LeafHash = hasher.HashLeaf(mapID, l.Index, 0, l.LeafValue) if err = tx.Set(ctx, l.Index, *l); err != nil { return nil, err diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index d8301c87eb..4029162f8d 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -400,7 +400,6 @@ func (s *SubtreeCache) newEmptySubtree(id storage.NodeID, px []byte) *storagepb. func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.PopulateSubtreeFunc { return func(st *storagepb.SubtreeProto) error { st.InternalNodes = make(map[string][]byte) - rootID := storage.NewNodeIDFromHash(st.Prefix) leaves := make([]merkle.HStar2LeafHash, 0, len(st.Leaves)) for k64, v := range st.Leaves { k, err := base64.StdEncoding.DecodeString(k64) @@ -410,22 +409,21 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop if k[0]%depthQuantum != 0 { return fmt.Errorf("unexpected non-leaf suffix found: %x", k) } + index := new(big.Int).SetBytes(k[1:]) // TODO: create a function for parsing k + index = index.Lsh(index, uint(hasher.BitLen()-len(index.Bytes()))) leaves = append(leaves, merkle.HStar2LeafHash{ + Index: index, LeafHash: v, - Index: new(big.Int).SetBytes(k[1:]), }) } hs2 := merkle.NewHStar2(treeID, hasher) - root, err := hs2.HStar2Nodes(st.Prefix, rootID.PrefixLenBits, int(st.Depth), leaves, - func(depth int, index *big.Int) ([]byte, error) { - return nil, nil - }, + root, err := hs2.HStar2Nodes(st.Prefix, int(st.Depth), leaves, nil, func(depth int, index *big.Int, h []byte) error { if depth == 0 { // no space for the root in the node cache return nil } - nodeID := storage.NewNodeIDFromRelativeBigInt(st.Prefix, int(st.Depth), depth, index, hasher.BitLen()) + nodeID := storage.NewNodeIDFromBigInt(depth, index, hasher.BitLen()) _, sfx := nodeID.Split(len(st.Prefix), int(st.Depth)) sfxKey := sfx.String() if glog.V(4) { diff --git a/storage/types.go b/storage/types.go index 73ba1bbd9c..04bbfa69c5 100644 --- a/storage/types.go +++ b/storage/types.go @@ -284,15 +284,18 @@ func (n *NodeID) CoordString() string { // Siblings returns the siblings of the given node. func (n *NodeID) Siblings() []NodeID { - r := make([]NodeID, n.PrefixLenBits, n.PrefixLenBits) l := n.PrefixLenBits + r := make([]NodeID, l) // Index of the bit to twiddle: bi := n.PathLenBits() - n.PrefixLenBits - for i := 0; i < len(r); i++ { + for i := range r { r[i].PrefixLenBits = l - i r[i].Path = make([]byte, len(n.Path)) copy(r[i].Path, n.Path) r[i].SetBit(bi, n.Bit(bi)^1) + for j := bi - 1; j >= 0; j-- { + r[i].SetBit(j, 0) + } bi++ } return r @@ -330,11 +333,6 @@ func (n *NodeID) Equivalent(other NodeID) bool { return n.String() == other.String() } -// Equal returns true iff a and b have the same string representation. -func Equal(a, b *NodeID) bool { - return a.String() == b.String() -} - // PopulateSubtreeFunc is a function which knows how to re-populate a subtree // from just its leaf nodes. type PopulateSubtreeFunc func(*storagepb.SubtreeProto) error From d13095db9801d0c35bdefd004afda4bdde3143f0 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Tue, 25 Jul 2017 15:37:11 +0100 Subject: [PATCH 04/15] Add MaskIndex --- merkle/common.go | 24 ++++++++++++++++++++++++ merkle/common_test.go | 30 ++++++++++++++++++++++++++++++ merkle/hstar2.go | 1 + merkle/hstar2_test.go | 6 +++--- 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/merkle/common.go b/merkle/common.go index 9f3db0bdd4..554c8d72d3 100644 --- a/merkle/common.go +++ b/merkle/common.go @@ -42,3 +42,27 @@ func flipBit(index []byte, i int) []byte { r[bIndex] ^= 1 << uint(i%8) return r } + +// Neighbor returns index with only the left i bits set and the i'th bit flipped. +func Neighbor(index []byte, i int) []byte { + r := flipBit(index, i) + return MaskIndex(r, len(index)*8-i) +} + +// leftmask contains bitmasks indexed such that the left x bits are set. It is +// indexed by byte position from 0-7 0 is special cased to 0xFF since 8 mod 8 +// is 0. leftmask is only used to mask the last byte. +var leftmask = [8]byte{0xFF, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE} + +// MaskIndex returns index with only the left depth bits set. +func MaskIndex(index []byte, depth int) []byte { + r := make([]byte, len(index)) + if depth > 0 { + // Copy the first depthBytes. + depthBytes := (depth + 7) >> 3 + copy(r, index[:depthBytes]) + // Mask off unwanted bits in the last byte. + r[depthBytes-1] = r[depthBytes-1] & leftmask[depth%8] + } + return r +} diff --git a/merkle/common_test.go b/merkle/common_test.go index 3522938020..7e97574b58 100644 --- a/merkle/common_test.go +++ b/merkle/common_test.go @@ -66,3 +66,33 @@ func TestFlipBit(t *testing.T) { } } } + +func TestMaskIndex(t *testing.T) { + for _, tc := range []struct { + index []byte + depth int + want []byte + }{ + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 0, want: h2b("0000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 1, want: h2b("8000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 2, want: h2b("C000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 3, want: h2b("E000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 4, want: h2b("F000000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 5, want: h2b("F800000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 6, want: h2b("FC00000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 7, want: h2b("FE00000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 8, want: h2b("FF00000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 9, want: h2b("FF80000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 10, want: h2b("FFC0000000000000000000000000000000000000")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 159, want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE")}, + {index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 160, want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 1, want: h2b("0000000000000000000000000000000000000000")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 17, want: h2b("0001000000000000000000000000000000000000")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 159, want: h2b("000102030405060708090A0B0C0D0E0F10111212")}, + {index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 160, want: h2b("000102030405060708090A0B0C0D0E0F10111213")}, + } { + if got, want := MaskIndex(tc.index, tc.depth), tc.want; !bytes.Equal(got, want) { + t.Errorf("maskIndex(%x, %v): %x, want %x", tc.index, tc.depth, got, want) + } + } +} diff --git a/merkle/hstar2.go b/merkle/hstar2.go index 3d882611e9..728445e910 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -142,6 +142,7 @@ func (s *HStar2) get(index *big.Int, depth int, getter SparseGetNodeFunc) ([]byt } height := s.hasher.BitLen() - depth indexBytes := PaddedBytes(index, s.hasher.Size()) + indexBytes = MaskIndex(indexBytes, depth) return s.hasher.HashEmpty(s.treeID, indexBytes, height), nil } diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index 8e8452b29f..1ce2ed7ccd 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -178,16 +178,16 @@ func TestHStar2OffsetRootKAT(t *testing.T) { if got, want := root, x.root; !bytes.Equal(got, want) { t.Errorf("HStar2Nodes(i: %v, size:%v): %x, want: %x", i, size, got, want) } + break } + break } } func TestHStar2NegativeTreeLevelOffset(t *testing.T) { s := NewHStar2(treeID, maphasher.Default) - _, err := s.HStar2Nodes(make([]byte, 31), 9, []HStar2LeafHash{}, - func(int, *big.Int) ([]byte, error) { return nil, nil }, - func(int, *big.Int, []byte) error { return nil }) + _, err := s.HStar2Nodes(make([]byte, 31), 9, []HStar2LeafHash{}, nil, nil) if got, want := err, ErrNegativeTreeLevelOffset; got != want { t.Fatalf("Hstar2Nodes(): %v, want %v", got, want) } From 4adb58f08fc3ae7099bf597ffec3fbfecd0844fe Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 27 Jul 2017 10:58:42 +0100 Subject: [PATCH 05/15] Use height in leaf hash calculation --- integration/maptest/map_test.go | 47 ++++++++++++++++++++------------- merkle/sparse_merkle_tree.go | 9 ++----- server/map_rpc_server.go | 2 +- storage/types.go | 6 +++++ 4 files changed, 38 insertions(+), 26 deletions(-) diff --git a/integration/maptest/map_test.go b/integration/maptest/map_test.go index f0b260d0ac..7c50e42bff 100644 --- a/integration/maptest/map_test.go +++ b/integration/maptest/map_test.go @@ -88,13 +88,14 @@ func verifyGetMapLeavesResponse(getResp *trillian.GetMapLeavesResponse, indexes leafHash := incl.GetLeaf().GetLeafHash() proof := incl.GetInclusion() - if got, want := leafHash, hasher.HashLeaf(treeID, index, hasher.BitLen(), leaf); !bytes.Equal(got, want) { + if got, want := leafHash, hasher.HashLeaf(treeID, index, 0, leaf); !bytes.Equal(got, want) { return fmt.Errorf("HashLeaf(%s): %x, want %x", leaf, got, want) } if err := merkle.VerifyMapInclusionProof(treeID, index, leafHash, rootHash, proof, hasher); err != nil { return fmt.Errorf("VerifyMapInclusionProof(%x): %v", index, err) } + break } return nil } @@ -124,29 +125,40 @@ func TestInclusion(t *testing.T) { HashStrategy trillian.HashStrategy leaves []*trillian.MapLeaf }{ - { - desc: "maphasher single", - HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + /* + { + desc: "maphasher single", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + }, + }, + { + desc: "maphasher multi", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, + }, + }, + { + desc: "CONIKS single", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + }, }, - }, + */ { - desc: "maphasher multi", - HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + desc: "CONIKS multi", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, leaves: []*trillian.MapLeaf{ {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, }, }, - { - desc: "CONIKS", - HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, - leaves: []*trillian.MapLeaf{ - {Index: h2b("4100000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - }, - }, } { tree, hasher, err := newTreeWithHasher(ctx, env, tc.HashStrategy) if err != nil { @@ -363,8 +375,7 @@ func TestNonExistentLeaf(t *testing.T) { t.Errorf("len(leaf): %v, want, %v", got, want) } - if got, want := leafHash, - hasher.HashLeaf(tree.TreeId, index, hasher.BitLen(), leaf); !bytes.Equal(got, want) { + if got, want := leafHash, hasher.HashLeaf(tree.TreeId, index, 0, leaf); !bytes.Equal(got, want) { t.Errorf("HashLeaf(%s): %x, want %x", leaf, got, want) } if err := merkle.VerifyMapInclusionProof(tree.TreeId, index, diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index 2e8383928d..6dea2f15a6 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -19,7 +19,6 @@ import ( "context" "errors" "fmt" - "log" "math/big" "sync" @@ -387,20 +386,16 @@ func (s SparseMerkleTreeReader) RootAtRevision(ctx context.Context, rev int64) ( func (s SparseMerkleTreeReader) InclusionProof(ctx context.Context, rev int64, index []byte) ([][]byte, error) { nid := storage.NewNodeIDFromHash(index) sibs := nid.Siblings() - log.Printf("Siblings: ") - for _, s := range sibs { - log.Printf(" %x", s.Path) - } nodes, err := s.tx.GetMerkleNodes(ctx, rev, sibs) if err != nil { return nil, err } nodeMap := make(map[string]*storage.Node) - log.Printf("Got Nodes: ") + glog.Infof("Got Nodes: ") for _, n := range nodes { n := n // need this or we'll end up with the same node hash repeated in the map - log.Printf(" %x, %d: %x", n.NodeID.Path, len(n.NodeID.String()), n.Hash) + glog.Infof(" %x, %d: %x", n.NodeID.Path, len(n.NodeID.String()), n.Hash) nodeMap[n.NodeID.String()] = &n } diff --git a/server/map_rpc_server.go b/server/map_rpc_server.go index eb881695d6..3e879df0d1 100644 --- a/server/map_rpc_server.go +++ b/server/map_rpc_server.go @@ -106,7 +106,7 @@ func (t *TrillianMapServer) GetLeaves(ctx context.Context, req *trillian.GetMapL leaf = &trillian.MapLeaf{ Index: index, LeafValue: nil, - LeafHash: hasher.HashLeaf(mapID, index, hasher.BitLen(), nil), + LeafHash: hasher.HashLeaf(mapID, index, 0, nil), } } diff --git a/storage/types.go b/storage/types.go index 04bbfa69c5..38f20191e5 100644 --- a/storage/types.go +++ b/storage/types.go @@ -298,6 +298,12 @@ func (n *NodeID) Siblings() []NodeID { } bi++ } + if glog.V(5) { + glog.Infof("[%d, %x].Siblings():", n.PrefixLenBits, n.Path) + for _, s := range r { + glog.Infof(" %x", s.Path) + } + } return r } From 2b3dfe27c16194dfb14e9a6de6710d6c1df3080e Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 27 Jul 2017 12:30:45 +0100 Subject: [PATCH 06/15] ParseSuffix --- storage/cache/subtree_cache.go | 7 ++++--- storage/suffix.go | 13 +++++++++++++ storage/suffix_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index 4029162f8d..adce7dff2f 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -402,14 +402,15 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop st.InternalNodes = make(map[string][]byte) leaves := make([]merkle.HStar2LeafHash, 0, len(st.Leaves)) for k64, v := range st.Leaves { - k, err := base64.StdEncoding.DecodeString(k64) + sfx, err := storage.ParseSuffix(k64) if err != nil { return err } - if k[0]%depthQuantum != 0 { + // TODO(gdbelvin): test against subtree depth. + if sfx.Bits%depthQuantum != 0 { return fmt.Errorf("unexpected non-leaf suffix found: %x", k) } - index := new(big.Int).SetBytes(k[1:]) // TODO: create a function for parsing k + index := new(big.Int).SetBytes(sfx.Path) index = index.Lsh(index, uint(hasher.BitLen()-len(index.Bytes()))) leaves = append(leaves, merkle.HStar2LeafHash{ Index: index, diff --git a/storage/suffix.go b/storage/suffix.go index 18cd60a1f7..ebf4a7d21b 100644 --- a/storage/suffix.go +++ b/storage/suffix.go @@ -37,3 +37,16 @@ func (s Suffix) String() string { r = append(r, s.Path...) return base64.StdEncoding.EncodeToString(r) } + +// ParseSuffix converts a suffix string back into a Suffix. +func ParseSuffix(s string) (Suffix, error) { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return Suffix{}, err + } + + return Suffix{ + Bits: byte(b[0]), + Path: b[1:], + }, nil +} diff --git a/storage/suffix_test.go b/storage/suffix_test.go index 5477aa053f..d637488a4b 100644 --- a/storage/suffix_test.go +++ b/storage/suffix_test.go @@ -29,6 +29,33 @@ const ( // storage/cache when merkle no longer depends on storage.NodeID ) +func TestParseSuffix(t *testing.T) { + for _, tc := range []struct { + prefix []byte + leafIndex int64 + want []byte + }{ + {h2b(""), 1, h2b("0801")}, + {h2b("00"), 1, h2b("0801")}, + } { + nodeID := NewNodeIDFromPrefix(tc.prefix, logStrataDepth, tc.leafIndex, logStrataDepth, maxLogDepth) + _, sfx := nodeID.Split(len(tc.prefix), logStrataDepth) + sfxKey := sfx.String() + + sfxP, err := ParseSuffix(sfxKey) + if err != nil { + t.Errorf("ParseSuffix(%s): %v", sfxKey, err) + continue + } + if got, want := sfx.Bits, sfxP.Bits; got != want { + t.Errorf("ParseSuffix(%s).Bits: %v, want %v", sfxKey, got, want) + } + if got, want := sfx.Path, sfxP.Path; !bytes.Equal(got, want) { + t.Errorf("ParseSuffix(%s).Bits: %x, want %x", sfxKey, got, want) + } + } +} + // TestSuffixKeyEquals ensures that NodeID.Split produces the same output as makeSuffixKey for the Log's use cases. func TestSuffixKeyEquals(t *testing.T) { for _, tc := range []struct { From 80fb29466666514a7017cfa10a6e52a5a026708c Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 27 Jul 2017 20:16:30 +0100 Subject: [PATCH 07/15] Tests pass - proper indexes to HStar2 --- integration/maptest/map_test.go | 51 +++++++++++++++++---------------- storage/cache/subtree_cache.go | 10 ++++--- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/integration/maptest/map_test.go b/integration/maptest/map_test.go index 7c50e42bff..3b1487b33f 100644 --- a/integration/maptest/map_test.go +++ b/integration/maptest/map_test.go @@ -95,7 +95,6 @@ func verifyGetMapLeavesResponse(getResp *trillian.GetMapLeavesResponse, indexes leafHash, rootHash, proof, hasher); err != nil { return fmt.Errorf("VerifyMapInclusionProof(%x): %v", index, err) } - break } return nil } @@ -126,39 +125,42 @@ func TestInclusion(t *testing.T) { leaves []*trillian.MapLeaf }{ /* - { - desc: "maphasher single", - HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - }, - }, - { - desc: "maphasher multi", - HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, - }, + { + desc: "maphasher single", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, }, + }, { - desc: "CONIKS single", - HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + desc: "maphasher multi", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, leaves: []*trillian.MapLeaf{ {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, }, }, */ { - desc: "CONIKS multi", + desc: "CONIKS single", HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, + //{Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000180000000000000000000000000000000000000000000000000"), LeafValue: []byte("Z")}, }, }, + /* + { + desc: "CONIKS multi", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, + }, + }, + */ } { tree, hasher, err := newTreeWithHasher(ctx, env, tc.HashStrategy) if err != nil { @@ -211,10 +213,11 @@ func TestInclusionBatch(t *testing.T) { HashStrategy trillian.HashStrategy batchSize, numBatches int }{ + { desc: "maphasher batch", HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - batchSize: 64, numBatches: 32, + batchSize: 1, numBatches: 1, }, // TODO(gdbelvin): investigate batches of size > 150. // We are currently getting DB connection starvation: Too many connections. @@ -225,7 +228,7 @@ func TestInclusionBatch(t *testing.T) { } if err := RunMapBatchTest(ctx, env, tree, tc.batchSize, tc.numBatches); err != nil { - t.Errorf("%v: %v", tc.desc, err) + t.Errorf("BatchSize: %v, Batches: %v: %v", tc.batchSize, tc.numBatches, err) } } } diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index adce7dff2f..1cdc52b65d 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -343,7 +343,7 @@ func (s *SubtreeCache) SetNodeHash(id storage.NodeID, h []byte, getSubtree GetSu if err != nil { glog.Errorf("base64.DecodeString(%v): %v", sfxKey, err) } - glog.Infof("SetNodeHash(pfx: %s, sfx: %x): %x", prefixKey, b, h) + glog.Infof("SetNodeHash(pfx: %x, sfx: %x): %x", prefixKey, b, h) } return nil } @@ -408,10 +408,12 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop } // TODO(gdbelvin): test against subtree depth. if sfx.Bits%depthQuantum != 0 { - return fmt.Errorf("unexpected non-leaf suffix found: %x", k) + return fmt.Errorf("unexpected non-leaf suffix found: %x", sfx.Bits) } index := new(big.Int).SetBytes(sfx.Path) - index = index.Lsh(index, uint(hasher.BitLen()-len(index.Bytes()))) + + treeOffset := hasher.BitLen() - (len(st.Prefix)*8 + int(st.Depth)) + index = index.Lsh(index, uint(treeOffset)) leaves = append(leaves, merkle.HStar2LeafHash{ Index: index, LeafHash: v, @@ -420,7 +422,7 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop hs2 := merkle.NewHStar2(treeID, hasher) root, err := hs2.HStar2Nodes(st.Prefix, int(st.Depth), leaves, nil, func(depth int, index *big.Int, h []byte) error { - if depth == 0 { + if depth == len(st.Prefix)*8 { // no space for the root in the node cache return nil } From 3866314227d004c871b0f44296208a4fc7a8fe7a Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Fri, 28 Jul 2017 15:17:51 +0100 Subject: [PATCH 08/15] NewNodeIDFromPrefixSuffix --- integration/maptest/map_test.go | 51 +++++++++++++++----------------- storage/cache/subtree_cache.go | 5 +--- storage/types.go | 19 ++++++++++++ storage/types_test.go | 52 +++++++++++++++++++++------------ 4 files changed, 76 insertions(+), 51 deletions(-) diff --git a/integration/maptest/map_test.go b/integration/maptest/map_test.go index 3b1487b33f..b613cacf3e 100644 --- a/integration/maptest/map_test.go +++ b/integration/maptest/map_test.go @@ -124,43 +124,38 @@ func TestInclusion(t *testing.T) { HashStrategy trillian.HashStrategy leaves []*trillian.MapLeaf }{ - /* - { - desc: "maphasher single", - HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - }, + { + desc: "maphasher single", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, }, - { - desc: "maphasher multi", - HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, - }, + }, + { + desc: "maphasher multi", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, }, - */ + }, { - desc: "CONIKS single", + desc: "CONIKS across subtrees", HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, leaves: []*trillian.MapLeaf{ - //{Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, {Index: h2b("0000000000000180000000000000000000000000000000000000000000000000"), LeafValue: []byte("Z")}, }, }, - /* - { - desc: "CONIKS multi", - HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, - leaves: []*trillian.MapLeaf{ - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, - {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, - }, + { + desc: "CONIKS multi", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, }, - */ + }, } { tree, hasher, err := newTreeWithHasher(ctx, env, tc.HashStrategy) if err != nil { diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index 1cdc52b65d..5d97d9560e 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -410,12 +410,9 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop if sfx.Bits%depthQuantum != 0 { return fmt.Errorf("unexpected non-leaf suffix found: %x", sfx.Bits) } - index := new(big.Int).SetBytes(sfx.Path) - treeOffset := hasher.BitLen() - (len(st.Prefix)*8 + int(st.Depth)) - index = index.Lsh(index, uint(treeOffset)) leaves = append(leaves, merkle.HStar2LeafHash{ - Index: index, + Index: storage.NewNodeIDFromPrefixSuffix(st.Prefix, sfx, hasher.BitLen()).BigInt(), LeafHash: v, }) } diff --git a/storage/types.go b/storage/types.go index 38f20191e5..53c847fd74 100644 --- a/storage/types.go +++ b/storage/types.go @@ -166,6 +166,8 @@ func NewNodeIDFromBigInt(depth int, index *big.Int, totalDepth int) NodeID { copy(path[unusedHighBytes:], index.Bytes()) // TODO(gdbelvin): consider masking off insignificant bits past depth. + glog.V(5).Infof("NewNodeIDFromBigInt(%v, %x, %v): %v, %x", + depth, index.Bytes(), totalDepth, depth, path) return NodeID{ Path: path, @@ -173,6 +175,11 @@ func NewNodeIDFromBigInt(depth int, index *big.Int, totalDepth int) NodeID { } } +// BigInt returns the big.Int for this node. +func (n NodeID) BigInt() *big.Int { + return new(big.Int).SetBytes(n.Path) +} + // NewNodeIDWithPrefix creates a new NodeID of nodeIDLen bits with the prefixLen MSBs set to prefix. // NewNodeIDWithPrefix places the lower prefixLenBits of prefix in the most significant bits of path. // Path will have enough bytes to hold maxLenBits @@ -307,6 +314,18 @@ func (n *NodeID) Siblings() []NodeID { return r } +// NewNodeIDFromPrefixSuffix undoes Split() and returns the NodeID. +func NewNodeIDFromPrefixSuffix(prefix []byte, suffix Suffix, maxPathBits int) NodeID { + path := make([]byte, maxPathBits/8) + copy(path, prefix) + copy(path[len(prefix):], suffix.Path) + + return NodeID{ + Path: path, + PrefixLenBits: len(prefix)*8 + int(suffix.Bits), + } +} + // Split splits a NodeID into a prefix and a suffix at prefixSplit func (n *NodeID) Split(prefixBytes, suffixBits int) ([]byte, Suffix) { if n.PrefixLenBits == 0 { diff --git a/storage/types_test.go b/storage/types_test.go index df904d8c87..ea0d959ab2 100644 --- a/storage/types_test.go +++ b/storage/types_test.go @@ -59,27 +59,28 @@ func TestSplit(t *testing.T) { outPrefix []byte outSuffixBits int outSuffix []byte + unusedBytes int }{ - {h2b("1234567f"), 32, 3, 8, h2b("123456"), 8, h2b("7f")}, - {h2b("123456ff"), 29, 3, 8, h2b("123456"), 5, h2b("f8")}, - {h2b("123456ff"), 25, 3, 8, h2b("123456"), 1, h2b("80")}, - {h2b("12345678"), 16, 1, 8, h2b("12"), 8, h2b("34")}, - {h2b("12345678"), 9, 1, 8, h2b("12"), 1, h2b("00")}, - {h2b("12345678"), 8, 0, 8, h2b(""), 8, h2b("12")}, - {h2b("12345678"), 7, 0, 8, h2b(""), 7, h2b("12")}, - {h2b("12345678"), 0, 0, 8, h2b(""), 0, h2b("00")}, - {h2b("70"), 2, 0, 8, h2b(""), 2, h2b("40")}, - {h2b("70"), 3, 0, 8, h2b(""), 3, h2b("60")}, - {h2b("70"), 4, 0, 8, h2b(""), 4, h2b("70")}, - {h2b("70"), 5, 0, 8, h2b(""), 5, h2b("70")}, - {h2b("0003"), 16, 1, 8, h2b("00"), 8, h2b("03")}, - {h2b("0003"), 15, 1, 8, h2b("00"), 7, h2b("02")}, - {h2b("0001000000000000"), 16, 1, 8, h2b("00"), 8, h2b("01")}, - {h2b("0100000000000000"), 8, 0, 8, h2b(""), 8, h2b("01")}, + {h2b("1234567f"), 32, 3, 8, h2b("123456"), 8, h2b("7f"), 0}, + {h2b("123456ff"), 29, 3, 8, h2b("123456"), 5, h2b("f8"), 0}, + {h2b("123456ff"), 25, 3, 8, h2b("123456"), 1, h2b("80"), 0}, + {h2b("12345678"), 16, 1, 8, h2b("12"), 8, h2b("34"), 2}, + {h2b("12345678"), 9, 1, 8, h2b("12"), 1, h2b("00"), 2}, + {h2b("12345678"), 8, 0, 8, h2b(""), 8, h2b("12"), 3}, + {h2b("12345678"), 7, 0, 8, h2b(""), 7, h2b("12"), 3}, + {h2b("12345678"), 0, 0, 8, h2b(""), 0, h2b("00"), 3}, + {h2b("70"), 2, 0, 8, h2b(""), 2, h2b("40"), 0}, + {h2b("70"), 3, 0, 8, h2b(""), 3, h2b("60"), 0}, + {h2b("70"), 4, 0, 8, h2b(""), 4, h2b("70"), 0}, + {h2b("70"), 5, 0, 8, h2b(""), 5, h2b("70"), 0}, + {h2b("0003"), 16, 1, 8, h2b("00"), 8, h2b("03"), 0}, + {h2b("0003"), 15, 1, 8, h2b("00"), 7, h2b("02"), 0}, + {h2b("0001000000000000"), 16, 1, 8, h2b("00"), 8, h2b("01"), 6}, + {h2b("0100000000000000"), 8, 0, 8, h2b(""), 8, h2b("01"), 7}, // Map subtree scenarios - {h2b("0100000000000000"), 16, 0, 16, h2b(""), 16, h2b("0100")}, - {h2b("0100000000000000"), 32, 0, 32, h2b(""), 32, h2b("01000000")}, - {h2b("0000000000000000000000000000000000000000000000000000000000000001"), 256, 10, 176, h2b("00000000000000000000"), 176, h2b("00000000000000000000000000000000000000000001")}, + {h2b("0100000000000000"), 16, 0, 16, h2b(""), 16, h2b("0100"), 6}, + {h2b("0100000000000000"), 32, 0, 32, h2b(""), 32, h2b("01000000"), 4}, + {h2b("0000000000000000000000000000000000000000000000000000000000000001"), 256, 10, 176, h2b("00000000000000000000"), 176, h2b("00000000000000000000000000000000000000000001"), 0}, } { n := NewNodeIDFromHash(tc.inPath) n.PrefixLenBits = tc.inPathLenBits @@ -98,6 +99,19 @@ func TestSplit(t *testing.T) { if got, want := s.Path, tc.outSuffix; !bytes.Equal(got, want) { t.Errorf("%d, %x.Split(%v, %v).Path: %x, want %x", tc.inPathLenBits, tc.inPath, tc.splitBytes, tc.suffixBits, got, want) + continue + } + + newNode := NewNodeIDFromPrefixSuffix(p, s, len(tc.inPath)*8) + want := []byte{} + want = append(want, tc.outPrefix...) + want = append(want, tc.outSuffix...) + want = append(want, make([]byte, tc.unusedBytes)...) + if got, want := newNode.Path, want; !bytes.Equal(got, want) { + t.Errorf("NewNodeIDFromPrefix(%x, %v).Path: %x, want %x", p, s, got, want) + } + if got, want := newNode.PrefixLenBits, n.PrefixLenBits; got != want { + t.Errorf("NewNodeIDFromPrefix(%x, %v).PrefixLenBits: %x, want %x", p, s, got, want) } } } From 7477bfe3eeb340aa7d7e8d975a5d76cf49e0678f Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Fri, 28 Jul 2017 15:25:58 +0100 Subject: [PATCH 09/15] Remove NewNodeIDFromRelativeBigInt --- storage/types.go | 22 ---------------------- storage/types_test.go | 33 --------------------------------- 2 files changed, 55 deletions(-) diff --git a/storage/types.go b/storage/types.go index 53c847fd74..f7480adb87 100644 --- a/storage/types.go +++ b/storage/types.go @@ -130,28 +130,6 @@ func NewNodeIDFromPrefix(prefix []byte, depth int, index int64, subDepth, totalD } } -// NewNodeIDFromRelativeBigInt returns a NodeID given by a subtree and a subtree index. -// depth is the number of levels down from the top of the subtree -// subIndex is the path from the root of the subtree to the desired node, and continuing down to the bottom of the subtree. -// subIndex = horizontal index << height. -func NewNodeIDFromRelativeBigInt(prefix []byte, subtreeDepth, depth int, subIndex *big.Int, totalDepth int) NodeID { - // Put prefix in the MSB bits of path. - path := make([]byte, totalDepth/8) - copy(path, prefix) - - // Copy subIndex into subPath, right justified. - subPath := path[len(prefix) : len(prefix)+subtreeDepth/8] - unusedSubBytes := len(subPath) - len(subIndex.Bytes()) - copy(subPath[unusedSubBytes:], subIndex.Bytes()) - - glog.V(5).Infof("NewNodeIDFromRelativeBigInt({%x, %v}, %v, %x, %v): %v, %x", - prefix, subtreeDepth, depth, subIndex.Bytes(), totalDepth, len(prefix)*8+depth, path) - return NodeID{ - Path: path, - PrefixLenBits: len(prefix)*8 + depth, - } -} - // NewNodeIDFromBigInt returns a NodeID of a big.Int with no prefix. // index contains the path's least significant bits. // depth indicates the number of bits from the most significant bit to treat as part of the path. diff --git a/storage/types_test.go b/storage/types_test.go index ea0d959ab2..077a5c9176 100644 --- a/storage/types_test.go +++ b/storage/types_test.go @@ -116,39 +116,6 @@ func TestSplit(t *testing.T) { } } -func TestNewNodeIDFromRelativeBigInt(t *testing.T) { - for _, tc := range []struct { - prefix []byte - depth int - index int64 - subDepth int - totalDepth int - wantPath []byte - wantDepth int - }{ - {prefix: h2b(""), depth: 8, index: 0, subDepth: 8, totalDepth: 64, wantPath: h2b("0000000000000000"), wantDepth: 8}, - {prefix: h2b(""), depth: 8, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("0100000000000000"), wantDepth: 8}, - {prefix: h2b("00"), depth: 7, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("0001000000000000"), wantDepth: 15}, - {prefix: h2b("00"), depth: 8, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("0001000000000000"), wantDepth: 16}, - {prefix: h2b("00"), depth: 16, index: 257, subDepth: 16, totalDepth: 64, wantPath: h2b("0001010000000000"), wantDepth: 24}, - {prefix: h2b("12345678"), depth: 8, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("1234567801000000"), wantDepth: 40}, - - {prefix: h2b("00"), subDepth: 248, depth: 247, index: 1, totalDepth: 256, wantPath: h2b("0000000000000000000000000000000000000000000000000000000000000001"), wantDepth: 255}, - {prefix: h2b("00000000000000000000"), subDepth: 176, depth: 176, index: 1, totalDepth: 256, wantPath: h2b("0000000000000000000000000000000000000000000000000000000000000001"), wantDepth: 256}, - } { - i := big.NewInt(tc.index) - n := NewNodeIDFromRelativeBigInt(tc.prefix, tc.subDepth, tc.depth, i, tc.totalDepth) - if got, want := n.Path, tc.wantPath; !bytes.Equal(got, want) { - t.Errorf("NewNodeIDFromRelativeBigInt(%x, %v, %v, %v, %v).Path: %x, want %x", - tc.prefix, tc.depth, tc.index, tc.subDepth, tc.totalDepth, got, want) - } - if got, want := n.PrefixLenBits, tc.wantDepth; got != want { - t.Errorf("NewNodeIDFromRelativeBigInt(%x, %v, %v, %v, %v).Depth: %v, want %v", - tc.prefix, tc.depth, tc.index, tc.subDepth, tc.totalDepth, got, want) - } - } -} - func TestNewNodeIDFromPrefix(t *testing.T) { for _, tc := range []struct { prefix []byte From 5b32b7278f1f470f17b2e891dc4c6be8d4e14c60 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Fri, 28 Jul 2017 16:19:02 +0100 Subject: [PATCH 10/15] Use NewFromPrefixSuffix --- integration/maptest/map_test.go | 2 +- merkle/hstar2.go | 30 ++++++++++-------------------- merkle/hstar2_test.go | 26 +++----------------------- merkle/sparse_merkle_tree.go | 10 ++++------ merkle/sparse_merkle_tree_test.go | 12 ++++-------- 5 files changed, 22 insertions(+), 58 deletions(-) diff --git a/integration/maptest/map_test.go b/integration/maptest/map_test.go index b613cacf3e..688e5a52d6 100644 --- a/integration/maptest/map_test.go +++ b/integration/maptest/map_test.go @@ -212,7 +212,7 @@ func TestInclusionBatch(t *testing.T) { { desc: "maphasher batch", HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, - batchSize: 1, numBatches: 1, + batchSize: 64, numBatches: 32, }, // TODO(gdbelvin): investigate batches of size > 150. // We are currently getting DB connection starvation: Too many connections. diff --git a/merkle/hstar2.go b/merkle/hstar2.go index 728445e910..96ec2829f0 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -22,13 +22,14 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/storage" ) var ( - // ErrNegativeTreeLevelOffset indicates a negative level was specified. - ErrNegativeTreeLevelOffset = errors.New("treeLevelOffset cannot be negative") - smtOne = big.NewInt(1) - smtZero = big.NewInt(0) + // ErrSubtreeOverrun indicates that a subtree exceeds the maximum tree depth. + ErrSubtreeOverrun = errors.New("subtree with prefix exceeds maximum tree size") + smtOne = big.NewInt(1) + smtZero = big.NewInt(0) ) // HStar2LeafHash represents a leaf for the HStar2 sparse Merkle tree @@ -85,11 +86,10 @@ func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2Lea depth := len(prefix) * 8 totalDepth := depth + subtreeDepth if totalDepth > s.hasher.BitLen() { - return nil, ErrNegativeTreeLevelOffset + return nil, ErrSubtreeOverrun } sort.Sort(ByIndex{values}) - offset := new(big.Int).SetBytes(prefix) - offset = offset.Lsh(offset, uint(s.hasher.BitLen()-depth)) // shift prefix into place. + offset := storage.NewNodeIDFromPrefixSuffix(prefix, storage.Suffix{}, s.hasher.BitLen()).BigInt() return s.hStar2b(depth, totalDepth, values, offset, get, set) } @@ -140,10 +140,10 @@ func (s *HStar2) get(index *big.Int, depth int, getter SparseGetNodeFunc) ([]byt return h, nil } } + // TODO(gdbelvin): Hashers should accept depth as their main argument. height := s.hasher.BitLen() - depth - indexBytes := PaddedBytes(index, s.hasher.Size()) - indexBytes = MaskIndex(indexBytes, depth) - return s.hasher.HashEmpty(s.treeID, indexBytes, height), nil + nodeID := storage.NewNodeIDFromBigInt(index.BitLen(), index, s.hasher.BitLen()) + return s.hasher.HashEmpty(s.treeID, nodeID.Path, height), nil } // set attempts to use setter if it not nil. @@ -170,13 +170,3 @@ type ByIndex struct{ Leaves } // Less returns true if i.Index < j.Index func (s ByIndex) Less(i, j int) bool { return s.Leaves[i].Index.Cmp(s.Leaves[j].Index) < 0 } - -// PaddedBytes takes a big.Int and returns it's value, left padded with zeros. -// e.g. 1 -> 0000000000000000000000000000000000000001 -func PaddedBytes(i *big.Int, size int) []byte { - b := i.Bytes() - ret := make([]byte, size) - padBytes := len(ret) - len(b) - copy(ret[padBytes:], b) - return ret -} diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index 1ce2ed7ccd..e94da2d082 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/merkle/maphasher" + "github.com/google/trillian/storage" "github.com/google/trillian/testonly" ) @@ -145,10 +146,8 @@ func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HS t.Fatalf("Failed to calculate root %v", err) } - index := new(big.Int).SetBytes(prefix) - index = index.Lsh(index, uint(hasher.BitLen()-prefixSize)) ret = append(ret, HStar2LeafHash{ - Index: index, + Index: storage.NewNodeIDFromPrefixSuffix(prefix, storage.Suffix{}, hasher.BitLen()).BigInt(), LeafHash: root, }) } @@ -178,9 +177,7 @@ func TestHStar2OffsetRootKAT(t *testing.T) { if got, want := root, x.root; !bytes.Equal(got, want) { t.Errorf("HStar2Nodes(i: %v, size:%v): %x, want: %x", i, size, got, want) } - break } - break } } @@ -188,24 +185,7 @@ func TestHStar2NegativeTreeLevelOffset(t *testing.T) { s := NewHStar2(treeID, maphasher.Default) _, err := s.HStar2Nodes(make([]byte, 31), 9, []HStar2LeafHash{}, nil, nil) - if got, want := err, ErrNegativeTreeLevelOffset; got != want { + if got, want := err, ErrSubtreeOverrun; got != want { t.Fatalf("Hstar2Nodes(): %v, want %v", got, want) } } - -func TestPaddedBytes(t *testing.T) { - size := 160 / 8 - for _, tc := range []struct { - i *big.Int - want []byte - }{ - {i: big.NewInt(0), want: h2b("0000000000000000000000000000000000000000")}, - {i: big.NewInt(1), want: h2b("0000000000000000000000000000000000000001")}, - {i: new(big.Int).SetBytes(h2b("00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")), want: h2b("00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")}, - {i: new(big.Int).SetBytes(h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")), want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")}, - } { - if got, want := PaddedBytes(tc.i, size), tc.want; !bytes.Equal(got, want) { - t.Errorf("PaddedBytes(%d): %x, want %x", tc.i, got, want) - } - } -} diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index 6dea2f15a6..0d6cf9f5ac 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -211,15 +211,15 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { s.root <- rootHashOrError{hash: nil, err: err} return } - index := new(big.Int).SetBytes(ih.index) - index = index.Lsh(index, uint(s.hasher.BitLen()-len(ih.index)*8)) + nodeID := storage.NewNodeIDFromPrefixSuffix(ih.index, storage.Suffix{}, s.hasher.BitLen()) + leaves = append(leaves, HStar2LeafHash{ - Index: index, + Index: nodeID.BigInt(), LeafHash: ih.hash, }) nodesToStore = append(nodesToStore, storage.Node{ - NodeID: storage.NewNodeIDFromHash(ih.index), + NodeID: nodeID, Hash: ih.hash, NodeRevision: s.treeRevision, }) @@ -392,10 +392,8 @@ func (s SparseMerkleTreeReader) InclusionProof(ctx context.Context, rev int64, i } nodeMap := make(map[string]*storage.Node) - glog.Infof("Got Nodes: ") for _, n := range nodes { n := n // need this or we'll end up with the same node hash repeated in the map - glog.Infof(" %x, %d: %x", n.NodeID.Path, len(n.NodeID.String()), n.Hash) nodeMap[n.NodeID.String()] = &n } diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index fa4801ab56..8ac9418b6c 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -437,14 +437,10 @@ func testSparseTreeFetches(ctx context.Context, t *testing.T, vec sparseTestVect id := sibs[j].String() pathNode := nodeID.String()[:len(id)] if _, ok := reads[pathNode]; ok { - // we're modifying both children of a - // node because two keys are - // intersecting, since both will be - // recalculated neither will be read - // from storage so we remove the - // previously set expectation for this - // node's sibling, and skip adding one - // for this node: + // we're modifying both children of a node because two keys are + // intersecting, since both will be recalculated neither will be read + // from storage so we remove the previously set expectation for this + // node's sibling, and skip adding one for this node: delete(reads, pathNode) continue } From 0248a224bd9c8dcc287268e2827061a35100aa32 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Mon, 31 Jul 2017 10:42:34 +0100 Subject: [PATCH 11/15] Increase test timeout to 10 min --- scripts/presubmit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/presubmit.sh b/scripts/presubmit.sh index 589d3132b7..c1068c247f 100755 --- a/scripts/presubmit.sh +++ b/scripts/presubmit.sh @@ -93,7 +93,7 @@ main() { go build ${go_dirs} echo 'running go test' - go test -cover -timeout=5m ${goflags} ${go_dirs} + go test -cover -timeout=10m ${goflags} ${go_dirs} fi if [[ "${run_linters}" -eq 1 ]]; then From e7cfd3715a86a22bb50ba1f6c1c7436457c35846 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Mon, 31 Jul 2017 11:06:04 +0100 Subject: [PATCH 12/15] move NodeID to node package --- node/nodeid.go | 311 +++++++++++++++++++ storage/types_test.go => node/nodeid_test.go | 0 {storage => node}/suffix.go | 2 +- {storage => node}/suffix_test.go | 2 +- storage/types.go | 294 +----------------- 5 files changed, 315 insertions(+), 294 deletions(-) create mode 100644 node/nodeid.go rename storage/types_test.go => node/nodeid_test.go (100%) rename {storage => node}/suffix.go (98%) rename {storage => node}/suffix_test.go (99%) diff --git a/node/nodeid.go b/node/nodeid.go new file mode 100644 index 0000000000..38d54870bb --- /dev/null +++ b/node/nodeid.go @@ -0,0 +1,311 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package node + +import ( + "bytes" + "encoding/binary" + "fmt" + "math/big" + + "github.com/golang/glog" +) + +// NodeID uniquely identifies a Node within a versioned MerkleTree. +type NodeID struct { + // path is effectively a BigEndian bit set, with path[0] being the MSB + // (identifying the root child), and successive bits identifying the lower + // level children down to the leaf. + Path []byte + // PrefixLenBits is the number of MSB in Path which are considered part of + // this NodeID. + // + // e.g. if Path contains two bytes, and PrefixLenBits is 9, then the 8 bits + // in Path[0] are included, along with the lowest bit of Path[1] + PrefixLenBits int +} + +// PathLenBits returns 8 * len(path). +func (n NodeID) PathLenBits() int { + return len(n.Path) * 8 +} + +// bytesForBits returns the number of bytes required to store numBits bits. +func bytesForBits(numBits int) int { + return (numBits + 7) >> 3 +} + +// NewNodeIDFromHash creates a new NodeID for the given Hash. +func NewNodeIDFromHash(h []byte) NodeID { + return NodeID{ + Path: h, + PrefixLenBits: len(h) * 8, + } +} + +// NewEmptyNodeID creates a new zero-length NodeID with sufficient underlying +// capacity to store a maximum of maxLenBits. +func NewEmptyNodeID(maxLenBits int) NodeID { + if got, want := maxLenBits%8, 0; got != want { + panic(fmt.Sprintf("storeage: NewEmptyNodeID() maxLenBits mod 8: %v, want %v", got, want)) + } + return NodeID{ + Path: make([]byte, maxLenBits/8), + PrefixLenBits: 0, + } +} + +// NewNodeIDFromPrefix returns a nodeID for a particular node within a subtree. +// Prefix is the prefix of the subtree. +// depth is the depth of index from the root of the subtree. +// index is the horizontal location of the subtree leaf. +// subDepth is the total number of levels in the subtree. +// totalDepth is the number of levels in the whole tree. +func NewNodeIDFromPrefix(prefix []byte, depth int, index int64, subDepth, totalDepth int) NodeID { + if got, want := totalDepth%8, 0; got != want || got < want { + panic(fmt.Sprintf("storage NewNodeFromPrefix(): totalDepth mod 8: %v, want %v", got, want)) + } + if got, want := subDepth%8, 0; got != want || got < want { + panic(fmt.Sprintf("storage NewNodeFromPrefix(): subDepth mod 8: %v, want %v", got, want)) + } + if got, want := depth, 0; got < want { + panic(fmt.Sprintf("storage NewNodeFromPrefix(): depth: %v, want >= %v", got, want)) + } + + // Put prefix in the MSB bits of path. + path := make([]byte, totalDepth/8) + copy(path, prefix) + + // Convert index into absolute coordinates for subtree. + height := subDepth - depth + subIndex := index << uint(height) // index is the horizontal index at the given height. + + // Copy subDepth/8 bytes of subIndex into path. + subPath := new(bytes.Buffer) + binary.Write(subPath, binary.BigEndian, uint64(subIndex)) + unusedHighBytes := 64/8 - subDepth/8 + copy(path[len(prefix):], subPath.Bytes()[unusedHighBytes:]) + + return NodeID{ + Path: path, + PrefixLenBits: len(prefix)*8 + depth, + } +} + +// NewNodeIDFromBigInt returns a NodeID of a big.Int with no prefix. +// index contains the path's least significant bits. +// depth indicates the number of bits from the most significant bit to treat as part of the path. +func NewNodeIDFromBigInt(depth int, index *big.Int, totalDepth int) NodeID { + if got, want := totalDepth%8, 0; got != want || got < want { + panic(fmt.Sprintf("storage NewNodeFromBitInt(): totalDepth mod 8: %v, want %v", got, want)) + } + + // Put index in the LSB bits of path. + path := make([]byte, totalDepth/8) + unusedHighBytes := len(path) - len(index.Bytes()) + copy(path[unusedHighBytes:], index.Bytes()) + + // TODO(gdbelvin): consider masking off insignificant bits past depth. + glog.V(5).Infof("NewNodeIDFromBigInt(%v, %x, %v): %v, %x", + depth, index.Bytes(), totalDepth, depth, path) + + return NodeID{ + Path: path, + PrefixLenBits: depth, + } +} + +// BigInt returns the big.Int for this node. +func (n NodeID) BigInt() *big.Int { + return new(big.Int).SetBytes(n.Path) +} + +// NewNodeIDWithPrefix creates a new NodeID of nodeIDLen bits with the prefixLen MSBs set to prefix. +// NewNodeIDWithPrefix places the lower prefixLenBits of prefix in the most significant bits of path. +// Path will have enough bytes to hold maxLenBits +// +func NewNodeIDWithPrefix(prefix uint64, prefixLenBits, nodeIDLenBits, maxLenBits int) NodeID { + if got, want := nodeIDLenBits%8, 0; got != want { + panic(fmt.Sprintf("nodeIDLenBits mod 8: %v, want %v", got, want)) + } + maxLenBytes := bytesForBits(maxLenBits) + p := NodeID{ + Path: make([]byte, maxLenBytes), + PrefixLenBits: nodeIDLenBits, + } + + bit := maxLenBits - prefixLenBits + for i := 0; i < prefixLenBits; i++ { + if prefix&1 != 0 { + p.SetBit(bit, 1) + } + bit++ + prefix >>= 1 + } + return p +} + +func bitLen(x int64) int { + r := 0 + for x > 0 { + r++ + x >>= 1 + } + return r +} + +// NewNodeIDForTreeCoords creates a new NodeID for a Tree node with a specified depth and +// index. +// This method is used exclusively by the Log, and, since the Log model grows upwards from the +// leaves, we modify the provided coords accordingly. +// +// depth is the Merkle tree level: 0 = leaves, and increases upwards towards the root. +// +// index is the horizontal index into the tree at level depth, so the returned +// NodeID will be zero padded on the right by depth places. +func NewNodeIDForTreeCoords(depth int64, index int64, maxPathBits int) (NodeID, error) { + bl := bitLen(index) + if index < 0 || depth < 0 || + bl > int(maxPathBits-int(depth)) || + maxPathBits%8 != 0 { + return NodeID{}, fmt.Errorf("depth/index combination out of range: depth=%d index=%d maxPathBits=%v", depth, index, maxPathBits) + } + // This node is effectively a prefix of the subtree underneath (for non-leaf + // depths), so we shift the index accordingly. + uidx := uint64(index) << uint(depth) + r := NewEmptyNodeID(maxPathBits) + for i := len(r.Path) - 1; uidx > 0 && i >= 0; i-- { + r.Path[i] = byte(uidx & 0xff) + uidx >>= 8 + } + // In the storage model nodes closer to the leaves have longer nodeIDs, so + // we "reverse" depth here: + r.PrefixLenBits = int(maxPathBits - int(depth)) + return r, nil +} + +// SetBit sets the ith bit to true if b is non-zero, and false otherwise. +func (n *NodeID) SetBit(i int, b uint) { + // TODO(al): investigate whether having lookup tables for these might be + // faster. + bIndex := (n.PathLenBits() - i - 1) / 8 + if b == 0 { + n.Path[bIndex] &= ^(1 << uint(i%8)) + } else { + n.Path[bIndex] |= (1 << uint(i%8)) + } +} + +// Bit returns 1 if the ith bit is true, and false otherwise. +func (n *NodeID) Bit(i int) uint { + if got, want := i, n.PathLenBits()-1; got > want { + panic(fmt.Sprintf("storage: Bit(%v) > (PathLenBits() -1): %v", got, want)) + } + bIndex := (n.PathLenBits() - i - 1) / 8 + return uint((n.Path[bIndex] >> uint(i%8)) & 0x01) +} + +// String returns a string representation of the binary value of the NodeID. +// The left-most bit is the MSB (i.e. nearer the root of the tree). +func (n *NodeID) String() string { + var r bytes.Buffer + limit := n.PathLenBits() - n.PrefixLenBits + for i := n.PathLenBits() - 1; i >= limit; i-- { + r.WriteRune(rune('0' + n.Bit(i))) + } + return r.String() +} + +// CoordString returns a string representation assuming that the NodeID represents a +// tree coordinate. Using this on a NodeID for a sparse Merkle tree will give incorrect +// results. Intended for debugging purposes, the format could change. +func (n *NodeID) CoordString() string { + d := uint64(n.PathLenBits() - n.PrefixLenBits) + i := uint64(0) + for _, p := range n.Path { + i = (i << uint64(8)) + uint64(p) + } + + return fmt.Sprintf("[d:%d, i:%d]", d, i>>d) +} + +// Siblings returns the siblings of the given node. +func (n *NodeID) Siblings() []NodeID { + l := n.PrefixLenBits + r := make([]NodeID, l) + // Index of the bit to twiddle: + bi := n.PathLenBits() - n.PrefixLenBits + for i := range r { + r[i].PrefixLenBits = l - i + r[i].Path = make([]byte, len(n.Path)) + copy(r[i].Path, n.Path) + r[i].SetBit(bi, n.Bit(bi)^1) + for j := bi - 1; j >= 0; j-- { + r[i].SetBit(j, 0) + } + bi++ + } + if glog.V(5) { + glog.Infof("[%d, %x].Siblings():", n.PrefixLenBits, n.Path) + for _, s := range r { + glog.Infof(" %x", s.Path) + } + } + return r +} + +// NewNodeIDFromPrefixSuffix undoes Split() and returns the NodeID. +func NewNodeIDFromPrefixSuffix(prefix []byte, suffix Suffix, maxPathBits int) NodeID { + path := make([]byte, maxPathBits/8) + copy(path, prefix) + copy(path[len(prefix):], suffix.Path) + + return NodeID{ + Path: path, + PrefixLenBits: len(prefix)*8 + int(suffix.Bits), + } +} + +// Split splits a NodeID into a prefix and a suffix at prefixSplit +func (n *NodeID) Split(prefixBytes, suffixBits int) ([]byte, Suffix) { + if n.PrefixLenBits == 0 { + return []byte{}, Suffix{Bits: 0, Path: []byte{0}} + } + a := make([]byte, len(n.Path)) + copy(a, n.Path) + + bits := n.PrefixLenBits - prefixBytes*8 + if bits > suffixBits { + panic(fmt.Sprintf("storage Split: %x(n.PrefixLenBits: %v - prefixBytes: %v *8) > %v", n.Path, n.PrefixLenBits, prefixBytes, suffixBits)) + } + if bits == 0 { + panic(fmt.Sprintf("storage Split: %x(n.PrefixLenBits: %v - prefixBytes: %v *8) == 0", n.Path, n.PrefixLenBits, prefixBytes)) + } + suffixBytes := bytesForBits(bits) + sfx := Suffix{ + Bits: byte(bits), + Path: a[prefixBytes : prefixBytes+suffixBytes], + } + maskIndex := (bits - 1) / 8 + maskLowBits := (sfx.Bits-1)%8 + 1 + sfx.Path[maskIndex] &= ((0x01 << maskLowBits) - 1) << uint(8-maskLowBits) + + return a[:prefixBytes], sfx +} + +// Equivalent return true iff the other represents the same path prefix as this NodeID. +func (n *NodeID) Equivalent(other NodeID) bool { + return n.String() == other.String() +} diff --git a/storage/types_test.go b/node/nodeid_test.go similarity index 100% rename from storage/types_test.go rename to node/nodeid_test.go diff --git a/storage/suffix.go b/node/suffix.go similarity index 98% rename from storage/suffix.go rename to node/suffix.go index ebf4a7d21b..1af6961aa7 100644 --- a/storage/suffix.go +++ b/node/suffix.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package node import ( "encoding/base64" diff --git a/storage/suffix_test.go b/node/suffix_test.go similarity index 99% rename from storage/suffix_test.go rename to node/suffix_test.go index d637488a4b..cd1612ef7c 100644 --- a/storage/suffix_test.go +++ b/node/suffix_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package node import ( "bytes" diff --git a/storage/types.go b/storage/types.go index f7480adb87..04a6234f04 100644 --- a/storage/types.go +++ b/storage/types.go @@ -15,12 +15,9 @@ package storage import ( - "bytes" - "encoding/binary" "fmt" - "math/big" - "github.com/golang/glog" + "github.com/google/trillian/node" "github.com/google/trillian/storage/storagepb" ) @@ -44,298 +41,11 @@ func (s Error) Error() string { // Node represents a single node in a Merkle tree. type Node struct { - NodeID NodeID + NodeID node.NodeID Hash []byte NodeRevision int64 } -// NodeID uniquely identifies a Node within a versioned MerkleTree. -type NodeID struct { - // path is effectively a BigEndian bit set, with path[0] being the MSB - // (identifying the root child), and successive bits identifying the lower - // level children down to the leaf. - Path []byte - // PrefixLenBits is the number of MSB in Path which are considered part of - // this NodeID. - // - // e.g. if Path contains two bytes, and PrefixLenBits is 9, then the 8 bits - // in Path[0] are included, along with the lowest bit of Path[1] - PrefixLenBits int -} - -// PathLenBits returns 8 * len(path). -func (n NodeID) PathLenBits() int { - return len(n.Path) * 8 -} - -// bytesForBits returns the number of bytes required to store numBits bits. -func bytesForBits(numBits int) int { - return (numBits + 7) >> 3 -} - -// NewNodeIDFromHash creates a new NodeID for the given Hash. -func NewNodeIDFromHash(h []byte) NodeID { - return NodeID{ - Path: h, - PrefixLenBits: len(h) * 8, - } -} - -// NewEmptyNodeID creates a new zero-length NodeID with sufficient underlying -// capacity to store a maximum of maxLenBits. -func NewEmptyNodeID(maxLenBits int) NodeID { - if got, want := maxLenBits%8, 0; got != want { - panic(fmt.Sprintf("storeage: NewEmptyNodeID() maxLenBits mod 8: %v, want %v", got, want)) - } - return NodeID{ - Path: make([]byte, maxLenBits/8), - PrefixLenBits: 0, - } -} - -// NewNodeIDFromPrefix returns a nodeID for a particular node within a subtree. -// Prefix is the prefix of the subtree. -// depth is the depth of index from the root of the subtree. -// index is the horizontal location of the subtree leaf. -// subDepth is the total number of levels in the subtree. -// totalDepth is the number of levels in the whole tree. -func NewNodeIDFromPrefix(prefix []byte, depth int, index int64, subDepth, totalDepth int) NodeID { - if got, want := totalDepth%8, 0; got != want || got < want { - panic(fmt.Sprintf("storage NewNodeFromPrefix(): totalDepth mod 8: %v, want %v", got, want)) - } - if got, want := subDepth%8, 0; got != want || got < want { - panic(fmt.Sprintf("storage NewNodeFromPrefix(): subDepth mod 8: %v, want %v", got, want)) - } - if got, want := depth, 0; got < want { - panic(fmt.Sprintf("storage NewNodeFromPrefix(): depth: %v, want >= %v", got, want)) - } - - // Put prefix in the MSB bits of path. - path := make([]byte, totalDepth/8) - copy(path, prefix) - - // Convert index into absolute coordinates for subtree. - height := subDepth - depth - subIndex := index << uint(height) // index is the horizontal index at the given height. - - // Copy subDepth/8 bytes of subIndex into path. - subPath := new(bytes.Buffer) - binary.Write(subPath, binary.BigEndian, uint64(subIndex)) - unusedHighBytes := 64/8 - subDepth/8 - copy(path[len(prefix):], subPath.Bytes()[unusedHighBytes:]) - - return NodeID{ - Path: path, - PrefixLenBits: len(prefix)*8 + depth, - } -} - -// NewNodeIDFromBigInt returns a NodeID of a big.Int with no prefix. -// index contains the path's least significant bits. -// depth indicates the number of bits from the most significant bit to treat as part of the path. -func NewNodeIDFromBigInt(depth int, index *big.Int, totalDepth int) NodeID { - if got, want := totalDepth%8, 0; got != want || got < want { - panic(fmt.Sprintf("storage NewNodeFromBitInt(): totalDepth mod 8: %v, want %v", got, want)) - } - - // Put index in the LSB bits of path. - path := make([]byte, totalDepth/8) - unusedHighBytes := len(path) - len(index.Bytes()) - copy(path[unusedHighBytes:], index.Bytes()) - - // TODO(gdbelvin): consider masking off insignificant bits past depth. - glog.V(5).Infof("NewNodeIDFromBigInt(%v, %x, %v): %v, %x", - depth, index.Bytes(), totalDepth, depth, path) - - return NodeID{ - Path: path, - PrefixLenBits: depth, - } -} - -// BigInt returns the big.Int for this node. -func (n NodeID) BigInt() *big.Int { - return new(big.Int).SetBytes(n.Path) -} - -// NewNodeIDWithPrefix creates a new NodeID of nodeIDLen bits with the prefixLen MSBs set to prefix. -// NewNodeIDWithPrefix places the lower prefixLenBits of prefix in the most significant bits of path. -// Path will have enough bytes to hold maxLenBits -// -func NewNodeIDWithPrefix(prefix uint64, prefixLenBits, nodeIDLenBits, maxLenBits int) NodeID { - if got, want := nodeIDLenBits%8, 0; got != want { - panic(fmt.Sprintf("nodeIDLenBits mod 8: %v, want %v", got, want)) - } - maxLenBytes := bytesForBits(maxLenBits) - p := NodeID{ - Path: make([]byte, maxLenBytes), - PrefixLenBits: nodeIDLenBits, - } - - bit := maxLenBits - prefixLenBits - for i := 0; i < prefixLenBits; i++ { - if prefix&1 != 0 { - p.SetBit(bit, 1) - } - bit++ - prefix >>= 1 - } - return p -} - -func bitLen(x int64) int { - r := 0 - for x > 0 { - r++ - x >>= 1 - } - return r -} - -// NewNodeIDForTreeCoords creates a new NodeID for a Tree node with a specified depth and -// index. -// This method is used exclusively by the Log, and, since the Log model grows upwards from the -// leaves, we modify the provided coords accordingly. -// -// depth is the Merkle tree level: 0 = leaves, and increases upwards towards the root. -// -// index is the horizontal index into the tree at level depth, so the returned -// NodeID will be zero padded on the right by depth places. -func NewNodeIDForTreeCoords(depth int64, index int64, maxPathBits int) (NodeID, error) { - bl := bitLen(index) - if index < 0 || depth < 0 || - bl > int(maxPathBits-int(depth)) || - maxPathBits%8 != 0 { - return NodeID{}, fmt.Errorf("depth/index combination out of range: depth=%d index=%d maxPathBits=%v", depth, index, maxPathBits) - } - // This node is effectively a prefix of the subtree underneath (for non-leaf - // depths), so we shift the index accordingly. - uidx := uint64(index) << uint(depth) - r := NewEmptyNodeID(maxPathBits) - for i := len(r.Path) - 1; uidx > 0 && i >= 0; i-- { - r.Path[i] = byte(uidx & 0xff) - uidx >>= 8 - } - // In the storage model nodes closer to the leaves have longer nodeIDs, so - // we "reverse" depth here: - r.PrefixLenBits = int(maxPathBits - int(depth)) - return r, nil -} - -// SetBit sets the ith bit to true if b is non-zero, and false otherwise. -func (n *NodeID) SetBit(i int, b uint) { - // TODO(al): investigate whether having lookup tables for these might be - // faster. - bIndex := (n.PathLenBits() - i - 1) / 8 - if b == 0 { - n.Path[bIndex] &= ^(1 << uint(i%8)) - } else { - n.Path[bIndex] |= (1 << uint(i%8)) - } -} - -// Bit returns 1 if the ith bit is true, and false otherwise. -func (n *NodeID) Bit(i int) uint { - if got, want := i, n.PathLenBits()-1; got > want { - panic(fmt.Sprintf("storage: Bit(%v) > (PathLenBits() -1): %v", got, want)) - } - bIndex := (n.PathLenBits() - i - 1) / 8 - return uint((n.Path[bIndex] >> uint(i%8)) & 0x01) -} - -// String returns a string representation of the binary value of the NodeID. -// The left-most bit is the MSB (i.e. nearer the root of the tree). -func (n *NodeID) String() string { - var r bytes.Buffer - limit := n.PathLenBits() - n.PrefixLenBits - for i := n.PathLenBits() - 1; i >= limit; i-- { - r.WriteRune(rune('0' + n.Bit(i))) - } - return r.String() -} - -// CoordString returns a string representation assuming that the NodeID represents a -// tree coordinate. Using this on a NodeID for a sparse Merkle tree will give incorrect -// results. Intended for debugging purposes, the format could change. -func (n *NodeID) CoordString() string { - d := uint64(n.PathLenBits() - n.PrefixLenBits) - i := uint64(0) - for _, p := range n.Path { - i = (i << uint64(8)) + uint64(p) - } - - return fmt.Sprintf("[d:%d, i:%d]", d, i>>d) -} - -// Siblings returns the siblings of the given node. -func (n *NodeID) Siblings() []NodeID { - l := n.PrefixLenBits - r := make([]NodeID, l) - // Index of the bit to twiddle: - bi := n.PathLenBits() - n.PrefixLenBits - for i := range r { - r[i].PrefixLenBits = l - i - r[i].Path = make([]byte, len(n.Path)) - copy(r[i].Path, n.Path) - r[i].SetBit(bi, n.Bit(bi)^1) - for j := bi - 1; j >= 0; j-- { - r[i].SetBit(j, 0) - } - bi++ - } - if glog.V(5) { - glog.Infof("[%d, %x].Siblings():", n.PrefixLenBits, n.Path) - for _, s := range r { - glog.Infof(" %x", s.Path) - } - } - return r -} - -// NewNodeIDFromPrefixSuffix undoes Split() and returns the NodeID. -func NewNodeIDFromPrefixSuffix(prefix []byte, suffix Suffix, maxPathBits int) NodeID { - path := make([]byte, maxPathBits/8) - copy(path, prefix) - copy(path[len(prefix):], suffix.Path) - - return NodeID{ - Path: path, - PrefixLenBits: len(prefix)*8 + int(suffix.Bits), - } -} - -// Split splits a NodeID into a prefix and a suffix at prefixSplit -func (n *NodeID) Split(prefixBytes, suffixBits int) ([]byte, Suffix) { - if n.PrefixLenBits == 0 { - return []byte{}, Suffix{Bits: 0, Path: []byte{0}} - } - a := make([]byte, len(n.Path)) - copy(a, n.Path) - - bits := n.PrefixLenBits - prefixBytes*8 - if bits > suffixBits { - panic(fmt.Sprintf("storage Split: %x(n.PrefixLenBits: %v - prefixBytes: %v *8) > %v", n.Path, n.PrefixLenBits, prefixBytes, suffixBits)) - } - if bits == 0 { - panic(fmt.Sprintf("storage Split: %x(n.PrefixLenBits: %v - prefixBytes: %v *8) == 0", n.Path, n.PrefixLenBits, prefixBytes)) - } - suffixBytes := bytesForBits(bits) - sfx := Suffix{ - Bits: byte(bits), - Path: a[prefixBytes : prefixBytes+suffixBytes], - } - maskIndex := (bits - 1) / 8 - maskLowBits := (sfx.Bits-1)%8 + 1 - sfx.Path[maskIndex] &= ((0x01 << maskLowBits) - 1) << uint(8-maskLowBits) - - return a[:prefixBytes], sfx -} - -// Equivalent return true iff the other represents the same path prefix as this NodeID. -func (n *NodeID) Equivalent(other NodeID) bool { - return n.String() == other.String() -} - // PopulateSubtreeFunc is a function which knows how to re-populate a subtree // from just its leaf nodes. type PopulateSubtreeFunc func(*storagepb.SubtreeProto) error From 10331669a13cbb43cf65e94dbe8a56454ae4652a Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Mon, 31 Jul 2017 11:36:37 +0100 Subject: [PATCH 13/15] storage.NodeID -> node.NodeID --- log/sequencer.go | 2 +- log/sequencer_test.go | 4 ++-- merkle/merkle_path.go | 4 ++-- merkle/sparse_merkle_tree.go | 4 ++-- merkle/sparse_merkle_tree_test.go | 22 ++++++++--------- node/suffix_test.go | 2 +- server/log_rpc_server_test.go | 4 ++-- server/proof_fetcher.go | 2 +- server/sequencer_manager_test.go | 2 +- storage/cache/gen.go | 2 +- storage/cache/mock_node_storage.go | 2 +- storage/cache/subtree_cache.go | 28 +++++++++++----------- storage/cache/subtree_cache_test.go | 14 +++++------ storage/memory/tree_storage.go | 14 +++++------ storage/mysql/storage_test.go | 4 ++-- storage/mysql/tree_storage.go | 12 +++++----- storage/testonly/fake_node_reader.go | 12 +++++----- storage/testonly/matchers.go | 8 +++---- storage/testonly/nodes.go | 4 ++-- storage/tools/dump_tree/dumplib/dumplib.go | 2 +- 20 files changed, 74 insertions(+), 74 deletions(-) diff --git a/log/sequencer.go b/log/sequencer.go index 46e2f0af95..d7d7033dea 100644 --- a/log/sequencer.go +++ b/log/sequencer.go @@ -139,7 +139,7 @@ func (s Sequencer) buildMerkleTreeFromStorageAtRoot(ctx context.Context, root tr glog.Warningf("%v: Failed to create nodeID: %v", root.LogId, err) return nil, err } - nodes, err := tx.GetMerkleNodes(ctx, root.TreeRevision, []storage.NodeID{nodeID}) + nodes, err := tx.GetMerkleNodes(ctx, root.TreeRevision, []node.NodeID{nodeID}) if err != nil { glog.Warningf("%v: Failed to get Merkle nodes: %v", root.LogId, err) diff --git a/log/sequencer_test.go b/log/sequencer_test.go index 674ede149e..70edd823ea 100644 --- a/log/sequencer_test.go +++ b/log/sequencer_test.go @@ -59,10 +59,10 @@ var testRoot16 = trillian.SignedLogRoot{ // These will be accepted in either order because of custom sorting in the mock var updatedNodes = []storage.Node{ - {NodeID: storage.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10}, PrefixLenBits: 64}, + {NodeID: node.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10}, PrefixLenBits: 64}, Hash: testonly.MustDecodeBase64("L5Iyd7aFOVewxiRm29xD+EU+jvEo4RfufBijKdflWMk="), NodeRevision: 6}, { - NodeID: storage.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, PrefixLenBits: 59}, + NodeID: node.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, PrefixLenBits: 59}, Hash: testonly.MustDecodeBase64("R57DrKTGuZdjCNXjv6InGrm4rABLOn9yWpdHmYOoLwU="), NodeRevision: 6}, } diff --git a/merkle/merkle_path.go b/merkle/merkle_path.go index 789de15242..a77721aee1 100644 --- a/merkle/merkle_path.go +++ b/merkle/merkle_path.go @@ -30,7 +30,7 @@ const vvLevel = 4 // NodeFetch bundles a nodeID with additional information on how to use the node to construct the // correct proof. type NodeFetch struct { - NodeID storage.NodeID + NodeID node.NodeID Rehash bool } @@ -435,7 +435,7 @@ func checkRecomputation(fetches []NodeFetch) error { // siblingIDSkipLevels creates a new NodeID for the supplied node, accounting for levels skipped // in storage. Note that it returns an ID for the node sibling so care should be taken to pass the // correct value for the node parameter. -func siblingIDSkipLevels(snapshot, lastNode int64, level int, node int64, maxBitLen int) (storage.NodeID, error) { +func siblingIDSkipLevels(snapshot, lastNode int64, level int, node int64, maxBitLen int) (node.NodeID, error) { l, sibling := skipMissingLevels(snapshot, lastNode, level, node) return storage.NewNodeIDForTreeCoords(int64(l), sibling, maxBitLen) } diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index 0d6cf9f5ac..034708fb16 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -232,7 +232,7 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { nodeID := storage.NewNodeIDFromBigInt(depth, index, s.hasher.BitLen()) glog.V(4).Infof("buildSubtree.get(%x, %d) nid: %x, %v", index.Bytes(), depth, nodeID.Path, nodeID.PrefixLenBits) - nodes, err := s.tx.GetMerkleNodes(ctx, s.treeRevision, []storage.NodeID{nodeID}) + nodes, err := s.tx.GetMerkleNodes(ctx, s.treeRevision, []node.NodeID{nodeID}) if err != nil { return nil, err } @@ -359,7 +359,7 @@ func NewSparseMerkleTreeWriter(ctx context.Context, treeID, rev int64, h hashers // revision, or ErrNoSuchRevision if the requested revision doesn't exist. func (s SparseMerkleTreeReader) RootAtRevision(ctx context.Context, rev int64) ([]byte, error) { rootNodeID := storage.NewEmptyNodeID(256) - nodes, err := s.tx.GetMerkleNodes(ctx, rev, []storage.NodeID{rootNodeID}) + nodes, err := s.tx.GetMerkleNodes(ctx, rev, []node.NodeID{rootNodeID}) if err != nil { return nil, err } diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index 8ac9418b6c..558b420f5e 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -43,11 +43,11 @@ var ( // These nodes were generated randomly and reviewed to ensure node IDs do not collide with // those fetched during the test. var inclusionProofIncorrectTestNodes = []storage.Node{ - {NodeID: storage.NodeID{Path: []uint8{0x2c, 0x8b, 0xcf, 0xe1, 0xc5, 0x71, 0xf4, 0x2d, 0xc2, 0xe9, 0x22, 0x7d, 0x91, 0xd5, 0x93, 0x70, 0x8f, 0x8c, 0x40, 0xca, 0xf, 0xd3, 0xd8, 0x4b, 0x43, 0x6a, 0x3, 0x2f, 0xf1, 0x4, 0x7, 0x9b}, PrefixLenBits: 174}, Hash: []uint8{0x4, 0x7b, 0xe5, 0xab, 0x12, 0x2d, 0x44, 0x98, 0xd8, 0xcc, 0xc7, 0x27, 0x4d, 0xc5, 0xda, 0x59, 0x38, 0xf5, 0x4d, 0x9c, 0x98, 0x33, 0x2a, 0x95, 0xb1, 0x20, 0xe2, 0x8c, 0x7, 0x5f, 0xb5, 0x9a}, NodeRevision: 34}, - {NodeID: storage.NodeID{Path: []uint8{0x7c, 0xf5, 0x65, 0xc6, 0xd5, 0xbe, 0x2d, 0x39, 0xff, 0xf4, 0x58, 0xc2, 0x9f, 0x4f, 0x9, 0x3c, 0x54, 0x62, 0xf5, 0x35, 0x19, 0x87, 0x56, 0xb5, 0x4c, 0x6c, 0x11, 0xf3, 0xd7, 0x2, 0xc, 0x80}, PrefixLenBits: 234}, Hash: []uint8{0xbc, 0x33, 0xbe, 0x74, 0x79, 0x43, 0x59, 0x83, 0x5d, 0x93, 0x87, 0x13, 0x22, 0x98, 0xa0, 0x69, 0xed, 0xa5, 0xca, 0xfb, 0x7c, 0x16, 0x91, 0x51, 0xa2, 0xb, 0x9f, 0x17, 0xe4, 0x3f, 0xe3, 0x3}, NodeRevision: 34}, - {NodeID: storage.NodeID{Path: []uint8{0x5f, 0xc6, 0x73, 0x1c, 0x5d, 0x57, 0x23, 0xdc, 0x6a, 0xd, 0x38, 0xcb, 0x41, 0x25, 0x97, 0x2, 0x63, 0x8d, 0xa, 0x2d, 0xbe, 0x8e, 0x88, 0xff, 0x9e, 0x54, 0x5b, 0xb4, 0x5d, 0x4e, 0x6e, 0x5b}, PrefixLenBits: 223}, Hash: []uint8{0xb6, 0xd4, 0xbd, 0x76, 0x5e, 0x9b, 0x80, 0x2f, 0x71, 0x32, 0x5e, 0xf8, 0x41, 0xea, 0x47, 0xc7, 0x4, 0x7d, 0xd, 0x64, 0xa8, 0xf6, 0x22, 0xe4, 0xb4, 0xe1, 0xef, 0x2f, 0x67, 0xf8, 0x8b, 0xaa}, NodeRevision: 34}, - {NodeID: storage.NodeID{Path: []uint8{0x30, 0xe, 0x65, 0x75, 0x4d, 0xd9, 0x7a, 0x1, 0xc5, 0x2b, 0x2a, 0x6f, 0x4b, 0x59, 0x5d, 0xa8, 0xeb, 0x65, 0x25, 0x3a, 0xc5, 0xf7, 0xd2, 0x4b, 0xcc, 0x54, 0xbf, 0xe8, 0x6e, 0xe8, 0x96, 0xb7}, PrefixLenBits: 156}, Hash: []uint8{0x74, 0x93, 0x28, 0x98, 0xbc, 0xd0, 0xfd, 0x28, 0xa9, 0x39, 0xb5, 0xb5, 0xe9, 0xcc, 0x17, 0xe0, 0xe2, 0xd, 0x16, 0x14, 0xfd, 0xb1, 0x67, 0x19, 0x31, 0x3, 0x73, 0x35, 0xb4, 0x1d, 0x6d, 0x1d}, NodeRevision: 34}, - {NodeID: storage.NodeID{Path: []uint8{0x8e, 0x3b, 0x81, 0xe4, 0x2f, 0xe6, 0xd6, 0x52, 0x9b, 0xbd, 0x36, 0xc5, 0x3, 0x52, 0xe9, 0x60, 0xbb, 0xcb, 0xc9, 0xbd, 0x57, 0x96, 0xaf, 0x18, 0xd4, 0x94, 0xdd, 0x8, 0xa2, 0x43, 0x1e, 0x10}, PrefixLenBits: 157}, Hash: []uint8{0xe0, 0xb6, 0xea, 0x8a, 0xf1, 0x57, 0x1e, 0x5c, 0xbe, 0xbe, 0xd9, 0x5b, 0x29, 0x5f, 0x3, 0x7c, 0x32, 0x33, 0x77, 0xf7, 0x1c, 0x9e, 0x19, 0x4d, 0xc6, 0xdb, 0x5, 0xf7, 0x3e, 0x6c, 0xcb, 0x85}, NodeRevision: 34}, + {NodeID: node.NodeID{Path: []uint8{0x2c, 0x8b, 0xcf, 0xe1, 0xc5, 0x71, 0xf4, 0x2d, 0xc2, 0xe9, 0x22, 0x7d, 0x91, 0xd5, 0x93, 0x70, 0x8f, 0x8c, 0x40, 0xca, 0xf, 0xd3, 0xd8, 0x4b, 0x43, 0x6a, 0x3, 0x2f, 0xf1, 0x4, 0x7, 0x9b}, PrefixLenBits: 174}, Hash: []uint8{0x4, 0x7b, 0xe5, 0xab, 0x12, 0x2d, 0x44, 0x98, 0xd8, 0xcc, 0xc7, 0x27, 0x4d, 0xc5, 0xda, 0x59, 0x38, 0xf5, 0x4d, 0x9c, 0x98, 0x33, 0x2a, 0x95, 0xb1, 0x20, 0xe2, 0x8c, 0x7, 0x5f, 0xb5, 0x9a}, NodeRevision: 34}, + {NodeID: node.NodeID{Path: []uint8{0x7c, 0xf5, 0x65, 0xc6, 0xd5, 0xbe, 0x2d, 0x39, 0xff, 0xf4, 0x58, 0xc2, 0x9f, 0x4f, 0x9, 0x3c, 0x54, 0x62, 0xf5, 0x35, 0x19, 0x87, 0x56, 0xb5, 0x4c, 0x6c, 0x11, 0xf3, 0xd7, 0x2, 0xc, 0x80}, PrefixLenBits: 234}, Hash: []uint8{0xbc, 0x33, 0xbe, 0x74, 0x79, 0x43, 0x59, 0x83, 0x5d, 0x93, 0x87, 0x13, 0x22, 0x98, 0xa0, 0x69, 0xed, 0xa5, 0xca, 0xfb, 0x7c, 0x16, 0x91, 0x51, 0xa2, 0xb, 0x9f, 0x17, 0xe4, 0x3f, 0xe3, 0x3}, NodeRevision: 34}, + {NodeID: node.NodeID{Path: []uint8{0x5f, 0xc6, 0x73, 0x1c, 0x5d, 0x57, 0x23, 0xdc, 0x6a, 0xd, 0x38, 0xcb, 0x41, 0x25, 0x97, 0x2, 0x63, 0x8d, 0xa, 0x2d, 0xbe, 0x8e, 0x88, 0xff, 0x9e, 0x54, 0x5b, 0xb4, 0x5d, 0x4e, 0x6e, 0x5b}, PrefixLenBits: 223}, Hash: []uint8{0xb6, 0xd4, 0xbd, 0x76, 0x5e, 0x9b, 0x80, 0x2f, 0x71, 0x32, 0x5e, 0xf8, 0x41, 0xea, 0x47, 0xc7, 0x4, 0x7d, 0xd, 0x64, 0xa8, 0xf6, 0x22, 0xe4, 0xb4, 0xe1, 0xef, 0x2f, 0x67, 0xf8, 0x8b, 0xaa}, NodeRevision: 34}, + {NodeID: node.NodeID{Path: []uint8{0x30, 0xe, 0x65, 0x75, 0x4d, 0xd9, 0x7a, 0x1, 0xc5, 0x2b, 0x2a, 0x6f, 0x4b, 0x59, 0x5d, 0xa8, 0xeb, 0x65, 0x25, 0x3a, 0xc5, 0xf7, 0xd2, 0x4b, 0xcc, 0x54, 0xbf, 0xe8, 0x6e, 0xe8, 0x96, 0xb7}, PrefixLenBits: 156}, Hash: []uint8{0x74, 0x93, 0x28, 0x98, 0xbc, 0xd0, 0xfd, 0x28, 0xa9, 0x39, 0xb5, 0xb5, 0xe9, 0xcc, 0x17, 0xe0, 0xe2, 0xd, 0x16, 0x14, 0xfd, 0xb1, 0x67, 0x19, 0x31, 0x3, 0x73, 0x35, 0xb4, 0x1d, 0x6d, 0x1d}, NodeRevision: 34}, + {NodeID: node.NodeID{Path: []uint8{0x8e, 0x3b, 0x81, 0xe4, 0x2f, 0xe6, 0xd6, 0x52, 0x9b, 0xbd, 0x36, 0xc5, 0x3, 0x52, 0xe9, 0x60, 0xbb, 0xcb, 0xc9, 0xbd, 0x57, 0x96, 0xaf, 0x18, 0xd4, 0x94, 0xdd, 0x8, 0xa2, 0x43, 0x1e, 0x10}, PrefixLenBits: 157}, Hash: []uint8{0xe0, 0xb6, 0xea, 0x8a, 0xf1, 0x57, 0x1e, 0x5c, 0xbe, 0xbe, 0xd9, 0x5b, 0x29, 0x5f, 0x3, 0x7c, 0x32, 0x33, 0x77, 0xf7, 0x1c, 0x9e, 0x19, 0x4d, 0xc6, 0xdb, 0x5, 0xf7, 0x3e, 0x6c, 0xcb, 0x85}, NodeRevision: 34}, } func maybeProfileCPU(t *testing.T) func() { @@ -98,7 +98,7 @@ func getSparseMerkleTreeWriterWithMockTX(ctx context.Context, ctrl *gomock.Contr type rootNodeMatcher struct{} func (r rootNodeMatcher) Matches(x interface{}) bool { - nodes, ok := x.([]storage.NodeID) + nodes, ok := x.([]node.NodeID) if !ok { return false } @@ -393,11 +393,11 @@ func TestSparseMerkleTreeWriter(t *testing.T) { } type nodeIDFuncMatcher struct { - f func(ids []storage.NodeID) bool + f func(ids []node.NodeID) bool } func (f nodeIDFuncMatcher) Matches(x interface{}) bool { - n, ok := x.([]storage.NodeID) + n, ok := x.([]node.NodeID) if !ok { return false } @@ -419,7 +419,7 @@ func testSparseTreeFetches(ctx context.Context, t *testing.T, vec sparseTestVect reads := make(map[string]string) readMutex := sync.Mutex{} - var leafNodeIDs []storage.NodeID + var leafNodeIDs []node.NodeID { readMutex.Lock() @@ -459,7 +459,7 @@ func testSparseTreeFetches(ctx context.Context, t *testing.T, vec sparseTestVect // Now, set up a mock call for GetMerkleNodes for the nodeIDs in the map // we've just created: - tx.EXPECT().GetMerkleNodes(ctx, int64(rev), nodeIDFuncMatcher{func(ids []storage.NodeID) bool { + tx.EXPECT().GetMerkleNodes(ctx, int64(rev), nodeIDFuncMatcher{func(ids []node.NodeID) bool { if len(ids) == 0 { return false } @@ -476,7 +476,7 @@ func testSparseTreeFetches(ctx context.Context, t *testing.T, vec sparseTestVect // rather than doing that we'll make a note of all the unexpected IDs here // instead, and we can then print them out later on. tx.EXPECT().GetMerkleNodes(ctx, int64(rev), gomock.Any()).AnyTimes().Do( - func(rev int64, a []storage.NodeID) { + func(rev int64, a []node.NodeID) { if a == nil { return } diff --git a/node/suffix_test.go b/node/suffix_test.go index cd1612ef7c..67d88d25b1 100644 --- a/node/suffix_test.go +++ b/node/suffix_test.go @@ -26,7 +26,7 @@ const ( logStrataDepth = 8 maxLogDepth = 64 // TODO(gdbelvin): remove these constants in favor of the real ones in - // storage/cache when merkle no longer depends on storage.NodeID + // storage/cache when merkle no longer depends on node.NodeID ) func TestParseSuffix(t *testing.T) { diff --git a/server/log_rpc_server_test.go b/server/log_rpc_server_test.go index c429495a12..eecca1e5eb 100644 --- a/server/log_rpc_server_test.go +++ b/server/log_rpc_server_test.go @@ -68,12 +68,12 @@ var ( getConsistencyProofRequest25 = trillian.GetConsistencyProofRequest{LogId: logID1, FirstTreeSize: 10, SecondTreeSize: 25} getConsistencyProofRequest7 = trillian.GetConsistencyProofRequest{LogId: logID1, FirstTreeSize: 4, SecondTreeSize: 7} - nodeIdsInclusionSize7Index2 = []storage.NodeID{ + nodeIdsInclusionSize7Index2 = []node.NodeID{ stestonly.MustCreateNodeIDForTreeCoords(0, 3, 64), stestonly.MustCreateNodeIDForTreeCoords(1, 0, 64), stestonly.MustCreateNodeIDForTreeCoords(2, 1, 64)} - nodeIdsConsistencySize4ToSize7 = []storage.NodeID{stestonly.MustCreateNodeIDForTreeCoords(2, 1, 64)} + nodeIdsConsistencySize4ToSize7 = []node.NodeID{stestonly.MustCreateNodeIDForTreeCoords(2, 1, 64)} ) func TestGetLeavesByIndexInvalidIndexRejected(t *testing.T) { diff --git a/server/proof_fetcher.go b/server/proof_fetcher.go index 847a3f1119..7ff044b1a5 100644 --- a/server/proof_fetcher.go +++ b/server/proof_fetcher.go @@ -101,7 +101,7 @@ func (r *rehasher) rehashedProof(leafIndex int64) (trillian.Proof, error) { // fetchNodes extracts the NodeIDs from a list of NodeFetch structs and passes them // to storage, returning the result after some additional validation checks. func fetchNodes(ctx context.Context, tx storage.NodeReader, treeRevision int64, fetches []merkle.NodeFetch) ([]storage.Node, error) { - proofNodeIDs := make([]storage.NodeID, 0, len(fetches)) + proofNodeIDs := make([]node.NodeID, 0, len(fetches)) for _, fetch := range fetches { proofNodeIDs = append(proofNodeIDs, fetch.NodeID) diff --git a/server/sequencer_manager_test.go b/server/sequencer_manager_test.go index 3c194524af..23494cec39 100644 --- a/server/sequencer_manager_test.go +++ b/server/sequencer_manager_test.go @@ -64,7 +64,7 @@ var testRoot0 = trillian.SignedLogRoot{ SignatureAlgorithm: sigpb.DigitallySigned_ECDSA, }, } -var updatedNodes0 = []storage.Node{{NodeID: storage.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, PrefixLenBits: 64}, Hash: testonly.MustDecodeBase64("bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0="), NodeRevision: 1}} +var updatedNodes0 = []storage.Node{{NodeID: node.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, PrefixLenBits: 64}, Hash: testonly.MustDecodeBase64("bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0="), NodeRevision: 1}} var updatedRoot = trillian.SignedLogRoot{ LogId: testLogID1, TimestampNanos: fakeTime.UnixNano(), diff --git a/storage/cache/gen.go b/storage/cache/gen.go index 68d298d4d9..febd041469 100644 --- a/storage/cache/gen.go +++ b/storage/cache/gen.go @@ -24,6 +24,6 @@ import ( // NodeStorage provides an interface for storing and retrieving subtrees. type NodeStorage interface { - GetSubtree(n storage.NodeID) (*storagepb.SubtreeProto, error) + GetSubtree(n node.NodeID) (*storagepb.SubtreeProto, error) SetSubtrees(s []*storagepb.SubtreeProto) error } diff --git a/storage/cache/mock_node_storage.go b/storage/cache/mock_node_storage.go index b8be7cbcf3..4c7cccbfdd 100644 --- a/storage/cache/mock_node_storage.go +++ b/storage/cache/mock_node_storage.go @@ -34,7 +34,7 @@ func (_m *MockNodeStorage) EXPECT() *MockNodeStorageMockRecorder { } // GetSubtree mocks base method -func (_m *MockNodeStorage) GetSubtree(_param0 storage.NodeID) (*storagepb.SubtreeProto, error) { +func (_m *MockNodeStorage) GetSubtree(_param0 node.NodeID) (*storagepb.SubtreeProto, error) { ret := _m.ctrl.Call(_m, "GetSubtree", _param0) ret0, _ := ret[0].(*storagepb.SubtreeProto) ret1, _ := ret[1].(error) diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index 5d97d9560e..7121668901 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -29,10 +29,10 @@ import ( ) // GetSubtreeFunc describes a function which can return a Subtree from storage. -type GetSubtreeFunc func(id storage.NodeID) (*storagepb.SubtreeProto, error) +type GetSubtreeFunc func(id node.NodeID) (*storagepb.SubtreeProto, error) // GetSubtreesFunc describes a function which can return a number of Subtrees from storage. -type GetSubtreesFunc func(ids []storage.NodeID) ([]*storagepb.SubtreeProto, error) +type GetSubtreesFunc func(ids []node.NodeID) ([]*storagepb.SubtreeProto, error) // SetSubtreesFunc describes a function which can store a collection of Subtrees into storage. type SetSubtreesFunc func(s []*storagepb.SubtreeProto) error @@ -132,7 +132,7 @@ func (s *SubtreeCache) stratumInfoForPrefixLength(numBits int) stratumInfo { // splitNodeID breaks a NodeID out into its prefix and suffix parts. // unless ID is 0 bits long, Suffix must always contain at least one bit. -func (s *SubtreeCache) splitNodeID(id storage.NodeID) ([]byte, storage.Suffix) { +func (s *SubtreeCache) splitNodeID(id node.NodeID) ([]byte, storage.Suffix) { sInfo := s.stratumInfoForPrefixLength(id.PrefixLenBits - 1) return id.Split(sInfo.prefixBytes, sInfo.depth) } @@ -140,12 +140,12 @@ func (s *SubtreeCache) splitNodeID(id storage.NodeID) ([]byte, storage.Suffix) { // preload calculates the set of subtrees required to know the hashes of the // passed in node IDs, uses getSubtrees to retrieve them, and finally populates // the cache structures with the data. -func (s *SubtreeCache) preload(ids []storage.NodeID, getSubtrees GetSubtreesFunc) error { +func (s *SubtreeCache) preload(ids []node.NodeID, getSubtrees GetSubtreesFunc) error { s.mutex.Lock() defer s.mutex.Unlock() // Figure out the set of subtrees we need: - want := make(map[string]*storage.NodeID) + want := make(map[string]*node.NodeID) for _, id := range ids { id := id px, _ := s.splitNodeID(id) @@ -163,7 +163,7 @@ func (s *SubtreeCache) preload(ids []storage.NodeID, getSubtrees GetSubtreesFunc return nil } - list := make([]storage.NodeID, 0, len(want)) + list := make([]node.NodeID, 0, len(want)) for _, v := range want { list = append(list, *v) } @@ -197,7 +197,7 @@ func (s *SubtreeCache) preload(ids []storage.NodeID, getSubtrees GetSubtreesFunc // GetNodes returns the requested nodes, calling the getSubtrees function if // they are not already cached. -func (s *SubtreeCache) GetNodes(ids []storage.NodeID, getSubtrees GetSubtreesFunc) ([]storage.Node, error) { +func (s *SubtreeCache) GetNodes(ids []node.NodeID, getSubtrees GetSubtreesFunc) ([]storage.Node, error) { if glog.V(4) { for _, n := range ids { glog.Infof("cache: GetNodes(%x, %d", n.Path, n.PrefixLenBits) @@ -211,11 +211,11 @@ func (s *SubtreeCache) GetNodes(ids []storage.NodeID, getSubtrees GetSubtreesFun for _, id := range ids { h, err := s.GetNodeHash( id, - func(n storage.NodeID) (*storagepb.SubtreeProto, error) { + func(n node.NodeID) (*storagepb.SubtreeProto, error) { // This should never happen - we should've already read all the data we // need above, in Preload() glog.Warningf("Unexpectedly reading from within GetNodeHash(): %s", n.String()) - ret, err := getSubtrees([]storage.NodeID{n}) + ret, err := getSubtrees([]node.NodeID{n}) if err != nil || len(ret) == 0 { return nil, err } @@ -239,14 +239,14 @@ func (s *SubtreeCache) GetNodes(ids []storage.NodeID, getSubtrees GetSubtreesFun } // GetNodeHash returns a single node hash from the cache. -func (s *SubtreeCache) GetNodeHash(id storage.NodeID, getSubtree GetSubtreeFunc) ([]byte, error) { +func (s *SubtreeCache) GetNodeHash(id node.NodeID, getSubtree GetSubtreeFunc) ([]byte, error) { s.mutex.RLock() defer s.mutex.RUnlock() return s.getNodeHashUnderLock(id, getSubtree) } // getNodeHashUnderLock must be called with s.mutex locked. -func (s *SubtreeCache) getNodeHashUnderLock(id storage.NodeID, getSubtree GetSubtreeFunc) ([]byte, error) { +func (s *SubtreeCache) getNodeHashUnderLock(id node.NodeID, getSubtree GetSubtreeFunc) ([]byte, error) { px, sx := s.splitNodeID(id) prefixKey := string(px) c := s.subtrees[prefixKey] @@ -304,7 +304,7 @@ func (s *SubtreeCache) getNodeHashUnderLock(id storage.NodeID, getSubtree GetSub } // SetNodeHash sets a node hash in the cache. -func (s *SubtreeCache) SetNodeHash(id storage.NodeID, h []byte, getSubtree GetSubtreeFunc) error { +func (s *SubtreeCache) SetNodeHash(id node.NodeID, h []byte, getSubtree GetSubtreeFunc) error { s.mutex.Lock() defer s.mutex.Unlock() px, sx := s.splitNodeID(id) @@ -379,7 +379,7 @@ func (s *SubtreeCache) Flush(setSubtrees SetSubtreesFunc) error { return setSubtrees(treesToWrite) } -func (s *SubtreeCache) newEmptySubtree(id storage.NodeID, px []byte) *storagepb.SubtreeProto { +func (s *SubtreeCache) newEmptySubtree(id node.NodeID, px []byte) *storagepb.SubtreeProto { sInfo := s.stratumInfoForPrefixLength(id.PrefixLenBits) glog.V(1).Infof("Creating new empty subtree for %x, with depth %d", px, sInfo.depth) // storage didn't have one for us, so we'll store an empty proto here @@ -402,7 +402,7 @@ func PopulateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.Pop st.InternalNodes = make(map[string][]byte) leaves := make([]merkle.HStar2LeafHash, 0, len(st.Leaves)) for k64, v := range st.Leaves { - sfx, err := storage.ParseSuffix(k64) + sfx, err := node.ParseSuffix(k64) if err != nil { return err } diff --git a/storage/cache/subtree_cache_test.go b/storage/cache/subtree_cache_test.go index 0894bb5bc9..58b84e998f 100644 --- a/storage/cache/subtree_cache_test.go +++ b/storage/cache/subtree_cache_test.go @@ -119,10 +119,10 @@ func TestCacheGetNodesReadsSubtrees(t *testing.T) { m := NewMockNodeStorage(mockCtrl) c := NewSubtreeCache(defaultLogStrata, PopulateMapSubtreeNodes(treeID, maphasher.Default), PrepareMapSubtreeWrite()) - nodeIDs := []storage.NodeID{ - storage.NewNodeIDFromHash([]byte("1234")), - storage.NewNodeIDFromHash([]byte("4567")), - storage.NewNodeIDFromHash([]byte("89ab")), + nodeIDs := []node.NodeID{ + node.NewNodeIDFromHash([]byte("1234")), + node.NewNodeIDFromHash([]byte("4567")), + node.NewNodeIDFromHash([]byte("89ab")), } // Set up the expected reads: @@ -143,7 +143,7 @@ func TestCacheGetNodesReadsSubtrees(t *testing.T) { nodeIDs, // Glue function to convert a call requesting multiple subtrees into a // sequence of calls to our mock storage: - func(ids []storage.NodeID) ([]*storagepb.SubtreeProto, error) { + func(ids []node.NodeID) ([]*storagepb.SubtreeProto, error) { ret := make([]*storagepb.SubtreeProto, 0) for _, i := range ids { r, err := m.GetSubtree(i) @@ -161,7 +161,7 @@ func TestCacheGetNodesReadsSubtrees(t *testing.T) { } } -func noFetch(storage.NodeID) (*storagepb.SubtreeProto, error) { +func noFetch(node.NodeID) (*storagepb.SubtreeProto, error) { return nil, errors.New("not supposed to read anything") } @@ -184,7 +184,7 @@ func TestCacheFlush(t *testing.T) { //e := nodeID e.PrefixLenBits = b expectedSetIDs[e.String()] = "expected" - m.EXPECT().GetSubtree(stestonly.NodeIDEq(e)).Do(func(n storage.NodeID) { + m.EXPECT().GetSubtree(stestonly.NodeIDEq(e)).Do(func(n node.NodeID) { t.Logf("read %v", n) }).Return((*storagepb.SubtreeProto)(nil), nil) } diff --git a/storage/memory/tree_storage.go b/storage/memory/tree_storage.go index ee9b0daba9..5a4e74121c 100644 --- a/storage/memory/tree_storage.go +++ b/storage/memory/tree_storage.go @@ -35,7 +35,7 @@ const degree = 8 // unseqKey formats a key for use in a tree's BTree store. // The associated Item value will be the stubtreeProto with the given nodeID // prefix. -func subtreeKey(treeID, rev int64, nodeID storage.NodeID) btree.Item { +func subtreeKey(treeID, rev int64, nodeID node.NodeID) btree.Item { return &kv{k: fmt.Sprintf("/%d/subtree/%s/%d", treeID, nodeID.String(), rev)} } @@ -162,8 +162,8 @@ type treeTX struct { unlock func() } -func (t *treeTX) getSubtree(ctx context.Context, treeRevision int64, nodeID storage.NodeID) (*storagepb.SubtreeProto, error) { - s, err := t.getSubtrees(ctx, treeRevision, []storage.NodeID{nodeID}) +func (t *treeTX) getSubtree(ctx context.Context, treeRevision int64, nodeID node.NodeID) (*storagepb.SubtreeProto, error) { + s, err := t.getSubtrees(ctx, treeRevision, []node.NodeID{nodeID}) if err != nil { return nil, err } @@ -177,7 +177,7 @@ func (t *treeTX) getSubtree(ctx context.Context, treeRevision int64, nodeID stor } } -func (t *treeTX) getSubtrees(ctx context.Context, treeRevision int64, nodeIDs []storage.NodeID) ([]*storagepb.SubtreeProto, error) { +func (t *treeTX) getSubtrees(ctx context.Context, treeRevision int64, nodeIDs []node.NodeID) ([]*storagepb.SubtreeProto, error) { if len(nodeIDs) == 0 { return nil, nil } @@ -228,20 +228,20 @@ func (t *treeTX) storeSubtrees(ctx context.Context, subtrees []*storagepb.Subtre // getSubtreesAtRev returns a GetSubtreesFunc which reads at the passed in rev. func (t *treeTX) getSubtreesAtRev(ctx context.Context, rev int64) cache.GetSubtreesFunc { - return func(ids []storage.NodeID) ([]*storagepb.SubtreeProto, error) { + return func(ids []node.NodeID) ([]*storagepb.SubtreeProto, error) { return t.getSubtrees(ctx, rev, ids) } } // GetMerkleNodes returns the requests nodes at (or below) the passed in treeRevision. -func (t *treeTX) GetMerkleNodes(ctx context.Context, treeRevision int64, nodeIDs []storage.NodeID) ([]storage.Node, error) { +func (t *treeTX) GetMerkleNodes(ctx context.Context, treeRevision int64, nodeIDs []node.NodeID) ([]storage.Node, error) { return t.subtreeCache.GetNodes(nodeIDs, t.getSubtreesAtRev(ctx, treeRevision)) } func (t *treeTX) SetMerkleNodes(ctx context.Context, nodes []storage.Node) error { for _, n := range nodes { err := t.subtreeCache.SetNodeHash(n.NodeID, n.Hash, - func(nID storage.NodeID) (*storagepb.SubtreeProto, error) { + func(nID node.NodeID) (*storagepb.SubtreeProto, error) { return t.getSubtree(ctx, t.writeRevision, nID) }) if err != nil { diff --git a/storage/mysql/storage_test.go b/storage/mysql/storage_test.go index 0a6c251538..78e4035869 100644 --- a/storage/mysql/storage_test.go +++ b/storage/mysql/storage_test.go @@ -42,7 +42,7 @@ func TestNodeRoundTrip(t *testing.T) { const writeRevision = int64(100) nodesToStore := createSomeNodes() - nodeIDsToRead := make([]storage.NodeID, len(nodesToStore)) + nodeIDsToRead := make([]node.NodeID, len(nodesToStore)) for i := range nodesToStore { nodeIDsToRead[i] = nodesToStore[i].NodeID } @@ -93,7 +93,7 @@ func TestLogNodeRoundTripMultiSubtree(t *testing.T) { if err != nil { t.Fatalf("failed to create test tree: %v", err) } - nodeIDsToRead := make([]storage.NodeID, len(nodesToStore)) + nodeIDsToRead := make([]node.NodeID, len(nodesToStore)) for i := range nodesToStore { nodeIDsToRead[i] = nodesToStore[i].NodeID } diff --git a/storage/mysql/tree_storage.go b/storage/mysql/tree_storage.go index f08e7e710c..13e7c0d6b2 100644 --- a/storage/mysql/tree_storage.go +++ b/storage/mysql/tree_storage.go @@ -168,8 +168,8 @@ type treeTX struct { writeRevision int64 } -func (t *treeTX) getSubtree(ctx context.Context, treeRevision int64, nodeID storage.NodeID) (*storagepb.SubtreeProto, error) { - s, err := t.getSubtrees(ctx, treeRevision, []storage.NodeID{nodeID}) +func (t *treeTX) getSubtree(ctx context.Context, treeRevision int64, nodeID node.NodeID) (*storagepb.SubtreeProto, error) { + s, err := t.getSubtrees(ctx, treeRevision, []node.NodeID{nodeID}) if err != nil { return nil, err } @@ -183,7 +183,7 @@ func (t *treeTX) getSubtree(ctx context.Context, treeRevision int64, nodeID stor } } -func (t *treeTX) getSubtrees(ctx context.Context, treeRevision int64, nodeIDs []storage.NodeID) ([]*storagepb.SubtreeProto, error) { +func (t *treeTX) getSubtrees(ctx context.Context, treeRevision int64, nodeIDs []node.NodeID) ([]*storagepb.SubtreeProto, error) { glog.V(4).Infof("getSubtrees(") if len(nodeIDs) == 0 { return nil, nil @@ -359,20 +359,20 @@ func (t *treeTX) GetTreeRevisionIncludingSize(ctx context.Context, treeSize int6 // getSubtreesAtRev returns a GetSubtreesFunc which reads at the passed in rev. func (t *treeTX) getSubtreesAtRev(ctx context.Context, rev int64) cache.GetSubtreesFunc { - return func(ids []storage.NodeID) ([]*storagepb.SubtreeProto, error) { + return func(ids []node.NodeID) ([]*storagepb.SubtreeProto, error) { return t.getSubtrees(ctx, rev, ids) } } // GetMerkleNodes returns the requests nodes at (or below) the passed in treeRevision. -func (t *treeTX) GetMerkleNodes(ctx context.Context, treeRevision int64, nodeIDs []storage.NodeID) ([]storage.Node, error) { +func (t *treeTX) GetMerkleNodes(ctx context.Context, treeRevision int64, nodeIDs []node.NodeID) ([]storage.Node, error) { return t.subtreeCache.GetNodes(nodeIDs, t.getSubtreesAtRev(ctx, treeRevision)) } func (t *treeTX) SetMerkleNodes(ctx context.Context, nodes []storage.Node) error { for _, n := range nodes { err := t.subtreeCache.SetNodeHash(n.NodeID, n.Hash, - func(nID storage.NodeID) (*storagepb.SubtreeProto, error) { + func(nID node.NodeID) (*storagepb.SubtreeProto, error) { return t.getSubtree(ctx, t.writeRevision, nID) }) if err != nil { diff --git a/storage/testonly/fake_node_reader.go b/storage/testonly/fake_node_reader.go index 22692baae8..8bfabb1373 100644 --- a/storage/testonly/fake_node_reader.go +++ b/storage/testonly/fake_node_reader.go @@ -33,7 +33,7 @@ import ( // NodeMapping is a struct we use because we can't use NodeIDs as map keys. Callers pass this // and FakeNodeReader internally manages derived keys. type NodeMapping struct { - NodeID storage.NodeID + NodeID node.NodeID Node storage.Node } @@ -76,7 +76,7 @@ func (f FakeNodeReader) GetTreeRevisionIncludingSize(treeSize int64) (int64, err } // GetMerkleNodes implements the corresponding NodeReader API. -func (f FakeNodeReader) GetMerkleNodes(treeRevision int64, NodeIDs []storage.NodeID) ([]storage.Node, error) { +func (f FakeNodeReader) GetMerkleNodes(treeRevision int64, NodeIDs []node.NodeID) ([]storage.Node, error) { if f.treeRevision > treeRevision { return nil, fmt.Errorf("GetMerkleNodes() got treeRevision:%d, want up to: %d", treeRevision, f.treeRevision) } @@ -95,7 +95,7 @@ func (f FakeNodeReader) GetMerkleNodes(treeRevision int64, NodeIDs []storage.Nod return nodes, nil } -func (f FakeNodeReader) hasID(nodeID storage.NodeID) bool { +func (f FakeNodeReader) hasID(nodeID node.NodeID) bool { _, ok := f.nodeMap[nodeID.String()] return ok } @@ -174,7 +174,7 @@ func NewMultiFakeNodeReaderFromLeaves(batches []LeafBatch) *MultiFakeNodeReader return NewMultiFakeNodeReader(readers) } -func (m MultiFakeNodeReader) readerForNodeID(nodeID storage.NodeID, revision int64) *FakeNodeReader { +func (m MultiFakeNodeReader) readerForNodeID(nodeID node.NodeID, revision int64) *FakeNodeReader { // Work backwards and use the first reader where the node is present and the revision is in range for i := len(m.readers) - 1; i >= 0; i-- { if m.readers[i].treeRevision <= revision && m.readers[i].hasID(nodeID) { @@ -197,7 +197,7 @@ func (m MultiFakeNodeReader) GetTreeRevisionIncludingSize(treeSize int64) (int64 } // GetMerkleNodes implements the corresponding NodeReader API. -func (m MultiFakeNodeReader) GetMerkleNodes(ctx context.Context, treeRevision int64, NodeIDs []storage.NodeID) ([]storage.Node, error) { +func (m MultiFakeNodeReader) GetMerkleNodes(ctx context.Context, treeRevision int64, NodeIDs []node.NodeID) ([]storage.Node, error) { // Find the correct reader for the supplied tree revision. This must be done for each node // as earlier revisions may still be relevant nodes := make([]storage.Node, 0, len(NodeIDs)) @@ -209,7 +209,7 @@ func (m MultiFakeNodeReader) GetMerkleNodes(ctx context.Context, treeRevision in fmt.Errorf("want nodeID: %v with revision <= %d but no reader has it\n%v", nID, treeRevision, m) } - node, err := reader.GetMerkleNodes(treeRevision, []storage.NodeID{nID}) + node, err := reader.GetMerkleNodes(treeRevision, []node.NodeID{nID}) if err != nil { return nil, err } diff --git a/storage/testonly/matchers.go b/storage/testonly/matchers.go index b10fa5e2a9..f091573410 100644 --- a/storage/testonly/matchers.go +++ b/storage/testonly/matchers.go @@ -25,7 +25,7 @@ import ( ) type subtreeHasPrefix struct { - expectedID storage.NodeID + expectedID node.NodeID } func (s subtreeHasPrefix) Matches(x interface{}) bool { @@ -42,11 +42,11 @@ func (s subtreeHasPrefix) String() string { } type nodeIDEq struct { - expectedID storage.NodeID + expectedID node.NodeID } func (m nodeIDEq) Matches(x interface{}) bool { - n, ok := x.(storage.NodeID) + n, ok := x.(node.NodeID) if !ok { return false } @@ -58,7 +58,7 @@ func (m nodeIDEq) String() string { } // NodeIDEq returns a matcher that expects the specified NodeID. -func NodeIDEq(n storage.NodeID) gomock.Matcher { +func NodeIDEq(n node.NodeID) gomock.Matcher { return nodeIDEq{n} } diff --git a/storage/testonly/nodes.go b/storage/testonly/nodes.go index f6d9433d13..91fb91f68d 100644 --- a/storage/testonly/nodes.go +++ b/storage/testonly/nodes.go @@ -20,8 +20,8 @@ import ( ) // MustCreateNodeIDForTreeCoords creates a NodeID for the given position in the tree. -func MustCreateNodeIDForTreeCoords(depth, index int64, maxPathBits int) storage.NodeID { - n, err := storage.NewNodeIDForTreeCoords(depth, index, maxPathBits) +func MustCreateNodeIDForTreeCoords(depth, index int64, maxPathBits int) node.NodeID { + n, err := node.NewNodeIDForTreeCoords(depth, index, maxPathBits) if err != nil { panic(err) } diff --git a/storage/tools/dump_tree/dumplib/dumplib.go b/storage/tools/dump_tree/dumplib/dumplib.go index eeeda215f8..9918ac1ad5 100644 --- a/storage/tools/dump_tree/dumplib/dumplib.go +++ b/storage/tools/dump_tree/dumplib/dumplib.go @@ -430,7 +430,7 @@ func traverseTreeStorage(ls storage.LogStorage, treeID int64, ts int, rev int64) glog.Fatalf("NewNodeIDForTreeCoords: (%d, %d): got: %v, want: no err", level, node, err) } - nodes, err := tx.GetMerkleNodes(context.TODO(), rev, []storage.NodeID{nodeID}) + nodes, err := tx.GetMerkleNodes(context.TODO(), rev, []node.NodeID{nodeID}) if err != nil { glog.Fatalf("GetMerkleNodes: %s: %v", nodeID.CoordString(), err) } From 8e0578a594fe3eb7dff12865b193fd7713ac4cf0 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Mon, 31 Jul 2017 11:58:42 +0100 Subject: [PATCH 14/15] internal variable name node -> index --- merkle/hstar2.go | 4 +-- merkle/hstar2_test.go | 2 +- merkle/merkle_path.go | 34 +++++++++++----------- merkle/sparse_merkle_tree.go | 4 +-- merkle/sparse_merkle_tree_test.go | 2 +- storage/cache/subtree_cache.go | 2 +- storage/memory/tree_debug.go | 2 +- storage/tools/dump_tree/dumplib/dumplib.go | 9 +++--- 8 files changed, 30 insertions(+), 29 deletions(-) diff --git a/merkle/hstar2.go b/merkle/hstar2.go index 96ec2829f0..a542ccdeb9 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -22,7 +22,7 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle/hashers" - "github.com/google/trillian/storage" + "github.com/google/trillian/node" ) var ( @@ -89,7 +89,7 @@ func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2Lea return nil, ErrSubtreeOverrun } sort.Sort(ByIndex{values}) - offset := storage.NewNodeIDFromPrefixSuffix(prefix, storage.Suffix{}, s.hasher.BitLen()).BigInt() + offset := node.NewNodeIDFromPrefixSuffix(prefix, node.Suffix{}, s.hasher.BitLen()).BigInt() return s.hStar2b(depth, totalDepth, values, offset, get, set) } diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index e94da2d082..c80d08a20b 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -147,7 +147,7 @@ func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HS } ret = append(ret, HStar2LeafHash{ - Index: storage.NewNodeIDFromPrefixSuffix(prefix, storage.Suffix{}, hasher.BitLen()).BigInt(), + Index: node.NewNodeIDFromPrefixSuffix(prefix, node.Suffix{}, hasher.BitLen()).BigInt(), LeafHash: root, }) } diff --git a/merkle/merkle_path.go b/merkle/merkle_path.go index a77721aee1..954f29e265 100644 --- a/merkle/merkle_path.go +++ b/merkle/merkle_path.go @@ -106,20 +106,20 @@ func snapshotConsistency(snapshot1, snapshot2, treeSize int64, maxBitLen int) ([ glog.V(vLevel).Infof("snapshotConsistency: %d -> %d", snapshot1, snapshot2) level := 0 - node := snapshot1 - 1 + index := snapshot1 - 1 // Compute the (compressed) path to the root of snapshot2. // Everything left of 'node' is equal in both trees; no need to record. - for (node & 1) != 0 { - glog.V(vvLevel).Infof("Move up: l:%d n:%d", level, node) - node >>= 1 + for (index & 1) != 0 { + glog.V(vvLevel).Infof("Move up: l:%d n:%d", level, index) + index >>= 1 level++ } - if node != 0 { - glog.V(vvLevel).Infof("Not root snapshot1: %d", node) - // Not at the root of snapshot 1, record the node - n, err := storage.NewNodeIDForTreeCoords(int64(level), node, maxBitLen) + if index != 0 { + glog.V(vvLevel).Infof("Not root snapshot1: %d", index) + // Not at the root of snapshot 1, record the index + n, err := node.NewNodeIDForTreeCoords(int64(level), index, maxBitLen) if err != nil { return nil, err } @@ -127,15 +127,15 @@ func snapshotConsistency(snapshot1, snapshot2, treeSize int64, maxBitLen int) ([ } // Now append the path from this node to the root of snapshot2. - p, err := pathFromNodeToRootAtSnapshot(node, level, snapshot2, treeSize, maxBitLen) + p, err := pathFromNodeToRootAtSnapshot(index, level, snapshot2, treeSize, maxBitLen) if err != nil { return nil, err } return append(proof, p...), nil } -func pathFromNodeToRootAtSnapshot(node int64, level int, snapshot, treeSize int64, maxBitLen int) ([]NodeFetch, error) { - glog.V(vLevel).Infof("pathFromNodeToRootAtSnapshot(%d, %d, %d, %d, %d)", node, level, snapshot, treeSize, maxBitLen) +func pathFromNodeToRootAtSnapshot(index int64, level int, snapshot, treeSize int64, maxBitLen int) ([]NodeFetch, error) { + glog.V(vLevel).Infof("pathFromNodeToRootAtSnapshot(%d, %d, %d, %d, %d)", index, level, snapshot, treeSize, maxBitLen) proof := make([]NodeFetch, 0, bitLen(snapshot)+1) if snapshot == 0 { @@ -147,7 +147,7 @@ func pathFromNodeToRootAtSnapshot(node int64, level int, snapshot, treeSize int6 // Move up, recording the sibling of the current node at each level. for lastNode != 0 { - sibling := node ^ 1 + sibling := index ^ 1 if sibling < lastNode { // The sibling is not the last node of the level in the snapshot tree glog.V(vvLevel).Infof("Not last: S:%d L:%d", sibling, level) @@ -166,7 +166,7 @@ func pathFromNodeToRootAtSnapshot(node int64, level int, snapshot, treeSize int6 // No recomputation required as we're using the tree in its current state // Account for non existent nodes - these can only be the rightmost node at an // intermediate (non leaf) level in the tree so will always be a right sibling. - n, err := siblingIDSkipLevels(snapshot, lastNode, level, node, maxBitLen) + n, err := siblingIDSkipLevels(snapshot, lastNode, level, index, maxBitLen) if err != nil { return nil, err } @@ -191,7 +191,7 @@ func pathFromNodeToRootAtSnapshot(node int64, level int, snapshot, treeSize int6 } // Sibling > lastNode so does not exist, move up - node >>= 1 + index >>= 1 lastNode >>= 1 level++ } @@ -435,7 +435,7 @@ func checkRecomputation(fetches []NodeFetch) error { // siblingIDSkipLevels creates a new NodeID for the supplied node, accounting for levels skipped // in storage. Note that it returns an ID for the node sibling so care should be taken to pass the // correct value for the node parameter. -func siblingIDSkipLevels(snapshot, lastNode int64, level int, node int64, maxBitLen int) (node.NodeID, error) { - l, sibling := skipMissingLevels(snapshot, lastNode, level, node) - return storage.NewNodeIDForTreeCoords(int64(l), sibling, maxBitLen) +func siblingIDSkipLevels(snapshot, lastNode int64, level int, index int64, maxBitLen int) (node.NodeID, error) { + l, sibling := skipMissingLevels(snapshot, lastNode, level, index) + return node.NewNodeIDForTreeCoords(int64(l), sibling, maxBitLen) } diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index 034708fb16..6867e618f0 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -211,7 +211,7 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { s.root <- rootHashOrError{hash: nil, err: err} return } - nodeID := storage.NewNodeIDFromPrefixSuffix(ih.index, storage.Suffix{}, s.hasher.BitLen()) + nodeID := node.NewNodeIDFromPrefixSuffix(ih.index, node.Suffix{}, s.hasher.BitLen()) leaves = append(leaves, HStar2LeafHash{ Index: nodeID.BigInt(), @@ -358,7 +358,7 @@ func NewSparseMerkleTreeWriter(ctx context.Context, treeID, rev int64, h hashers // RootAtRevision returns the sparse Merkle tree root hash at the specified // revision, or ErrNoSuchRevision if the requested revision doesn't exist. func (s SparseMerkleTreeReader) RootAtRevision(ctx context.Context, rev int64) ([]byte, error) { - rootNodeID := storage.NewEmptyNodeID(256) + rootNodeID := node.NewEmptyNodeID(256) nodes, err := s.tx.GetMerkleNodes(ctx, rev, []node.NodeID{rootNodeID}) if err != nil { return nil, err diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index 558b420f5e..d455360d55 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -121,7 +121,7 @@ func randomBytes(t *testing.T, n int) []byte { func getRandomRootNode(t *testing.T, rev int64) storage.Node { return storage.Node{ - NodeID: storage.NewEmptyNodeID(0), + NodeID: node.NewEmptyNodeID(0), Hash: randomBytes(t, 32), NodeRevision: rev, } diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index 7121668901..bc62ce2e9d 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -132,7 +132,7 @@ func (s *SubtreeCache) stratumInfoForPrefixLength(numBits int) stratumInfo { // splitNodeID breaks a NodeID out into its prefix and suffix parts. // unless ID is 0 bits long, Suffix must always contain at least one bit. -func (s *SubtreeCache) splitNodeID(id node.NodeID) ([]byte, storage.Suffix) { +func (s *SubtreeCache) splitNodeID(id node.NodeID) ([]byte, node.Suffix) { sInfo := s.stratumInfoForPrefixLength(id.PrefixLenBits - 1) return id.Split(sInfo.prefixBytes, sInfo.depth) } diff --git a/storage/memory/tree_debug.go b/storage/memory/tree_debug.go index 82899c94a0..208ed47d1a 100644 --- a/storage/memory/tree_debug.go +++ b/storage/memory/tree_debug.go @@ -38,7 +38,7 @@ func Dump(t *btree.BTree) { func DumpSubtrees(ls storage.LogStorage, treeID int64, callback func(string, *storagepb.SubtreeProto)) { m := ls.(*memoryLogStorage) tree := m.trees[treeID] - pi := subtreeKey(treeID, 0, storage.NewEmptyNodeID(64)) + pi := subtreeKey(treeID, 0, node.NewEmptyNodeID(64)) tree.store.AscendGreaterOrEqual(pi, func(bi btree.Item) bool { i := bi.(*kv) diff --git a/storage/tools/dump_tree/dumplib/dumplib.go b/storage/tools/dump_tree/dumplib/dumplib.go index 9918ac1ad5..726af7b560 100644 --- a/storage/tools/dump_tree/dumplib/dumplib.go +++ b/storage/tools/dump_tree/dumplib/dumplib.go @@ -42,6 +42,7 @@ import ( "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/merkle/rfc6962" "github.com/google/trillian/monitoring" + "github.com/google/trillian/node" "github.com/google/trillian/quota" "github.com/google/trillian/storage" "github.com/google/trillian/storage/cache" @@ -422,12 +423,12 @@ func traverseTreeStorage(ls storage.LogStorage, treeID int64, ts int, rev int64) } for level := int64(0); level < levels; level++ { - for node := int64(0); node < nodesAtLevel; node++ { + for index := int64(0); index < nodesAtLevel; index++ { // We're going to request one node at a time, which would normally be slow but we have // the tree in RAM so it's not a real problem. - nodeID, err := storage.NewNodeIDForTreeCoords(level, node, 64) + nodeID, err := node.NewNodeIDForTreeCoords(level, index, 64) if err != nil { - glog.Fatalf("NewNodeIDForTreeCoords: (%d, %d): got: %v, want: no err", level, node, err) + glog.Fatalf("NewNodeIDForTreeCoords: (%d, %d): got: %v, want: no err", level, index, err) } nodes, err := tx.GetMerkleNodes(context.TODO(), rev, []node.NodeID{nodeID}) @@ -438,7 +439,7 @@ func traverseTreeStorage(ls storage.LogStorage, treeID int64, ts int, rev int64) glog.Fatalf("GetMerkleNodes: %s: want 1 node got: %v", nodeID.CoordString(), nodes) } - fmt.Fprintf(out, "%6d %6d -> %s\n", level, node, hex.EncodeToString(nodes[0].Hash)) + fmt.Fprintf(out, "%6d %6d -> %s\n", level, index, hex.EncodeToString(nodes[0].Hash)) } nodesAtLevel = nodesAtLevel >> 1 From 3d51cc3e37adb14257832fc19efd7cc48ea60055 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Mon, 31 Jul 2017 12:13:44 +0100 Subject: [PATCH 15/15] govendor --- crypto/keys/mock_keys.go | 3 ++- log/sequencer.go | 1 + log/sequencer_test.go | 1 + merkle/hstar2.go | 1 + merkle/hstar2_test.go | 2 +- merkle/merkle_path.go | 1 + merkle/sparse_merkle_tree.go | 1 + merkle/sparse_merkle_tree_test.go | 1 + quota/mock_quota.go | 3 ++- server/log_rpc_server_test.go | 1 + server/mock_log_operation.go | 3 ++- server/proof_fetcher.go | 1 + server/sequencer_manager_test.go | 1 + storage/cache/gen.go | 2 +- storage/cache/mock_node_storage.go | 5 +++-- storage/cache/subtree_cache.go | 1 + storage/cache/subtree_cache_test.go | 1 + storage/memory/tree_debug.go | 1 + storage/memory/tree_storage.go | 1 + storage/mock_storage.go | 5 +++-- storage/mysql/storage_test.go | 1 + storage/mysql/tree_storage.go | 1 + storage/testonly/fake_node_reader.go | 1 + storage/testonly/matchers.go | 1 + storage/testonly/nodes.go | 4 +--- 25 files changed, 32 insertions(+), 12 deletions(-) diff --git a/crypto/keys/mock_keys.go b/crypto/keys/mock_keys.go index e5c4ff211c..2b92906740 100644 --- a/crypto/keys/mock_keys.go +++ b/crypto/keys/mock_keys.go @@ -6,10 +6,11 @@ package keys import ( context "context" crypto "crypto" + reflect "reflect" + gomock "github.com/golang/mock/gomock" proto "github.com/golang/protobuf/proto" keyspb "github.com/google/trillian/crypto/keyspb" - reflect "reflect" ) // MockSignerFactory is a mock of SignerFactory interface diff --git a/log/sequencer.go b/log/sequencer.go index d7d7033dea..b34ab8e02d 100644 --- a/log/sequencer.go +++ b/log/sequencer.go @@ -30,6 +30,7 @@ import ( "github.com/google/trillian/merkle" "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/monitoring" + "github.com/google/trillian/node" "github.com/google/trillian/quota" "github.com/google/trillian/storage" "github.com/google/trillian/util" diff --git a/log/sequencer_test.go b/log/sequencer_test.go index 70edd823ea..1272eabf27 100644 --- a/log/sequencer_test.go +++ b/log/sequencer_test.go @@ -29,6 +29,7 @@ import ( "github.com/google/trillian/crypto/keys" "github.com/google/trillian/crypto/sigpb" "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/node" "github.com/google/trillian/quota" "github.com/google/trillian/storage" stestonly "github.com/google/trillian/storage/testonly" diff --git a/merkle/hstar2.go b/merkle/hstar2.go index a542ccdeb9..313f0bcae1 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -23,6 +23,7 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/node" + "github.com/google/trillian/storage" ) var ( diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index c80d08a20b..894d2b17f0 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -22,7 +22,7 @@ import ( "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/merkle/maphasher" - "github.com/google/trillian/storage" + "github.com/google/trillian/node" "github.com/google/trillian/testonly" ) diff --git a/merkle/merkle_path.go b/merkle/merkle_path.go index 954f29e265..9347765001 100644 --- a/merkle/merkle_path.go +++ b/merkle/merkle_path.go @@ -20,6 +20,7 @@ import ( "github.com/golang/glog" terr "github.com/google/trillian/errors" + "github.com/google/trillian/node" "github.com/google/trillian/storage" ) diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index 6867e618f0..613d1f5876 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -24,6 +24,7 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/node" "github.com/google/trillian/storage" ) diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index d455360d55..e612c10ce0 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -31,6 +31,7 @@ import ( "github.com/golang/glog" "github.com/golang/mock/gomock" "github.com/google/trillian/merkle/maphasher" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/testonly" ) diff --git a/quota/mock_quota.go b/quota/mock_quota.go index 9a838e3d86..b195316385 100644 --- a/quota/mock_quota.go +++ b/quota/mock_quota.go @@ -5,8 +5,9 @@ package quota import ( context "context" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + gomock "github.com/golang/mock/gomock" ) // MockManager is a mock of Manager interface diff --git a/server/log_rpc_server_test.go b/server/log_rpc_server_test.go index eecca1e5eb..e266cad3bf 100644 --- a/server/log_rpc_server_test.go +++ b/server/log_rpc_server_test.go @@ -25,6 +25,7 @@ import ( "github.com/google/trillian" "github.com/google/trillian/extension" "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/node" "github.com/google/trillian/storage" stestonly "github.com/google/trillian/storage/testonly" "github.com/kylelemons/godebug/pretty" diff --git a/server/mock_log_operation.go b/server/mock_log_operation.go index d1da16a4c7..a7caa36967 100644 --- a/server/mock_log_operation.go +++ b/server/mock_log_operation.go @@ -5,8 +5,9 @@ package server import ( context "context" - gomock "github.com/golang/mock/gomock" reflect "reflect" + + gomock "github.com/golang/mock/gomock" ) // MockLogOperation is a mock of LogOperation interface diff --git a/server/proof_fetcher.go b/server/proof_fetcher.go index 7ff044b1a5..2dc8c46387 100644 --- a/server/proof_fetcher.go +++ b/server/proof_fetcher.go @@ -21,6 +21,7 @@ import ( "github.com/google/trillian" "github.com/google/trillian/merkle" "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/node" "github.com/google/trillian/storage" ) diff --git a/server/sequencer_manager_test.go b/server/sequencer_manager_test.go index 23494cec39..b7b5fb69ff 100644 --- a/server/sequencer_manager_test.go +++ b/server/sequencer_manager_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/trillian/crypto/sigpb" "github.com/google/trillian/extension" "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/node" "github.com/google/trillian/quota" "github.com/google/trillian/storage" stestonly "github.com/google/trillian/storage/testonly" diff --git a/storage/cache/gen.go b/storage/cache/gen.go index febd041469..5732844efc 100644 --- a/storage/cache/gen.go +++ b/storage/cache/gen.go @@ -18,7 +18,7 @@ package cache //go:generate mockgen -self_package github.com/google/trillian/storage/cache -package cache -imports github.com/google/trillian/storage/storagepb -destination mock_node_storage.go github.com/google/trillian/storage/cache NodeStorage import ( - "github.com/google/trillian/storage" + "github.com/google/trillian/node" "github.com/google/trillian/storage/storagepb" ) diff --git a/storage/cache/mock_node_storage.go b/storage/cache/mock_node_storage.go index 4c7cccbfdd..f3595fbdcd 100644 --- a/storage/cache/mock_node_storage.go +++ b/storage/cache/mock_node_storage.go @@ -4,10 +4,11 @@ package cache import ( + reflect "reflect" + gomock "github.com/golang/mock/gomock" - storage "github.com/google/trillian/storage" + "github.com/google/trillian/node" storagepb "github.com/google/trillian/storage/storagepb" - reflect "reflect" ) // MockNodeStorage is a mock of NodeStorage interface diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index bc62ce2e9d..f49cff0489 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -24,6 +24,7 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle" "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/storage/storagepb" ) diff --git a/storage/cache/subtree_cache_test.go b/storage/cache/subtree_cache_test.go index 58b84e998f..3db5dc819d 100644 --- a/storage/cache/subtree_cache_test.go +++ b/storage/cache/subtree_cache_test.go @@ -23,6 +23,7 @@ import ( "github.com/google/trillian/merkle" "github.com/google/trillian/merkle/maphasher" "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/storage/storagepb" "github.com/google/trillian/testonly" diff --git a/storage/memory/tree_debug.go b/storage/memory/tree_debug.go index 208ed47d1a..04fac1ceb6 100644 --- a/storage/memory/tree_debug.go +++ b/storage/memory/tree_debug.go @@ -17,6 +17,7 @@ package memory import ( "github.com/golang/glog" "github.com/google/btree" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/storage/storagepb" ) diff --git a/storage/memory/tree_storage.go b/storage/memory/tree_storage.go index 5a4e74121c..0f7f4ce765 100644 --- a/storage/memory/tree_storage.go +++ b/storage/memory/tree_storage.go @@ -25,6 +25,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/google/btree" "github.com/google/trillian" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/storage/cache" "github.com/google/trillian/storage/storagepb" diff --git a/storage/mock_storage.go b/storage/mock_storage.go index 08290ef1fb..b79c210fa0 100644 --- a/storage/mock_storage.go +++ b/storage/mock_storage.go @@ -5,10 +5,11 @@ package storage import ( context "context" - gomock "github.com/golang/mock/gomock" - trillian "github.com/google/trillian" reflect "reflect" time "time" + + gomock "github.com/golang/mock/gomock" + trillian "github.com/google/trillian" ) // MockAdminStorage is a mock of AdminStorage interface diff --git a/storage/mysql/storage_test.go b/storage/mysql/storage_test.go index 78e4035869..3ef26a1b87 100644 --- a/storage/mysql/storage_test.go +++ b/storage/mysql/storage_test.go @@ -29,6 +29,7 @@ import ( "github.com/google/trillian" "github.com/google/trillian/merkle" "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/node" "github.com/google/trillian/storage" storageto "github.com/google/trillian/storage/testonly" ) diff --git a/storage/mysql/tree_storage.go b/storage/mysql/tree_storage.go index 13e7c0d6b2..743efd0d8a 100644 --- a/storage/mysql/tree_storage.go +++ b/storage/mysql/tree_storage.go @@ -25,6 +25,7 @@ import ( "github.com/golang/glog" "github.com/golang/protobuf/proto" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/storage/cache" "github.com/google/trillian/storage/storagepb" diff --git a/storage/testonly/fake_node_reader.go b/storage/testonly/fake_node_reader.go index 8bfabb1373..be7fe6dd2c 100644 --- a/storage/testonly/fake_node_reader.go +++ b/storage/testonly/fake_node_reader.go @@ -22,6 +22,7 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle" "github.com/google/trillian/merkle/rfc6962" + "github.com/google/trillian/node" "github.com/google/trillian/storage" ) diff --git a/storage/testonly/matchers.go b/storage/testonly/matchers.go index f091573410..a2b5b31f72 100644 --- a/storage/testonly/matchers.go +++ b/storage/testonly/matchers.go @@ -20,6 +20,7 @@ import ( "sort" "github.com/golang/mock/gomock" + "github.com/google/trillian/node" "github.com/google/trillian/storage" "github.com/google/trillian/storage/storagepb" ) diff --git a/storage/testonly/nodes.go b/storage/testonly/nodes.go index 91fb91f68d..84ed95e2ba 100644 --- a/storage/testonly/nodes.go +++ b/storage/testonly/nodes.go @@ -15,9 +15,7 @@ // Package testonly holds test-specific code for Trillian storage layers. package testonly -import ( - "github.com/google/trillian/storage" -) +import "github.com/google/trillian/node" // MustCreateNodeIDForTreeCoords creates a NodeID for the given position in the tree. func MustCreateNodeIDForTreeCoords(depth, index int64, maxPathBits int) node.NodeID {