diff --git a/integration/maptest/map_test.go b/integration/maptest/map_test.go index 5eee5c2c97..860d643e1a 100644 --- a/integration/maptest/map_test.go +++ b/integration/maptest/map_test.go @@ -231,6 +231,23 @@ func TestInclusion(t *testing.T) { {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, }, }, + { + desc: "CONIKS across subtrees", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000180000000000000000000000000000000000000000000000000"), LeafValue: []byte("Z")}, + }, + }, + { + desc: "CONIKS multi", + HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256, + leaves: []*trillian.MapLeaf{ + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")}, + {Index: h2b("0000000000000000000000000000000000000000000000000000000000000003"), LeafValue: nil}, + }, + }, } { tree, hasher, err := newTreeWithHasher(ctx, env, tc.HashStrategy) if err != nil { @@ -282,22 +299,35 @@ func TestInclusionBatch(t *testing.T) { desc string HashStrategy trillian.HashStrategy batchSize, numBatches int + large bool }{ + + { + desc: "maphasher short batch", + HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, + batchSize: 10, numBatches: 10, + large: false, + }, { desc: "maphasher batch", HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER, batchSize: 64, numBatches: 32, + large: true, }, // TODO(gdbelvin): investigate batches of size > 150. // We are currently getting DB connection starvation: Too many connections. } { + if testing.Short() && tc.large { + t.Logf("testing.Short() is true. Skipping %v", tc.desc) + continue + } tree, _, err := newTreeWithHasher(ctx, env, tc.HashStrategy) if err != nil { t.Errorf("%v: newTreeWithHasher(%v): %v", tc.desc, tc.HashStrategy, err) } if err := RunMapBatchTest(ctx, env, tree, tc.batchSize, tc.numBatches); err != nil { - t.Errorf("%v: %v", tc.desc, err) + t.Errorf("BatchSize: %v, Batches: %v: %v", tc.batchSize, tc.numBatches, err) } } } diff --git a/merkle/hstar2.go b/merkle/hstar2.go index 1cbfee32ed..96ec2829f0 100644 --- a/merkle/hstar2.go +++ b/merkle/hstar2.go @@ -22,13 +22,14 @@ import ( "github.com/golang/glog" "github.com/google/trillian/merkle/hashers" + "github.com/google/trillian/storage" ) var ( - // ErrNegativeTreeLevelOffset indicates a negative level was specified. - ErrNegativeTreeLevelOffset = errors.New("treeLevelOffset cannot be negative") - smtOne = big.NewInt(1) - smtZero = big.NewInt(0) + // ErrSubtreeOverrun indicates that a subtree exceeds the maximum tree depth. + ErrSubtreeOverrun = errors.New("subtree with prefix exceeds maximum tree size") + smtOne = big.NewInt(1) + smtZero = big.NewInt(0) ) // HStar2LeafHash represents a leaf for the HStar2 sparse Merkle tree @@ -54,15 +55,11 @@ func NewHStar2(treeID int64, hasher hashers.MapHasher) HStar2 { } } -// HStar2Root calculates the root of a sparse Merkle tree of depth n which contains -// the given set of non-null leaves. -func (s *HStar2) HStar2Root(n int, values []HStar2LeafHash) ([]byte, error) { +// HStar2Root calculates the root of a sparse Merkle tree of a given depth +// which contains the given set of non-null leaves. +func (s *HStar2) HStar2Root(depth int, values []HStar2LeafHash) ([]byte, error) { sort.Sort(ByIndex{values}) - return s.hStar2b(n, values, smtZero, - func(depth int, index *big.Int) ([]byte, error) { - return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth), nil - }, - func(int, *big.Int, []byte) error { return nil }) + return s.hStar2b(0, depth, values, smtZero, nil, nil) } // SparseGetNodeFunc should return any pre-existing node hash for the node address. @@ -76,78 +73,87 @@ type SparseSetNodeFunc func(depth int, index *big.Int, hash []byte) error // internal node values. Values must not contain multiple leaves for the same // index. // -// The treeLevelOffset argument is used when the tree to be calculated is part -// of a larger tree. It identifes the level in the larger tree at which the -// root of the subtree being calculated is found. -// e.g. Imagine a tree 256 levels deep, and that you already (somehow) happen -// to have the intermediate hash values for the non-null nodes 8 levels below -// the root already calculated (i.e. you just need to calculate the top 8 -// levels of a 256-level tree). To do this, you'd set treeDepth=8, and -// treeLevelOffset=248 (256-8). -func (s *HStar2) HStar2Nodes(treeDepth, treeLevelOffset int, values []HStar2LeafHash, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { +// prefix is the location of this subtree within the larger tree. Root is at nil. +// subtreeDepth is the number of levels in this subtree. +func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2LeafHash, + get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { if glog.V(3) { - glog.Infof("HStar2Nodes(%v, %v, %v)", treeDepth, treeLevelOffset, len(values)) + glog.Infof("HStar2Nodes(%x, %v, %v)", prefix, subtreeDepth, len(values)) for _, v := range values { glog.Infof(" %x: %x", v.Index.Bytes(), v.LeafHash) } } - if treeLevelOffset < 0 { - return nil, ErrNegativeTreeLevelOffset + depth := len(prefix) * 8 + totalDepth := depth + subtreeDepth + if totalDepth > s.hasher.BitLen() { + return nil, ErrSubtreeOverrun } sort.Sort(ByIndex{values}) - return s.hStar2b(treeDepth, values, smtZero, - func(depth int, index *big.Int) ([]byte, error) { - // if we've got a function for getting existing node values, try it: - h, err := get(treeDepth-depth, index) - if err != nil { - return nil, err - } - // if we got a value then we'll use that - if h != nil { - return h, nil - } - // otherwise just return the null hash for this level - return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth+treeLevelOffset), nil - }, - func(depth int, index *big.Int, hash []byte) error { - return set(treeDepth-depth, index, hash) - }) + offset := storage.NewNodeIDFromPrefixSuffix(prefix, storage.Suffix{}, s.hasher.BitLen()).BigInt() + return s.hStar2b(depth, totalDepth, values, offset, get, set) } -// hStar2b is the recursive implementation for calculating a sparse Merkle tree -// root value. -func (s *HStar2) hStar2b(n int, values []HStar2LeafHash, offset *big.Int, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { - if n == 0 { +// hStar2b computes a sparse Merkle tree root value recursively. +func (s *HStar2) hStar2b(depth, maxDepth int, values []HStar2LeafHash, offset *big.Int, + get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) { + if depth == maxDepth { switch { case len(values) == 0: - return get(n, offset) - case len(values) != 1: + return s.get(offset, depth, get) + case len(values) == 1: + return values[0].LeafHash, nil + default: return nil, fmt.Errorf("hStar2b base case: len(values): %d, want 1", len(values)) } - return values[0].LeafHash, nil } if len(values) == 0 { - return get(n, offset) + return s.get(offset, depth, get) } - split := new(big.Int).Lsh(smtOne, uint(n-1)) + bitsLeft := s.hasher.BitLen() - depth + split := new(big.Int).Lsh(smtOne, uint(bitsLeft-1)) split.Add(split, offset) i := sort.Search(len(values), func(i int) bool { return values[i].Index.Cmp(split) >= 0 }) - lhs, err := s.hStar2b(n-1, values[:i], offset, get, set) + lhs, err := s.hStar2b(depth+1, maxDepth, values[:i], offset, get, set) if err != nil { return nil, err } - rhs, err := s.hStar2b(n-1, values[i:], split, get, set) + rhs, err := s.hStar2b(depth+1, maxDepth, values[i:], split, get, set) if err != nil { return nil, err } h := s.hasher.HashChildren(lhs, rhs) - if set != nil { - set(n, offset, h) - } + s.set(offset, depth, h, set) return h, nil } +// get attempts to use getter. If getter fails, returns the HashEmpty value. +func (s *HStar2) get(index *big.Int, depth int, getter SparseGetNodeFunc) ([]byte, error) { + // if we've got a function for getting existing node values, try it: + if getter != nil { + h, err := getter(depth, index) + if err != nil { + return nil, err + } + // if we got a value then we'll use that + if h != nil { + return h, nil + } + } + // TODO(gdbelvin): Hashers should accept depth as their main argument. + height := s.hasher.BitLen() - depth + nodeID := storage.NewNodeIDFromBigInt(index.BitLen(), index, s.hasher.BitLen()) + return s.hasher.HashEmpty(s.treeID, nodeID.Path, height), nil +} + +// set attempts to use setter if it not nil. +func (s *HStar2) set(index *big.Int, depth int, hash []byte, setter SparseSetNodeFunc) error { + if setter != nil { + return setter(depth, index, hash) + } + return nil +} + // HStar2LeafHash sorting boilerplate below. // Leaves is a slice of HStar2LeafHash @@ -164,13 +170,3 @@ type ByIndex struct{ Leaves } // Less returns true if i.Index < j.Index func (s ByIndex) Less(i, j int) bool { return s.Leaves[i].Index.Cmp(s.Leaves[j].Index) < 0 } - -// PaddedBytes takes a big.Int and returns it's value, left padded with zeros. -// e.g. 1 -> 0000000000000000000000000000000000000001 -func PaddedBytes(i *big.Int, size int) []byte { - b := i.Bytes() - ret := make([]byte, size) - padBytes := len(ret) - len(b) - copy(ret[padBytes:], b) - return ret -} diff --git a/merkle/hstar2_test.go b/merkle/hstar2_test.go index f306668947..df2b6dc030 100644 --- a/merkle/hstar2_test.go +++ b/merkle/hstar2_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/trillian/merkle/hashers" "github.com/google/trillian/merkle/maphasher" + "github.com/google/trillian/storage" "github.com/google/trillian/testonly" ) @@ -87,7 +88,7 @@ func TestHStar2SimpleDataSetKAT(t *testing.T) { continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root: \n%x, want:\n%x", got, want) + t.Errorf("Root: %x, want: %x", got, want) } } } @@ -107,7 +108,7 @@ func TestHStar2GetSet(t *testing.T) { if len(values) != 1 { t.Fatalf("Should only have 1 leaf per run, got %d", len(values)) } - root, err := s.HStar2Nodes(s.hasher.BitLen(), 0, values, + root, err := s.HStar2Nodes(nil, s.hasher.BitLen(), values, func(depth int, index *big.Int) ([]byte, error) { return cache[fmt.Sprintf("%x/%d", index, depth)], nil }, @@ -120,7 +121,7 @@ func TestHStar2GetSet(t *testing.T) { continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root:\n%x, want:\n%x", got, want) + t.Errorf("Root: %x, want: %x", got, want) } } } @@ -130,20 +131,25 @@ func TestHStar2GetSet(t *testing.T) { // 256-prefixSize, and can be passed in as leaves to top-subtree calculation. func rootsForTrimmedKeys(t *testing.T, prefixSize int, lh []HStar2LeafHash) []HStar2LeafHash { var ret []HStar2LeafHash - s := NewHStar2(treeID, maphasher.Default) + hasher := maphasher.Default + s := NewHStar2(treeID, hasher) for i := range lh { - prefix := new(big.Int).Rsh(lh[i].Index, uint(s.hasher.BitLen()-prefixSize)) - b := lh[i].Index.Bytes() - // ensure we've got any chopped of leading zero bytes - for len(b) < 32 { - b = append([]byte{0}, b...) + subtreeDepth := s.hasher.BitLen() - prefixSize + prefix := lh[i].Index.Bytes() + // Left pad prefix with zeros back out to 32 bytes. + for len(prefix) < 32 { + prefix = append([]byte{0}, prefix...) } - lh[i].Index.SetBytes(b[prefixSize/8:]) - root, err := s.HStar2Root(s.hasher.BitLen()-prefixSize, []HStar2LeafHash{lh[i]}) + prefix = prefix[:prefixSize/8] // We only want the first prefixSize bytes. + root, err := s.HStar2Nodes(prefix, subtreeDepth, []HStar2LeafHash{lh[i]}, nil, nil) if err != nil { t.Fatalf("Failed to calculate root %v", err) } - ret = append(ret, HStar2LeafHash{prefix, root}) + + ret = append(ret, HStar2LeafHash{ + Index: storage.NewNodeIDFromPrefixSuffix(prefix, storage.Suffix{}, hasher.BitLen()).BigInt(), + LeafHash: root, + }) } return ret } @@ -163,15 +169,13 @@ func TestHStar2OffsetRootKAT(t *testing.T) { leaves := createHStar2Leaves(treeID, maphasher.Default, iv...) intermediates := rootsForTrimmedKeys(t, size, leaves) - root, err := s.HStar2Nodes(size, s.hasher.BitLen()-size, intermediates, - func(int, *big.Int) ([]byte, error) { return nil, nil }, - func(int, *big.Int, []byte) error { return nil }) + root, err := s.HStar2Nodes(nil, size, intermediates, nil, nil) if err != nil { t.Errorf("Failed to calculate root at iteration %d: %v", i, err) continue } if got, want := root, x.root; !bytes.Equal(got, want) { - t.Errorf("Root: %x, want: %x", got, want) + t.Errorf("HStar2Nodes(i: %v, size:%v): %x, want: %x", i, size, got, want) } } } @@ -180,27 +184,8 @@ func TestHStar2OffsetRootKAT(t *testing.T) { func TestHStar2NegativeTreeLevelOffset(t *testing.T) { s := NewHStar2(treeID, maphasher.Default) - _, err := s.HStar2Nodes(32, -1, []HStar2LeafHash{}, - func(int, *big.Int) ([]byte, error) { return nil, nil }, - func(int, *big.Int, []byte) error { return nil }) - if got, want := err, ErrNegativeTreeLevelOffset; got != want { + _, err := s.HStar2Nodes(make([]byte, 31), 9, []HStar2LeafHash{}, nil, nil) + if got, want := err, ErrSubtreeOverrun; got != want { t.Fatalf("Hstar2Nodes(): %v, want %v", got, want) } } - -func TestPaddedBytes(t *testing.T) { - size := 160 / 8 - for _, tc := range []struct { - i *big.Int - want []byte - }{ - {i: big.NewInt(0), want: h2b("0000000000000000000000000000000000000000")}, - {i: big.NewInt(1), want: h2b("0000000000000000000000000000000000000001")}, - {i: new(big.Int).SetBytes(h2b("00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")), want: h2b("00FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")}, - {i: new(big.Int).SetBytes(h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")), want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F")}, - } { - if got, want := PaddedBytes(tc.i, size), tc.want; !bytes.Equal(got, want) { - t.Errorf("PaddedBytes(%d): %x, want %x", tc.i, got, want) - } - } -} diff --git a/merkle/sparse_merkle_tree.go b/merkle/sparse_merkle_tree.go index a38903b04d..704f1bd52b 100644 --- a/merkle/sparse_merkle_tree.go +++ b/merkle/sparse_merkle_tree.go @@ -56,6 +56,7 @@ type indexAndHash struct { // rootHashOrError represents a (sub-)tree root hash, or an error which // prevented the calculation from completing. +// TODO(gdbelvin): represent an empty subtree with a nil hash? type rootHashOrError struct { hash []byte err error @@ -109,7 +110,7 @@ type subtreeWriter struct { tx storage.TreeTX treeRevision int64 - treeHasher hashers.MapHasher + hasher hashers.MapHasher getSubtree getSubtreeFunc } @@ -151,30 +152,31 @@ func (s *subtreeWriter) getOrCreateChildSubtree(ctx context.Context, childPrefix return subtree, nil } -// SetLeaf sets a single leaf hash for incorporation into the sparse Merkle -// tree. +// SetLeaf sets a single leaf hash for incorporation into the sparse Merkle tree. +// index is the full path of the leaf, starting from the root (not the subtree's root). func (s *subtreeWriter) SetLeaf(ctx context.Context, index []byte, hash []byte) error { - indexLen := len(index) * 8 + depth := len(index) * 8 + absSubtreeDepth := len(s.prefix)*8 + s.subtreeDepth switch { - case indexLen < s.subtreeDepth: - return fmt.Errorf("index length %d is < our depth %d", indexLen, s.subtreeDepth) + case depth < absSubtreeDepth: + return fmt.Errorf("depth: %d, want >= %d", depth, absSubtreeDepth) - case indexLen > s.subtreeDepth: - childPrefix := index[:s.subtreeDepth/8] + case depth > absSubtreeDepth: + childPrefix := index[:absSubtreeDepth/8] subtree, err := s.getOrCreateChildSubtree(ctx, childPrefix) if err != nil { return err } - return subtree.SetLeaf(ctx, index[s.subtreeDepth/8:], hash) + return subtree.SetLeaf(ctx, index, hash) - case indexLen == s.subtreeDepth: - s.leafQueue <- func() (*indexAndHash, error) { return &indexAndHash{index: index, hash: hash}, nil } + default: // depth == absSubtreeDepth: + s.leafQueue <- func() (*indexAndHash, error) { + return &indexAndHash{index: index, hash: hash}, nil + } return nil } - - return fmt.Errorf("internal logic error in SetLeaf. index length: %d, subtreeDepth: %d", indexLen, s.subtreeDepth) } // CalculateRoot initiates the process of calculating the subtree root. @@ -209,25 +211,27 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { s.root <- rootHashOrError{hash: nil, err: err} return } - leaves = append(leaves, HStar2LeafHash{Index: new(big.Int).SetBytes(ih.index), LeafHash: ih.hash}) + nodeID := storage.NewNodeIDFromPrefixSuffix(ih.index, storage.Suffix{}, s.hasher.BitLen()) + + leaves = append(leaves, HStar2LeafHash{ + Index: nodeID.BigInt(), + LeafHash: ih.hash, + }) nodesToStore = append(nodesToStore, storage.Node{ - NodeID: storage.NewNodeIDFromHash(bytes.Join([][]byte{s.prefix, ih.index}, []byte{})), + NodeID: nodeID, Hash: ih.hash, NodeRevision: s.treeRevision, }) - } // calculate new root, and intermediate nodes: - hs2 := NewHStar2(s.treeID, s.treeHasher) - treeDepthOffset := (s.treeHasher.Size()-len(s.prefix))*8 - s.subtreeDepth - totalDepth := len(s.prefix)*8 + s.subtreeDepth - root, err := hs2.HStar2Nodes(s.subtreeDepth, treeDepthOffset, leaves, - func(height int, index *big.Int) ([]byte, error) { - nodeID := storage.NewNodeIDFromRelativeBigInt(s.prefix, s.subtreeDepth, height, index, totalDepth) + hs2 := NewHStar2(s.treeID, s.hasher) + root, err := hs2.HStar2Nodes(s.prefix, s.subtreeDepth, leaves, + func(depth int, index *big.Int) ([]byte, error) { + nodeID := storage.NewNodeIDFromBigInt(depth, index, s.hasher.BitLen()) glog.V(4).Infof("buildSubtree.get(%x, %d) nid: %x, %v", - index.Bytes(), height, nodeID.Path, nodeID.PrefixLenBits) + index.Bytes(), depth, nodeID.Path, nodeID.PrefixLenBits) nodes, err := s.tx.GetMerkleNodes(ctx, s.treeRevision, []storage.NodeID{nodeID}) if err != nil { return nil, err @@ -235,23 +239,23 @@ func (s *subtreeWriter) buildSubtree(ctx context.Context) { if len(nodes) == 0 { return nil, nil } - if expected, got := nodeID, nodes[0].NodeID; !expected.Equivalent(got) { - return nil, fmt.Errorf("expected node ID %s from storage, but got %s", expected.String(), got.String()) + if got, want := nodes[0].NodeID, nodeID; !got.Equivalent(want) { + return nil, fmt.Errorf("got node %s from storage, want %s", got, want) } - if expected, got := s.treeRevision, nodes[0].NodeRevision; got > expected { - return nil, fmt.Errorf("expected node revision <= %d, but got %d", expected, got) + if got, want := nodes[0].NodeRevision, s.treeRevision; got > want { + return nil, fmt.Errorf("got node revision %d, want <= %d", got, want) } return nodes[0].Hash, nil }, - func(height int, index *big.Int, h []byte) error { + func(depth int, index *big.Int, h []byte) error { // Don't store the root node of the subtree - that's part of the parent // tree. - if height == 0 && len(s.prefix) > 0 { + if depth == len(s.prefix)*8 && len(s.prefix) > 0 { return nil } - nodeID := storage.NewNodeIDFromRelativeBigInt(s.prefix, s.subtreeDepth, height, index, totalDepth) + nodeID := storage.NewNodeIDFromBigInt(depth, index, s.hasher.BitLen()) glog.V(4).Infof("buildSubtree.set(%x, %v) nid: %x, %v : %x", - index.Bytes(), height, nodeID.Path, nodeID.PrefixLenBits, h) + index.Bytes(), depth, nodeID.Path, nodeID.PrefixLenBits, h) nodesToStore = append(nodesToStore, storage.Node{ NodeID: nodeID, @@ -321,7 +325,7 @@ func newLocalSubtreeWriter(ctx context.Context, treeID, rev int64, prefix []byte root: make(chan rootHashOrError, 1), children: make(map[string]Subtree), tx: tx, - treeHasher: h, + hasher: h, getSubtree: func(ctx context.Context, p []byte) (Subtree, error) { myPrefix := bytes.Join([][]byte{prefix, p}, []byte{}) return newLocalSubtreeWriter(ctx, treeID, rev, myPrefix, depths[1:], newTX, h) @@ -388,8 +392,10 @@ func (s SparseMerkleTreeReader) InclusionProof(ctx context.Context, rev int64, i } nodeMap := make(map[string]*storage.Node) + glog.Infof("Got Nodes: ") for _, n := range nodes { n := n // need this or we'll end up with the same node hash repeated in the map + glog.Infof(" %x, %d: %x", n.NodeID.Path, len(n.NodeID.String()), n.Hash) nodeMap[n.NodeID.String()] = &n } diff --git a/merkle/sparse_merkle_tree_test.go b/merkle/sparse_merkle_tree_test.go index b0202a1cc7..2262907e6e 100644 --- a/merkle/sparse_merkle_tree_test.go +++ b/merkle/sparse_merkle_tree_test.go @@ -369,8 +369,8 @@ func testSparseTreeCalculatedRootWithWriter(ctx context.Context, t *testing.T, r if err != nil { t.Fatalf("Failed to commit map changes: %v", err) } - if expected, got := vec.expectedRoot, root; !bytes.Equal(expected, got) { - t.Errorf("Expected root:\n%s, but got root:\n%s", base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(got)) + if got, want := root, vec.expectedRoot; !bytes.Equal(got, want) { + t.Errorf("got root: %x, want %x", got, want) } } diff --git a/scripts/presubmit.sh b/scripts/presubmit.sh index 589d3132b7..b547329350 100755 --- a/scripts/presubmit.sh +++ b/scripts/presubmit.sh @@ -93,7 +93,7 @@ main() { go build ${go_dirs} echo 'running go test' - go test -cover -timeout=5m ${goflags} ${go_dirs} + go test -cover -timeout=5m -short ${goflags} ${go_dirs} fi if [[ "${run_linters}" -eq 1 ]]; then diff --git a/storage/cache/map_subtree_cache.go b/storage/cache/map_subtree_cache.go index ee17dce6c9..c691f2e649 100644 --- a/storage/cache/map_subtree_cache.go +++ b/storage/cache/map_subtree_cache.go @@ -39,33 +39,30 @@ func NewMapSubtreeCache(mapStrata []int, treeID int64, hasher hashers.MapHasher) func populateMapSubtreeNodes(treeID int64, hasher hashers.MapHasher) storage.PopulateSubtreeFunc { return func(st *storagepb.SubtreeProto) error { st.InternalNodes = make(map[string][]byte) - rootID := storage.NewNodeIDFromHash(st.Prefix) leaves := make([]merkle.HStar2LeafHash, 0, len(st.Leaves)) for k64, v := range st.Leaves { - k, err := base64.StdEncoding.DecodeString(k64) + sfx, err := storage.ParseSuffix(k64) if err != nil { return err } - if k[0]%depthQuantum != 0 { - return fmt.Errorf("unexpected non-leaf suffix found: %x", k) + // TODO(gdbelvin): test against subtree depth. + if sfx.Bits%depthQuantum != 0 { + return fmt.Errorf("unexpected non-leaf suffix found: %x", sfx.Bits) } + leaves = append(leaves, merkle.HStar2LeafHash{ + Index: storage.NewNodeIDFromPrefixSuffix(st.Prefix, sfx, hasher.BitLen()).BigInt(), LeafHash: v, - Index: new(big.Int).SetBytes(k[1:]), }) } hs2 := merkle.NewHStar2(treeID, hasher) - offset := hasher.BitLen() - rootID.PrefixLenBits - int(st.Depth) - root, err := hs2.HStar2Nodes(int(st.Depth), offset, leaves, - func(depth int, index *big.Int) ([]byte, error) { - return nil, nil - }, + root, err := hs2.HStar2Nodes(st.Prefix, int(st.Depth), leaves, nil, func(depth int, index *big.Int, h []byte) error { - if depth == 0 && len(st.Prefix) > 0 { + if depth == len(st.Prefix)*8 && len(st.Prefix) > 0 { // no space for the root in the node cache return nil } - nodeID := storage.NewNodeIDFromRelativeBigInt(st.Prefix, int(st.Depth), depth, index, hasher.BitLen()) + nodeID := storage.NewNodeIDFromBigInt(depth, index, hasher.BitLen()) _, sfx := nodeID.Split(len(st.Prefix), int(st.Depth)) sfxKey := sfx.String() if glog.V(4) { diff --git a/storage/cache/subtree_cache.go b/storage/cache/subtree_cache.go index b066958ca5..37d5985506 100644 --- a/storage/cache/subtree_cache.go +++ b/storage/cache/subtree_cache.go @@ -340,7 +340,7 @@ func (s *SubtreeCache) SetNodeHash(id storage.NodeID, h []byte, getSubtree GetSu if err != nil { glog.Errorf("base64.DecodeString(%v): %v", sfxKey, err) } - glog.Infof("SetNodeHash(pfx: %s, sfx: %x): %x", prefixKey, b, h) + glog.Infof("SetNodeHash(pfx: %x, sfx: %x): %x", prefixKey, b, h) } return nil } diff --git a/storage/suffix.go b/storage/suffix.go index 18cd60a1f7..ebf4a7d21b 100644 --- a/storage/suffix.go +++ b/storage/suffix.go @@ -37,3 +37,16 @@ func (s Suffix) String() string { r = append(r, s.Path...) return base64.StdEncoding.EncodeToString(r) } + +// ParseSuffix converts a suffix string back into a Suffix. +func ParseSuffix(s string) (Suffix, error) { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return Suffix{}, err + } + + return Suffix{ + Bits: byte(b[0]), + Path: b[1:], + }, nil +} diff --git a/storage/suffix_test.go b/storage/suffix_test.go index 5477aa053f..1881c6b1a2 100644 --- a/storage/suffix_test.go +++ b/storage/suffix_test.go @@ -29,6 +29,66 @@ const ( // storage/cache when merkle no longer depends on storage.NodeID ) +//h2b6 takes a hex string and emits a base64 string +func h2b6(h string) string { + return base64.StdEncoding.EncodeToString(h2b(h)) +} + +func TestParseSuffix(t *testing.T) { + for _, tc := range []struct { + suffix string + wantBits byte + wantPath []byte + wantErr bool + }{ + {h2b6("0100"), 1, h2b("00"), false}, + {h2b6("0801"), 8, h2b("01"), false}, + {"----", 1, h2b("00"), true}, + } { + sfx, err := ParseSuffix(tc.suffix) + if got, want := err != nil, tc.wantErr; got != want { + t.Errorf("ParseSuffix(%s): %v, wantErr: %v", tc.suffix, err, want) + continue + } + if err != nil { + continue + } + if got, want := sfx.Bits, tc.wantBits; got != want { + t.Errorf("ParseSuffix(%s).Bits: %v, want %v", tc.suffix, got, want) + } + if got, want := sfx.Path, tc.wantPath; !bytes.Equal(got, want) { + t.Errorf("ParseSuffix(%s).Path: %x, want %x", tc.suffix, got, want) + } + } +} + +func TestSplitParseSuffixRoundtrip(t *testing.T) { + for _, tc := range []struct { + prefix []byte + leafIndex int64 + want []byte + }{ + {h2b(""), 1, h2b("0801")}, + {h2b("00"), 1, h2b("0801")}, + } { + nodeID := NewNodeIDFromPrefix(tc.prefix, logStrataDepth, tc.leafIndex, logStrataDepth, maxLogDepth) + _, sfx := nodeID.Split(len(tc.prefix), logStrataDepth) + sfxKey := sfx.String() + + sfxP, err := ParseSuffix(sfxKey) + if err != nil { + t.Errorf("ParseSuffix(%s): %v", sfxKey, err) + continue + } + if got, want := sfx.Bits, sfxP.Bits; got != want { + t.Errorf("ParseSuffix(%s).Bits: %v, want %v", sfxKey, got, want) + } + if got, want := sfx.Path, sfxP.Path; !bytes.Equal(got, want) { + t.Errorf("ParseSuffix(%s).Path: %x, want %x", sfxKey, got, want) + } + } +} + // TestSuffixKeyEquals ensures that NodeID.Split produces the same output as makeSuffixKey for the Log's use cases. func TestSuffixKeyEquals(t *testing.T) { for _, tc := range []struct { diff --git a/storage/types.go b/storage/types.go index ee08149dd5..da27d1076f 100644 --- a/storage/types.go +++ b/storage/types.go @@ -130,28 +130,6 @@ func NewNodeIDFromPrefix(prefix []byte, depth int, index int64, subDepth, totalD } } -// NewNodeIDFromRelativeBigInt returns a NodeID given by a subtree and a subtree index. -// depth is the number of levels down from the top of the subtree -// subIndex is the path from the root of the subtree to the desired node, and continuing down to the bottom of the subtree. -// subIndex = horizontal index << height. -func NewNodeIDFromRelativeBigInt(prefix []byte, subtreeDepth, depth int, subIndex *big.Int, totalDepth int) NodeID { - // Put prefix in the MSB bits of path. - path := make([]byte, totalDepth/8) - copy(path, prefix) - - // Copy subIndex into subPath, right justified. - subPath := path[len(prefix) : len(prefix)+subtreeDepth/8] - unusedSubBytes := len(subPath) - len(subIndex.Bytes()) - copy(subPath[unusedSubBytes:], subIndex.Bytes()) - - glog.V(5).Infof("NewNodeIDFromRelativeBigInt({%x, %v}, %v, %x, %v): %v, %x", - prefix, subtreeDepth, depth, subIndex.Bytes(), totalDepth, len(prefix)*8+depth, path) - return NodeID{ - Path: path, - PrefixLenBits: len(prefix)*8 + depth, - } -} - // NewNodeIDFromBigInt returns a NodeID of a big.Int with no prefix. // index contains the path's least significant bits. // depth indicates the number of bits from the most significant bit to treat as part of the path. @@ -166,6 +144,8 @@ func NewNodeIDFromBigInt(depth int, index *big.Int, totalDepth int) NodeID { copy(path[unusedHighBytes:], index.Bytes()) // TODO(gdbelvin): consider masking off insignificant bits past depth. + glog.V(5).Infof("NewNodeIDFromBigInt(%v, %x, %v): %v, %x", + depth, index.Bytes(), totalDepth, depth, path) return NodeID{ Path: path, @@ -173,6 +153,11 @@ func NewNodeIDFromBigInt(depth int, index *big.Int, totalDepth int) NodeID { } } +// BigInt returns the big.Int for this node. +func (n NodeID) BigInt() *big.Int { + return new(big.Int).SetBytes(n.Path) +} + // NewNodeIDWithPrefix creates a new NodeID of nodeIDLen bits with the prefixLen MSBs set to prefix. // NewNodeIDWithPrefix places the lower prefixLenBits of prefix in the most significant bits of path. // Path will have enough bytes to hold maxLenBits @@ -337,6 +322,18 @@ func (n *NodeID) Siblings() []NodeID { return sibs } +// NewNodeIDFromPrefixSuffix undoes Split() and returns the NodeID. +func NewNodeIDFromPrefixSuffix(prefix []byte, suffix Suffix, maxPathBits int) NodeID { + path := make([]byte, maxPathBits/8) + copy(path, prefix) + copy(path[len(prefix):], suffix.Path) + + return NodeID{ + Path: path, + PrefixLenBits: len(prefix)*8 + int(suffix.Bits), + } +} + // Split splits a NodeID into a prefix and a suffix at prefixSplit func (n *NodeID) Split(prefixBytes, suffixBits int) ([]byte, Suffix) { if n.PrefixLenBits == 0 { diff --git a/storage/types_test.go b/storage/types_test.go index 5fd0c16f59..b7bdba462f 100644 --- a/storage/types_test.go +++ b/storage/types_test.go @@ -90,27 +90,28 @@ func TestSplit(t *testing.T) { outPrefix []byte outSuffixBits int outSuffix []byte + unusedBytes int }{ - {h2b("1234567f"), 32, 3, 8, h2b("123456"), 8, h2b("7f")}, - {h2b("123456ff"), 29, 3, 8, h2b("123456"), 5, h2b("f8")}, - {h2b("123456ff"), 25, 3, 8, h2b("123456"), 1, h2b("80")}, - {h2b("12345678"), 16, 1, 8, h2b("12"), 8, h2b("34")}, - {h2b("12345678"), 9, 1, 8, h2b("12"), 1, h2b("00")}, - {h2b("12345678"), 8, 0, 8, h2b(""), 8, h2b("12")}, - {h2b("12345678"), 7, 0, 8, h2b(""), 7, h2b("12")}, - {h2b("12345678"), 0, 0, 8, h2b(""), 0, h2b("00")}, - {h2b("70"), 2, 0, 8, h2b(""), 2, h2b("40")}, - {h2b("70"), 3, 0, 8, h2b(""), 3, h2b("60")}, - {h2b("70"), 4, 0, 8, h2b(""), 4, h2b("70")}, - {h2b("70"), 5, 0, 8, h2b(""), 5, h2b("70")}, - {h2b("0003"), 16, 1, 8, h2b("00"), 8, h2b("03")}, - {h2b("0003"), 15, 1, 8, h2b("00"), 7, h2b("02")}, - {h2b("0001000000000000"), 16, 1, 8, h2b("00"), 8, h2b("01")}, - {h2b("0100000000000000"), 8, 0, 8, h2b(""), 8, h2b("01")}, + {h2b("1234567f"), 32, 3, 8, h2b("123456"), 8, h2b("7f"), 0}, + {h2b("123456ff"), 29, 3, 8, h2b("123456"), 5, h2b("f8"), 0}, + {h2b("123456ff"), 25, 3, 8, h2b("123456"), 1, h2b("80"), 0}, + {h2b("12345678"), 16, 1, 8, h2b("12"), 8, h2b("34"), 2}, + {h2b("12345678"), 9, 1, 8, h2b("12"), 1, h2b("00"), 2}, + {h2b("12345678"), 8, 0, 8, h2b(""), 8, h2b("12"), 3}, + {h2b("12345678"), 7, 0, 8, h2b(""), 7, h2b("12"), 3}, + {h2b("12345678"), 0, 0, 8, h2b(""), 0, h2b("00"), 3}, + {h2b("70"), 2, 0, 8, h2b(""), 2, h2b("40"), 0}, + {h2b("70"), 3, 0, 8, h2b(""), 3, h2b("60"), 0}, + {h2b("70"), 4, 0, 8, h2b(""), 4, h2b("70"), 0}, + {h2b("70"), 5, 0, 8, h2b(""), 5, h2b("70"), 0}, + {h2b("0003"), 16, 1, 8, h2b("00"), 8, h2b("03"), 0}, + {h2b("0003"), 15, 1, 8, h2b("00"), 7, h2b("02"), 0}, + {h2b("0001000000000000"), 16, 1, 8, h2b("00"), 8, h2b("01"), 6}, + {h2b("0100000000000000"), 8, 0, 8, h2b(""), 8, h2b("01"), 7}, // Map subtree scenarios - {h2b("0100000000000000"), 16, 0, 16, h2b(""), 16, h2b("0100")}, - {h2b("0100000000000000"), 32, 0, 32, h2b(""), 32, h2b("01000000")}, - {h2b("0000000000000000000000000000000000000000000000000000000000000001"), 256, 10, 176, h2b("00000000000000000000"), 176, h2b("00000000000000000000000000000000000000000001")}, + {h2b("0100000000000000"), 16, 0, 16, h2b(""), 16, h2b("0100"), 6}, + {h2b("0100000000000000"), 32, 0, 32, h2b(""), 32, h2b("01000000"), 4}, + {h2b("0000000000000000000000000000000000000000000000000000000000000001"), 256, 10, 176, h2b("00000000000000000000"), 176, h2b("00000000000000000000000000000000000000000001"), 0}, } { n := NewNodeIDFromHash(tc.inPath) n.PrefixLenBits = tc.inPathLenBits @@ -129,39 +130,19 @@ func TestSplit(t *testing.T) { if got, want := s.Path, tc.outSuffix; !bytes.Equal(got, want) { t.Errorf("%d, %x.Split(%v, %v).Path: %x, want %x", tc.inPathLenBits, tc.inPath, tc.splitBytes, tc.suffixBits, got, want) + continue } - } -} -func TestNewNodeIDFromRelativeBigInt(t *testing.T) { - for _, tc := range []struct { - prefix []byte - depth int - index int64 - subDepth int - totalDepth int - wantPath []byte - wantDepth int - }{ - {prefix: h2b(""), depth: 8, index: 0, subDepth: 8, totalDepth: 64, wantPath: h2b("0000000000000000"), wantDepth: 8}, - {prefix: h2b(""), depth: 8, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("0100000000000000"), wantDepth: 8}, - {prefix: h2b("00"), depth: 7, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("0001000000000000"), wantDepth: 15}, - {prefix: h2b("00"), depth: 8, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("0001000000000000"), wantDepth: 16}, - {prefix: h2b("00"), depth: 16, index: 257, subDepth: 16, totalDepth: 64, wantPath: h2b("0001010000000000"), wantDepth: 24}, - {prefix: h2b("12345678"), depth: 8, index: 1, subDepth: 8, totalDepth: 64, wantPath: h2b("1234567801000000"), wantDepth: 40}, - - {prefix: h2b("00"), subDepth: 248, depth: 247, index: 1, totalDepth: 256, wantPath: h2b("0000000000000000000000000000000000000000000000000000000000000001"), wantDepth: 255}, - {prefix: h2b("00000000000000000000"), subDepth: 176, depth: 176, index: 1, totalDepth: 256, wantPath: h2b("0000000000000000000000000000000000000000000000000000000000000001"), wantDepth: 256}, - } { - i := big.NewInt(tc.index) - n := NewNodeIDFromRelativeBigInt(tc.prefix, tc.subDepth, tc.depth, i, tc.totalDepth) - if got, want := n.Path, tc.wantPath; !bytes.Equal(got, want) { - t.Errorf("NewNodeIDFromRelativeBigInt(%x, %v, %v, %v, %v).Path: %x, want %x", - tc.prefix, tc.depth, tc.index, tc.subDepth, tc.totalDepth, got, want) + newNode := NewNodeIDFromPrefixSuffix(p, s, len(tc.inPath)*8) + want := []byte{} + want = append(want, tc.outPrefix...) + want = append(want, tc.outSuffix...) + want = append(want, make([]byte, tc.unusedBytes)...) + if got, want := newNode.Path, want; !bytes.Equal(got, want) { + t.Errorf("NewNodeIDFromPrefix(%x, %v).Path: %x, want %x", p, s, got, want) } - if got, want := n.PrefixLenBits, tc.wantDepth; got != want { - t.Errorf("NewNodeIDFromRelativeBigInt(%x, %v, %v, %v, %v).Depth: %v, want %v", - tc.prefix, tc.depth, tc.index, tc.subDepth, tc.totalDepth, got, want) + if got, want := newNode.PrefixLenBits, n.PrefixLenBits; got != want { + t.Errorf("NewNodeIDFromPrefix(%x, %v).PrefixLenBits: %x, want %x", p, s, got, want) } } }