Move NodeID to it's own package

integration/maptest/map_test.go

@@ -88,7 +88,7 @@ func verifyGetMapLeavesResponse(getResp *trillian.GetMapLeavesResponse, indexes
 		leafHash := incl.GetLeaf().GetLeafHash()
 		proof := incl.GetInclusion()
 
-		if got, want := leafHash, hasher.HashLeaf(treeID, index, hasher.BitLen(), leaf); !bytes.Equal(got, want) {
+		if got, want := leafHash, hasher.HashLeaf(treeID, index, 0, leaf); !bytes.Equal(got, want) {
 			return fmt.Errorf("HashLeaf(%s): %x, want %x", leaf, got, want)
 		}
 		if err := merkle.VerifyMapInclusionProof(treeID, index,
@@ -140,6 +140,22 @@ func TestInclusion(t *testing.T) {
 				{Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")},
 			},
 		},
+		{
+			desc:         "CONIKS across subtrees",
+			HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256,
+			leaves: []*trillian.MapLeaf{
+				{Index: h2b("0000000000000180000000000000000000000000000000000000000000000000"), LeafValue: []byte("Z")},
+			},
+		},
+		{
+			desc:         "CONIKS multi",
+			HashStrategy: trillian.HashStrategy_CONIKS_SHA512_256,
+			leaves: []*trillian.MapLeaf{
+				{Index: h2b("0000000000000000000000000000000000000000000000000000000000000000"), LeafValue: []byte("A")},
+				{Index: h2b("0000000000000000000000000000000000000000000000000000000000000001"), LeafValue: []byte("B")},
+				{Index: h2b("0000000000000000000000000000000000000000000000000000000000000002"), LeafValue: []byte("C")},
+			},
+		},
 	} {
 		tree, hasher, err := newTreeWithHasher(ctx, env, tc.HashStrategy)
 		if err != nil {
@@ -192,6 +208,7 @@ func TestInclusionBatch(t *testing.T) {
 		HashStrategy          trillian.HashStrategy
 		batchSize, numBatches int
 	}{
+
 		{
 			desc:         "maphasher batch",
 			HashStrategy: trillian.HashStrategy_TEST_MAP_HASHER,
@@ -206,7 +223,7 @@ func TestInclusionBatch(t *testing.T) {
 		}
 
 		if err := RunMapBatchTest(ctx, env, tree, tc.batchSize, tc.numBatches); err != nil {
-			t.Errorf("%v: %v", tc.desc, err)
+			t.Errorf("BatchSize: %v, Batches: %v: %v", tc.batchSize, tc.numBatches, err)
 		}
 	}
 }
@@ -356,8 +373,7 @@ func TestNonExistentLeaf(t *testing.T) {
 				t.Errorf("len(leaf): %v, want, %v", got, want)
 			}
 
-			if got, want := leafHash,
-				hasher.HashLeaf(tree.TreeId, index, hasher.BitLen(), leaf); !bytes.Equal(got, want) {
+			if got, want := leafHash, hasher.HashLeaf(tree.TreeId, index, 0, leaf); !bytes.Equal(got, want) {
 				t.Errorf("HashLeaf(%s): %x, want %x", leaf, got, want)
 			}
 			if err := merkle.VerifyMapInclusionProof(tree.TreeId, index,

log/sequencer.go

@@ -30,6 +30,7 @@ import (
 	"github.com/google/trillian/merkle"
 	"github.com/google/trillian/merkle/hashers"
 	"github.com/google/trillian/monitoring"
+	"github.com/google/trillian/node"
 	"github.com/google/trillian/quota"
 	"github.com/google/trillian/storage"
 	"github.com/google/trillian/util"
@@ -139,7 +140,7 @@ func (s Sequencer) buildMerkleTreeFromStorageAtRoot(ctx context.Context, root tr
 			glog.Warningf("%v: Failed to create nodeID: %v", root.LogId, err)
 			return nil, err
 		}
-		nodes, err := tx.GetMerkleNodes(ctx, root.TreeRevision, []storage.NodeID{nodeID})
+		nodes, err := tx.GetMerkleNodes(ctx, root.TreeRevision, []node.NodeID{nodeID})
 
 		if err != nil {
 			glog.Warningf("%v: Failed to get Merkle nodes: %v", root.LogId, err)

log/sequencer_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/google/trillian/crypto/keys"
 	"github.com/google/trillian/crypto/sigpb"
 	"github.com/google/trillian/merkle/rfc6962"
+	"github.com/google/trillian/node"
 	"github.com/google/trillian/quota"
 	"github.com/google/trillian/storage"
 	stestonly "github.com/google/trillian/storage/testonly"
@@ -59,10 +60,10 @@ var testRoot16 = trillian.SignedLogRoot{
 
 // These will be accepted in either order because of custom sorting in the mock
 var updatedNodes = []storage.Node{
-	{NodeID: storage.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10}, PrefixLenBits: 64},
+	{NodeID: node.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10}, PrefixLenBits: 64},
 		Hash: testonly.MustDecodeBase64("L5Iyd7aFOVewxiRm29xD+EU+jvEo4RfufBijKdflWMk="), NodeRevision: 6},
 	{
-		NodeID: storage.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, PrefixLenBits: 59},
+		NodeID: node.NodeID{Path: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, PrefixLenBits: 59},
 		Hash:   testonly.MustDecodeBase64("R57DrKTGuZdjCNXjv6InGrm4rABLOn9yWpdHmYOoLwU="), NodeRevision: 6},
 }
 

merkle/common.go

@@ -32,3 +32,37 @@ func bit(index []byte, i int) uint {
 	bIndex := (IndexBits - i - 1) / 8
 	return uint((index[bIndex] >> uint(i%8)) & 0x01)
 }
+
+// flipBit returns index with the i'th bit from the right flipped.
+func flipBit(index []byte, i int) []byte {
+	r := make([]byte, len(index))
+	copy(r, index)
+	IndexBits := len(index) * 8
+	bIndex := (IndexBits - i - 1) / 8
+	r[bIndex] ^= 1 << uint(i%8)
+	return r
+}
+
+// Neighbor returns index with only the left i bits set and the i'th bit flipped.
+func Neighbor(index []byte, i int) []byte {
+	r := flipBit(index, i)
+	return MaskIndex(r, len(index)*8-i)
+}
+
+// leftmask contains bitmasks indexed such that the left x bits are set. It is
+// indexed by byte position from 0-7 0 is special cased to 0xFF since 8 mod 8
+// is 0. leftmask is only used to mask the last byte.
+var leftmask = [8]byte{0xFF, 0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE}
+
+// MaskIndex returns index with only the left depth bits set.
+func MaskIndex(index []byte, depth int) []byte {
+	r := make([]byte, len(index))
+	if depth > 0 {
+		// Copy the first depthBytes.
+		depthBytes := (depth + 7) >> 3
+		copy(r, index[:depthBytes])
+		// Mask off unwanted bits in the last byte.
+		r[depthBytes-1] = r[depthBytes-1] & leftmask[depth%8]
+	}
+	return r
+}

merkle/common_test.go

@@ -15,6 +15,7 @@
 package merkle
 
 import (
+	"bytes"
 	"testing"
 )
 
@@ -41,3 +42,57 @@ func TestBit(t *testing.T) {
 		}
 	}
 }
+
+func TestFlipBit(t *testing.T) {
+	for _, tc := range []struct {
+		index []byte
+		i     int
+		want  []byte
+	}{
+		{index: h2b("00"), i: 0, want: h2b("01")},
+		{index: h2b("00"), i: 7, want: h2b("80")},
+		{index: h2b("000b"), i: 0, want: h2b("000a")},
+		{index: h2b("000b"), i: 1, want: h2b("0009")},
+		{index: h2b("000b"), i: 2, want: h2b("000f")},
+		{index: h2b("000b"), i: 3, want: h2b("0003")},
+		{index: h2b("0001"), i: 0, want: h2b("0000")},
+		{index: h2b("8000"), i: 15, want: h2b("0000")},
+		{index: h2b("0000000000000001"), i: 0, want: h2b("0000000000000000")},
+		{index: h2b("0000000000010000"), i: 16, want: h2b("0000000000000000")},
+		{index: h2b("8000000000000000"), i: 63, want: h2b("0000000000000000")},
+	} {
+		if got, want := flipBit(tc.index, tc.i), tc.want; !bytes.Equal(got, want) {
+			t.Errorf("flipBit(%x, %d): %x, want %x", tc.index, tc.i, got, want)
+		}
+	}
+}
+
+func TestMaskIndex(t *testing.T) {
+	for _, tc := range []struct {
+		index []byte
+		depth int
+		want  []byte
+	}{
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 0, want: h2b("0000000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 1, want: h2b("8000000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 2, want: h2b("C000000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 3, want: h2b("E000000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 4, want: h2b("F000000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 5, want: h2b("F800000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 6, want: h2b("FC00000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 7, want: h2b("FE00000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 8, want: h2b("FF00000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 9, want: h2b("FF80000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 10, want: h2b("FFC0000000000000000000000000000000000000")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 159, want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE")},
+		{index: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), depth: 160, want: h2b("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")},
+		{index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 1, want: h2b("0000000000000000000000000000000000000000")},
+		{index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 17, want: h2b("0001000000000000000000000000000000000000")},
+		{index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 159, want: h2b("000102030405060708090A0B0C0D0E0F10111212")},
+		{index: h2b("000102030405060708090A0B0C0D0E0F10111213"), depth: 160, want: h2b("000102030405060708090A0B0C0D0E0F10111213")},
+	} {
+		if got, want := MaskIndex(tc.index, tc.depth), tc.want; !bytes.Equal(got, want) {
+			t.Errorf("maskIndex(%x, %v): %x, want %x", tc.index, tc.depth, got, want)
+		}
+	}
+}

merkle/hstar2.go

@@ -22,13 +22,15 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/google/trillian/merkle/hashers"
+	"github.com/google/trillian/node"
+	"github.com/google/trillian/storage"
 )
 
 var (
-	// ErrNegativeTreeLevelOffset indicates a negative level was specified.
-	ErrNegativeTreeLevelOffset = errors.New("treeLevelOffset cannot be negative")
-	smtOne                     = big.NewInt(1)
-	smtZero                    = big.NewInt(0)
+	// ErrSubtreeOverrun indicates that a subtree exceeds the maximum tree depth.
+	ErrSubtreeOverrun = errors.New("subtree with prefix exceeds maximum tree size")
+	smtOne            = big.NewInt(1)
+	smtZero           = big.NewInt(0)
 )
 
 // HStar2LeafHash represents a leaf for the HStar2 sparse Merkle tree
@@ -54,15 +56,11 @@ func NewHStar2(treeID int64, hasher hashers.MapHasher) HStar2 {
 	}
 }
 
-// HStar2Root calculates the root of a sparse Merkle tree of depth n which contains
-// the given set of non-null leaves.
-func (s *HStar2) HStar2Root(n int, values []HStar2LeafHash) ([]byte, error) {
+// HStar2Root calculates the root of a sparse Merkle tree of a given depth
+// which contains the given set of non-null leaves.
+func (s *HStar2) HStar2Root(depth int, values []HStar2LeafHash) ([]byte, error) {
 	sort.Sort(ByIndex{values})
-	return s.hStar2b(n, values, smtZero,
-		func(depth int, index *big.Int) ([]byte, error) {
-			return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth), nil
-		},
-		func(int, *big.Int, []byte) error { return nil })
+	return s.hStar2b(0, depth, values, smtZero, nil, nil)
 }
 
 // SparseGetNodeFunc should return any pre-existing node hash for the node address.
@@ -76,78 +74,87 @@ type SparseSetNodeFunc func(depth int, index *big.Int, hash []byte) error
 // internal node values. Values must not contain multiple leaves for the same
 // index.
 //
-// The treeLevelOffset argument is used when the tree to be calculated is part
-// of a larger tree. It identifes the level in the larger tree at which the
-// root of the subtree being calculated is found.
-// e.g. Imagine a tree 256 levels deep, and that you already (somehow) happen
-// to have the intermediate hash values for the non-null nodes 8 levels below
-// the root already calculated (i.e. you just need to calculate the top 8
-// levels of a 256-level tree).  To do this, you'd set treeDepth=8, and
-// treeLevelOffset=248 (256-8).
-func (s *HStar2) HStar2Nodes(treeDepth, treeLevelOffset int, values []HStar2LeafHash, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) {
+// prefix is the location of this subtree within the larger tree. Root is at nil.
+// subtreeDepth is the number of levels in this subtree.
+func (s *HStar2) HStar2Nodes(prefix []byte, subtreeDepth int, values []HStar2LeafHash,
+	get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) {
 	if glog.V(3) {
-		glog.Infof("HStar2Nodes(%v, %v, %v)", treeDepth, treeLevelOffset, len(values))
+		glog.Infof("HStar2Nodes(%x, %v, %v)", prefix, subtreeDepth, len(values))
 		for _, v := range values {
 			glog.Infof("  %x: %x", v.Index.Bytes(), v.LeafHash)
 		}
 	}
-	if treeLevelOffset < 0 {
-		return nil, ErrNegativeTreeLevelOffset
+	depth := len(prefix) * 8
+	totalDepth := depth + subtreeDepth
+	if totalDepth > s.hasher.BitLen() {
+		return nil, ErrSubtreeOverrun
 	}
 	sort.Sort(ByIndex{values})
-	return s.hStar2b(treeDepth, values, smtZero,
-		func(depth int, index *big.Int) ([]byte, error) {
-			// if we've got a function for getting existing node values, try it:
-			h, err := get(treeDepth-depth, index)
-			if err != nil {
-				return nil, err
-			}
-			// if we got a value then we'll use that
-			if h != nil {
-				return h, nil
-			}
-			// otherwise just return the null hash for this level
-			return s.hasher.HashEmpty(s.treeID, PaddedBytes(index, s.hasher.Size()), depth+treeLevelOffset), nil
-		},
-		func(depth int, index *big.Int, hash []byte) error {
-			return set(treeDepth-depth, index, hash)
-		})
+	offset := node.NewNodeIDFromPrefixSuffix(prefix, node.Suffix{}, s.hasher.BitLen()).BigInt()
+	return s.hStar2b(depth, totalDepth, values, offset, get, set)
 }
 
-// hStar2b is the recursive implementation for calculating a sparse Merkle tree
-// root value.
-func (s *HStar2) hStar2b(n int, values []HStar2LeafHash, offset *big.Int, get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) {
-	if n == 0 {
+// hStar2b computes a sparse Merkle tree root value recursively.
+func (s *HStar2) hStar2b(depth, maxDepth int, values []HStar2LeafHash, offset *big.Int,
+	get SparseGetNodeFunc, set SparseSetNodeFunc) ([]byte, error) {
+	if depth == maxDepth {
 		switch {
 		case len(values) == 0:
-			return get(n, offset)
-		case len(values) != 1:
+			return s.get(offset, depth, get)
+		case len(values) == 1:
+			return values[0].LeafHash, nil
+		default:
 			return nil, fmt.Errorf("hStar2b base case: len(values): %d, want 1", len(values))
 		}
-		return values[0].LeafHash, nil
 	}
 	if len(values) == 0 {
-		return get(n, offset)
+		return s.get(offset, depth, get)
 	}
 
-	split := new(big.Int).Lsh(smtOne, uint(n-1))
+	bitsLeft := s.hasher.BitLen() - depth
+	split := new(big.Int).Lsh(smtOne, uint(bitsLeft-1))
 	split.Add(split, offset)
 	i := sort.Search(len(values), func(i int) bool { return values[i].Index.Cmp(split) >= 0 })
-	lhs, err := s.hStar2b(n-1, values[:i], offset, get, set)
+	lhs, err := s.hStar2b(depth+1, maxDepth, values[:i], offset, get, set)
 	if err != nil {
 		return nil, err
 	}
-	rhs, err := s.hStar2b(n-1, values[i:], split, get, set)
+	rhs, err := s.hStar2b(depth+1, maxDepth, values[i:], split, get, set)
 	if err != nil {
 		return nil, err
 	}
 	h := s.hasher.HashChildren(lhs, rhs)
-	if set != nil {
-		set(n, offset, h)
-	}
+	s.set(offset, depth, h, set)
 	return h, nil
 }
 
+// get attempts to use getter. If getter fails, returns the HashEmpty value.
+func (s *HStar2) get(index *big.Int, depth int, getter SparseGetNodeFunc) ([]byte, error) {
+	// if we've got a function for getting existing node values, try it:
+	if getter != nil {
+		h, err := getter(depth, index)
+		if err != nil {
+			return nil, err
+		}
+		// if we got a value then we'll use that
+		if h != nil {
+			return h, nil
+		}
+	}
+	// TODO(gdbelvin): Hashers should accept depth as their main argument.
+	height := s.hasher.BitLen() - depth
+	nodeID := storage.NewNodeIDFromBigInt(index.BitLen(), index, s.hasher.BitLen())
+	return s.hasher.HashEmpty(s.treeID, nodeID.Path, height), nil
+}
+
+// set attempts to use setter if it not nil.
+func (s *HStar2) set(index *big.Int, depth int, hash []byte, setter SparseSetNodeFunc) error {
+	if setter != nil {
+		return setter(depth, index, hash)
+	}
+	return nil
+}
+
 // HStar2LeafHash sorting boilerplate below.
 
 // Leaves is a slice of HStar2LeafHash
@@ -164,13 +171,3 @@ type ByIndex struct{ Leaves }
 
 // Less returns true if i.Index < j.Index
 func (s ByIndex) Less(i, j int) bool { return s.Leaves[i].Index.Cmp(s.Leaves[j].Index) < 0 }
-
-// PaddedBytes takes a big.Int and returns it's value, left padded with zeros.
-// e.g. 1 -> 0000000000000000000000000000000000000001
-func PaddedBytes(i *big.Int, size int) []byte {
-	b := i.Bytes()
-	ret := make([]byte, size)
-	padBytes := len(ret) - len(b)
-	copy(ret[padBytes:], b)
-	return ret
-}