diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 9559221c3..d59f068bd 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -13,3 +13,4 @@ Antonio Marcedone Cesar Ghali Daniel Ziegler Gary Belvin +Ismail Khoffi diff --git a/cmd/keytransparency-monitor/main.go b/cmd/keytransparency-monitor/main.go index b76685507..c422008c4 100644 --- a/cmd/keytransparency-monitor/main.go +++ b/cmd/keytransparency-monitor/main.go @@ -42,7 +42,10 @@ import ( spb "github.com/google/keytransparency/impl/proto/keytransparency_v1_service" mopb "github.com/google/keytransparency/impl/proto/monitor_v1_service" mupb "github.com/google/keytransparency/impl/proto/mutation_v1_service" - _ "github.com/google/trillian/merkle/coniks" // Register coniks + tlogcli "github.com/google/trillian/client" + "github.com/google/trillian/crypto/keys/der" + _ "github.com/google/trillian/merkle/coniks" // Register coniks + "github.com/google/trillian/merkle/hashers" _ "github.com/google/trillian/merkle/objhasher" // Register objhasher ) @@ -141,13 +144,22 @@ func main() { // Insert handlers for other http paths here. mux := http.NewServeMux() mux.Handle("/", gwmux) + logHasher, err := hashers.NewLogHasher(logTree.GetHashStrategy()) + if err != nil { + glog.Fatalf("Could not initialize log hasher: %v", err) + } + logPubKey, err := der.UnmarshalPublicKey(logTree.GetPublicKey().GetDer()) + if err != nil { + glog.Fatalf("Failed parsing Log public key: %v", err) + } + logVerifier := tlogcli.NewLogVerifier(logHasher, logPubKey) - // initialize the mutations API client and feed the responses it got - // into the monitor: - mon, err := cmon.New(logTree, mapTree, crypto.NewSHA256Signer(key), store) + mon, err := cmon.New(logVerifier, mapTree, crypto.NewSHA256Signer(key), store) if err != nil { glog.Exitf("Failed to initialize monitor: %v", err) } + // initialize the mutations API client and feed the responses it got + // into the monitor: mutCli := client.New(mcc, *pollPeriod) responses, errs := mutCli.StartPolling(1) go func() { diff --git a/core/monitor/monitor.go b/core/monitor/monitor.go index 419936b28..f1981ec3b 100644 --- a/core/monitor/monitor.go +++ b/core/monitor/monitor.go @@ -25,39 +25,39 @@ import ( ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" "github.com/google/trillian" + "github.com/google/trillian/client" tcrypto "github.com/google/trillian/crypto" - "github.com/google/trillian/merkle" + "github.com/google/trillian/crypto/keys/der" "github.com/google/trillian/merkle/hashers" ) // Monitor holds the internal state for a monitor accessing the mutations API // and for verifying its responses. type Monitor struct { - hasher hashers.MapHasher - logPubKey crypto.PublicKey + mapID int64 + mapHasher hashers.MapHasher mapPubKey crypto.PublicKey - logVerifier merkle.LogVerifier + logVerifier client.LogVerifier signer *tcrypto.Signer - // TODO(ismail): update last trusted signed log root - //trusted trillian.SignedLogRoot - store *storage.Storage + trusted *trillian.SignedLogRoot + store *storage.Storage } // New creates a new instance of the monitor. -func New(logTree, mapTree *trillian.Tree, signer *tcrypto.Signer, store *storage.Storage) (*Monitor, error) { - logHasher, err := hashers.NewLogHasher(logTree.GetHashStrategy()) - if err != nil { - return nil, fmt.Errorf("Failed creating LogHasher: %v", err) - } +func New(logverifierCli client.LogVerifier, mapTree *trillian.Tree, signer *tcrypto.Signer, store *storage.Storage) (*Monitor, error) { mapHasher, err := hashers.NewMapHasher(mapTree.GetHashStrategy()) if err != nil { return nil, fmt.Errorf("Failed creating MapHasher: %v", err) } + mapPubKey, err := der.UnmarshalPublicKey(mapTree.GetPublicKey().GetDer()) + if err != nil { + return nil, fmt.Errorf("Could not unmarshal map public key: %v", err) + } return &Monitor{ - hasher: mapHasher, - logVerifier: merkle.NewLogVerifier(logHasher), - logPubKey: logTree.GetPublicKey(), - mapPubKey: mapTree.GetPublicKey(), + logVerifier: logverifierCli, + mapID: mapTree.TreeId, + mapHasher: mapHasher, + mapPubKey: mapPubKey, signer: signer, store: store, }, nil @@ -70,7 +70,7 @@ func (m *Monitor) Process(resp *ktpb.GetMutationsResponse) error { var smr *trillian.SignedMapRoot var err error seen := time.Now().Unix() - errs := m.verifyMutationsResponse(resp) + errs := m.VerifyMutationsResponse(resp) if len(errs) == 0 { glog.Infof("Successfully verified mutations response for epoch: %v", resp.Epoch) smr, err = m.signMapRoot(resp) diff --git a/core/monitor/verify.go b/core/monitor/verify.go index 1deadfb45..66648ebbf 100644 --- a/core/monitor/verify.go +++ b/core/monitor/verify.go @@ -18,14 +18,190 @@ package monitor import ( + "bytes" + "encoding/json" + "errors" + "math/big" + + "github.com/google/keytransparency/core/mutator/entry" + + "github.com/golang/glog" + tcrypto "github.com/google/trillian/crypto" + "github.com/google/trillian/merkle" + "github.com/google/trillian/storage" + ktpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" ) -// verifyMutationsResponse verifies a response received by the GetMutations API. +var ( + // ErrInconsistentProofs occurs when the server returned different hashes + // for the same inclusion proof node in the tree. + ErrInconsistentProofs = errors.New("inconsistent inclusion proofs") + // ErrInvalidLogConsistencyProof occurs when the log consistency proof does + // not verify. + ErrInvalidLogConsistencyProof = errors.New("invalid log consistency proof") + // ErrInvalidLogInclusion occurs if the inclusion proof for the signed map + // root into the log does not verify. + ErrInvalidLogInclusion = errors.New("invalid log inclusion proof") + // ErrInvalidLogSignature occurs if the log roots signature does not verify. + ErrInvalidLogSignature = errors.New("invalid signature on log root") + // ErrInvalidMapSignature occurs if the map roots signature does not verify. + ErrInvalidMapSignature = errors.New("invalid signature on map root") + // ErrInvalidMutation occurs when verification failed because of an invalid + // mutation. + ErrInvalidMutation = errors.New("invalid mutation") + // ErrNotMatchingMapRoot occurs when the reconstructed root differs from the + // one we received from the server. + ErrNotMatchingMapRoot = errors.New("recreated root does not match") +) + +// VerifyMutationsResponse verifies a response received by the GetMutations API. // Additionally to the response it takes a complete list of mutations. The list // of received mutations may differ from those included in the initial response -// because of the max. page size. If any verification check failed it returns -// an error. -func (m *Monitor) verifyMutationsResponse(in *ktpb.GetMutationsResponse) []error { +// because of the max. page size. +func (m *Monitor) VerifyMutationsResponse(in *ktpb.GetMutationsResponse) []error { + errList := make([]error, 0) + + if m.trusted == nil { + m.trusted = in.GetLogRoot() + } + + if err := m.logVerifier.VerifyRoot(m.trusted, in.GetLogRoot(), in.GetLogConsistency()); err != nil { + // this could be one of ErrInvalidLogSignature, ErrInvalidLogConsistencyProof + errList = append(errList, err) + } + // updated trusted log root + m.trusted = in.GetLogRoot() + + b, err := json.Marshal(in.GetSmr()) + if err != nil { + glog.Errorf("json.Marshal(): %v", err) + errList = append(errList, ErrInvalidMapSignature) + } + leafIndex := in.GetSmr().GetMapRevision() + treeSize := in.GetLogRoot().GetTreeSize() + err = m.logVerifier.VerifyInclusionAtIndex(in.GetLogRoot(), b, leafIndex, in.GetLogInclusion()) + if err != nil { + glog.Errorf("m.logVerifier.VerifyInclusionAtIndex((%v, %v, _): %v", leafIndex, treeSize, err) + errList = append(errList, ErrInvalidLogInclusion) + } + + // copy of singed map root + smr := *in.GetSmr() + // reset to the state before it was signed: + smr.Signature = nil + // verify signature on map root: + if err := tcrypto.VerifyObject(m.mapPubKey, smr, in.GetSmr().GetSignature()); err != nil { + glog.Infof("couldn't verify signature on map root: %v", err) + errList = append(errList, ErrInvalidMapSignature) + } + + // we need the old root for verifying the inclusion of the old leafs in the + // previous epoch. Storage always stores the mutations response independent + // from if the checks succeeded or not. + var oldRoot []byte + // mutations happen after epoch 1 which is stored in storage: + if m.store.LatestEpoch() > 0 { + // retrieve the old root hash from storage + monRes, err := m.store.Get(in.Epoch - 1) + if err != nil { + glog.Infof("Could not retrieve previous monitoring result: %v", err) + } + oldRoot = monRes.Response.GetSmr().GetRootHash() + + if err := m.verifyMutations(in.GetMutations(), oldRoot, + in.GetSmr().GetRootHash(), in.GetSmr().GetMapId()); len(err) > 0 { + errList = append(errList, err...) + } + } + + return errList +} + +func (m *Monitor) verifyMutations(muts []*ktpb.Mutation, oldRoot, expectedNewRoot []byte, mapID int64) []error { + errList := make([]error, 0) + mutator := entry.New() + oldProofNodes := make(map[string][]byte) + newLeaves := make([]merkle.HStar2LeafHash, 0, len(muts)) + glog.Infof("verifyMutations() called with %v mutations.", len(muts)) + + for _, mut := range muts { + oldLeaf, err := entry.FromLeafValue(mut.GetProof().GetLeaf().GetLeafValue()) + if err != nil { + errList = append(errList, ErrInvalidMutation) + } + + // verify that the provided leaf’s inclusion proof goes to epoch e-1: + index := mut.GetProof().GetLeaf().GetIndex() + leaf := mut.GetProof().GetLeaf().GetLeafValue() + if err := merkle.VerifyMapInclusionProof(mapID, index, + leaf, oldRoot, mut.GetProof().GetInclusion(), m.mapHasher); err != nil { + glog.Infof("VerifyMapInclusionProof(%x): %v", index, err) + errList = append(errList, ErrInvalidMutation) + } + + // compute the new leaf + newLeaf, err := mutator.Mutate(oldLeaf, mut.GetUpdate()) + if err != nil { + glog.Infof("Mutation did not verify: %v", err) + errList = append(errList, ErrInvalidMutation) + } + newLeafnID := storage.NewNodeIDFromPrefixSuffix(index, storage.Suffix{}, m.mapHasher.BitLen()) + newLeafHash := m.mapHasher.HashLeaf(mapID, index, newLeaf) + newLeaves = append(newLeaves, merkle.HStar2LeafHash{ + Index: newLeafnID.BigInt(), + LeafHash: newLeafHash, + }) + + // store the proof hashes locally to recompute the tree below: + sibIDs := newLeafnID.Siblings() + proofs := mut.GetProof().GetInclusion() + for level, sibID := range sibIDs { + proof := proofs[level] + if p, ok := oldProofNodes[sibID.String()]; ok { + // sanity check: for each mut overlapping proof nodes should be + // equal: + if !bytes.Equal(p, proof) { + // this is really odd and should never happen + errList = append(errList, ErrInconsistentProofs) + } + } else { + if len(proof) > 0 { + oldProofNodes[sibID.String()] = proof + } + } + } + } + + if err := m.validateMapRoot(expectedNewRoot, mapID, newLeaves, oldProofNodes); err != nil { + errList = append(errList, err) + } + + return errList +} + +func (m *Monitor) validateMapRoot(expectedRoot []byte, mapID int64, mutatedLeaves []merkle.HStar2LeafHash, oldProofNodes map[string][]byte) error { + // compute the new root using local intermediate hashes from epoch e + // (above proof hashes): + hs2 := merkle.NewHStar2(mapID, m.mapHasher) + newRoot, err := hs2.HStar2Nodes([]byte{}, m.mapHasher.BitLen(), mutatedLeaves, + func(depth int, index *big.Int) ([]byte, error) { + nID := storage.NewNodeIDFromBigInt(depth, index, m.mapHasher.BitLen()) + if p, ok := oldProofNodes[nID.String()]; ok { + return p, nil + } + return nil, nil + }, nil) + + if err != nil { + glog.Errorf("hs2.HStar2Nodes(_): %v", err) + return ErrNotMatchingMapRoot + } + + // verify rootHash + if !bytes.Equal(newRoot, expectedRoot) { + return ErrNotMatchingMapRoot + } + return nil } diff --git a/core/mutation/mutation.go b/core/mutation/mutation.go index a8fa2e548..f190270fb 100644 --- a/core/mutation/mutation.go +++ b/core/mutation/mutation.go @@ -106,7 +106,13 @@ func (s *Server) GetMutations(ctx context.Context, in *tpb.GetMutationsRequest) } // Get leaf proofs. // TODO: allow leaf proofs to be optional. - proofs, err := s.inclusionProofs(ctx, indexes, in.Epoch) + var epoch int64 + if in.Epoch > 1 { + epoch = in.Epoch - 1 + } else { + epoch = 1 + } + proofs, err := s.inclusionProofs(ctx, indexes, epoch) if err != nil { return nil, err } @@ -114,10 +120,10 @@ func (s *Server) GetMutations(ctx context.Context, in *tpb.GetMutationsRequest) mutations[i].Proof = p } - // MapRevisions start at 1. Log leave's index starts at 0. + // MapRevisions start at 1. Log leave's index starts at 1. // MapRevision should be at least 1 since the Signer is // supposed to create at least one revision on startup. - respEpoch := resp.GetMapRoot().GetMapRevision() - 1 + respEpoch := resp.GetMapRoot().GetMapRevision() // Fetch log proofs. logRoot, logConsistency, logInclusion, err := s.logProofs(ctx, in.GetFirstTreeSize(), respEpoch) if err != nil { diff --git a/impl/monitor/client/client.go b/impl/monitor/client/client.go index 18245eb76..13e15be5e 100644 --- a/impl/monitor/client/client.go +++ b/impl/monitor/client/client.go @@ -64,9 +64,9 @@ func (c *Client) StartPolling(startEpoch int64) (<-chan *ktpb.GetMutationsRespon glog.Infof("Polling: %v", now) // time out if we exceed the poll period: ctx, _ := context.WithTimeout(context.Background(), c.pollPeriod) - monitorResp, err := c.pollMutations(ctx, epoch) + monitorResp, err := c.PollMutations(ctx, epoch) if err != nil { - glog.Infof("pollMutations(_): %v", err) + glog.Infof("PollMutations(_): %v", err) errChan <- err } else { // only write to results channel and increment epoch @@ -80,7 +80,9 @@ func (c *Client) StartPolling(startEpoch int64) (<-chan *ktpb.GetMutationsRespon return response, errChan } -func (c *Client) pollMutations(ctx context.Context, +// PollMutations polls GetMutationsResponses from the configured +// key-transparency server's mutations API. +func (c *Client) PollMutations(ctx context.Context, queryEpoch int64, opts ...grpc.CallOption) (*ktpb.GetMutationsResponse, error) { response, err := c.client.GetMutations(ctx, &ktpb.GetMutationsRequest{ diff --git a/integration/monitor_test.go b/integration/monitor_test.go new file mode 100644 index 000000000..2b3ecc5a1 --- /dev/null +++ b/integration/monitor_test.go @@ -0,0 +1,118 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integration + +import ( + "context" + "testing" + "time" + + "github.com/google/keytransparency/core/monitor" + "github.com/google/keytransparency/core/monitor/storage" + kpb "github.com/google/keytransparency/core/proto/keytransparency_v1_types" + "github.com/google/keytransparency/impl/monitor/client" + spb "github.com/google/keytransparency/impl/proto/keytransparency_v1_service" + mupb "github.com/google/keytransparency/impl/proto/mutation_v1_service" + "github.com/google/trillian/crypto" + "github.com/google/trillian/crypto/keys/pem" + + "github.com/google/keytransparency/cmd/keytransparency-client/grpcc" + "github.com/google/keytransparency/core/crypto/signatures" + "github.com/google/keytransparency/core/fake" +) + +const ( + monitorPrivKey = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIAV7H3qRi/cj/w04vEQBFjLdhcXRbZR4ouT5zaAy1XUHoAoGCCqGSM49 +AwEHoUQDQgAEqUDbATN2maGIm6YQLpjx67bYN1hxPPdF0VrPTZe36yQhH+GCwZQV +amFdON6OhjYnBmJWe4fVnbxny0PfpkvXtg== +-----END EC PRIVATE KEY-----` +) + +func TestMonitor(t *testing.T) { + bctx := context.Background() + env := NewEnv(t) + defer env.Close(t) + env.Client.RetryCount = 0 + c := spb.NewKeyTransparencyServiceClient(env.Conn) + // setup monitor: + resp, err := c.GetDomainInfo(bctx, &kpb.GetDomainInfoRequest{}) + if err != nil { + t.Fatalf("Couldn't retrieve domain info: %v", err) + } + signer, err := pem.UnmarshalPrivateKey(monitorPrivKey, "") + if err != nil { + t.Fatalf("Couldn't create signer: %v", err) + } + logTree := resp.Log + mapTree := resp.Map + _ = logTree + store := storage.New() + // TODO(ismail): setup and use a real logVerifier instead: + mon, err := monitor.New(fake.NewFakeTrillianLogVerifier(), mapTree, crypto.NewSHA256Signer(signer), store) + if err != nil { + t.Fatalf("Couldn't create monitor: %v", err) + } + mcc := mupb.NewMutationServiceClient(env.Conn) + mutCli := client.New(mcc, time.Second) + + for _, tc := range []struct { + // the userIDs to update, if no userIDs are provided, no update request + // will be send before querying + userIDs []string + updateData []byte + signers []signatures.Signer + authorizedKeys []*kpb.PublicKey + // the epoch to query after sending potential updates + queryEpoch int64 + }{ + // query first epoch, don't update + {[]string{}, nil, nil, nil, 1}, + // create one mutation and new epoch (not forced like in sequencer): + {[]string{"test@test.com"}, []byte("testData"), []signatures.Signer{createSigner(t, testPrivKey1)}, []*kpb.PublicKey{getAuthorizedKey(testPubKey1)}, 2}, + // create several mutations and new epoch + {[]string{"test@test.com", "test2@test2.com"}, []byte("more update data"), []signatures.Signer{createSigner(t, testPrivKey1)}, []*kpb.PublicKey{getAuthorizedKey(testPubKey1)}, 3}, + } { + for _, userID := range tc.userIDs { + _, err = env.Client.Update(GetNewOutgoingContextWithFakeAuth(userID), + userID, appID, tc.updateData, tc.signers, tc.authorizedKeys) + if err != grpcc.ErrRetry { + t.Fatalf("Could not send update request: %v", err) + } + } + + if err := env.Signer.CreateEpoch(bctx, false); err != nil { + t.Fatalf("CreateEpoch(_): %v", err) + } + + mutResp, err := mutCli.PollMutations(bctx, tc.queryEpoch) + if err != nil { + t.Fatalf("Could not query mutations: %v", err) + } + + if err := mon.Process(mutResp); err != nil { + t.Fatalf("Monitor could not process mutations: %v", err) + } + + mresp, err := store.Get(tc.queryEpoch) + if err != nil { + t.Fatalf("Could not read monitoring response: %v", err) + } + + for _, err := range mresp.Errors { + t.Errorf("Got error: %v", err) + } + } +} diff --git a/integration/testutil.go b/integration/testutil.go index 5a0198f3e..9b5d53412 100644 --- a/integration/testutil.go +++ b/integration/testutil.go @@ -43,14 +43,13 @@ import ( _ "github.com/mattn/go-sqlite3" // Use sqlite database for testing. + cmutation "github.com/google/keytransparency/core/mutation" + "github.com/google/keytransparency/impl/mutation" pb "github.com/google/keytransparency/impl/proto/keytransparency_v1_service" + mpb "github.com/google/keytransparency/impl/proto/mutation_v1_service" stestonly "github.com/google/trillian/storage/testonly" ) -const ( - logID = 0 -) - // NewDB creates a new in-memory database for testing. func NewDB(t testing.TB) *sql.DB { db, err := sql.Open("sqlite3", "file:dummy.db?mode=memory&cache=shared") @@ -130,12 +129,20 @@ func NewEnv(t *testing.T) *Env { t.Fatalf("CreateTree(): %v", err) } mapID := tree.TreeId - mapPubKey, err := der.UnmarshalPublicKey(tree.GetPublicKey().GetDer()) if err != nil { t.Fatalf("Failed to load signing keypair: %v", err) } + // Configure log. + logTree, err := mapEnv.AdminClient.CreateTree(ctx, &trillian.CreateTreeRequest{ + Tree: stestonly.LogTree, + }) + if err != nil { + t.Fatalf("CreateTree(): %v", err) + } + logID := logTree.GetTreeId() + // Common data structures. mutations, err := mutations.New(sqldb, mapID) if err != nil { @@ -152,15 +159,15 @@ func NewEnv(t *testing.T) *Env { t.Fatalf("Failed to create committer: %v", err) } authz := authorization.New() - tlog := fake.NewFakeTrillianLogClient() - tadmin := trillian.NewTrillianAdminClient(nil) factory := transaction.NewFactory(sqldb) - server := keyserver.New(logID, tlog, mapID, mapEnv.MapClient, tadmin, commitments, + server := keyserver.New(logID, tlog, mapID, mapEnv.MapClient, mapEnv.AdminClient, commitments, vrfPriv, mutator, auth, authz, factory, mutations) s := grpc.NewServer() + msrv := mutation.New(cmutation.New(logID, mapID, tlog, mapEnv.MapClient, mutations, factory)) pb.RegisterKeyTransparencyServiceServer(s, server) + mpb.RegisterMutationServiceServer(s, msrv) // Signer signer := sequencer.New(mapID, mapEnv.MapClient, logID, tlog, mutator, mutations, factory)