From 594cbdb7b2f44441b7a59ebb6034ace46b81e5a3 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 17 Aug 2017 14:32:58 +0100 Subject: [PATCH 1/5] Use raw trillian.Log connection in Signer Previously, the signer used a verifying trillian log client in the signer. This: - Required a separate build step to acquire the verifying key - Was tied to a confusing and only partially implemented multi-tennant configuration system. This commit simplifies the situation to make a pure grpc call to the trillian log backend, simplifying debugging, and makes things consistent consistent with the way we are querying the trillian map server, and is also consistent with the way the Certificate Transparency frontends treat their own trillian backends. --- README.md | 1 - cmd/keytransparency-signer/main.go | 26 +++++++------------------- core/fake/trillian_log_client.go | 2 +- core/signer/signer.go | 30 +++++++++++++++++++----------- integration/testutil.go | 9 +-------- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 65f008ec6..de4ae2c4c 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,6 @@ source scripts/configure_trillian.sh && createLog && createMap ``` 3. Run Key Transparency -- `docker-compose build kt-signer` - `docker-compose up -d` - `docker-compose logs --tail=0 --follow` - [https://localhost:8080/v1/users/foo@bar.com?app_id=app1](https://localhost:8080/v1/users/foo@bar.com?app_id=app1) diff --git a/cmd/keytransparency-signer/main.go b/cmd/keytransparency-signer/main.go index ee89763a8..30e880a80 100644 --- a/cmd/keytransparency-signer/main.go +++ b/cmd/keytransparency-signer/main.go @@ -20,12 +20,9 @@ import ( "net/http" "time" - "github.com/google/keytransparency/core/admin" - "github.com/google/keytransparency/core/appender" "github.com/google/keytransparency/core/mutator/entry" "github.com/google/keytransparency/core/signer" - "github.com/google/keytransparency/impl/config" "github.com/google/keytransparency/impl/sql/engine" "github.com/google/keytransparency/impl/sql/mutations" "github.com/google/keytransparency/impl/transaction" @@ -41,18 +38,14 @@ import ( var ( metricsAddr = flag.String("metrics-addr", ":8081", "The ip:port to publish metrics on") serverDBPath = flag.String("db", "db", "Database connection string") - domain = flag.String("domain", "example.com", "Distinguished name for this key server") minEpochDuration = flag.Duration("min-period", time.Second*60, "Minimum time between epoch creation (create epochs only if there where mutations). Expected to be smaller than max-period.") maxEpochDuration = flag.Duration("max-period", time.Hour*12, "Maximum time between epoch creation (independent from mutations). This value should about half the time guaranteed by the policy.") - // Info to connect to sparse merkle tree database. + // Info to connect to the trillian map and log. mapID = flag.Int64("map-id", 0, "ID for backend map") mapURL = flag.String("map-url", "", "URL of Trilian Map Server") - - // Info to send Signed Map Heads to a Trillian Log. - logID = flag.Int64("log-id", 0, "Trillian Log ID") - logURL = flag.String("log-url", "", "URL of Trillian Log Server for Signed Map Heads") - logPubKey = flag.String("log-key", "", "File path to public key of the Trillian Log") + logID = flag.Int64("log-id", 0, "Trillian Log ID") + logURL = flag.String("log-url", "", "URL of Trillian Log Server for Signed Map Heads") ) func openDB() *sql.DB { @@ -86,17 +79,12 @@ func main() { tmap := trillian.NewTrillianMapClient(mconn) // Connection to append only log - tlog, err := config.LogClient(*logID, *logURL, *logPubKey) + lconn, err := grpc.Dial(*logURL, grpc.WithInsecure()) if err != nil { - glog.Exitf("LogClient(%v, %v, %v): %v", *logID, *logURL, *logPubKey, err) + glog.Exitf("Failed to connect to %v: %v", *logURL, err) } + tlog := trillian.NewTrillianLogClient(lconn) - // Create signer helper objects. - static := admin.NewStatic() - if err := static.AddLog(*logID, tlog); err != nil { - glog.Exitf("static.AddLog(%v): %v", *mapID, err) - } - sths := appender.NewTrillian(static) // TODO: add mutations and mutator to admin. mutations, err := mutations.New(sqldb, *mapID) if err != nil { @@ -112,7 +100,7 @@ func main() { } }() - signer := signer.New(*domain, *mapID, tmap, *logID, sths, mutator, mutations, factory) + signer := signer.New(*mapID, tmap, *logID, tlog, mutator, mutations, factory) glog.Infof("Signer starting") signer.StartSigning(context.Background(), *minEpochDuration, *maxEpochDuration) glog.Errorf("Signer exiting") diff --git a/core/fake/trillian_log_client.go b/core/fake/trillian_log_client.go index 306aabf1b..3c6a9d441 100644 --- a/core/fake/trillian_log_client.go +++ b/core/fake/trillian_log_client.go @@ -28,7 +28,7 @@ func NewFakeTrillianLogClient() trillian.TrillianLogClient { } func (l *logServer) QueueLeaf(ctx context.Context, in *trillian.QueueLeafRequest, opts ...grpc.CallOption) (*trillian.QueueLeafResponse, error) { - panic("not implemented") + return nil, nil } func (l *logServer) QueueLeaves(ctx context.Context, in *trillian.QueueLeavesRequest, opts ...grpc.CallOption) (*trillian.QueueLeavesResponse, error) { diff --git a/core/signer/signer.go b/core/signer/signer.go index e57e44bc8..5538cff8b 100644 --- a/core/signer/signer.go +++ b/core/signer/signer.go @@ -15,11 +15,11 @@ package signer import ( + "encoding/json" "fmt" "math" "time" - "github.com/google/keytransparency/core/appender" "github.com/google/keytransparency/core/mutator" "github.com/google/keytransparency/core/mutator/entry" "github.com/google/keytransparency/core/transaction" @@ -63,31 +63,28 @@ func init() { // Signer processes mutations and sends them to the trillian map. type Signer struct { - realm string mapID int64 tmap trillian.TrillianMapClient logID int64 - sths appender.Remote + tlog trillian.TrillianLogClient mutator mutator.Mutator mutations mutator.Mutation factory transaction.Factory } // New creates a new instance of the signer. -func New(realm string, - mapID int64, +func New(mapID int64, tmap trillian.TrillianMapClient, logID int64, - sths appender.Remote, + tlog trillian.TrillianLogClient, mutator mutator.Mutator, mutations mutator.Mutation, factory transaction.Factory) *Signer { return &Signer{ - realm: realm, mapID: mapID, tmap: tmap, logID: logID, - sths: sths, + tlog: tlog, mutator: mutator, mutations: mutations, factory: factory, @@ -310,9 +307,20 @@ func (s *Signer) CreateEpoch(ctx context.Context, forceNewEpoch bool) error { glog.V(2).Infof("CreateEpoch: SetLeaves:{Revision: %v, HighestFullyCompletedSeq: %v}", revision, seq) // Put SignedMapHead in an append only log. - if err := s.sths.Write(ctx, s.logID, revision, setResp.GetMapRoot()); err != nil { - // TODO(gdbelvin): If the log doesn't do this, we need to generate an emergency alert. - return fmt.Errorf("sths.Write(%v, %v): %v", s.logID, revision, err) + smrJSON, err := json.Marshal(setResp.GetMapRoot()) + if err != nil { + return err + } + // TODO(gbelvin): Add leaf at a specific index. trillian#423 + // TODO(gdbelvin): If the log doesn't do this, we need to generate an emergency alert. + if _, err := s.tlog.QueueLeaf(ctx, &trillian.QueueLeafRequest{ + LogId: s.logID, + Leaf: &trillian.LogLeaf{ + LeafValue: smrJSON, + }, + }); err != nil { + return fmt.Errorf("trillianLog.QueueLeaf(logID: %v, leaf: %v): %v", + s.logID, smrJSON, err) } mutationsCtr.Add(float64(len(mutations))) diff --git a/integration/testutil.go b/integration/testutil.go index be62c987a..64f1b62e1 100644 --- a/integration/testutil.go +++ b/integration/testutil.go @@ -22,8 +22,6 @@ import ( "testing" "github.com/google/keytransparency/cmd/keytransparency-client/grpcc" - "github.com/google/keytransparency/core/admin" - "github.com/google/keytransparency/core/appender" "github.com/google/keytransparency/core/authentication" "github.com/google/keytransparency/core/crypto/vrf" "github.com/google/keytransparency/core/crypto/vrf/p256" @@ -178,12 +176,7 @@ func NewEnv(t *testing.T) *Env { pb.RegisterKeyTransparencyServiceServer(s, server) // Signer - admin := admin.NewStatic() - if err := admin.AddLog(logID, fake.NewFakeVerifyingLogClient()); err != nil { - t.Fatalf("failed to add log to admin: %v", err) - } - sthsLog := appender.NewTrillian(admin) - signer := signer.New("", mapID, mapEnv.MapClient, logID, sthsLog, mutator, mutations, factory) + signer := signer.New(mapID, mapEnv.MapClient, logID, tlog, mutator, mutations, factory) addr, lis := Listen(t) go s.Serve(lis) From bcd17bd42ae1cf5b2ffaf105d3234413cbc929dc Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 17 Aug 2017 14:42:48 +0100 Subject: [PATCH 2/5] Remove unused appender and admin interfaces The appender interface is a defunct wrapper for the trillian log. The admin interface is a defunct api for multi-tennant configuration. --- core/admin/interface.go | 23 -------- core/admin/static.go | 56 ------------------- core/admin/static_test.go | 44 --------------- core/appender/appender.go | 59 -------------------- core/appender/trillian.go | 98 ---------------------------------- core/appender/trillian_test.go | 61 --------------------- 6 files changed, 341 deletions(-) delete mode 100644 core/admin/interface.go delete mode 100644 core/admin/static.go delete mode 100644 core/admin/static_test.go delete mode 100644 core/appender/appender.go delete mode 100644 core/appender/trillian.go delete mode 100644 core/appender/trillian_test.go diff --git a/core/admin/interface.go b/core/admin/interface.go deleted file mode 100644 index 87787360a..000000000 --- a/core/admin/interface.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package admin supports configuring Key Transparency with multiple Trillian backends. -package admin - -import "github.com/google/trillian/client" - -// Admin returns objects for specific domains. -type Admin interface { - LogClient(logID int64) (client.VerifyingLogClient, error) -} diff --git a/core/admin/static.go b/core/admin/static.go deleted file mode 100644 index 85ee36632..000000000 --- a/core/admin/static.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package admin - -import ( - "fmt" - - "github.com/google/trillian/client" -) - -// Static implements an admin interface for a static set of backends. -// Updates require a restart. -type Static struct { - backends map[int64]*row -} - -// row represents one domain backend. -type row struct { - log client.VerifyingLogClient -} - -// NewStatic returns an admin interface which returns trillian objects. -func NewStatic() *Static { - return &Static{ - backends: make(map[int64]*row), - } -} - -// AddLog adds a particular log to Static. -func (s *Static) AddLog(logID int64, log client.VerifyingLogClient) error { - s.backends[logID] = &row{ - log: log, - } - return nil -} - -// LogClient returns the log client for logID. -func (s *Static) LogClient(logID int64) (client.VerifyingLogClient, error) { - r, ok := s.backends[logID] - if !ok { - return nil, fmt.Errorf("No backend found for logID: %v", logID) - } - return r.log, nil -} diff --git a/core/admin/static_test.go b/core/admin/static_test.go deleted file mode 100644 index 9eeb67fcd..000000000 --- a/core/admin/static_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package admin - -import ( - "testing" - - "github.com/google/keytransparency/core/fake" -) - -func TestStatic(t *testing.T) { - admin := NewStatic() - client := fake.NewFakeVerifyingLogClient() - - for _, tc := range []struct { - logID int64 - add, want bool - }{ - {logID: 0, add: false, want: false}, - {logID: 1, add: true, want: true}, - } { - if tc.add { - if err := admin.AddLog(tc.logID, client); err != nil { - t.Errorf("AddLog(): %v, want nil", err) - } - } - _, err := admin.LogClient(tc.logID) - if got, want := err == nil, tc.want; got != want { - t.Errorf("LogClient(%v): %v, want nil? %v", tc.logID, err, want) - } - } -} diff --git a/core/appender/appender.go b/core/appender/appender.go deleted file mode 100644 index 4151b13b5..000000000 --- a/core/appender/appender.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package appender - -import ( - "github.com/google/keytransparency/core/transaction" - - "golang.org/x/net/context" -) - -// Appender is an append only interface into a data structure. -type Appender interface { - // Adds an object to the append-only data structure. - Append(ctx context.Context, txn transaction.Txn, epoch int64, obj interface{}) error - - // Epoch retrieves a specific object. - // Returns obj and a serialized ct.SignedCertificateTimestamp - Epoch(ctx context.Context, epoch int64, obj interface{}) ([]byte, error) - - // Latest returns the latest object. - // Returns epoch, obj, and a serialized ct.SignedCertificateTimestamp - Latest(ctx context.Context, obj interface{}) (int64, []byte, error) -} - -// Local stores a list of items that have been sequenced. -type Local interface { - // Write writes an object at a given epoch. - Write(txn transaction.Txn, logID, epoch int64, obj interface{}) error - - // Read retrieves a specific object at a given epoch. - Read(txn transaction.Txn, logID, epoch int64, obj interface{}) error - - // Latest returns the latest object and its epoch. - Latest(txn transaction.Txn, logID int64, obj interface{}) (int64, error) -} - -// Remote stores a list of items in a remote service. -type Remote interface { - // Write writes an object at a given epoch. - Write(ctx context.Context, logID, epoch int64, obj interface{}) error - - // Read retrieves a specific object at a given epoch. - Read(ctx context.Context, logID, epoch int64, obj interface{}) error - - // Latest returns the latest object and its epoch. - Latest(ctx context.Context, logID int64, obj interface{}) (int64, error) -} diff --git a/core/appender/trillian.go b/core/appender/trillian.go deleted file mode 100644 index c7c37c291..000000000 --- a/core/appender/trillian.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package appender - -import ( - "database/sql" - "encoding/json" - "fmt" - - "github.com/google/keytransparency/core/admin" - - "golang.org/x/net/context" -) - -// Trillian sends sequenced items to a Trillian log. -type Trillian struct { - admin admin.Admin -} - -// NewTrillian creates a new client to a Trillian Log. -func NewTrillian(admin admin.Admin) Remote { - return &Trillian{ - admin: admin, - } -} - -// Append sends obj to Trillian as a json object at the given epoch index. -func (t *Trillian) Write(ctx context.Context, logID, epoch int64, obj interface{}) error { - log, err := t.admin.LogClient(logID) - if err != nil { - return err - } - b, err := json.Marshal(obj) - if err != nil { - return err - } - // TODO(gbelvin): Add leaf at a specific index. trillian#423 - // Insert index = epoch -1. MapRevisions start at 1. Log leaves start at 0. - if err := log.AddLeaf(ctx, b); err != nil { - return err - } - return nil -} - -// Epoch sets object to the value at a particular index. Returns associated data with that index, an SCT. -// Trillian does not return SCTs so this implementation always returns nil. -func (t *Trillian) Read(ctx context.Context, logID, epoch int64, obj interface{}) error { - log, err := t.admin.LogClient(logID) - if err != nil { - return err - } - - leaves, err := log.ListByIndex(ctx, epoch, 1) - if err != nil { - return err - } - if len(leaves) != 1 { - return fmt.Errorf("Leaf not returned") - } - // Unmarshal leaf into obj. - if err := json.Unmarshal(leaves[0].LeafValue, &obj); err != nil { - return err - } - return nil -} - -// Latest retrieves the last object. Returns sql.ErrNoRows if empty. -func (t *Trillian) Latest(ctx context.Context, logID int64, obj interface{}) (int64, error) { - log, err := t.admin.LogClient(logID) - if err != nil { - return 0, err - } - - if err := log.UpdateRoot(ctx); err != nil { - return 0, err - } - epoch := log.Root().TreeSize - 1 - if epoch < 0 { - return 0, sql.ErrNoRows - } - if err := t.Read(ctx, logID, epoch, obj); err != nil { - return 0, err - } - return epoch, nil - -} diff --git a/core/appender/trillian_test.go b/core/appender/trillian_test.go deleted file mode 100644 index 4b5b9c768..000000000 --- a/core/appender/trillian_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2016 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package appender - -import ( - "context" - "testing" - - "github.com/google/keytransparency/core/admin" - "github.com/google/keytransparency/core/fake" -) - -func TestLatest(t *testing.T) { - ctx := context.Background() - fakeLog := fake.NewFakeVerifyingLogClient() - admin := admin.NewStatic() - if err := admin.AddLog(0, fakeLog); err != nil { - t.Fatalf("failed to add log to admin: %v", err) - } - a := NewTrillian(admin) - - for _, tc := range []struct { - logID int64 - epoch int64 - data []byte - want int64 - }{ - {0, 0, []byte("foo"), 0}, - {0, 1, []byte("foo"), 1}, - {0, 2, []byte("foo"), 2}, - } { - if err := a.Write(ctx, tc.logID, tc.epoch, tc.data); err != nil { - t.Errorf("Write(%v, %v): %v, want nil", tc.epoch, tc.data, err) - } - - var obj []byte - if err := a.Read(ctx, tc.logID, tc.epoch, &obj); err != nil { - t.Errorf("Read(%v): %v, want nil", tc.epoch, err) - } - - epoch, err := a.Latest(ctx, tc.logID, &obj) - if err != nil { - t.Errorf("Latest(): %v, want nil", err) - } - if got := epoch; got != tc.want { - t.Errorf("Latest(): %v, want %v", got, tc.want) - } - } -} From c35d1c7641f59ceed5b7c0b352b27224b22e4128 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 17 Aug 2017 14:01:20 +0100 Subject: [PATCH 3/5] Initialize log with empty SignedMapRoot This brings the relationship between MapRevisions and Log indexes back into parity. Previously log index = map revision -1 Now, log index = map revision Fixes #703 --- cmd/keytransparency-signer/Dockerfile | 3 +- core/client/kt/verify.go | 2 +- core/keyserver/keyserver.go | 6 ++-- core/signer/signer.go | 48 ++++++++++++++++++++++++++- docker-compose.yml | 1 - 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/cmd/keytransparency-signer/Dockerfile b/cmd/keytransparency-signer/Dockerfile index ab7f047c0..760980fa6 100644 --- a/cmd/keytransparency-signer/Dockerfile +++ b/cmd/keytransparency-signer/Dockerfile @@ -16,7 +16,6 @@ ENV MIN_SIGN_PERIOD=5s \ ENV VERBOSITY=0 -ADD keytransparency/genfiles/* /kt/ ADD ./keytransparency /go/src/github.com/google/keytransparency ADD ./trillian /go/src/github.com/google/trillian WORKDIR /go/src/github.com/google/keytransparency @@ -27,7 +26,7 @@ RUN go get -tags="mysql" ./cmd/keytransparency-signer ENTRYPOINT /go/bin/keytransparency-signer \ --db="${DB_USER}:${DB_PASSWORD}@tcp(${DB_HOST})/${DB_DATABASE}" \ --min-period="$MIN_SIGN_PERIOD" --max-period="$MAX_SIGN_PERIOD" \ - --log-id="$LOG_ID" --log-url="$LOG_URL" --log-key="$LOG_KEY" \ + --log-id="$LOG_ID" --log-url="$LOG_URL" \ --map-id="$MAP_ID" --map-url="$MAP_URL" \ --alsologtostderr --v=${VERBOSITY} diff --git a/core/client/kt/verify.go b/core/client/kt/verify.go index 2de32604a..d11d0e290 100644 --- a/core/client/kt/verify.go +++ b/core/client/kt/verify.go @@ -139,7 +139,7 @@ func (v *Verifier) VerifyGetEntryResponse(ctx context.Context, userID, appID str if err != nil { return fmt.Errorf("json.Marshal(): %v", err) } - logLeafIndex := in.GetSmr().GetMapRevision() - 1 + logLeafIndex := in.GetSmr().GetMapRevision() if err := v.logVerifier.VerifyInclusionAtIndex(trusted, b, logLeafIndex, in.GetLogInclusion()); err != nil { return fmt.Errorf("VerifyInclusionAtIndex(%s, %v, _): %v", diff --git a/core/keyserver/keyserver.go b/core/keyserver/keyserver.go index 3136a62a2..0f4221474 100644 --- a/core/keyserver/keyserver.go +++ b/core/keyserver/keyserver.go @@ -173,10 +173,8 @@ func (s *Server) getEntry(ctx context.Context, userID, appID string, firstTreeSi &trillian.GetInclusionProofRequest{ LogId: s.logID, // SignedMapRoot must be placed in the log at MapRevision. - // MapRevisions start at 1. Log leaves start at 0. - // MapRevision should be at least 1 since the Signer is - // supposed to create at least one revision on startup. - LeafIndex: getResp.GetMapRoot().GetMapRevision() - 1, + // MapRevisions start at 1. Log leaves start at 1. + LeafIndex: getResp.GetMapRoot().GetMapRevision(), TreeSize: secondTreeSize, }) if err != nil { diff --git a/core/signer/signer.go b/core/signer/signer.go index 5538cff8b..6a84f0a04 100644 --- a/core/signer/signer.go +++ b/core/signer/signer.go @@ -15,6 +15,7 @@ package signer import ( + "crypto/sha256" "encoding/json" "fmt" "math" @@ -91,9 +92,52 @@ func New(mapID int64, } } +// Initialize inserts the object hash of an empty struct into the log if it is empty. +// This keeps the log leaves in-sync with the map which starts off with an +// empty log root at map revision 0. +func (s *Signer) Initialize(ctx context.Context) error { + logRoot, err := s.tlog.GetLatestSignedLogRoot(ctx, &trillian.GetLatestSignedLogRootRequest{ + LogId: s.logID, + }) + if err != nil { + return fmt.Errorf("GetLatestSignedLogRoot(%v): %v", s.logID, err) + } + mapRoot, err := s.tmap.GetSignedMapRoot(ctx, &trillian.GetSignedMapRootRequest{ + MapId: s.mapID, + }) + if err != nil { + return fmt.Errorf("GetSignedMapRoot(%v): %v", s.mapID, err) + } + + // If the tree is empty and the map is empty, + // add the empty map root to the log. + if logRoot.GetSignedLogRoot().GetTreeSize() == 0 && + mapRoot.GetMapRoot().GetMapRevision() == 0 { + smrJSON, err := json.Marshal(mapRoot.GetMapRoot()) + idHash := sha256.Sum256(smrJSON) + if err != nil { + return err + } + if _, err := s.tlog.QueueLeaf(ctx, &trillian.QueueLeafRequest{ + LogId: s.logID, + Leaf: &trillian.LogLeaf{ + LeafValue: smrJSON, + LeafIdentityHash: idHash[:], + }, + }); err != nil { + return fmt.Errorf("trillian.QueueLeaf(logID: %v, leaf: %v): %v", + s.logID, smrJSON, err) + } + } + return nil +} + // StartSigning advance epochs once per minInterval, if there were mutations, // and at least once per maxElapsed minIntervals. func (s *Signer) StartSigning(ctx context.Context, minInterval, maxInterval time.Duration) { + if err := s.Initialize(ctx); err != nil { + glog.Errorf("Initialize() failed: %v", err) + } var rootResp *trillian.GetSignedMapRootResponse ctxTime, cancel := context.WithTimeout(ctx, minInterval) rootResp, err := s.tmap.GetSignedMapRoot(ctxTime, &trillian.GetSignedMapRootRequest{ @@ -311,12 +355,14 @@ func (s *Signer) CreateEpoch(ctx context.Context, forceNewEpoch bool) error { if err != nil { return err } + idHash := sha256.Sum256(smrJSON) // TODO(gbelvin): Add leaf at a specific index. trillian#423 // TODO(gdbelvin): If the log doesn't do this, we need to generate an emergency alert. if _, err := s.tlog.QueueLeaf(ctx, &trillian.QueueLeafRequest{ LogId: s.logID, Leaf: &trillian.LogLeaf{ - LeafValue: smrJSON, + LeafValue: smrJSON, + LeafIdentityHash: idHash[:], }, }); err != nil { return fmt.Errorf("trillianLog.QueueLeaf(logID: %v, leaf: %v): %v", diff --git a/docker-compose.yml b/docker-compose.yml index 861bf56ce..8be70825a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,7 +143,6 @@ services: DB_DATABASE: test DB_USER: test DB_PASSWORD: zaphod - LOG_KEY: /kt/trillian-log.pem MIN_SIGN_PERIOD: 5s MAX_SIGN_PERIOD: 5m VERBOSITY: 5 From bdffb7dc5e092ff5a5f4ba76ef1c4f9dee65dac1 Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Thu, 17 Aug 2017 17:40:18 +0100 Subject: [PATCH 4/5] remove duplicate code --- core/signer/signer.go | 46 ++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/core/signer/signer.go b/core/signer/signer.go index 6a84f0a04..341e3ca31 100644 --- a/core/signer/signer.go +++ b/core/signer/signer.go @@ -113,21 +113,9 @@ func (s *Signer) Initialize(ctx context.Context) error { // add the empty map root to the log. if logRoot.GetSignedLogRoot().GetTreeSize() == 0 && mapRoot.GetMapRoot().GetMapRevision() == 0 { - smrJSON, err := json.Marshal(mapRoot.GetMapRoot()) - idHash := sha256.Sum256(smrJSON) - if err != nil { + if err := queueLogLeaf(ctx, s.tlog, s.logID, mapRoot.GetMapRoot()); err != nil { return err } - if _, err := s.tlog.QueueLeaf(ctx, &trillian.QueueLeafRequest{ - LogId: s.logID, - Leaf: &trillian.LogLeaf{ - LeafValue: smrJSON, - LeafIdentityHash: idHash[:], - }, - }); err != nil { - return fmt.Errorf("trillian.QueueLeaf(logID: %v, leaf: %v): %v", - s.logID, smrJSON, err) - } } return nil } @@ -351,28 +339,36 @@ func (s *Signer) CreateEpoch(ctx context.Context, forceNewEpoch bool) error { glog.V(2).Infof("CreateEpoch: SetLeaves:{Revision: %v, HighestFullyCompletedSeq: %v}", revision, seq) // Put SignedMapHead in an append only log. - smrJSON, err := json.Marshal(setResp.GetMapRoot()) + if err := queueLogLeaf(ctx, s.tlog, s.logID, setResp.GetMapRoot()); err != nil { + // TODO(gdbelvin): If the log doesn't do this, we need to generate an emergency alert. + return err + } + + mutationsCtr.Add(float64(len(mutations))) + indexCtr.Add(float64(len(indexes))) + mapUpdateHist.Observe(mapSetEnd.Sub(mapSetStart).Seconds()) + createEpochHist.Observe(time.Since(start).Seconds()) + glog.Infof("CreatedEpoch: rev: %v, root: %x", revision, setResp.GetMapRoot().GetRootHash()) + return nil +} + +// TODO(gbelvin): Add leaf at a specific index. trillian#423 +func queueLogLeaf(ctx context.Context, tlog trillian.TrillianLogClient, logID int64, smr *trillian.SignedMapRoot) error { + smrJSON, err := json.Marshal(smr) if err != nil { return err } idHash := sha256.Sum256(smrJSON) - // TODO(gbelvin): Add leaf at a specific index. trillian#423 - // TODO(gdbelvin): If the log doesn't do this, we need to generate an emergency alert. - if _, err := s.tlog.QueueLeaf(ctx, &trillian.QueueLeafRequest{ - LogId: s.logID, + + if _, err := tlog.QueueLeaf(ctx, &trillian.QueueLeafRequest{ + LogId: logID, Leaf: &trillian.LogLeaf{ LeafValue: smrJSON, LeafIdentityHash: idHash[:], }, }); err != nil { return fmt.Errorf("trillianLog.QueueLeaf(logID: %v, leaf: %v): %v", - s.logID, smrJSON, err) + logID, smrJSON, err) } - - mutationsCtr.Add(float64(len(mutations))) - indexCtr.Add(float64(len(indexes))) - mapUpdateHist.Observe(mapSetEnd.Sub(mapSetStart).Seconds()) - createEpochHist.Observe(time.Since(start).Seconds()) - glog.Infof("CreatedEpoch: rev: %v, root: %x", revision, setResp.GetMapRoot().GetRootHash()) return nil } From 4fccc2ec51523c64c085352ad82fb09c8f168afa Mon Sep 17 00:00:00 2001 From: Gary Belvin Date: Fri, 18 Aug 2017 08:45:07 +0100 Subject: [PATCH 5/5] nit --- core/signer/signer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/signer/signer.go b/core/signer/signer.go index 341e3ca31..9412e1c94 100644 --- a/core/signer/signer.go +++ b/core/signer/signer.go @@ -352,7 +352,7 @@ func (s *Signer) CreateEpoch(ctx context.Context, forceNewEpoch bool) error { return nil } -// TODO(gbelvin): Add leaf at a specific index. trillian#423 +// TODO(gdbelvin): Add leaf at a specific index. trillian#423 func queueLogLeaf(ctx context.Context, tlog trillian.TrillianLogClient, logID int64, smr *trillian.SignedMapRoot) error { smrJSON, err := json.Marshal(smr) if err != nil {