From 1133c78a109225e666fc8461987a89c4fdbbed8b Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 14 Dec 2014 18:05:00 +0100 Subject: [PATCH 1/3] Key votes storage benchmark --- _research/vote_storage/benchmark_test.go | 115 +++++++++++++++++++++++ _vagrant/Vagrantfile | 5 + 2 files changed, 120 insertions(+) create mode 100644 _research/vote_storage/benchmark_test.go diff --git a/_research/vote_storage/benchmark_test.go b/_research/vote_storage/benchmark_test.go new file mode 100644 index 0000000..adab145 --- /dev/null +++ b/_research/vote_storage/benchmark_test.go @@ -0,0 +1,115 @@ +package vote_storage_test + +import ( + "math/rand" + "testing" + + "github.com/dancannon/gorethink" + "github.com/dchest/uniuri" +) + +var ( + session *gorethink.Session + key2find string + table2search string +) + +func init() { + var err error + session, err = gorethink.Connect(gorethink.ConnectOpts{ + Address: "127.0.0.1:28015", + }) + if err != nil { + panic(err) + } + + key2find = uniuri.New() + + // Create a new table + gorethink.Db("test").TableDrop("benchmark_keys_list").Run(session) + gorethink.Db("test").TableCreate("benchmark_keys_list").Run(session) + + var klist []*KeysList + + // Populate with sample data + for n := 0; n < 300; n++ { + keys := rndStringSlice(999) + keys = randomlyInsert(keys, key2find) + + y := uniuri.New() + if n == 153 { + table2search = y + } + + klist = append(klist, &KeysList{ + ID: y, + Voted: keys, + }) + } + + gorethink.Db("test").Table("benchmark_keys_list").Insert(klist).Run(session) +} + +func rndStringSlice(count int) []string { + var r []string + for i := 0; i < count; i++ { + r = append(r, uniuri.New()) + } + return r +} + +func randomlyInsert(s []string, x string) []string { + i := rand.Intn(len(s) - 1) + + s = append(s[:i], append([]string{x}, s[i:]...)...) + + return s +} + +type KeysList struct { + ID string `gorethink:"id"` + Voted []string `gorethink:"voted"` +} + +func BenchmarkContains(b *testing.B) { + for n := 0; n < b.N; n++ { + contains, err := gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").Contains(key2find).Run(session) + if err != nil { + b.Log(err) + b.Fail() + } + + var res bool + err = contains.One(&res) + if err != nil { + b.Log(err) + b.Fail() + } + if !res { + b.Log("invalid response") + b.Fail() + } + } +} + +func BenchmarkAppend(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").Append(uniuri.New()).Run(session) + if err != nil { + b.Log(err) + b.Fail() + } + } +} + +func BenchmarkDelete(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").DeleteAt( + gorethink.Expr(gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").IndexesOf(key2find).AtIndex(0)), + ).Run(session) + if err != nil { + b.Log(err) + b.Fail() + } + } +} diff --git a/_vagrant/Vagrantfile b/_vagrant/Vagrantfile index 8b5447e..53ea370 100644 --- a/_vagrant/Vagrantfile +++ b/_vagrant/Vagrantfile @@ -15,6 +15,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # redis config.vm.network "forwarded_port", guest: 6379, host: 6379 + config.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 4 + end + # load ansible playbook config.vm.provision "shell", path: "deploy.sh" end From 0a1040f167fdc271dc44f26572ead0f2d93ffae5 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 14 Dec 2014 18:41:03 +0100 Subject: [PATCH 2/3] models.Key change --- models/account.go | 12 +++--------- models/key.go | 25 ++++++++++++------------- routes/keys.go | 35 +++++++++++++++++++++++++++-------- utils/pgp.go | 21 +++++++++++++++++++++ 4 files changed, 63 insertions(+), 30 deletions(-) create mode 100644 utils/pgp.go diff --git a/models/account.go b/models/account.go index aa834f1..b2cbef5 100644 --- a/models/account.go +++ b/models/account.go @@ -14,17 +14,11 @@ type Account struct { Billing BillingData `json:"billing" gorethink:"billing"` // Password is the password used to login to the account. - // It's hashed and salted using a cryptographically strong method (bcrypt|scrypt). + // It's hashed and salted using scrypt. Password string `json:"-" gorethink:"password"` - // PgpExpDate is an RFC3339-encoded string containing the expiry date of the user's public key - PgpExpDate string `json:"pgp_exp_date" gorethink:"pgp_exp_date"` - - // PgpFingerprint is a SHA-512 hash of the user's public key - PgpFingerprint string `json:"pgp_fingerprint" gorethink:"pgp_fingerprint"` - - // PgpPublicKey is a copy of the user's current public key. It can also be found in the 'keys' db. - PgpPublicKey string `json:"pgp_public_key" gorethink:"pgp_public_key"` + // PGPKey is the fingerprint of account's default key + PGPKey string `json:"pgp_key" gorethink:"pgp_key"` // Settings contains data needed to customize the user experience. // TODO Work in progress diff --git a/models/key.go b/models/key.go index f0c6d86..67c7f25 100644 --- a/models/key.go +++ b/models/key.go @@ -1,19 +1,18 @@ package models type Key struct { - Resource - Expiring + Resource // ID is the fingerprint, Name is empty + Expiring // ExpiryDate is either empty or expiring, user can set it - // ID is the fingerprint + //Body []byte `json:"body" gorethink:"body"` // Raw key contents - Image []byte `json:"image" gorethink:"image"` - - // the actual key - Key string `json:"key" gorethink:"key"` - - OwnerName string `json:"owner_name" gorethink:"owner_name"` - - // the actual id - KeyID string `json:"key_id" gorethink:"key_id"` - KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` + Headers map[string]string `json:"headers" gorethink:"headers"` // Headers passed with the key + Email string `json:"email" gorethink:"email"` // Address associated with the key + Algorithm string `json:"algorithm" gorethink:"algorithm"`// Algorithm of the key + Length uint16 `json:"length" gorethink:"length"`// Length of the key + Key string `json:"key" gorethink:"key"` // Armor-encoded key + KeyID string `json:"key_id" gorethink:"key_id"` // PGP key ID + KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` // Shorter version of above + Reliability int `json:"reliability" gorethink:"reliability"` // Reliability algorithm cached result + MasterKey string `json:"master_key" gorethink:"mater_key"` // MasterKey's ID - no idea how it works } diff --git a/routes/keys.go b/routes/keys.go index 980f705..5da2b52 100644 --- a/routes/keys.go +++ b/routes/keys.go @@ -9,6 +9,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/zenazn/goji/web" "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" "github.com/lavab/api/env" "github.com/lavab/api/models" @@ -59,8 +60,7 @@ func KeysList(w http.ResponseWriter, r *http.Request) { // KeysCreateRequest contains the data passed to the KeysCreate endpoint. type KeysCreateRequest struct { - Key string `json:"key" schema:"key"` // gpg armored key - Image string `json:"image" schema:"image"` // todo + Key string `json:"key" schema:"key"` // gpg armored key } // KeysCreateResponse contains the result of the KeysCreate request. @@ -105,6 +105,21 @@ func KeysCreate(c web.C, w http.ResponseWriter, r *http.Request) { return } + // Parse using armor pkg + block, err := armor.Decode(strings.NewReader(input.Key)) + if err != nil { + utils.JSONResponse(w, 409, &KeysCreateResponse{ + Success: false, + Message: "Invalid key format", + }) + + env.Log.WithFields(logrus.Fields{ + "error": err, + "list": entityList, + }).Warn("Cannot parse an armored key #2") + return + } + // Get the account from db account, err := env.Accounts.GetAccount(session.Owner) if err != nil { @@ -132,17 +147,21 @@ func KeysCreate(c web.C, w http.ResponseWriter, r *http.Request) { // Allocate a new key key := &models.Key{ Resource: models.MakeResource( - session.Owner, + account.ID, fmt.Sprintf( - "%d/%s public key", + "%s/%d/%s public key", + utils.GetAlgorithmName(publicKey.PrimaryKey.PubKeyAlgo), bitLength, publicKey.PrimaryKey.KeyIdString(), ), ), - OwnerName: account.Name, - Key: input.Key, - KeyID: publicKey.PrimaryKey.KeyIdString(), - KeyIDShort: publicKey.PrimaryKey.KeyIdShortString(), + Headers: block.Header, + Algorithm: utils.GetAlgorithmName(publicKey.PrimaryKey.PubKeyAlgo), + Length: bitLength, + Key: input.Key, + KeyID: publicKey.PrimaryKey.KeyIdString(), + KeyIDShort: publicKey.PrimaryKey.KeyIdShortString(), + Reliability: 0, } // Update id as we can't do it directly during allocation diff --git a/utils/pgp.go b/utils/pgp.go new file mode 100644 index 0000000..b806532 --- /dev/null +++ b/utils/pgp.go @@ -0,0 +1,21 @@ +package utils + +import "golang.org/x/crypto/openpgp/packet" + +// GetAlgorithmName returns algorithm's name depending on its ID +func GetAlgorithmName(id packet.PublicKeyAlgorithm) string { + switch id { + case 1, 2, 3: + return "RSA" + case 16: + return "ElGamal" + case 17: + return "DSA" + case 18: + return "ECDH" + case 19: + return "ECDSA" + default: + return "unknown" + } +} From 44d0b68d31504d3027be270ee2b2f366e71b35b6 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 14 Dec 2014 19:30:00 +0100 Subject: [PATCH 3/3] Fixing before merge; changed GET /keys to work with new logic --- .travis.yml | 14 -------------- db/table_keys.go | 4 ++-- models/key.go | 18 +++++++++--------- routes/keys.go | 13 +++++++++++-- 4 files changed, 22 insertions(+), 27 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 84a85c5..0000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: go - -go: - - 1.3.1 - -services: - - redis-server - -before_script: - - source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list - - wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - - - sudo apt-get update - - sudo apt-get install rethinkdb - - rethinkdb --bind all & diff --git a/db/table_keys.go b/db/table_keys.go index 225a830..8679e87 100644 --- a/db/table_keys.go +++ b/db/table_keys.go @@ -8,10 +8,10 @@ type KeysTable struct { RethinkCRUD } -func (k *KeysTable) FindByName(name string) ([]*models.Key, error) { +func (k *KeysTable) FindByOwner(id string) ([]*models.Key, error) { var results []*models.Key - if err := k.FindByAndFetch("owner_name", name, &results); err != nil { + if err := k.FindByAndFetch("owner", id, &results); err != nil { return nil, err } diff --git a/models/key.go b/models/key.go index 67c7f25..b114ab8 100644 --- a/models/key.go +++ b/models/key.go @@ -6,13 +6,13 @@ type Key struct { //Body []byte `json:"body" gorethink:"body"` // Raw key contents - Headers map[string]string `json:"headers" gorethink:"headers"` // Headers passed with the key - Email string `json:"email" gorethink:"email"` // Address associated with the key - Algorithm string `json:"algorithm" gorethink:"algorithm"`// Algorithm of the key - Length uint16 `json:"length" gorethink:"length"`// Length of the key - Key string `json:"key" gorethink:"key"` // Armor-encoded key - KeyID string `json:"key_id" gorethink:"key_id"` // PGP key ID - KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` // Shorter version of above - Reliability int `json:"reliability" gorethink:"reliability"` // Reliability algorithm cached result - MasterKey string `json:"master_key" gorethink:"mater_key"` // MasterKey's ID - no idea how it works + Headers map[string]string `json:"headers" gorethink:"headers"` // Headers passed with the key + Email string `json:"email" gorethink:"email"` // Address associated with the key + Algorithm string `json:"algorithm" gorethink:"algorithm"` // Algorithm of the key + Length uint16 `json:"length" gorethink:"length"` // Length of the key + Key string `json:"key" gorethink:"key"` // Armor-encoded key + KeyID string `json:"key_id" gorethink:"key_id"` // PGP key ID + KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` // Shorter version of above + Reliability int `json:"reliability" gorethink:"reliability"` // Reliability algorithm cached result + MasterKey string `json:"master_key" gorethink:"mater_key"` // MasterKey's ID - no idea how it works } diff --git a/routes/keys.go b/routes/keys.go index 5da2b52..4328583 100644 --- a/routes/keys.go +++ b/routes/keys.go @@ -35,8 +35,17 @@ func KeysList(w http.ResponseWriter, r *http.Request) { return } + account, err := env.Accounts.FindAccountByName(user) + if err != nil { + utils.JSONResponse(w, 409, &KeysListResponse{ + Success: false, + Message: "Invalid username", + }) + return + } + // Find all keys owner by user - keys, err := env.Keys.FindByName(user) + keys, err := env.Keys.FindByOwner(account.ID) if err != nil { utils.JSONResponse(w, 500, &KeysListResponse{ Success: false, @@ -149,7 +158,7 @@ func KeysCreate(c web.C, w http.ResponseWriter, r *http.Request) { Resource: models.MakeResource( account.ID, fmt.Sprintf( - "%s/%d/%s public key", + "%s/%d/%s", utils.GetAlgorithmName(publicKey.PrimaryKey.PubKeyAlgo), bitLength, publicKey.PrimaryKey.KeyIdString(),