From 769a8e19f20071e3ba934fdb6d97ec2739a6bd73 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 8 Jan 2015 22:16:32 +0100 Subject: [PATCH 1/4] labels stuff --- db/setup.go | 15 ++++--- db/table_emails.go | 57 +++++++++++++++++++++++++ db/table_labels.go | 102 +++++++++++++++++++++++++++++++++++++++++++++ env/env.go | 2 + models/email.go | 2 + setup/setup.go | 8 ++++ 6 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 db/table_labels.go diff --git a/db/setup.go b/db/setup.go index 51e84c1..75483a5 100644 --- a/db/setup.go +++ b/db/setup.go @@ -12,15 +12,14 @@ var ( // Indexes of tables in the database var tableIndexes = map[string][]string{ - "tokens": []string{"user", "user_id"}, "accounts": []string{"name"}, - "emails": []string{"user_id"}, - "drafts": []string{"user_id"}, - "contacts": []string{}, - "threads": []string{"user_id"}, - "labels": []string{}, - "keys": []string{}, - "reservations": []string{}, + "contacts": []string{"owner"}, + "emails": []string{"owner", "label_ids"}, + "keys": []string{"owner", "key_id"}, + "labels": []string{"owner"}, + "reservations": []string{"email", "name"}, + "threads": []string{"owner"}, + "tokens": []string{"owner"}, } // List of names of databases diff --git a/db/table_emails.go b/db/table_emails.go index 4e0134e..fee1855 100644 --- a/db/table_emails.go +++ b/db/table_emails.go @@ -102,3 +102,60 @@ func (e *EmailsTable) List( return resp, nil } + +func (e *EmailsTable) GetByLabel(label string) ([]*models.Email, error) { + var result []*models.Email + + cursor, err := e.GetTable().Filter(func(row gorethink.Term) gorethink.Term { + return row.Field("label_ids").Contains(label) + }).GetAll().Run(e.GetSession()) + if err != nil { + return nil, err + } + + err = cursor.All(&result) + if err != nil { + return nil, err + } + + return result, nil +} + +func (e *EmailsTable) CountByLabel(label string) (int, error) { + var result int + + cursor, err := e.GetTable().Filter(func(row gorethink.Term) gorethink.Term { + return row.Field("label_ids").Contains(label) + }).Count().Run(e.GetSession()) + if err != nil { + return 0, err + } + + err = cursor.One(&result) + if err != nil { + return 0, err + } + + return result, nil +} + +func (e *EmailsTable) CountByLabelUnread(label string) (int, error) { + var result int + + cursor, err := e.GetTable().Filter(func(row gorethink.Term) gorethink.Term { + return gorethink.And( + row.Field("label_ids").Contains(label), + row.Field("is_read").Eq(false), + ) + }).Count().Run(e.GetSession()) + if err != nil { + return 0, err + } + + err = cursor.One(&result) + if err != nil { + return 0, err + } + + return result, nil +} diff --git a/db/table_labels.go b/db/table_labels.go new file mode 100644 index 0000000..9fa5a71 --- /dev/null +++ b/db/table_labels.go @@ -0,0 +1,102 @@ +package db + +import ( + "time" + + "github.com/dancannon/gorethink" + + "github.com/lavab/api/cache" + "github.com/lavab/api/models" +) + +type LabelsTable struct { + RethinkCRUD + Cache cache.Cache + Expires time.Duration +} + +func (l *LabelsTable) Insert(data interface{}) error { + if err := l.RethinkCRUD.Insert(data); err != nil { + return err + } + + label, ok := data.(*models.Token) + if !ok { + return nil + } + + return l.Cache.Set(l.RethinkCRUD.GetTableName()+":"+label.ID, label, l.Expires) +} + +// Update clears all updated keys +func (l *LabelsTable) Update(data interface{}) error { + if err := l.RethinkCRUD.Update(data); err != nil { + return err + } + + return l.Cache.DeleteMask(l.RethinkCRUD.GetTableName() + ":*") +} + +// UpdateID updates the specified label and updates cache +func (l *LabelsTable) UpdateID(id string, data interface{}) error { + if err := l.RethinkCRUD.UpdateID(id, data); err != nil { + return err + } + + label, err := l.GetLabel(id) + if err != nil { + return err + } + + return l.Cache.Set(l.RethinkCRUD.GetTableName()+":"+id, label, l.Expires) +} + +// Delete removes from db and cache using filter +func (l *LabelsTable) Delete(cond interface{}) error { + result, err := l.GetTable().Filter(cond).Delete(gorethink.DeleteOpts{ + ReturnChanges: true, + }).RunWrite(l.GetSession()) + if err != nil { + return err + } + + var ids []interface{} + for _, change := range result.Changes { + ids = append(ids, l.RethinkCRUD.GetTableName()+":"+change.OldValue.(map[string]interface{})["id"].(string)) + } + + return l.Cache.DeleteMulti(ids...) +} + +// DeleteID removes from db and cache using id query +func (l *LabelsTable) DeleteID(id string) error { + if err := l.RethinkCRUD.DeleteID(id); err != nil { + return err + } + + return l.Cache.Delete(l.RethinkCRUD.GetTableName() + ":" + id) +} + +// FindFetchOne tries cache and then tries using DefaultCRUD's fetch operation +func (l *LabelsTable) FindFetchOne(id string, value interface{}) error { + if err := l.Cache.Get(id, value); err == nil { + return nil + } + + err := l.RethinkCRUD.FindFetchOne(id, value) + if err != nil { + return err + } + + return l.Cache.Set(l.RethinkCRUD.GetTableName()+":"+id, value, l.Expires) +} + +func (l *LabelsTable) GetLabel(id string) (*models.Label, error) { + var result models.Label + + if err := l.FindFetchOne(id, &result); err != nil { + return nil, err + } + + return &result, nil +} diff --git a/env/env.go b/env/env.go index ec7444a..a1fa521 100644 --- a/env/env.go +++ b/env/env.go @@ -31,6 +31,8 @@ var ( Reservations *db.ReservationsTable // Emails is the global instance of EmailsTable Emails *db.EmailsTable + // Labels is the global instance of LabelsTable + Labels *db.LabelsTable // Factors contains all currently registered factors Factors map[string]factor.Factor // NATS is the encoded connection to the NATS queue diff --git a/models/email.go b/models/email.go index 03bd9e3..823704a 100644 --- a/models/email.go +++ b/models/email.go @@ -31,4 +31,6 @@ type Email struct { ThreadID string `json:"thread_id" gorethink:"thread_id"` Status string `json:"status" gorethink:"status"` + + IsRead string `json:"is_read" gorethink:"is_read"` } diff --git a/setup/setup.go b/setup/setup.go index bcf241f..a58ec4d 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -154,6 +154,14 @@ func PrepareMux(flags *env.Flags) *web.Mux { "emails", ), } + env.Labels = &db.LabelsTable{ + RethinkCRUD: db.NewCRUDTable( + rethinkSession, + rethinkOpts.Database, + "labels", + ), + Cache: redis, + } // NATS queue connection nc, err := nats.Connect(flags.NATSAddress) From 6ec558c21661e145ea4c6c96af3f5d9a35e1073d Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 8 Jan 2015 23:19:49 +0100 Subject: [PATCH 2/4] Moar labels stuff --- db/table_labels.go | 29 +++++++++++++++++++++++++++++ models/label.go | 9 ++------- routes/contacts.go | 2 +- routes/labels.go | 34 +++++++++++++++++++++++++++------- 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/db/table_labels.go b/db/table_labels.go index 9fa5a71..f28da14 100644 --- a/db/table_labels.go +++ b/db/table_labels.go @@ -11,6 +11,7 @@ import ( type LabelsTable struct { RethinkCRUD + Emails *EmailsTable Cache cache.Cache Expires time.Duration } @@ -98,5 +99,33 @@ func (l *LabelsTable) GetLabel(id string) (*models.Label, error) { return nil, err } + totalCount, err := l.Emails.CountByLabel(result.ID) + if err != nil { + return nil, err + } + + result.EmailsTotal = totalCount + + unreadCount, err := l.Emails.CountByLabelUnread(result.ID) + if err != nil { + return nil, err + } + + result.EmailsUnread = unreadCount + return &result, nil } + +// GetOwnedBy returns all labels owned by id +func (l *LabelsTable) GetOwnedBy(id string) ([]*models.Label, error) { + var result []*models.Label + + err := l.WhereAndFetch(map[string]interface{}{ + "owner": id, + }, &result) + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/models/label.go b/models/label.go index b36320a..6daa02c 100644 --- a/models/label.go +++ b/models/label.go @@ -16,11 +16,6 @@ type Label struct { // Examples: inbox, trash, spam, drafts, starred, etc. Builtin bool `json:"builtin" gorethink:"builtin"` - // EmailsUnread is the number of unread emails that have a particular label applied. - // Storing this for each label eliminates the need of db lookups for this commonly needed information. - EmailsUnread int `json:"emails_unread" gorethink:"emails_unread"` - - // EmailsTotal is the number of emails that have a particular label applied. - // Storing this for each label eliminates the need of db lookups for this commonly needed information. - EmailsTotal int `json:"emails_total" gorethink:"emails_total"` + EmailsUnread int `json:"emails_unread" gorethink:"-"` + EmailsTotal int `json:"emails_total" gorethink:"-"` } diff --git a/routes/contacts.go b/routes/contacts.go index 6118540..93e9364 100644 --- a/routes/contacts.go +++ b/routes/contacts.go @@ -37,7 +37,7 @@ func ContactsList(c web.C, w http.ResponseWriter, r *http.Request) { return } - utils.JSONResponse(w, 501, &ContactsListResponse{ + utils.JSONResponse(w, 200, &ContactsListResponse{ Success: true, Contacts: &contacts, }) diff --git a/routes/labels.go b/routes/labels.go index 45d0e97..cd3b2ef 100644 --- a/routes/labels.go +++ b/routes/labels.go @@ -3,20 +3,40 @@ package routes import ( "net/http" + "github.com/Sirupsen/logrus" + "github.com/lavab/api/env" + "github.com/lavab/api/models" "github.com/lavab/api/utils" + "github.com/zenazn/goji/web" ) // LabelsListResponse contains the result of the LabelsList request. type LabelsListResponse struct { - Success bool `json:"success"` - Message string `json:"message"` + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Labels *[]*models.Label `json:"labels,omitempty"` } -// LabelsList does *something* - TODO -func LabelsList(w http.ResponseWriter, r *http.Request) { - utils.JSONResponse(w, 501, &LabelsListResponse{ - Success: false, - Message: "Sorry, not implemented yet", +// LabelsList fetches all labels +func LabelsList(c web.C, w http.ResponseWriter, r *http.Request) { + session := c.Env["token"].(*models.Token) + + labels, err := env.Labels.GetOwnedBy(session.Owner) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Error("Unable to fetch labels") + + utils.JSONResponse(w, 500, &LabelsListResponse{ + Success: false, + Message: "Internal error (code LA/LI/01)", + }) + return + } + + utils.JSONResponse(w, 200, &LabelsListResponse{ + Success: true, + Labels: &labels, }) } From eaab6ab80261ed7f220dc05ace895054486b5a52 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 11 Jan 2015 00:11:53 +0100 Subject: [PATCH 3/4] New invite tokens feature --- models/token.go | 2 ++ routes/accounts.go | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/models/token.go b/models/token.go index 90c415a..c1c93f3 100644 --- a/models/token.go +++ b/models/token.go @@ -7,6 +7,8 @@ type Token struct { // Type describes the token's purpose: auth, invite, confirm, upgrade. Type string `json:"type" gorethink:"type"` + + Email string `json:"email,omitempty" gorethink:"email"` } // MakeToken creates a generic token. diff --git a/routes/accounts.go b/routes/accounts.go index bf3ede9..6665753 100644 --- a/routes/accounts.go +++ b/routes/accounts.go @@ -183,6 +183,13 @@ func AccountsCreate(w http.ResponseWriter, r *http.Request) { return } + // Both username and password are filled, so we can create a new account. + account := &models.Account{ + Resource: models.MakeResource("", input.Username), + Type: "beta", + AltEmail: input.AltEmail, + } + // Check "invited" for token validity if requestType == "invited" { // Fetch the token from the database @@ -216,17 +223,12 @@ func AccountsCreate(w http.ResponseWriter, r *http.Request) { }) return } + + account.AltEmail = token.Email } // TODO: sanitize user name (i.e. remove caps, periods) - // Both username and password are filled, so we can create a new account. - account := &models.Account{ - Resource: models.MakeResource("", input.Username), - Type: "beta", - AltEmail: input.AltEmail, - } - // Set the password err = account.SetPassword(input.Password) if err != nil { From 35394cbc525ec5a2b761a9035b10b6701dd1cdb8 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 11 Jan 2015 00:53:23 +0100 Subject: [PATCH 4/4] Labels implementation --- db/table_emails.go | 28 ++++++++++++++++++++---- db/table_labels.go | 54 +++++++++++++++++++++++++++++++++++----------- db/table_tokens.go | 2 +- routes/accounts.go | 31 ++++++++++++++++++++++++++ routes/emails.go | 3 ++- setup/setup.go | 3 ++- 6 files changed, 101 insertions(+), 20 deletions(-) diff --git a/db/table_emails.go b/db/table_emails.go index fee1855..c7430bb 100644 --- a/db/table_emails.go +++ b/db/table_emails.go @@ -52,11 +52,31 @@ func (e *EmailsTable) List( sort []string, offset int, limit int, + label string, ) ([]*models.Email, error) { - // Filter by owner's ID - term := e.GetTable().Filter(map[string]interface{}{ - "owner": owner, - }) + + var term gorethink.Term + + if owner != "" && label != "" { + term = e.GetTable().Filter(func(row gorethink.Term) gorethink.Term { + return gorethink.And( + row.Field("owner").Eq(owner), + row.Field("label_ids").Contains(label), + ) + }) + } + + if owner != "" && label == "" { + term = e.GetTable().Filter(map[string]interface{}{ + "owner": owner, + }) + } + + if owner == "" && label != "" { + term = e.GetTable().Filter(func(row gorethink.Term) gorethink.Term { + return row.Field("label_ids").Contains(label) + }) + } // If sort array has contents, parse them and add to the term if sort != nil && len(sort) > 0 { diff --git a/db/table_labels.go b/db/table_labels.go index f28da14..ff518f5 100644 --- a/db/table_labels.go +++ b/db/table_labels.go @@ -71,30 +71,28 @@ func (l *LabelsTable) Delete(cond interface{}) error { // DeleteID removes from db and cache using id query func (l *LabelsTable) DeleteID(id string) error { - if err := l.RethinkCRUD.DeleteID(id); err != nil { + label, err := l.GetLabel(id) + if err != nil { return err } - return l.Cache.Delete(l.RethinkCRUD.GetTableName() + ":" + id) -} - -// FindFetchOne tries cache and then tries using DefaultCRUD's fetch operation -func (l *LabelsTable) FindFetchOne(id string, value interface{}) error { - if err := l.Cache.Get(id, value); err == nil { - return nil - } - - err := l.RethinkCRUD.FindFetchOne(id, value) - if err != nil { + if err := l.RethinkCRUD.DeleteID(l.RethinkCRUD.GetTableName() + ":" + id); err != nil { return err } - return l.Cache.Set(l.RethinkCRUD.GetTableName()+":"+id, value, l.Expires) + l.Cache.Delete(l.RethinkCRUD.GetTableName() + ":" + id) + l.Cache.Delete(l.RethinkCRUD.GetTableName() + ":owner:" + label.Owner) + + return nil } func (l *LabelsTable) GetLabel(id string) (*models.Label, error) { var result models.Label + if err := l.Cache.Get(l.RethinkCRUD.GetTableName()+":"+id, &result); err == nil { + return &result, nil + } + if err := l.FindFetchOne(id, &result); err != nil { return nil, err } @@ -113,6 +111,11 @@ func (l *LabelsTable) GetLabel(id string) (*models.Label, error) { result.EmailsUnread = unreadCount + err = l.Cache.Set(l.RethinkCRUD.GetTableName()+":"+id, result, l.Expires) + if err != nil { + return nil, err + } + return &result, nil } @@ -120,6 +123,10 @@ func (l *LabelsTable) GetLabel(id string) (*models.Label, error) { func (l *LabelsTable) GetOwnedBy(id string) ([]*models.Label, error) { var result []*models.Label + if err := l.Cache.Get(l.RethinkCRUD.GetTableName()+":owner:"+id, &result); err == nil { + return result, nil + } + err := l.WhereAndFetch(map[string]interface{}{ "owner": id, }, &result) @@ -127,5 +134,26 @@ func (l *LabelsTable) GetOwnedBy(id string) ([]*models.Label, error) { return nil, err } + for i := range result { + totalCount, err := l.Emails.CountByLabel(result[i].ID) + if err != nil { + return nil, err + } + + result[i].EmailsTotal = totalCount + + unreadCount, err := l.Emails.CountByLabelUnread(result[i].ID) + if err != nil { + return nil, err + } + + result[i].EmailsTotal = unreadCount + } + + err = l.Cache.Set(l.RethinkCRUD.GetTableName()+":owner:"+id, result, l.Expires) + if err != nil { + return nil, err + } + return result, nil } diff --git a/db/table_tokens.go b/db/table_tokens.go index 5703067..468c491 100644 --- a/db/table_tokens.go +++ b/db/table_tokens.go @@ -81,7 +81,7 @@ func (t *TokensTable) DeleteID(id string) error { // FindFetchOne tries cache and then tries using DefaultCRUD's fetch operation func (t *TokensTable) FindFetchOne(id string, value interface{}) error { - if err := t.Cache.Get(id, value); err == nil { + if err := t.Cache.Get(t.RethinkCRUD.GetTableName()+":"+id, value); err == nil { return nil } diff --git a/routes/accounts.go b/routes/accounts.go index 6665753..4e5b867 100644 --- a/routes/accounts.go +++ b/routes/accounts.go @@ -266,6 +266,37 @@ func AccountsCreate(w http.ResponseWriter, r *http.Request) { return } + // Create labels + err = env.Labels.Insert([]*models.Label{ + &models.Label{ + Resource: models.MakeResource(account.ID, "Inbox"), + Builtin: true, + }, + &models.Label{ + Resource: models.MakeResource(account.ID, "Trash"), + Builtin: true, + }, + &models.Label{ + Resource: models.MakeResource(account.ID, "Spam"), + Builtin: true, + }, + &models.Label{ + Resource: models.MakeResource(account.ID, "Starred"), + Builtin: true, + }, + }) + if err != nil { + utils.JSONResponse(w, 500, &AccountsCreateResponse{ + Success: false, + Message: "Internal server error - AC/CR/03", + }) + + env.Log.WithFields(logrus.Fields{ + "error": err.Error(), + }).Error("Could not insert labels into the database") + return + } + // Send the email if classic and return a response if requestType == "classic" { // TODO: Send emails diff --git a/routes/emails.go b/routes/emails.go index 8f0222b..9d46926 100644 --- a/routes/emails.go +++ b/routes/emails.go @@ -36,6 +36,7 @@ func EmailsList(c web.C, w http.ResponseWriter, r *http.Request) { sortRaw = query.Get("sort") offsetRaw = query.Get("offset") limitRaw = query.Get("limit") + label = query.Get("label") sort []string offset int limit int @@ -80,7 +81,7 @@ func EmailsList(c web.C, w http.ResponseWriter, r *http.Request) { } // Get contacts from the database - emails, err := env.Emails.List(session.Owner, sort, offset, limit) + emails, err := env.Emails.List(session.Owner, sort, offset, limit, label) if err != nil { env.Log.WithFields(logrus.Fields{ "error": err, diff --git a/setup/setup.go b/setup/setup.go index a58ec4d..b3d1a7f 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -160,7 +160,8 @@ func PrepareMux(flags *env.Flags) *web.Mux { rethinkOpts.Database, "labels", ), - Cache: redis, + Emails: env.Emails, + Cache: redis, } // NATS queue connection