From 69d32cb91a85eb03bbb97ec66b814e7f613c8e8e Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Wed, 10 Dec 2014 00:28:08 +0100 Subject: [PATCH 01/30] Redis connection --- cache/cache.go | 9 ++++++ cache/redis_cache.go | 76 ++++++++++++++++++++++++++++++++++++++++++++ env/config.go | 6 +++- main.go | 18 +++++++++-- setup/setup.go | 17 ++++++++-- 5 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 cache/cache.go create mode 100644 cache/redis_cache.go diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..cc5215c --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,9 @@ +package cache + +// Cache is the basic interface for cache implementations +type Cache interface { + Get(key string, pointer interface{}) error + Set(key string, value interface{}) error + Delete(key string) error + Exists(key string) error +} diff --git a/cache/redis_cache.go b/cache/redis_cache.go new file mode 100644 index 0000000..1a2e5d1 --- /dev/null +++ b/cache/redis_cache.go @@ -0,0 +1,76 @@ +package cache + +import ( + "time" + + "github.com/garyburd/redigo/redis" +) + +// RedisCache is an implementation of Cache that uses Redis as a backend +type RedisCache struct { + pool *redis.Pool +} + +// RedisCacheOpts is used to pass options to NewRedisCache +type RedisCacheOpts struct { + Address string + Database int + Password string + MaxIdle int + IdleTimeout time.Duration +} + +// NewRedisCache creates a new cache with a redis backend +func NewRedisCache(options *RedisCacheOpts) (*RedisCache, error) { + // Default values + if options.MaxIdle == 0 { + options.MaxIdle = 3 + } + if options.IdleTimeout == 0 { + options.IdleTimeout = 240 * time.Second + } + + // Create a new redis pool + pool := &redis.Pool{ + MaxIdle: options.MaxIdle, + IdleTimeout: options.IdleTimeout, + Dial: func() (redis.Conn, error) { + c, err := redis.Dial("tcp", options.Address) + if err != nil { + return nil, err + } + + if options.Password != "" { + if _, err := c.Do("AUTH", options.Password); err != nil { + c.Close() + return nil, err + } + } + + if options.Database != 0 { + if _, err := c.Do("SELECT", options.Database); err != nil { + c.Close() + return nil, err + } + } + + return c, err + }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + _, err := c.Do("PING") + return err + }, + } + + // Test the pool + conn := pool.Get() + defer conn.Close() + if err := pool.TestOnBorrow(conn, time.Now()); err != nil { + return nil, err + } + + // Return a new cache struct + return &RedisCache{ + pool: pool, + }, nil +} diff --git a/env/config.go b/env/config.go index 93ea52a..69b1ff9 100644 --- a/env/config.go +++ b/env/config.go @@ -11,7 +11,11 @@ type Flags struct { ClassicRegistration bool UsernameReservation bool - RethinkDBURL string + RedisAddress string + RedisDatabase int + RedisPassword string + + RethinkDBAddress string RethinkDBKey string RethinkDBDatabase string } diff --git a/main.go b/main.go index db34c48..792507c 100644 --- a/main.go +++ b/main.go @@ -27,8 +27,18 @@ var ( sessionDuration = flag.Int("session_duration", 72, "Session duration expressed in hours") classicRegistration = flag.Bool("classic_registration", false, "Classic registration enabled?") usernameReservation = flag.Bool("username_reservation", false, "Username reservation enabled?") + // Cache-related flags + redisAddress = flag.String("redis_address", func() string { + address := os.Getenv("REDIS_PORT_6379_TCP_ADDR") + if address == "" { + address = "127.0.0.1" + } + return address + ":28015" + }(), "Address of the redis server") + redisDatabase = flag.Int("redis_db", 0, "Index of redis database to use") + redisPassword = flag.String("redis_password", "", "Password of the redis server") // Database-related flags - rethinkdbURL = flag.String("rethinkdb_url", func() string { + rethinkdbAddress = flag.String("rethinkdb_address", func() string { address := os.Getenv("RETHINKDB_PORT_28015_TCP_ADDR") if address == "" { address = "127.0.0.1" @@ -60,7 +70,11 @@ func main() { ClassicRegistration: *classicRegistration, UsernameReservation: *usernameReservation, - RethinkDBURL: *rethinkdbURL, + RedisAddress: *redisAddress, + RedisDatabase: *redisDatabase, + RedisPassword: *redisPassword, + + RethinkDBAddress: *rethinkdbAddress, RethinkDBKey: *rethinkdbKey, RethinkDBDatabase: *rethinkdbDatabase, } diff --git a/setup/setup.go b/setup/setup.go index ab93a54..bacedf6 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -8,6 +8,7 @@ import ( "github.com/zenazn/goji/web" "github.com/zenazn/goji/web/middleware" + "github.com/lavab/api/cache" "github.com/lavab/api/db" "github.com/lavab/api/env" "github.com/lavab/api/routes" @@ -31,14 +32,26 @@ func PrepareMux(flags *env.Flags) *web.Mux { // Pass it to the environment package env.Log = log + // Initialize the cache + redis, err := cache.NewRedisCache(&cache.RedisCacheOpts{ + Address: flags.RedisAddress, + Database: flags.RedisDatabase, + Password: flags.RedisPassword, + }) + if err != nil { + log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to connect to the redis server") + } + // Set up the database rethinkOpts := gorethink.ConnectOpts{ - Address: flags.RethinkDBURL, + Address: flags.RethinkDBAddress, AuthKey: flags.RethinkDBKey, MaxIdle: 10, IdleTimeout: time.Second * 10, } - err := db.Setup(rethinkOpts) + err = db.Setup(rethinkOpts) if err != nil { log.WithFields(logrus.Fields{ "error": err, From 79a7b5b006a77cd60951454d0a30675cbe4a1a05 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Wed, 10 Dec 2014 01:01:25 +0100 Subject: [PATCH 02/30] changing port --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 792507c..e7393c0 100644 --- a/main.go +++ b/main.go @@ -33,7 +33,7 @@ var ( if address == "" { address = "127.0.0.1" } - return address + ":28015" + return address + ":6379" }(), "Address of the redis server") redisDatabase = flag.Int("redis_db", 0, "Index of redis database to use") redisPassword = flag.String("redis_password", "", "Password of the redis server") From f4ee3f0777fe79de6943476dddeea233321e4340 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Wed, 10 Dec 2014 14:14:59 +0100 Subject: [PATCH 03/30] Continuing the implementation --- cache/cache.go | 4 +++- cache/redis_cache.go | 16 ++++++++++++++++ env/env.go | 3 +++ setup/setup.go | 2 ++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cache/cache.go b/cache/cache.go index cc5215c..8805f65 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,9 +1,11 @@ package cache +import "time" + // Cache is the basic interface for cache implementations type Cache interface { Get(key string, pointer interface{}) error - Set(key string, value interface{}) error + Set(key string, value interface{}, expires time.Duration) error Delete(key string) error Exists(key string) error } diff --git a/cache/redis_cache.go b/cache/redis_cache.go index 1a2e5d1..ee6af48 100644 --- a/cache/redis_cache.go +++ b/cache/redis_cache.go @@ -74,3 +74,19 @@ func NewRedisCache(options *RedisCacheOpts) (*RedisCache, error) { pool: pool, }, nil } + +func (r *RedisCache) Get(key string, pointer interface{}) error { + return nil +} + +func (r *RedisCache) Set(key string, value interface{}, expires time.Duration) error { + return nil +} + +func (r *RedisCache) Delete(key string) error { + return nil +} + +func (r *RedisCache) Exists(key string) error { + return nil +} diff --git a/env/env.go b/env/env.go index 0a32a7c..bfdac13 100644 --- a/env/env.go +++ b/env/env.go @@ -4,6 +4,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/dancannon/gorethink" + "github.com/lavab/api/cache" "github.com/lavab/api/db" ) @@ -14,6 +15,8 @@ var ( Log *logrus.Logger // Rethink contains the RethinkDB session used in the API Rethink *gorethink.Session + // Cache is the global instance of the cache interface + Cache cache.Cache // Accounts is the global instance of AccountsTable Accounts *db.AccountsTable // Tokens is the global instance of TokensTable diff --git a/setup/setup.go b/setup/setup.go index bacedf6..8ef00e5 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -44,6 +44,8 @@ func PrepareMux(flags *env.Flags) *web.Mux { }).Fatal("Unable to connect to the redis server") } + env.Cache = redis + // Set up the database rethinkOpts := gorethink.ConnectOpts{ Address: flags.RethinkDBAddress, From 2d3c9abcac671d0366662797ee7d9295b82d6987 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Wed, 10 Dec 2014 21:06:54 +0100 Subject: [PATCH 04/30] Added vagrant for easier testing --- .gitignore | 1 + _vagrant/Vagrantfile | 20 + _vagrant/deploy.sh | 14 + _vagrant/playbook.yml | 46 +++ _vagrant/redis.conf | 827 ++++++++++++++++++++++++++++++++++++++++ _vagrant/rethinkdb.conf | 68 ++++ routes/init_test.go | 6 +- setup/setup_test.go | 2 +- 8 files changed, 981 insertions(+), 3 deletions(-) create mode 100644 _vagrant/Vagrantfile create mode 100644 _vagrant/deploy.sh create mode 100644 _vagrant/playbook.yml create mode 100644 _vagrant/redis.conf create mode 100644 _vagrant/rethinkdb.conf diff --git a/.gitignore b/.gitignore index 9bb4877..1e6f0b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ *.exe +_vagrant/.vagrant diff --git a/_vagrant/Vagrantfile b/_vagrant/Vagrantfile new file mode 100644 index 0000000..8b5447e --- /dev/null +++ b/_vagrant/Vagrantfile @@ -0,0 +1,20 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "ubuntu/trusty64" + + # rethinkdb + config.vm.network "forwarded_port", guest: 8080, host: 8080 + config.vm.network "forwarded_port", guest: 28015, host: 28015 + config.vm.network "forwarded_port", guest: 29015, host: 29015 + + # redis + config.vm.network "forwarded_port", guest: 6379, host: 6379 + + # load ansible playbook + config.vm.provision "shell", path: "deploy.sh" +end diff --git a/_vagrant/deploy.sh b/_vagrant/deploy.sh new file mode 100644 index 0000000..95804d3 --- /dev/null +++ b/_vagrant/deploy.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +export DEBIAN_FRONTEND=noninteractive + +apt-get update -qq +apt-get install -y python2.7 python-pip + +pip install ansible + +mkdir /etc/ansible +echo "localhost" > /etc/ansible/hosts + +cd /vagrant +ansible-playbook -vvvv playbook.yml diff --git a/_vagrant/playbook.yml b/_vagrant/playbook.yml new file mode 100644 index 0000000..72f0e27 --- /dev/null +++ b/_vagrant/playbook.yml @@ -0,0 +1,46 @@ +--- +- hosts: 127.0.0.1 + connection: local + sudo: true + tasks: + # install rethinkdb + - name: add rethinkdb sources + shell: . /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | tee /etc/apt/sources.list.d/rethinkdb.list + - name: add rethinkdb key + shell: wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | apt-key add - + - name: update apt cache + apt: update_cache=yes + - name: install rethinkdb + apt: pkg=rethinkdb state=present + # configure rethinkdb + - name: ensure group is present + group: name=rethinkdb state=present + - name: ensure user is present + user: name=rethinkdb state=present + - name: copy master config file + copy: src=./rethinkdb.conf dest=/etc/rethinkdb/instances.d/rethinkdb.conf owner=rethinkdb group=rethinkdb mode=664 + - name: start rethinkdb + service: name=rethinkdb state=restarted + # install redis + - name: prepare a directory + shell: mkdir /tmp/redis-stable + - name: download redis + get_url: url=http://download.redis.io/redis-stable.tar.gz dest=/tmp/redis-stable.tar.gz + - name: unpack redis + unarchive: src=/tmp/redis-stable.tar.gz dest=/tmp/redis-stable copy=no + - name: make redis + shell: 'cd /tmp/redis-stable/redis-stable && make && make install' + - name: ensure group is present + group: name=redis state=present + - name: ensure user is present + user: name=redis state=present + - name: prepare config and storage directories for redis + shell: mkdir /etc/redis; mkdir /var/redis; sudo mkdir /var/redis/6379 + - name: prepare the config and an init script for redis + shell: sudo cp /tmp/redis-stable/redis-stable/utils/redis_init_script /etc/init.d/redis_6379 + - name: upload the config to the vm + copy: src=./redis.conf dest=/etc/redis/6379.conf owner=redis group=redis mode=664 + - name: put the prepared config on the server + shell: sudo update-rc.d redis_6379 defaults + - name: start rethinkdb + service: name=rethinkdb state=restarted diff --git a/_vagrant/redis.conf b/_vagrant/redis.conf new file mode 100644 index 0000000..d176fe0 --- /dev/null +++ b/_vagrant/redis.conf @@ -0,0 +1,827 @@ +# Redis configuration file example + +# Note on units: when memory size is needed, it is possible to specify +# it in the usual form of 1k 5GB 4M and so forth: +# +# 1k => 1000 bytes +# 1kb => 1024 bytes +# 1m => 1000000 bytes +# 1mb => 1024*1024 bytes +# 1g => 1000000000 bytes +# 1gb => 1024*1024*1024 bytes +# +# units are case insensitive so 1GB 1Gb 1gB are all the same. + +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis servers but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################ GENERAL ##################################### + +# By default Redis does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. +daemonize yes + +# When running daemonized, Redis writes a pid file in /var/run/redis.pid by +# default. You can specify a custom pid file location here. +pidfile /var/run/redis_6379.pid + +# Accept connections on the specified port, default is 6379. +# If port 0 is specified Redis will not listen on a TCP socket. +port 6379 + +# TCP listen() backlog. +# +# In high requests-per-second environments you need an high backlog in order +# to avoid slow clients connections issues. Note that the Linux kernel +# will silently truncate it to the value of /proc/sys/net/core/somaxconn so +# make sure to raise both the value of somaxconn and tcp_max_syn_backlog +# in order to get the desired effect. +tcp-backlog 511 + +# By default Redis listens for connections from all the network interfaces +# available on the server. It is possible to listen to just one or multiple +# interfaces using the "bind" configuration directive, followed by one or +# more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 + +# Specify the path for the Unix socket that will be used to listen for +# incoming connections. There is no default, so Redis will not listen +# on a unix socket when not specified. +# +# unixsocket /tmp/redis.sock +# unixsocketperm 700 + +# Close the connection after a client is idle for N seconds (0 to disable) +timeout 0 + +# TCP keepalive. +# +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: +# +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. +tcp-keepalive 0 + +# Specify the server verbosity level. +# This can be one of: +# debug (a lot of information, useful for development/testing) +# verbose (many rarely useful info, but not a mess like the debug level) +# notice (moderately verbose, what you want in production probably) +# warning (only very important / critical messages are logged) +loglevel notice + +# Specify the log file name. Also the empty string can be used to force +# Redis to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile /var/log/redis_6379.log + +# To enable logging to the system logger, just set 'syslog-enabled' to yes, +# and optionally update the other syslog parameters to suit your needs. +# syslog-enabled no + +# Specify the syslog identity. +# syslog-ident redis + +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# syslog-facility local0 + +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases'-1 +databases 16 + +################################ SNAPSHOTTING ################################ +# +# Save the DB on disk: +# +# save +# +# Will save the DB if both the given number of seconds and the given +# number of write operations against the DB occurred. +# +# In the example below the behaviour will be to save: +# after 900 sec (15 min) if at least 1 key changed +# after 300 sec (5 min) if at least 10 keys changed +# after 60 sec if at least 10000 keys changed +# +# Note: you can disable saving completely by commenting out all "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" + +save 900 1 +save 300 10 +save 60 10000 + +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in a hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# disaster will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usual even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + +# Compress string objects using LZF when dump .rdb databases? +# For default that's set to 'yes' as it's almost always a win. +# If you want to save some CPU in the saving child set it to 'no' but +# the dataset will likely be bigger if you have compressible values or keys. +rdbcompression yes + +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + +# The filename where to dump the DB +dbfilename dump.rdb + +# The working directory. +# +# The DB will be written inside this directory, with the filename specified +# above using the 'dbfilename' configuration directive. +# +# The Append Only File will also be created inside this directory. +# +# Note that you must specify a directory here, not a file name. +dir /var/redis/6379 + +################################# REPLICATION ################################# + +# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# another Redis server. A few things to understand ASAP about Redis replication. +# +# 1) Redis replication is asynchronous, but you can configure a master to +# stop accepting writes if it appears to be not connected with at least +# a given number of slaves. +# 2) Redis slaves are able to perform a partial resynchronization with the +# master if the replication link is lost for a relatively small amount of +# time. You may want to configure the replication backlog size (see the next +# sections of this file) with a sensible value depending on your needs. +# 3) Replication is automatic and does not need user intervention. After a +# network partition slaves automatically try to reconnect to masters +# and resynchronize with them. +# +# slaveof + +# If the master is password protected (using the "requirepass" configuration +# directive below) it is possible to tell the slave to authenticate before +# starting the replication synchronization process, otherwise the master will +# refuse the slave request. +# +# masterauth + +# When a slave loses its connection with the master, or when the replication +# is still in progress, the slave can act in two different ways: +# +# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# still reply to client requests, possibly with out of date data, or the +# data set may just be empty if this is the first synchronization. +# +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# an error "SYNC with master in progress" to all the kind of commands +# but to INFO and SLAVEOF. +# +slave-serve-stale-data yes + +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it because of a +# misconfiguration. +# +# Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. +slave-read-only yes + +# Replication SYNC strategy: disk or socket. +# +# ------------------------------------------------------- +# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY +# ------------------------------------------------------- +# +# New slaves and reconnecting slaves that are not able to continue the replication +# process just receiving differences, need to do what is called a "full +# synchronization". An RDB file is transmitted from the master to the slaves. +# The transmission can happen in two different ways: +# +# 1) Disk-backed: The Redis master creates a new process that writes the RDB +# file on disk. Later the file is transferred by the parent +# process to the slaves incrementally. +# 2) Diskless: The Redis master creates a new process that directly writes the +# RDB file to slave sockets, without touching the disk at all. +# +# With disk-backed replication, while the RDB file is generated, more slaves +# can be queued and served with the RDB file as soon as the current child producing +# the RDB file finishes its work. With diskless replication instead once +# the transfer starts, new slaves arriving will be queued and a new transfer +# will start when the current one terminates. +# +# When diskless replication is used, the master waits a configurable amount of +# time (in seconds) before starting the transfer in the hope that multiple slaves +# will arrive and the transfer can be parallelized. +# +# With slow disks and fast (large bandwidth) networks, diskless replication +# works better. +repl-diskless-sync no + +# When diskless replication is enabled, it is possible to configure the delay +# the server waits in order to spawn the child that trnasfers the RDB via socket +# to the slaves. +# +# This is important since once the transfer starts, it is not possible to serve +# new slaves arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more slaves arrive. +# +# The delay is specified in seconds, and by default is 5 seconds. To disable +# it entirely just set it to 0 seconds and the transfer will start ASAP. +repl-diskless-sync-delay 5 + +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The bigger the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + +# The slave priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a slave to promote into a +# master if the master is no longer working correctly. +# +# A slave with a low priority number is considered better for promotion, so +# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# pick the one with priority 10, that is the lowest. +# +# However a special priority of 0 marks the slave as not able to perform the +# role of master, so a slave with priority of 0 will never be selected by +# Redis Sentinel for promotion. +# +# By default the priority is 100. +slave-priority 100 + +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEE that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. + +################################## SECURITY ################################### + +# Require clients to issue AUTH before processing any other +# commands. This might be useful in environments in which you do not trust +# others with access to the host running redis-server. +# +# This should stay commented out for backward compatibility and because most +# people do not need auth (e.g. they run their own servers). +# +# Warning: since Redis is pretty fast an outside user can try up to +# 150k passwords per second against a good box. This means that you should +# use a very strong password otherwise it will be very easy to break. +# +# requirepass foobared + +# Command renaming. +# +# It is possible to change the name of dangerous commands in a shared +# environment. For instance the CONFIG command may be renamed into something +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. +# +# Example: +# +# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 +# +# It is also possible to completely kill a command by renaming it into +# an empty string: +# +# rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. + +################################### LIMITS #################################### + +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able to configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# +# Once the limit is reached Redis will close all the new connections sending +# an error 'max number of clients reached'. +# +# maxclients 10000 + +# Don't use more memory than the specified amount of bytes. +# When the memory limit is reached Redis will try to remove keys +# according to the eviction policy selected (see maxmemory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# a hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). +# +# maxmemory + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select among five behaviors: +# +# volatile-lru -> remove the key with an expire set using an LRU algorithm +# allkeys-lru -> remove any key according to the LRU algorithm +# volatile-random -> remove a random key with an expire set +# allkeys-random -> remove a random key, any key +# volatile-ttl -> remove the key with the nearest expire time (minor TTL) +# noeviction -> don't expire at all, just return an error on write operations +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy volatile-lru + +# LRU and minimal TTL algorithms are not precise algorithms but approximated +# algorithms (in order to save memory), so you can select as well the sample +# size to check. For instance for default Redis will check three keys and +# pick the one that was used less recently, you can change the sample size +# using the following configuration directive. +# +# maxmemory-samples 3 + +############################## APPEND ONLY MODE ############################### + +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. + +appendonly no + +# The name of the append only file (default: "appendonly.aof") + +appendfilename "appendonly.aof" + +# The fsync() call tells the Operating System to actually write data on disk +# instead of waiting for more data in the output buffer. Some OS will really flush +# data on disk, some other OS will just try to do it ASAP. +# +# Redis supports three different modes: +# +# no: don't fsync, just let the OS flush the data when it wants. Faster. +# always: fsync after every write to the append only log. Slow, Safest. +# everysec: fsync only one time every second. Compromise. +# +# The default is "everysec", as that's usually the right compromise between +# speed and data safety. It's up to you to understand if you can relax this to +# "no" that will let the operating system flush the output buffer when +# it wants, for better performances (but if you can live with the idea of +# some data loss consider the default persistence mode that's snapshotting), +# or on the contrary, use "always" that's very slow but a bit safer than +# everysec. +# +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# +# If unsure, use "everysec". + +# appendfsync always +appendfsync everysec +# appendfsync no + +# When the AOF fsync policy is set to always or everysec, and a background +# saving process (a background save or AOF log background rewriting) is +# performing a lot of I/O against the disk, in some Linux configurations +# Redis may block too long on the fsync() call. Note that there is no fix for +# this currently, as even performing fsync in a different thread will block +# our synchronous write(2) call. +# +# In order to mitigate this problem it's possible to use the following option +# that will prevent fsync() from being called in the main process while a +# BGSAVE or BGREWRITEAOF is in progress. +# +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the +# default Linux settings). +# +# If you have latency problems turn this to "yes". Otherwise leave it as +# "no" that is the safest pick from the point of view of durability. + +no-appendfsync-on-rewrite no + +# Automatic rewrite of the append only file. +# Redis is able to automatically rewrite the log file implicitly calling +# BGREWRITEAOF when the AOF log size grows by the specified percentage. +# +# This is how it works: Redis remembers the size of the AOF file after the +# latest rewrite (if no rewrite has happened since the restart, the size of +# the AOF at startup is used). +# +# This base size is compared to the current size. If the current size is +# bigger than the specified percentage, the rewrite is triggered. Also +# you need to specify a minimal size for the AOF file to be rewritten, this +# is useful to avoid rewriting the AOF file even if the percentage increase +# is reached but it is still pretty small. +# +# Specify a percentage of zero in order to disable the automatic AOF +# rewrite feature. + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb + +# An AOF file may be found to be truncated at the end during the Redis +# startup process, when the AOF data gets loaded back into memory. +# This may happen when the system where Redis is running +# crashes, especially when an ext4 filesystem is mounted without the +# data=ordered option (however this can't happen when Redis itself +# crashes or aborts but the operating system still works correctly). +# +# Redis can either exit with an error when this happens, or load as much +# data as possible (the default now) and start if the AOF file is found +# to be truncated at the end. The following option controls this behavior. +# +# If aof-load-truncated is set to yes, a truncated AOF file is loaded and +# the Redis server starts emitting a log to inform the user of the event. +# Otherwise if the option is set to no, the server aborts with an error +# and refuses to start. When the option is set to no, the user requires +# to fix the AOF file using the "redis-check-aof" utility before to restart +# the server. +# +# Note that if the AOF file will be found to be corrupted in the middle +# the server will still exit with an error. This option only applies when +# Redis will try to read more data from the AOF file but not enough bytes +# will be found. +aof-load-truncated yes + +################################ LUA SCRIPTING ############################### + +# Max execution time of a Lua script in milliseconds. +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceeds the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write command was +# already issued by the script but the user doesn't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################## SLOW LOG ################################### + +# The Redis Slow Log is a system to log queries that exceeded a specified +# execution time. The execution time does not include the I/O operations +# like talking with the client, sending the reply and so forth, +# but just the time needed to actually execute the command (this is the only +# stage of command execution where the thread is blocked and can not serve +# other requests in the meantime). +# +# You can configure the slow log with two parameters: one tells Redis +# what is the execution time, in microseconds, to exceed in order for the +# command to get logged, and the other parameter is the length of the +# slow log. When a new command is logged the oldest one is removed from the +# queue of logged commands. + +# The following time is expressed in microseconds, so 1000000 is equivalent +# to one second. Note that a negative number disables the slow log, while +# a value of zero forces the logging of every command. +slowlog-log-slower-than 10000 + +# There is no limit to this length. Just be aware that it will consume memory. +# You can reclaim memory used by the slow log with SLOWLOG RESET. +slowlog-max-len 128 + +################################ LATENCY MONITOR ############################## + +# The Redis latency monitoring subsystem samples different operations +# at runtime in order to collect data related to possible sources of +# latency of a Redis instance. +# +# Via the LATENCY command this information is available to the user that can +# print graphs and obtain reports. +# +# The system only logs operations that were performed in a time equal or +# greater than the amount of milliseconds specified via the +# latency-monitor-threshold configuration directive. When its value is set +# to zero, the latency monitor is turned off. +# +# By default latency monitoring is disabled since it is mostly not needed +# if you don't have latency issues, and collecting data has a performance +# impact, that while very small, can be measured under big load. Latency +# monitoring can easily be enalbed at runtime using the command +# "CONFIG SET latency-monitor-threshold " if needed. +latency-monitor-threshold 0 + +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/notifications +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# of zero or multiple characters. The empty string means that notifications +# are disabled. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" + +############################### ADVANCED CONFIG ############################### + +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 + +# Similarly to hashes, small lists are also encoded in a special way in order +# to save a lot of space. The special representation is only used when +# you are under the following limits: +list-max-ziplist-entries 512 +list-max-ziplist-value 64 + +# Sets have a special encoding in just one case: when a set is composed +# of just strings that happen to be integers in radix 10 in the range +# of 64 bit signed integers. +# The following configuration setting sets the limit in the size of the +# set in order to use this special memory saving encoding. +set-max-intset-entries 512 + +# Similarly to hashes and lists, sorted sets are also specially encoded in +# order to save a lot of space. This encoding is only used when the length and +# elements of a sorted set are below the following limits: +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +# HyperLogLog sparse representation bytes limit. The limit includes the +# 16 bytes header. When an HyperLogLog using the sparse representation crosses +# this limit, it is converted into the dense representation. +# +# A value greater than 16000 is totally useless, since at that point the +# dense representation is more memory efficient. +# +# The suggested value is ~ 3000 in order to have the benefits of +# the space efficient encoding without slowing down too much PFADD, +# which is O(N) with the sparse encoding. The value can be raised to +# ~ 10000 when CPU is not a concern, but space is, and the data set is +# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. +hll-sparse-max-bytes 3000 + +# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in +# order to help rehashing the main Redis hash table (the one mapping top-level +# keys to values). The hash table implementation Redis uses (see dict.c) +# performs a lazy rehashing: the more operation you run into a hash table +# that is rehashing, the more rehashing "steps" are performed, so if the +# server is idle the rehashing is never complete and some more memory is used +# by the hash table. +# +# The default is to use this millisecond 10 times every second in order to +# actively rehash the main dictionaries, freeing memory when possible. +# +# If unsure: +# use "activerehashing no" if you have hard latency requirements and it is +# not a good thing in your environment that Redis can reply from time to time +# to queries with 2 milliseconds delay. +# +# use "activerehashing yes" if you don't have such hard requirements but +# want to free memory asap when possible. +activerehashing yes + +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients including MONITOR clients +# slave -> slave clients +# pubsub -> clients subscribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled by setting them to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeout, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are performed with the same frequency, but Redis checks for +# tasks to perform according to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes diff --git a/_vagrant/rethinkdb.conf b/_vagrant/rethinkdb.conf new file mode 100644 index 0000000..4e6cce2 --- /dev/null +++ b/_vagrant/rethinkdb.conf @@ -0,0 +1,68 @@ +# +# RethinkDB instance configuration sample +# +# - Give this file the extension .conf and put it in /etc/rethinkdb/instances.d in order to enable it. +# - See http://www.rethinkdb.com/docs/guides/startup/ for the complete documentation +# - Uncomment an option to change its value. +# + +############################### +## RethinkDB configuration +############################### + +### Process options + +## User and group used to run rethinkdb +## Command line default: do not change user or group +## Init script default: rethinkdb user and group +# runuser=rethinkdb +# rungroup=rethinkdb + +## Stash the pid in this file when the process is running +## Command line default: none +## Init script default: /var/run/rethinkdb//pid_file (where is the name of this config file without the extension) +# pid-file=/var/run/rethinkdb/rethinkdb.pid + +### File path options + +## Directory to store data and metadata +## Command line default: ./rethinkdb_data +## Init script default: /var/lib/rethinkdb// (where is the name of this file without the extension) +# directory=/var/lib/rethinkdb/default + +### Network options + +## Address of local interfaces to listen on when accepting connections +## May be 'all' or an IP address, loopback addresses are enabled by default +## Default: all local addresses +bind=all + +## The port for rethinkdb protocol for client drivers +## Default: 28015 + port-offset +# driver-port=28015 + +## The port for receiving connections from other nodes +## Default: 29015 + port-offset +# cluster-port=29015 + +## The host:port of a node that rethinkdb will connect to +## This option can be specified multiple times. +## Default: none +# join=example.com:29015 + +## All ports used locally will have this value added +## Default: 0 +# port-offset=0 + +### Web options + +## Port for the http admin console +## Default: 8080 + port-offset +# http-port=8080 + +### CPU options + +## The number of cores to use +## Default: total number of cores of the CPU +# cores=2 + diff --git a/routes/init_test.go b/routes/init_test.go index 6d1d576..8818f23 100644 --- a/routes/init_test.go +++ b/routes/init_test.go @@ -27,14 +27,16 @@ func init() { ClassicRegistration: true, UsernameReservation: true, - RethinkDBURL: "127.0.0.1:28015", + RedisAddress: "127.0.0.1:6379", + + RethinkDBAddress: "127.0.0.1:28015", RethinkDBKey: "", RethinkDBDatabase: "test", } // Connect to the RethinkDB server rdbSession, err := gorethink.Connect(gorethink.ConnectOpts{ - Address: env.Config.RethinkDBURL, + Address: env.Config.RethinkDBAddress, AuthKey: env.Config.RethinkDBKey, MaxIdle: 10, IdleTimeout: time.Second * 10, diff --git a/setup/setup_test.go b/setup/setup_test.go index ebbe49b..34af288 100644 --- a/setup/setup_test.go +++ b/setup/setup_test.go @@ -18,7 +18,7 @@ func TestSetup(t *testing.T) { SessionDuration: 72, ClassicRegistration: true, - RethinkDBURL: "127.0.0.1:28015", + RethinkDBAddress: "127.0.0.1:28015", RethinkDBKey: "", RethinkDBDatabase: "test", } From 369509c29250956e6b25ea3b00d6cea60a203341 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Wed, 10 Dec 2014 22:42:27 +0100 Subject: [PATCH 05/30] Fixed some small issues that made tests impossible --- _vagrant/playbook.yml | 4 ++-- routes/init_test.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/_vagrant/playbook.yml b/_vagrant/playbook.yml index 72f0e27..55e9e82 100644 --- a/_vagrant/playbook.yml +++ b/_vagrant/playbook.yml @@ -42,5 +42,5 @@ copy: src=./redis.conf dest=/etc/redis/6379.conf owner=redis group=redis mode=664 - name: put the prepared config on the server shell: sudo update-rc.d redis_6379 defaults - - name: start rethinkdb - service: name=rethinkdb state=restarted + - name: start redis_6379 + service: name=redis_6379 state=restarted diff --git a/routes/init_test.go b/routes/init_test.go index 8818f23..a57c5a7 100644 --- a/routes/init_test.go +++ b/routes/init_test.go @@ -1,6 +1,7 @@ package routes_test import ( + "fmt" "net/http/httptest" "time" @@ -48,7 +49,7 @@ func init() { // Clear the test database err = gorethink.DbDrop("test").Exec(rdbSession) if err != nil { - panic("removing the test database should not return an error, got " + err.Error()) + fmt.Println("removing the test database should not return an error, got " + err.Error()) } // Disconnect From c159a6fff3c2095d9a0e82f755e3dc23b48f094d Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:18:09 +0100 Subject: [PATCH 06/30] Cache code --- cache/cache.go | 4 ++- cache/redis_cache.go | 78 +++++++++++++++++++++++++++++++++++++--- db/default_crud.go | 5 +++ db/rethink_crud.go | 12 ++++--- db/table_tokens.go | 84 ++++++++++++++++++++++++++++++++++++++++++++ setup/setup.go | 1 + 6 files changed, 173 insertions(+), 11 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 8805f65..a95ad01 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -7,5 +7,7 @@ type Cache interface { Get(key string, pointer interface{}) error Set(key string, value interface{}, expires time.Duration) error Delete(key string) error - Exists(key string) error + DeleteMask(mask string) error + DeleteMulti(keys ...interface{}) error + Exists(key string) (bool, error) } diff --git a/cache/redis_cache.go b/cache/redis_cache.go index ee6af48..d83e968 100644 --- a/cache/redis_cache.go +++ b/cache/redis_cache.go @@ -1,11 +1,22 @@ package cache import ( + "bytes" + "encoding/gob" "time" "github.com/garyburd/redigo/redis" ) +// Scripts! +var ( + scriptDeleteMask = redis.NewScript(0, ` + for _, k in ipairs( redis.call( 'keys', ARGV[1] ) ) do + redis.call( 'del', k ) + end + `) +) + // RedisCache is an implementation of Cache that uses Redis as a backend type RedisCache struct { pool *redis.Pool @@ -75,18 +86,75 @@ func NewRedisCache(options *RedisCacheOpts) (*RedisCache, error) { }, nil } +// Get retrieves data from the database and then decodes it into the passed pointer. func (r *RedisCache) Get(key string, pointer interface{}) error { - return nil + conn := r.pool.Get() + defer conn.Close() + + // Perform the get + data, err := redis.Bytes(conn.Do("GET", key)) + if err != nil { + return err + } + + // Initialize a new decoder + dec := gob.NewDecoder(bytes.NewReader(data)) + + // Decode it into pointer + return dec.Decode(pointer) } +// Set encodes passed value and sends it to redis func (r *RedisCache) Set(key string, value interface{}, expires time.Duration) error { - return nil + conn := r.pool.Get() + defer conn.Close() + + // Initialize a new encoder + var buffer bytes.Buffer + enc := gob.NewEncoder(&buffer) + + // Encode the value + if err := enc.Encode(value); err != nil { + return err + } + + // Save it into redis + if expires == 0 { + _, err := conn.Do("SET", key, buffer.Bytes()) + return err + } + + _, err := conn.Do("SETEX", key, expires.Seconds(), buffer.Bytes()) + return err } +// Delete removes data in redis by key func (r *RedisCache) Delete(key string) error { - return nil + conn := r.pool.Get() + defer conn.Close() + _, err := redis.Int(conn.Do("DEL", key)) + return err } -func (r *RedisCache) Exists(key string) error { - return nil +// DeleteMask removes data using KEYS masks +func (r *RedisCache) DeleteMask(mask string) error { + conn := r.pool.Get() + defer conn.Close() + _, err := scriptDeleteMask.Do(conn, mask) + return err +} + +// DeleteMulti removes multiple keys +func (r *RedisCache) DeleteMulti(keys ...interface{}) error { + conn := r.pool.Get() + defer conn.Close() + _, err := redis.Int(conn.Do("DEL", keys...)) + return err +} + +// Exists performs a check whether a key exists +func (r *RedisCache) Exists(key string) (bool, error) { + conn := r.pool.Get() + defer conn.Close() + return redis.Bool(conn.Do("EXISTS", key)) } diff --git a/db/default_crud.go b/db/default_crud.go index c3976c5..8cdfef5 100644 --- a/db/default_crud.go +++ b/db/default_crud.go @@ -35,6 +35,11 @@ func (d *Default) GetTable() gorethink.Term { return gorethink.Table(d.table) } +// GetSession returns the current session +func (d *Default) GetSession() *gorethink.Session { + return d.session +} + // Insert inserts a document into the database func (d *Default) Insert(data interface{}) error { _, err := d.GetTable().Insert(data).RunWrite(d.session) diff --git a/db/rethink_crud.go b/db/rethink_crud.go index 8c669ef..adaeaed 100644 --- a/db/rethink_crud.go +++ b/db/rethink_crud.go @@ -1,13 +1,15 @@ package db import ( - rethink "github.com/dancannon/gorethink" + "github.com/dancannon/gorethink" ) // RethinkTable contains the most basic table functions type RethinkTable interface { GetTableName() string GetDBName() string + GetTable() gorethink.Term + GetSession() *gorethink.Session } // RethinkCreator contains a function to create new instances in the table @@ -17,19 +19,19 @@ type RethinkCreator interface { // RethinkReader allows fetching resources from the database type RethinkReader interface { - Find(id string) (*rethink.Cursor, error) + Find(id string) (*gorethink.Cursor, error) FindFetchOne(id string, value interface{}) error - FindBy(key string, value interface{}) (*rethink.Cursor, error) + FindBy(key string, value interface{}) (*gorethink.Cursor, error) FindByAndFetch(key string, value interface{}, results interface{}) error FindByAndFetchOne(key string, value interface{}, result interface{}) error FindByAndCount(key string, value interface{}) (int, error) - Where(filter map[string]interface{}) (*rethink.Cursor, error) + Where(filter map[string]interface{}) (*gorethink.Cursor, error) WhereAndFetch(filter map[string]interface{}, results interface{}) error WhereAndFetchOne(filter map[string]interface{}, result interface{}) error - FindByIndex(index string, values ...interface{}) (*rethink.Cursor, error) + FindByIndex(index string, values ...interface{}) (*gorethink.Cursor, error) FindByIndexFetch(results interface{}, index string, values ...interface{}) error FindByIndexFetchOne(result interface{}, index string, values ...interface{}) error } diff --git a/db/table_tokens.go b/db/table_tokens.go index 8de409e..5703067 100644 --- a/db/table_tokens.go +++ b/db/table_tokens.go @@ -1,12 +1,96 @@ package db import ( + "time" + + "github.com/dancannon/gorethink" + + "github.com/lavab/api/cache" "github.com/lavab/api/models" ) // TokensTable implements the CRUD interface for tokens type TokensTable struct { RethinkCRUD + Cache cache.Cache + Expires time.Duration +} + +// Insert monkey-patches the DefaultCRUD method and introduces caching +func (t *TokensTable) Insert(data interface{}) error { + if err := t.RethinkCRUD.Insert(data); err != nil { + return err + } + + token, ok := data.(*models.Token) + if !ok { + return nil + } + + return t.Cache.Set(t.RethinkCRUD.GetTableName()+":"+token.ID, token, t.Expires) +} + +// Update clears all updated keys +func (t *TokensTable) Update(data interface{}) error { + if err := t.RethinkCRUD.Update(data); err != nil { + return err + } + + return t.Cache.DeleteMask(t.RethinkCRUD.GetTableName() + ":*") +} + +// UpdateID updates the specified token and updates cache +func (t *TokensTable) UpdateID(id string, data interface{}) error { + if err := t.RethinkCRUD.UpdateID(id, data); err != nil { + return err + } + + token, err := t.GetToken(id) + if err != nil { + return err + } + + return t.Cache.Set(t.RethinkCRUD.GetTableName()+":"+id, token, t.Expires) +} + +// Delete removes from db and cache using filter +func (t *TokensTable) Delete(cond interface{}) error { + result, err := t.GetTable().Filter(cond).Delete(gorethink.DeleteOpts{ + ReturnChanges: true, + }).RunWrite(t.GetSession()) + if err != nil { + return err + } + + var ids []interface{} + for _, change := range result.Changes { + ids = append(ids, t.RethinkCRUD.GetTableName()+":"+change.OldValue.(map[string]interface{})["id"].(string)) + } + + return t.Cache.DeleteMulti(ids...) +} + +// DeleteID removes from db and cache using id query +func (t *TokensTable) DeleteID(id string) error { + if err := t.RethinkCRUD.DeleteID(id); err != nil { + return err + } + + return t.Cache.Delete(t.RethinkCRUD.GetTableName() + ":" + id) +} + +// FindFetchOne tries cache and then tries using DefaultCRUD's fetch operation +func (t *TokensTable) FindFetchOne(id string, value interface{}) error { + if err := t.Cache.Get(id, value); err == nil { + return nil + } + + err := t.RethinkCRUD.FindFetchOne(id, value) + if err != nil { + return err + } + + return t.Cache.Set(t.RethinkCRUD.GetTableName()+":"+id, value, t.Expires) } // GetToken returns a token with specified name diff --git a/setup/setup.go b/setup/setup.go index 8ef00e5..07bb0fc 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -79,6 +79,7 @@ func PrepareMux(flags *env.Flags) *web.Mux { rethinkOpts.Database, "tokens", ), + Cache: redis, } env.Accounts = &db.AccountsTable{ RethinkCRUD: db.NewCRUDTable( From d08fd276ba5d029d9d8432c721564cca6ff07bfe Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:22:36 +0100 Subject: [PATCH 07/30] Added redis to the travis tests --- .travis.yml | 3 +++ setup/setup_test.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1c03fcf..84a85c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ language: go go: - 1.3.1 +services: + - redis-server + before_script: - source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list - wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - diff --git a/setup/setup_test.go b/setup/setup_test.go index 34af288..406ecbe 100644 --- a/setup/setup_test.go +++ b/setup/setup_test.go @@ -18,6 +18,8 @@ func TestSetup(t *testing.T) { SessionDuration: 72, ClassicRegistration: true, + RedisAddress: "127.0.0.1:6379", + RethinkDBAddress: "127.0.0.1:28015", RethinkDBKey: "", RethinkDBDatabase: "test", From 50885df410258ec141b6fc7b6b4a9506ffc30e71 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:38:20 +0100 Subject: [PATCH 08/30] Added a CircleCI build script --- circle.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 circle.yml diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..a7bb463 --- /dev/null +++ b/circle.yml @@ -0,0 +1,15 @@ +machine: + timezone: + Europe/Berlin + +dependencies: + pre: + - source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list + - wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - + - sudo apt-get update + - sudo apt-get install rethinkdb + - wget http://download.redis.io/releases/redis-2.8.18.tar.gz + - tar xvzf redis-2.8.18.tar.gz + - cd redis-2.8.18.tar.gz && make + - rethinkdb --bind all & + - src/redis-server & From f262dfc692331eefa5b8c46a13e379652befffbd Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:40:35 +0100 Subject: [PATCH 09/30] Fixing typo in circle.yml --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index a7bb463..1772094 100644 --- a/circle.yml +++ b/circle.yml @@ -10,6 +10,6 @@ dependencies: - sudo apt-get install rethinkdb - wget http://download.redis.io/releases/redis-2.8.18.tar.gz - tar xvzf redis-2.8.18.tar.gz - - cd redis-2.8.18.tar.gz && make + - cd redis-2.8.18 && make - rethinkdb --bind all & - src/redis-server & From abefed5a9db67d8d27acd05cb3731cce38a231ad Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:44:30 +0100 Subject: [PATCH 10/30] Trying to get services running --- circle.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index 1772094..d83be8a 100644 --- a/circle.yml +++ b/circle.yml @@ -11,5 +11,8 @@ dependencies: - wget http://download.redis.io/releases/redis-2.8.18.tar.gz - tar xvzf redis-2.8.18.tar.gz - cd redis-2.8.18 && make - - rethinkdb --bind all & - - src/redis-server & + post: + - rethinkdb --bind all + background: true + - src/redis-server + background: true From a9d229a4d21d3b4c8941eb1e6c9721c808f7bb37 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:45:17 +0100 Subject: [PATCH 11/30] atom pls --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index d83be8a..cd94c12 100644 --- a/circle.yml +++ b/circle.yml @@ -13,6 +13,6 @@ dependencies: - cd redis-2.8.18 && make post: - rethinkdb --bind all - background: true + background: true - src/redis-server - background: true + background: true From 446793c97de82d17168cf59d69f973d28cf8cfde Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 20:45:58 +0100 Subject: [PATCH 12/30] circleci pls --- circle.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index cd94c12..f8adbd3 100644 --- a/circle.yml +++ b/circle.yml @@ -12,7 +12,7 @@ dependencies: - tar xvzf redis-2.8.18.tar.gz - cd redis-2.8.18 && make post: - - rethinkdb --bind all - background: true - - src/redis-server - background: true + - rethinkdb --bind all: + background: true + - src/redis-server: + background: true From 0ab5ba3f1b9038264adfc5710f144570d312cc9b Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Thu, 11 Dec 2014 21:01:56 +0100 Subject: [PATCH 13/30] Custom test command --- circle.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/circle.yml b/circle.yml index f8adbd3..3250787 100644 --- a/circle.yml +++ b/circle.yml @@ -16,3 +16,7 @@ dependencies: background: true - src/redis-server: background: true + +test: + override: + - go test -v ./... From 1133c78a109225e666fc8461987a89c4fdbbed8b Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 14 Dec 2014 18:05:00 +0100 Subject: [PATCH 14/30] Key votes storage benchmark --- _research/vote_storage/benchmark_test.go | 115 +++++++++++++++++++++++ _vagrant/Vagrantfile | 5 + 2 files changed, 120 insertions(+) create mode 100644 _research/vote_storage/benchmark_test.go diff --git a/_research/vote_storage/benchmark_test.go b/_research/vote_storage/benchmark_test.go new file mode 100644 index 0000000..adab145 --- /dev/null +++ b/_research/vote_storage/benchmark_test.go @@ -0,0 +1,115 @@ +package vote_storage_test + +import ( + "math/rand" + "testing" + + "github.com/dancannon/gorethink" + "github.com/dchest/uniuri" +) + +var ( + session *gorethink.Session + key2find string + table2search string +) + +func init() { + var err error + session, err = gorethink.Connect(gorethink.ConnectOpts{ + Address: "127.0.0.1:28015", + }) + if err != nil { + panic(err) + } + + key2find = uniuri.New() + + // Create a new table + gorethink.Db("test").TableDrop("benchmark_keys_list").Run(session) + gorethink.Db("test").TableCreate("benchmark_keys_list").Run(session) + + var klist []*KeysList + + // Populate with sample data + for n := 0; n < 300; n++ { + keys := rndStringSlice(999) + keys = randomlyInsert(keys, key2find) + + y := uniuri.New() + if n == 153 { + table2search = y + } + + klist = append(klist, &KeysList{ + ID: y, + Voted: keys, + }) + } + + gorethink.Db("test").Table("benchmark_keys_list").Insert(klist).Run(session) +} + +func rndStringSlice(count int) []string { + var r []string + for i := 0; i < count; i++ { + r = append(r, uniuri.New()) + } + return r +} + +func randomlyInsert(s []string, x string) []string { + i := rand.Intn(len(s) - 1) + + s = append(s[:i], append([]string{x}, s[i:]...)...) + + return s +} + +type KeysList struct { + ID string `gorethink:"id"` + Voted []string `gorethink:"voted"` +} + +func BenchmarkContains(b *testing.B) { + for n := 0; n < b.N; n++ { + contains, err := gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").Contains(key2find).Run(session) + if err != nil { + b.Log(err) + b.Fail() + } + + var res bool + err = contains.One(&res) + if err != nil { + b.Log(err) + b.Fail() + } + if !res { + b.Log("invalid response") + b.Fail() + } + } +} + +func BenchmarkAppend(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").Append(uniuri.New()).Run(session) + if err != nil { + b.Log(err) + b.Fail() + } + } +} + +func BenchmarkDelete(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").DeleteAt( + gorethink.Expr(gorethink.Db("test").Table("benchmark_keys_list").Get(table2search).Field("voted").IndexesOf(key2find).AtIndex(0)), + ).Run(session) + if err != nil { + b.Log(err) + b.Fail() + } + } +} diff --git a/_vagrant/Vagrantfile b/_vagrant/Vagrantfile index 8b5447e..53ea370 100644 --- a/_vagrant/Vagrantfile +++ b/_vagrant/Vagrantfile @@ -15,6 +15,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # redis config.vm.network "forwarded_port", guest: 6379, host: 6379 + config.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 4 + end + # load ansible playbook config.vm.provision "shell", path: "deploy.sh" end From 0a1040f167fdc271dc44f26572ead0f2d93ffae5 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 14 Dec 2014 18:41:03 +0100 Subject: [PATCH 15/30] models.Key change --- models/account.go | 12 +++--------- models/key.go | 25 ++++++++++++------------- routes/keys.go | 35 +++++++++++++++++++++++++++-------- utils/pgp.go | 21 +++++++++++++++++++++ 4 files changed, 63 insertions(+), 30 deletions(-) create mode 100644 utils/pgp.go diff --git a/models/account.go b/models/account.go index aa834f1..b2cbef5 100644 --- a/models/account.go +++ b/models/account.go @@ -14,17 +14,11 @@ type Account struct { Billing BillingData `json:"billing" gorethink:"billing"` // Password is the password used to login to the account. - // It's hashed and salted using a cryptographically strong method (bcrypt|scrypt). + // It's hashed and salted using scrypt. Password string `json:"-" gorethink:"password"` - // PgpExpDate is an RFC3339-encoded string containing the expiry date of the user's public key - PgpExpDate string `json:"pgp_exp_date" gorethink:"pgp_exp_date"` - - // PgpFingerprint is a SHA-512 hash of the user's public key - PgpFingerprint string `json:"pgp_fingerprint" gorethink:"pgp_fingerprint"` - - // PgpPublicKey is a copy of the user's current public key. It can also be found in the 'keys' db. - PgpPublicKey string `json:"pgp_public_key" gorethink:"pgp_public_key"` + // PGPKey is the fingerprint of account's default key + PGPKey string `json:"pgp_key" gorethink:"pgp_key"` // Settings contains data needed to customize the user experience. // TODO Work in progress diff --git a/models/key.go b/models/key.go index f0c6d86..67c7f25 100644 --- a/models/key.go +++ b/models/key.go @@ -1,19 +1,18 @@ package models type Key struct { - Resource - Expiring + Resource // ID is the fingerprint, Name is empty + Expiring // ExpiryDate is either empty or expiring, user can set it - // ID is the fingerprint + //Body []byte `json:"body" gorethink:"body"` // Raw key contents - Image []byte `json:"image" gorethink:"image"` - - // the actual key - Key string `json:"key" gorethink:"key"` - - OwnerName string `json:"owner_name" gorethink:"owner_name"` - - // the actual id - KeyID string `json:"key_id" gorethink:"key_id"` - KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` + Headers map[string]string `json:"headers" gorethink:"headers"` // Headers passed with the key + Email string `json:"email" gorethink:"email"` // Address associated with the key + Algorithm string `json:"algorithm" gorethink:"algorithm"`// Algorithm of the key + Length uint16 `json:"length" gorethink:"length"`// Length of the key + Key string `json:"key" gorethink:"key"` // Armor-encoded key + KeyID string `json:"key_id" gorethink:"key_id"` // PGP key ID + KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` // Shorter version of above + Reliability int `json:"reliability" gorethink:"reliability"` // Reliability algorithm cached result + MasterKey string `json:"master_key" gorethink:"mater_key"` // MasterKey's ID - no idea how it works } diff --git a/routes/keys.go b/routes/keys.go index 980f705..5da2b52 100644 --- a/routes/keys.go +++ b/routes/keys.go @@ -9,6 +9,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/zenazn/goji/web" "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/armor" "github.com/lavab/api/env" "github.com/lavab/api/models" @@ -59,8 +60,7 @@ func KeysList(w http.ResponseWriter, r *http.Request) { // KeysCreateRequest contains the data passed to the KeysCreate endpoint. type KeysCreateRequest struct { - Key string `json:"key" schema:"key"` // gpg armored key - Image string `json:"image" schema:"image"` // todo + Key string `json:"key" schema:"key"` // gpg armored key } // KeysCreateResponse contains the result of the KeysCreate request. @@ -105,6 +105,21 @@ func KeysCreate(c web.C, w http.ResponseWriter, r *http.Request) { return } + // Parse using armor pkg + block, err := armor.Decode(strings.NewReader(input.Key)) + if err != nil { + utils.JSONResponse(w, 409, &KeysCreateResponse{ + Success: false, + Message: "Invalid key format", + }) + + env.Log.WithFields(logrus.Fields{ + "error": err, + "list": entityList, + }).Warn("Cannot parse an armored key #2") + return + } + // Get the account from db account, err := env.Accounts.GetAccount(session.Owner) if err != nil { @@ -132,17 +147,21 @@ func KeysCreate(c web.C, w http.ResponseWriter, r *http.Request) { // Allocate a new key key := &models.Key{ Resource: models.MakeResource( - session.Owner, + account.ID, fmt.Sprintf( - "%d/%s public key", + "%s/%d/%s public key", + utils.GetAlgorithmName(publicKey.PrimaryKey.PubKeyAlgo), bitLength, publicKey.PrimaryKey.KeyIdString(), ), ), - OwnerName: account.Name, - Key: input.Key, - KeyID: publicKey.PrimaryKey.KeyIdString(), - KeyIDShort: publicKey.PrimaryKey.KeyIdShortString(), + Headers: block.Header, + Algorithm: utils.GetAlgorithmName(publicKey.PrimaryKey.PubKeyAlgo), + Length: bitLength, + Key: input.Key, + KeyID: publicKey.PrimaryKey.KeyIdString(), + KeyIDShort: publicKey.PrimaryKey.KeyIdShortString(), + Reliability: 0, } // Update id as we can't do it directly during allocation diff --git a/utils/pgp.go b/utils/pgp.go new file mode 100644 index 0000000..b806532 --- /dev/null +++ b/utils/pgp.go @@ -0,0 +1,21 @@ +package utils + +import "golang.org/x/crypto/openpgp/packet" + +// GetAlgorithmName returns algorithm's name depending on its ID +func GetAlgorithmName(id packet.PublicKeyAlgorithm) string { + switch id { + case 1, 2, 3: + return "RSA" + case 16: + return "ElGamal" + case 17: + return "DSA" + case 18: + return "ECDH" + case 19: + return "ECDSA" + default: + return "unknown" + } +} From 44d0b68d31504d3027be270ee2b2f366e71b35b6 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 14 Dec 2014 19:30:00 +0100 Subject: [PATCH 16/30] Fixing before merge; changed GET /keys to work with new logic --- .travis.yml | 14 -------------- db/table_keys.go | 4 ++-- models/key.go | 18 +++++++++--------- routes/keys.go | 13 +++++++++++-- 4 files changed, 22 insertions(+), 27 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 84a85c5..0000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: go - -go: - - 1.3.1 - -services: - - redis-server - -before_script: - - source /etc/lsb-release && echo "deb http://download.rethinkdb.com/apt $DISTRIB_CODENAME main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list - - wget -qO- http://download.rethinkdb.com/apt/pubkey.gpg | sudo apt-key add - - - sudo apt-get update - - sudo apt-get install rethinkdb - - rethinkdb --bind all & diff --git a/db/table_keys.go b/db/table_keys.go index 225a830..8679e87 100644 --- a/db/table_keys.go +++ b/db/table_keys.go @@ -8,10 +8,10 @@ type KeysTable struct { RethinkCRUD } -func (k *KeysTable) FindByName(name string) ([]*models.Key, error) { +func (k *KeysTable) FindByOwner(id string) ([]*models.Key, error) { var results []*models.Key - if err := k.FindByAndFetch("owner_name", name, &results); err != nil { + if err := k.FindByAndFetch("owner", id, &results); err != nil { return nil, err } diff --git a/models/key.go b/models/key.go index 67c7f25..b114ab8 100644 --- a/models/key.go +++ b/models/key.go @@ -6,13 +6,13 @@ type Key struct { //Body []byte `json:"body" gorethink:"body"` // Raw key contents - Headers map[string]string `json:"headers" gorethink:"headers"` // Headers passed with the key - Email string `json:"email" gorethink:"email"` // Address associated with the key - Algorithm string `json:"algorithm" gorethink:"algorithm"`// Algorithm of the key - Length uint16 `json:"length" gorethink:"length"`// Length of the key - Key string `json:"key" gorethink:"key"` // Armor-encoded key - KeyID string `json:"key_id" gorethink:"key_id"` // PGP key ID - KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` // Shorter version of above - Reliability int `json:"reliability" gorethink:"reliability"` // Reliability algorithm cached result - MasterKey string `json:"master_key" gorethink:"mater_key"` // MasterKey's ID - no idea how it works + Headers map[string]string `json:"headers" gorethink:"headers"` // Headers passed with the key + Email string `json:"email" gorethink:"email"` // Address associated with the key + Algorithm string `json:"algorithm" gorethink:"algorithm"` // Algorithm of the key + Length uint16 `json:"length" gorethink:"length"` // Length of the key + Key string `json:"key" gorethink:"key"` // Armor-encoded key + KeyID string `json:"key_id" gorethink:"key_id"` // PGP key ID + KeyIDShort string `json:"key_id_short" gorethink:"key_id_short"` // Shorter version of above + Reliability int `json:"reliability" gorethink:"reliability"` // Reliability algorithm cached result + MasterKey string `json:"master_key" gorethink:"mater_key"` // MasterKey's ID - no idea how it works } diff --git a/routes/keys.go b/routes/keys.go index 5da2b52..4328583 100644 --- a/routes/keys.go +++ b/routes/keys.go @@ -35,8 +35,17 @@ func KeysList(w http.ResponseWriter, r *http.Request) { return } + account, err := env.Accounts.FindAccountByName(user) + if err != nil { + utils.JSONResponse(w, 409, &KeysListResponse{ + Success: false, + Message: "Invalid username", + }) + return + } + // Find all keys owner by user - keys, err := env.Keys.FindByName(user) + keys, err := env.Keys.FindByOwner(account.ID) if err != nil { utils.JSONResponse(w, 500, &KeysListResponse{ Success: false, @@ -149,7 +158,7 @@ func KeysCreate(c web.C, w http.ResponseWriter, r *http.Request) { Resource: models.MakeResource( account.ID, fmt.Sprintf( - "%s/%d/%s public key", + "%s/%d/%s", utils.GetAlgorithmName(publicKey.PrimaryKey.PubKeyAlgo), bitLength, publicKey.PrimaryKey.KeyIdString(), From 0747039b18048dc865ebb782635c8ed6cd9eab50 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Mon, 15 Dec 2014 23:18:22 +0100 Subject: [PATCH 17/30] YubiCloud support added-ish --- .gitignore | 1 + env/config.go | 3 +++ env/env.go | 3 +++ factor/authenticator.go | 38 ++++++++++++++++++++++++++++++++ factor/method.go | 7 ++++++ factor/yubicloud.go | 48 +++++++++++++++++++++++++++++++++++++++++ main.go | 10 ++++++++- models/account.go | 3 +++ routes/tokens.go | 40 ++++++++++++++++++++++++++++++---- setup/setup.go | 16 ++++++++++++++ 10 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 factor/authenticator.go create mode 100644 factor/method.go create mode 100644 factor/yubicloud.go diff --git a/.gitignore b/.gitignore index 1e6f0b7..4140b23 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ *.exe _vagrant/.vagrant +.config.conf diff --git a/env/config.go b/env/config.go index 69b1ff9..c07ee6a 100644 --- a/env/config.go +++ b/env/config.go @@ -18,4 +18,7 @@ type Flags struct { RethinkDBAddress string RethinkDBKey string RethinkDBDatabase string + + YubiCloudID string + YubiCloudKey string } diff --git a/env/env.go b/env/env.go index bfdac13..e7f5ec5 100644 --- a/env/env.go +++ b/env/env.go @@ -6,6 +6,7 @@ import ( "github.com/lavab/api/cache" "github.com/lavab/api/db" + "github.com/lavab/api/factor" ) var ( @@ -27,4 +28,6 @@ var ( Contacts *db.ContactsTable // Reservations is the global instance of ReservationsTable Reservations *db.ReservationsTable + // Factors contains all currently registered factors + Factors map[string]factor.Factor ) diff --git a/factor/authenticator.go b/factor/authenticator.go new file mode 100644 index 0000000..0c5b42e --- /dev/null +++ b/factor/authenticator.go @@ -0,0 +1,38 @@ +package factor + +import ( + "github.com/gokyle/hotp" +) + +type Authenticator struct { + length int +} + +func NewAuthenticator(length int) *Authenticator { + return &Authenticator{ + length: length, + } +} + +func (a *Authenticator) Type() string { + return "authenticator" +} + +func (a *Authenticator) Request(data string) (string, error) { + otp, err := hotp.GenerateHOTP(a.length, false) + if err != nil { + return "", err + } + + return otp.URL(data), nil +} + +func (a *Authenticator) Verify(data string, input string) (bool, error) { + // obviously broken + hotp, err := hotp.Unmarshal([]byte(data)) + if err != nil { + return false, err + } + + return hotp.Check(input), nil +} diff --git a/factor/method.go b/factor/method.go new file mode 100644 index 0000000..e13f320 --- /dev/null +++ b/factor/method.go @@ -0,0 +1,7 @@ +package factor + +type Factor interface { + Type() string + Request(data string) (string, error) + Verify(data string, input string) (bool, error) +} diff --git a/factor/yubicloud.go b/factor/yubicloud.go new file mode 100644 index 0000000..9f37334 --- /dev/null +++ b/factor/yubicloud.go @@ -0,0 +1,48 @@ +package factor + +import "github.com/GeertJohan/yubigo" + +// YubiCloud is an implementation of Factor to authenticate with YubiCloud +type YubiCloud struct { + client *yubigo.YubiAuth +} + +// NewYubiCloud set ups a new Factor that supports authing using YubiCloud +func NewYubiCloud(id string, key string) (*YubiCloud, error) { + client, err := yubigo.NewYubiAuth(id, key) + if err != nil { + return nil, err + } + + return &YubiCloud{ + client: client, + }, nil +} + +// Type returns factor's type +func (y *YubiCloud) Type() string { + return "yubicloud" +} + +// Request does nothing in this driver +func (y *YubiCloud) Request(data string) (string, error) { + return "", nil +} + +// Verify checks if the token is valid +func (y *YubiCloud) Verify(data string, input string) (bool, error) { + if input[:12] != data { + return false, nil + } + + _, ok, err := y.client.Verify(input) + if err != nil { + return false, err + } + + if !ok { + return false, nil + } + + return true, nil +} diff --git a/main.go b/main.go index e7393c0..7eb6bbd 100644 --- a/main.go +++ b/main.go @@ -18,9 +18,11 @@ import ( // https://github.com/unrolled/secure var ( + // Enable namsral/flag functionality + configFlag = flag.String("config", "", "config file to load") // General flags bindAddress = flag.String("bind", ":5000", "Network address used to bind") - apiVersion = flag.String("version", "v0", "Shown API version") + apiVersion = flag.String("api_version", "v0", "Shown API version") logFormatterType = flag.String("log", "text", "Log formatter type. Either \"json\" or \"text\"") forceColors = flag.Bool("force_colors", false, "Force colored prompt?") // Registration settings @@ -53,6 +55,9 @@ var ( } return database }(), "Database name on the RethinkDB server") + // YubiCloud params + yubiCloudID = flag.String("yubicloud_id", "", "YubiCloud API id") + yubiCloudKey = flag.String("yubicloud_key", "", "YubiCloud API key") ) func main() { @@ -77,6 +82,9 @@ func main() { RethinkDBAddress: *rethinkdbAddress, RethinkDBKey: *rethinkdbKey, RethinkDBDatabase: *rethinkdbDatabase, + + YubiCloudID: *yubiCloudID, + YubiCloudKey: *yubiCloudKey, } // Generate a mux diff --git a/models/account.go b/models/account.go index b2cbef5..ed77e33 100644 --- a/models/account.go +++ b/models/account.go @@ -34,6 +34,9 @@ type Account struct { AltEmail string `json:"alt_email" gorethink:"alt_email"` + FactorType string `json:"-" gorethink:"factor_type"` + FactorValue string `json:"-" gorethink:"factor_value"` + Status string `json:"status" gorethink:"status"` } diff --git a/routes/tokens.go b/routes/tokens.go index 3b11567..87d24b8 100644 --- a/routes/tokens.go +++ b/routes/tokens.go @@ -38,13 +38,16 @@ type TokensCreateRequest struct { Username string `json:"username" schema:"username"` Password string `json:"password" schema:"password"` Type string `json:"type" schema:"type"` + Token string `json:"token" schema:"token"` } // TokensCreateResponse contains the result of the TokensCreate request. type TokensCreateResponse struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Token *models.Token `json:"token,omitempty"` + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Token *models.Token `json:"token,omitempty"` + FactorType string `json:"factor_type,omitempty"` + FactorRequest string `json:"factor_request,omitempty"` } // TokensCreate allows logging in to an account. @@ -104,12 +107,41 @@ func TokensCreate(w http.ResponseWriter, r *http.Request) { } } + // Check for 2nd factor + if user.FactorType != "" { + factor, ok := env.Factors[user.FactorType] + if ok { + if input.Token == "" { + req, err := factor.Request(user.ID) + if err == nil { + utils.JSONResponse(w, 403, &TokensCreateResponse{ + Success: false, + Message: "Factor token was not passed", + FactorType: user.FactorType, + FactorRequest: req, + }) + return + } + } else { + ok, err := factor.Verify(user.FactorValue, input.Token) + if !ok || err != nil { + utils.JSONResponse(w, 403, &TokensCreateResponse{ + Success: false, + Message: "Invalid token passed", + FactorType: user.FactorType, + }) + return + } + } + } + } + // Calculate the expiry date expDate := time.Now().Add(time.Hour * time.Duration(env.Config.SessionDuration)) // Create a new token token := &models.Token{ - Expiring: models.Expiring{expDate}, + Expiring: models.Expiring{ExpiryDate: expDate}, Resource: models.MakeResource(user.ID, "Auth token expiring on "+expDate.Format(time.RFC3339)), Type: input.Type, } diff --git a/setup/setup.go b/setup/setup.go index 07bb0fc..eabe49d 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -11,6 +11,7 @@ import ( "github.com/lavab/api/cache" "github.com/lavab/api/db" "github.com/lavab/api/env" + "github.com/lavab/api/factor" "github.com/lavab/api/routes" "github.com/lavab/glogrus" ) @@ -111,6 +112,21 @@ func PrepareMux(flags *env.Flags) *web.Mux { ), } + // Initialize factors + env.Factors = make(map[string]factor.Factor) + if flags.YubiCloudID != "" { + yubicloud, err := factor.NewYubiCloud(flags.YubiCloudID, flags.YubiCloudKey) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to initiate YubiCloud") + } + env.Factors[yubicloud.Type()] = yubicloud + } + + authenticator := factor.NewAuthenticator(6) + env.Factors[authenticator.Type()] = authenticator + // Create a new goji mux mux := web.New() From 518045d8f24572c078651ff377e987f23755278e Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Wed, 17 Dec 2014 14:10:57 +0100 Subject: [PATCH 18/30] Added support for multiple Yubikeys --- factor/authenticator.go | 4 ++-- factor/method.go | 2 +- factor/yubicloud.go | 14 ++++++++++++-- models/account.go | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/factor/authenticator.go b/factor/authenticator.go index 0c5b42e..ba747b3 100644 --- a/factor/authenticator.go +++ b/factor/authenticator.go @@ -27,9 +27,9 @@ func (a *Authenticator) Request(data string) (string, error) { return otp.URL(data), nil } -func (a *Authenticator) Verify(data string, input string) (bool, error) { +func (a *Authenticator) Verify(data []string, input string) (bool, error) { // obviously broken - hotp, err := hotp.Unmarshal([]byte(data)) + hotp, err := hotp.Unmarshal([]byte(data[0])) if err != nil { return false, err } diff --git a/factor/method.go b/factor/method.go index e13f320..df869ed 100644 --- a/factor/method.go +++ b/factor/method.go @@ -3,5 +3,5 @@ package factor type Factor interface { Type() string Request(data string) (string, error) - Verify(data string, input string) (bool, error) + Verify(data []string, input string) (bool, error) } diff --git a/factor/yubicloud.go b/factor/yubicloud.go index 9f37334..8bd0927 100644 --- a/factor/yubicloud.go +++ b/factor/yubicloud.go @@ -30,8 +30,18 @@ func (y *YubiCloud) Request(data string) (string, error) { } // Verify checks if the token is valid -func (y *YubiCloud) Verify(data string, input string) (bool, error) { - if input[:12] != data { +func (y *YubiCloud) Verify(data []string, input string) (bool, error) { + publicKey := input[:12] + + found := false + for _, prefix := range data { + if publicKey == prefix { + found = true + break + } + } + + if !found { return false, nil } diff --git a/models/account.go b/models/account.go index ed77e33..6e5f6f9 100644 --- a/models/account.go +++ b/models/account.go @@ -34,8 +34,8 @@ type Account struct { AltEmail string `json:"alt_email" gorethink:"alt_email"` - FactorType string `json:"-" gorethink:"factor_type"` - FactorValue string `json:"-" gorethink:"factor_value"` + FactorType string `json:"-" gorethink:"factor_type"` + FactorValue []string `json:"-" gorethink:"factor_value"` Status string `json:"status" gorethink:"status"` } From 0596b0099e72f62f23ed5e56bbe310b90094686a Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sat, 20 Dec 2014 00:10:07 +0100 Subject: [PATCH 19/30] queue basis --- _vagrant/Vagrantfile | 66 +++++++++++++++++++++++++++++++++++--------- env/config.go | 2 ++ env/env.go | 7 +++++ main.go | 10 +++++++ setup/setup.go | 38 +++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 13 deletions(-) diff --git a/_vagrant/Vagrantfile b/_vagrant/Vagrantfile index 53ea370..e14720f 100644 --- a/_vagrant/Vagrantfile +++ b/_vagrant/Vagrantfile @@ -5,21 +5,61 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "ubuntu/trusty64" - # rethinkdb - config.vm.network "forwarded_port", guest: 8080, host: 8080 - config.vm.network "forwarded_port", guest: 28015, host: 28015 - config.vm.network "forwarded_port", guest: 29015, host: 29015 + #config.vm.define "redisthink" do |rethinkdb| + #rethinkdb.vm.box = "ubuntu/trusty64" - # redis - config.vm.network "forwarded_port", guest: 6379, host: 6379 + # rethinkdb + #rethinkdb.vm.network "forwarded_port", guest: 8080, host: 8080 + #rethinkdb.vm.network "forwarded_port", guest: 28015, host: 28015 + #rethinkdb.vm.network "forwarded_port", guest: 29015, host: 29015 - config.vm.provider "virtualbox" do |v| - v.memory = 2048 - v.cpus = 4 - end + # redis + #rethinkdb.vm.network "forwarded_port", guest: 6379, host: 6379 + + #rethinkdb.vm.provider "virtualbox" do |v| + #v.memory = 2048 + #v.cpus = 4 + #end + + # load ansible playbook + #rethinkdb.vm.provision "shell", path: "deploy.sh" + #end + + config.vm.define "docker" do |docker| + docker.vm.box = "ubuntu/trusty64" + + docker.vm.network "forwarded_port", guest: 4160, host: 4160 + docker.vm.network "forwarded_port", guest: 4161, host: 4161 + docker.vm.network "forwarded_port", guest: 4150, host: 4150 + docker.vm.network "forwarded_port", guest: 4151, host: 4151 + docker.vm.network "forwarded_port", guest: 6379, host: 6379 + docker.vm.network "forwarded_port", guest: 8080, host: 8080 + docker.vm.network "forwarded_port", guest: 28015, host: 28015 + docker.vm.network "forwarded_port", guest: 29015, host: 29015 - # load ansible playbook - config.vm.provision "shell", path: "deploy.sh" + docker.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + v.customize ["modifyvm", :id, "--natdnsproxy1", "on"] + end + + docker.vm.provision "docker" do |d| + d.pull_images "nsqio/nsqlookupd" + d.run "nsqio/nsqlookupd", + args: "--name lookupd -p 4160:4160 -p 4161:4161" + + d.pull_images "nsqio/nsqd" + d.run "nsqio/nsqd", + args: "--name nsqd -p 4150:4150 -p 4151:4151", + cmd: "--broadcast-address=172.17.42.1 --lookupd-tcp-address=172.17.42.1:4160" + + d.pull_images "dockerfile/rethinkdb" + d.run "dockerfile/rethinkdb", + args: "--name rethinkdb -p 8080:8080 -p 28015:28015 -p 29015:29015" + + d.pull_images "dockerfile/redis" + d.run "dockerfile/redis", + args: "--name redis -p 6379:6379" + end + end end diff --git a/env/config.go b/env/config.go index c07ee6a..22daddf 100644 --- a/env/config.go +++ b/env/config.go @@ -19,6 +19,8 @@ type Flags struct { RethinkDBKey string RethinkDBDatabase string + NSQAddress string + YubiCloudID string YubiCloudKey string } diff --git a/env/env.go b/env/env.go index e7f5ec5..b2e3e62 100644 --- a/env/env.go +++ b/env/env.go @@ -2,6 +2,7 @@ package env import ( "github.com/Sirupsen/logrus" + "github.com/bitly/go-nsq" "github.com/dancannon/gorethink" "github.com/lavab/api/cache" @@ -30,4 +31,10 @@ var ( Reservations *db.ReservationsTable // Factors contains all currently registered factors Factors map[string]factor.Factor + // NSQProducer is used for email sending + NSQProducer *nsq.Producer + // DeliveryConsumer is the NSQ consumer used for email delivery confirmations + DeliveryConsumer *nsq.Consumer + // ReceiptConsumer is the NSQ consumer for new email receipt handling + ReceiptConsumer *nsq.Consumer ) diff --git a/main.go b/main.go index 7eb6bbd..122ada8 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,14 @@ var ( } return database }(), "Database name on the RethinkDB server") + // NSQ address + nsqAddress = flag.String("nsq_address", func() string { + address := os.Getenv("NSQLOOKUPD_PORT_4160_TCP_ADDR") + if address == "" { + address = "127.0.0.1" + } + return address + ":4160" + }(), "Address of the NSQ server") // YubiCloud params yubiCloudID = flag.String("yubicloud_id", "", "YubiCloud API id") yubiCloudKey = flag.String("yubicloud_key", "", "YubiCloud API key") @@ -83,6 +91,8 @@ func main() { RethinkDBKey: *rethinkdbKey, RethinkDBDatabase: *rethinkdbDatabase, + NSQAddress: *nsqAddress, + YubiCloudID: *yubiCloudID, YubiCloudKey: *yubiCloudKey, } diff --git a/setup/setup.go b/setup/setup.go index eabe49d..b09b094 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -4,6 +4,7 @@ import ( "time" "github.com/Sirupsen/logrus" + "github.com/bitly/go-nsq" "github.com/dancannon/gorethink" "github.com/zenazn/goji/web" "github.com/zenazn/goji/web/middleware" @@ -112,6 +113,43 @@ func PrepareMux(flags *env.Flags) *web.Mux { ), } + // Initialize the NSQ connections + nsqProducer, err := nsq.NewProducer(flags.NSQAddress, nsq.NewConfig()) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to create a NSQProducer") + } + env.NSQProducer = nsqProducer + + deliveryConsumer, err := nsq.NewConsumer("delivery", "confirmation", nsq.NewConfig()) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to create a DeliveryConsumer") + } + err = deliveryConsumer.ConnectToNSQLookupd(flags.NSQAddress) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to connect to nsqlookupd") + } + env.DeliveryConsumer = deliveryConsumer + + receiptConsumer, err := nsq.NewConsumer("receipt", "notification", nsq.NewConfig()) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to create a DeliveryConsumer") + } + err = receiptConsumer.ConnectToNSQLookupd(flags.NSQAddress) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to connect to nsqlookupd") + } + env.ReceiptConsumer = receiptConsumer + // Initialize factors env.Factors = make(map[string]factor.Factor) if flags.YubiCloudID != "" { From 943d36df4ca7e05f3de777243ff626f4d884ba75 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 21 Dec 2014 01:02:14 +0100 Subject: [PATCH 20/30] A large chunk of base emails code --- _vagrant/Vagrantfile | 17 ++----- db/table_emails.go | 42 ++++++++++++++++ env/config.go | 2 +- env/env.go | 12 ++--- main.go | 12 ++--- models/base_encrypted.go | 4 +- models/email.go | 2 + routes/emails.go | 105 +++++++++++++++++++++++++++++++++++++-- setup/setup.go | 56 ++++++++++----------- 9 files changed, 192 insertions(+), 60 deletions(-) create mode 100644 db/table_emails.go diff --git a/_vagrant/Vagrantfile b/_vagrant/Vagrantfile index e14720f..36d0121 100644 --- a/_vagrant/Vagrantfile +++ b/_vagrant/Vagrantfile @@ -29,10 +29,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "docker" do |docker| docker.vm.box = "ubuntu/trusty64" - docker.vm.network "forwarded_port", guest: 4160, host: 4160 - docker.vm.network "forwarded_port", guest: 4161, host: 4161 - docker.vm.network "forwarded_port", guest: 4150, host: 4150 - docker.vm.network "forwarded_port", guest: 4151, host: 4151 + docker.vm.network "forwarded_port", guest: 4222, host: 4222 + docker.vm.network "forwarded_port", guest: 8333, host: 8333 docker.vm.network "forwarded_port", guest: 6379, host: 6379 docker.vm.network "forwarded_port", guest: 8080, host: 8080 docker.vm.network "forwarded_port", guest: 28015, host: 28015 @@ -44,14 +42,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end docker.vm.provision "docker" do |d| - d.pull_images "nsqio/nsqlookupd" - d.run "nsqio/nsqlookupd", - args: "--name lookupd -p 4160:4160 -p 4161:4161" - - d.pull_images "nsqio/nsqd" - d.run "nsqio/nsqd", - args: "--name nsqd -p 4150:4150 -p 4151:4151", - cmd: "--broadcast-address=172.17.42.1 --lookupd-tcp-address=172.17.42.1:4160" + d.pull_images "apcera/gnatsd" + d.run "apcera/gnatsd", + args: "--name lookupd -p 4222:4222 -p 8333:8333" d.pull_images "dockerfile/rethinkdb" d.run "dockerfile/rethinkdb", diff --git a/db/table_emails.go b/db/table_emails.go new file mode 100644 index 0000000..0a14eb9 --- /dev/null +++ b/db/table_emails.go @@ -0,0 +1,42 @@ +package db + +import ( + "github.com/lavab/api/models" +) + +// Emails implements the CRUD interface for tokens +type EmailsTable struct { + RethinkCRUD +} + +// GetEmail returns a token with specified name +func (c *EmailsTable) GetEmail(id string) (*models.Email, error) { + var result models.Email + + if err := c.FindFetchOne(id, &result); err != nil { + return nil, err + } + + return &result, nil +} + +// GetOwnedBy returns all contacts owned by id +func (c *EmailsTable) GetOwnedBy(id string) ([]*models.Email, error) { + var result []*models.Email + + err := c.WhereAndFetch(map[string]interface{}{ + "owner": id, + }, &result) + if err != nil { + return nil, err + } + + return result, nil +} + +// DeleteOwnedBy deletes all contacts owned by id +func (c *EmailsTable) DeleteOwnedBy(id string) error { + return c.Delete(map[string]interface{}{ + "owner": id, + }) +} diff --git a/env/config.go b/env/config.go index 22daddf..6b655df 100644 --- a/env/config.go +++ b/env/config.go @@ -19,7 +19,7 @@ type Flags struct { RethinkDBKey string RethinkDBDatabase string - NSQAddress string + NATSAddress string YubiCloudID string YubiCloudKey string diff --git a/env/env.go b/env/env.go index b2e3e62..ec7444a 100644 --- a/env/env.go +++ b/env/env.go @@ -2,7 +2,7 @@ package env import ( "github.com/Sirupsen/logrus" - "github.com/bitly/go-nsq" + "github.com/apcera/nats" "github.com/dancannon/gorethink" "github.com/lavab/api/cache" @@ -29,12 +29,10 @@ var ( Contacts *db.ContactsTable // Reservations is the global instance of ReservationsTable Reservations *db.ReservationsTable + // Emails is the global instance of EmailsTable + Emails *db.EmailsTable // Factors contains all currently registered factors Factors map[string]factor.Factor - // NSQProducer is used for email sending - NSQProducer *nsq.Producer - // DeliveryConsumer is the NSQ consumer used for email delivery confirmations - DeliveryConsumer *nsq.Consumer - // ReceiptConsumer is the NSQ consumer for new email receipt handling - ReceiptConsumer *nsq.Consumer + // NATS is the encoded connection to the NATS queue + NATS *nats.EncodedConn ) diff --git a/main.go b/main.go index 122ada8..787ee23 100644 --- a/main.go +++ b/main.go @@ -55,14 +55,14 @@ var ( } return database }(), "Database name on the RethinkDB server") - // NSQ address - nsqAddress = flag.String("nsq_address", func() string { - address := os.Getenv("NSQLOOKUPD_PORT_4160_TCP_ADDR") + // NATS address + natsAddress = flag.String("nats_address", func() string { + address := os.Getenv("NATS_PORT_4222_TCP_ADDR") if address == "" { address = "127.0.0.1" } - return address + ":4160" - }(), "Address of the NSQ server") + return "nats://" + address + ":4222" + }(), "Address of the NATS server") // YubiCloud params yubiCloudID = flag.String("yubicloud_id", "", "YubiCloud API id") yubiCloudKey = flag.String("yubicloud_key", "", "YubiCloud API key") @@ -91,7 +91,7 @@ func main() { RethinkDBKey: *rethinkdbKey, RethinkDBDatabase: *rethinkdbDatabase, - NSQAddress: *nsqAddress, + NATSAddress: *natsAddress, YubiCloudID: *yubiCloudID, YubiCloudKey: *yubiCloudKey, diff --git a/models/base_encrypted.go b/models/base_encrypted.go index 44eec0b..0597cd1 100644 --- a/models/base_encrypted.go +++ b/models/base_encrypted.go @@ -5,8 +5,8 @@ type Encrypted struct { // Encoding tells the reader how to decode the data; can be "json", "protobuf", maybe more in the future Encoding string `json:"encoding" gorethink:"encoding"` - // PgpFingerprints contains the fingerprints of the PGP public keys used to encrypt the data. - PgpFingerprints []string `json:"pgp_fingerprints" gorethink:"pgp_fingerprints"` + // PGPFingerprints contains the fingerprints of the PGP public keys used to encrypt the data. + PGPFingerprints []string `json:"pgp_fingerprints" gorethink:"pgp_fingerprints"` // Data is the raw, PGP-encrypted data Data string `json:"raw" gorethink:"raw"` diff --git a/models/email.go b/models/email.go index 610570c..10aaa5e 100644 --- a/models/email.go +++ b/models/email.go @@ -23,4 +23,6 @@ type Email struct { // ThreadID ThreadID string `json:"thread_id" gorethink:"thread_id"` + + Status string `json:"status" gorethink:"status"` } diff --git a/routes/emails.go b/routes/emails.go index deb3dd2..3cce604 100644 --- a/routes/emails.go +++ b/routes/emails.go @@ -3,10 +3,19 @@ package routes import ( "net/http" + "github.com/Sirupsen/logrus" + "github.com/ugorji/go/codec" + "github.com/zenazn/goji/web" + + "github.com/lavab/api/env" "github.com/lavab/api/models" "github.com/lavab/api/utils" ) +var ( + msgpackCodec codec.MsgpackHandle +) + // EmailsListResponse contains the result of the EmailsList request. type EmailsListResponse struct { Success bool `json:"success"` @@ -24,6 +33,18 @@ func EmailsList(w http.ResponseWriter, r *http.Request) { }) } +type EmailsCreateRequest struct { + To []string `json:"to"` + BCC []string `json:"bcc"` + ReplyTo string `json:"reply_to"` + ThreadID string `json:"thread_id"` + Title string `json:"title"` + Body string `json:"body"` + Preview string `json:"preview"` + Attachments []string `json:"attachments"` + PGPFingerprints []string `json:"pgp_fingerprints"` +} + // EmailsCreateResponse contains the result of the EmailsCreate request. type EmailsCreateResponse struct { Success bool `json:"success"` @@ -32,10 +53,88 @@ type EmailsCreateResponse struct { } // EmailsCreate sends a new email -func EmailsCreate(w http.ResponseWriter, r *http.Request) { - utils.JSONResponse(w, 200, &EmailsCreateResponse{ +func EmailsCreate(c web.C, w http.ResponseWriter, r *http.Request) { + // Decode the request + var input EmailsCreateRequest + err := utils.ParseRequest(r, &input) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Warn("Unable to decode a request") + + utils.JSONResponse(w, 400, &EmailsCreateResponse{ + Success: false, + Message: "Invalid input format", + }) + return + } + + // Fetch the current session from the middleware + session := c.Env["token"].(*models.Token) + + // Ensure that the input data isn't empty + if len(input.To) == 0 || input.Title == "" || input.Body == "" { + utils.JSONResponse(w, 400, &EmailsCreateResponse{ + Success: false, + Message: "Invalid request", + }) + return + } + + // Create a new email struct + email := &models.Email{ + Resource: models.MakeResource(session.Owner, input.Title), + AttachmentIDs: input.Attachments, + Body: models.Encrypted{ + Encoding: "json", + PGPFingerprints: input.PGPFingerprints, + Data: input.Body, + Schema: "email_body", + VersionMajor: 1, + VersionMinor: 0, + }, + Preview: models.Encrypted{ + Encoding: "json", + PGPFingerprints: input.PGPFingerprints, + Data: input.Preview, + Schema: "email_preview", + VersionMajor: 1, + VersionMinor: 0, + }, + ThreadID: input.ThreadID, + Status: "queued", + } + + // Insert the email into the database + if err := env.Emails.Insert(email); err != nil { + utils.JSONResponse(w, 500, &EmailsCreateResponse{ + Success: false, + Message: "internal server error - EM/CR/01", + }) + + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Error("Could not insert an email into the database") + return + } + + // Add a send request to the queue + err = env.NATS.Publish("send", email.ID) + if err != nil { + utils.JSONResponse(w, 500, &EmailsCreateResponse{ + Success: false, + Message: "internal server error - EM/CR/03", + }) + + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Error("Could not publish an email send request") + return + } + + utils.JSONResponse(w, 201, &EmailsCreateResponse{ Success: true, - Created: []string{"123"}, + Created: []string{email.ID}, }) } diff --git a/setup/setup.go b/setup/setup.go index b09b094..f2215d3 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -1,10 +1,11 @@ package setup import ( + "fmt" "time" "github.com/Sirupsen/logrus" - "github.com/bitly/go-nsq" + "github.com/apcera/nats" "github.com/dancannon/gorethink" "github.com/zenazn/goji/web" "github.com/zenazn/goji/web/middleware" @@ -112,43 +113,40 @@ func PrepareMux(flags *env.Flags) *web.Mux { "reservations", ), } - - // Initialize the NSQ connections - nsqProducer, err := nsq.NewProducer(flags.NSQAddress, nsq.NewConfig()) - if err != nil { - env.Log.WithFields(logrus.Fields{ - "error": err, - }).Fatal("Unable to create a NSQProducer") + env.Emails = &db.EmailsTable{ + RethinkCRUD: db.NewCRUDTable( + rethinkSession, + rethinkOpts.Database, + "emails", + ), } - env.NSQProducer = nsqProducer - deliveryConsumer, err := nsq.NewConsumer("delivery", "confirmation", nsq.NewConfig()) + // NATS queue connection + nc, err := nats.Connect(flags.NATSAddress) if err != nil { env.Log.WithFields(logrus.Fields{ - "error": err, - }).Fatal("Unable to create a DeliveryConsumer") + "error": err, + "address": flags.NATSAddress, + }).Fatal("Unable to connect to NATS") } - err = deliveryConsumer.ConnectToNSQLookupd(flags.NSQAddress) - if err != nil { - env.Log.WithFields(logrus.Fields{ - "error": err, - }).Fatal("Unable to connect to nsqlookupd") - } - env.DeliveryConsumer = deliveryConsumer - receiptConsumer, err := nsq.NewConsumer("receipt", "notification", nsq.NewConfig()) - if err != nil { - env.Log.WithFields(logrus.Fields{ - "error": err, - }).Fatal("Unable to create a DeliveryConsumer") - } - err = receiptConsumer.ConnectToNSQLookupd(flags.NSQAddress) + c, err := nats.NewEncodedConn(nc, "json") if err != nil { env.Log.WithFields(logrus.Fields{ - "error": err, - }).Fatal("Unable to connect to nsqlookupd") + "error": err, + "address": flags.NATSAddress, + }).Fatal("Unable to initialize a JSON NATS connection") } - env.ReceiptConsumer = receiptConsumer + + c.Subscribe("delivery", func(s string) { + fmt.Printf("Received a message: %s\n", s) + }) + + c.Subscribe("receipt", func(s string) { + fmt.Printf("Received a message: %s\n", s) + }) + + env.NATS = c // Initialize factors env.Factors = make(map[string]factor.Factor) From 91106bd703e6a7d9c46a8657c33bc6ec805efaea Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Mon, 22 Dec 2014 00:07:06 +0100 Subject: [PATCH 21/30] websockets --- _research/ws_client/index.html | 44 +++++++++++++++++++++++++++ _vagrant/Vagrantfile | 2 +- setup/setup.go | 55 ++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 _research/ws_client/index.html diff --git a/_research/ws_client/index.html b/_research/ws_client/index.html new file mode 100644 index 0000000..e11663a --- /dev/null +++ b/_research/ws_client/index.html @@ -0,0 +1,44 @@ + + + + + lavab client + + + + +
+
+ + + + + + + diff --git a/_vagrant/Vagrantfile b/_vagrant/Vagrantfile index 36d0121..15c639f 100644 --- a/_vagrant/Vagrantfile +++ b/_vagrant/Vagrantfile @@ -44,7 +44,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| docker.vm.provision "docker" do |d| d.pull_images "apcera/gnatsd" d.run "apcera/gnatsd", - args: "--name lookupd -p 4222:4222 -p 8333:8333" + args: "--name gnatsd -p 4222:4222 -p 8333:8333" d.pull_images "dockerfile/rethinkdb" d.run "dockerfile/rethinkdb", diff --git a/setup/setup.go b/setup/setup.go index f2215d3..d1768c6 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -1,12 +1,19 @@ package setup import ( + "bufio" "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" "time" "github.com/Sirupsen/logrus" "github.com/apcera/nats" "github.com/dancannon/gorethink" + "github.com/googollee/go-socket.io" + "github.com/rs/cors" "github.com/zenazn/goji/web" "github.com/zenazn/goji/web/middleware" @@ -174,6 +181,10 @@ func PrepareMux(flags *env.Flags) *web.Mux { mux.Use(middleware.RequestID) mux.Use(glogrus.NewGlogrus(log, "api")) mux.Use(middleware.Recoverer) + mux.Use(cors.New(cors.Options{ + AllowCredentials: true, + AllowedOrigins: []string{"*"}, + }).Handler) mux.Use(middleware.AutomaticOptions) // Set up an auth'd mux @@ -229,6 +240,50 @@ func PrepareMux(flags *env.Flags) *web.Mux { mux.Get("/keys/:id", routes.KeysGet) auth.Post("/keys/:id/vote", routes.KeysVote) + // WebSockets handler + ws, err := socketio.NewServer(nil) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Fatal("Unable to create a socket.io server") + } + ws.On("connection", func(so socketio.Socket) { + env.Log.WithFields(logrus.Fields{ + "id": so.Id(), + }).Info("New WebSockets connection") + + so.On("request", func(id string, method string, path string, data string, headers map[string]string) { + w := httptest.NewRecorder() + r, err := http.NewRequest(method, "http://api.lavaboom.io"+path, strings.NewReader(data)) + if err != nil { + so.Emit("error", err.Error()) + return + } + + for key, value := range headers { + r.Header.Set(key, value) + } + + mux.ServeHTTP(w, r) + + resp, err := http.ReadResponse(bufio.NewReader(w.Body), r) + if err != nil { + so.Emit("error", err.Error()) + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + so.Emit("error", err.Error()) + return + } + + so.Emit("response", id, resp.StatusCode, resp.Header, body) + }) + }) + + mux.Handle("/socket.io/", ws) + // Merge the muxes mux.Handle("/*", auth) From 76c1a2d7490c5a888723d42347f528c5615a0dc8 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Fri, 26 Dec 2014 19:15:47 +0100 Subject: [PATCH 22/30] SockJS support --- _research/ws_client/index.html | 28 +++++------ setup/setup.go | 88 ++++++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 19 deletions(-) diff --git a/_research/ws_client/index.html b/_research/ws_client/index.html index e11663a..f0c88d4 100644 --- a/_research/ws_client/index.html +++ b/_research/ws_client/index.html @@ -17,27 +17,23 @@ - + diff --git a/setup/setup.go b/setup/setup.go index d1768c6..698c422 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -2,6 +2,7 @@ package setup import ( "bufio" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -16,6 +17,7 @@ import ( "github.com/rs/cors" "github.com/zenazn/goji/web" "github.com/zenazn/goji/web/middleware" + "gopkg.in/igm/sockjs-go.v2/sockjs" "github.com/lavab/api/cache" "github.com/lavab/api/db" @@ -182,8 +184,7 @@ func PrepareMux(flags *env.Flags) *web.Mux { mux.Use(glogrus.NewGlogrus(log, "api")) mux.Use(middleware.Recoverer) mux.Use(cors.New(cors.Options{ - AllowCredentials: true, - AllowedOrigins: []string{"*"}, + AllowedOrigins: []string{"*"}, }).Handler) mux.Use(middleware.AutomaticOptions) @@ -282,7 +283,88 @@ func PrepareMux(flags *env.Flags) *web.Mux { }) }) - mux.Handle("/socket.io/", ws) + mux.Handle("/ws/*", sockjs.NewHandler("/ws", sockjs.DefaultOptions, func(session sockjs.Session) { + // A new goroutine seems to be spawned for each new session + for { + // Read a message from the input + msg, err := session.Recv() + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while reading from a WebSocket") + break + } + + // Decode the message + var input struct { + ID string `json:"id"` + Method string `json:"method"` + Path string `json:"path"` + Body string `json:"body"` + Headers map[string]string `json:"headers"` + } + err = json.Unmarshal([]byte(msg), &input) + if err != nil { + // Return an error response + resp, _ := json.Marshal(map[string]interface{}{ + "error": err, + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + continue + } + + // Perform the request + w := httptest.NewRecorder() + r, err := http.NewRequest(input.Method, "http://api.lavaboom.io"+input.Path, strings.NewReader(input.Body)) + if err != nil { + // Return an error response + resp, _ := json.Marshal(map[string]interface{}{ + "error": err.Error(), + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + continue + } + + r.RequestURI = input.Path + + for key, value := range input.Headers { + r.Header.Set(key, value) + } + + mux.ServeHTTP(w, r) + + // Return the final response + result, _ := json.Marshal(map[string]interface{}{ + "id": input.ID, + "status": w.Code, + "header": w.HeaderMap, + "body": w.Body.String(), + }) + err = session.Send(string(result)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + } + })) // Merge the muxes mux.Handle("/*", auth) From 4a55c84b496fdcec56d48120e535c2da33279fc1 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sat, 27 Dec 2014 12:47:08 +0100 Subject: [PATCH 23/30] Added message type to data sent back to WS client --- setup/setup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/setup.go b/setup/setup.go index 698c422..c169fa5 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -350,6 +350,7 @@ func PrepareMux(flags *env.Flags) *web.Mux { // Return the final response result, _ := json.Marshal(map[string]interface{}{ + "type": "response", "id": input.ID, "status": w.Code, "header": w.HeaderMap, From d8d0ecec2749596086e8a6991ac8e01be81b053c Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sat, 27 Dec 2014 12:50:17 +0100 Subject: [PATCH 24/30] Fixed tests and changed status badges in README.md --- README.md | 4 ++-- circle.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 923460f..fa5f6f7 100644 --- a/README.md +++ b/README.md @@ -56,5 +56,5 @@ version=v0 ## Build status: - - `master` - [![Build Status](https://magnum.travis-ci.com/lavab/api.svg?token=kJbppXeTxzqpCVvt4t5X&branch=master)](https://magnum.travis-ci.com/lavab/api) - - `develop` - [![Build Status](https://magnum.travis-ci.com/lavab/api.svg?token=kJbppXeTxzqpCVvt4t5X&branch=develop)](https://magnum.travis-ci.com/lavab/api) \ No newline at end of file + - `master` - [![Circle CI](https://circleci.com/gh/lavab/api/tree/master.svg?style=svg&circle-token=4a52d619a03d0249906195d6447ceb60a475c0c5)](https://circleci.com/gh/lavab/api/tree/master) + - `develop` - [![Circle CI](https://circleci.com/gh/lavab/api/tree/develop.svg?style=svg&circle-token=4a52d619a03d0249906195d6447ceb60a475c0c5)](https://circleci.com/gh/lavab/api/tree/develop) diff --git a/circle.yml b/circle.yml index 3250787..e0897b4 100644 --- a/circle.yml +++ b/circle.yml @@ -11,11 +11,14 @@ dependencies: - wget http://download.redis.io/releases/redis-2.8.18.tar.gz - tar xvzf redis-2.8.18.tar.gz - cd redis-2.8.18 && make + - go get github.com/apcera/gnatsd post: - rethinkdb --bind all: background: true - src/redis-server: background: true + - gnatsd: + background: true test: override: From 5ea87c34188c886eae248317a2a838ad68259c0a Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sat, 27 Dec 2014 12:56:41 +0100 Subject: [PATCH 25/30] Fixing configuration for automatic tests --- routes/init_test.go | 2 ++ setup/setup_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/routes/init_test.go b/routes/init_test.go index a57c5a7..fb0e4e7 100644 --- a/routes/init_test.go +++ b/routes/init_test.go @@ -30,6 +30,8 @@ func init() { RedisAddress: "127.0.0.1:6379", + NATSAddress: "nats://127.0.0.1:4222", + RethinkDBAddress: "127.0.0.1:28015", RethinkDBKey: "", RethinkDBDatabase: "test", diff --git a/setup/setup_test.go b/setup/setup_test.go index 406ecbe..c455a0c 100644 --- a/setup/setup_test.go +++ b/setup/setup_test.go @@ -20,6 +20,8 @@ func TestSetup(t *testing.T) { RedisAddress: "127.0.0.1:6379", + NATSAddress: "nats://127.0.0.1:4222", + RethinkDBAddress: "127.0.0.1:28015", RethinkDBKey: "", RethinkDBDatabase: "test", From ff64a723e71a2e3aa0a609d3c0abfe6e4c6f0a41 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 28 Dec 2014 01:12:11 +0100 Subject: [PATCH 26/30] GET /emails --- db/table_emails.go | 78 ++++++++++++++++++++++++++++++---- routes/emails.go | 103 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 165 insertions(+), 16 deletions(-) diff --git a/db/table_emails.go b/db/table_emails.go index 0a14eb9..4e0134e 100644 --- a/db/table_emails.go +++ b/db/table_emails.go @@ -1,6 +1,8 @@ package db import ( + "github.com/dancannon/gorethink" + "github.com/lavab/api/models" ) @@ -10,21 +12,21 @@ type EmailsTable struct { } // GetEmail returns a token with specified name -func (c *EmailsTable) GetEmail(id string) (*models.Email, error) { +func (e *EmailsTable) GetEmail(id string) (*models.Email, error) { var result models.Email - if err := c.FindFetchOne(id, &result); err != nil { + if err := e.FindFetchOne(id, &result); err != nil { return nil, err } return &result, nil } -// GetOwnedBy returns all contacts owned by id -func (c *EmailsTable) GetOwnedBy(id string) ([]*models.Email, error) { +// GetOwnedBy returns all emails owned by id +func (e *EmailsTable) GetOwnedBy(id string) ([]*models.Email, error) { var result []*models.Email - err := c.WhereAndFetch(map[string]interface{}{ + err := e.WhereAndFetch(map[string]interface{}{ "owner": id, }, &result) if err != nil { @@ -34,9 +36,69 @@ func (c *EmailsTable) GetOwnedBy(id string) ([]*models.Email, error) { return result, nil } -// DeleteOwnedBy deletes all contacts owned by id -func (c *EmailsTable) DeleteOwnedBy(id string) error { - return c.Delete(map[string]interface{}{ +// DeleteOwnedBy deletes all emails owned by id +func (e *EmailsTable) DeleteOwnedBy(id string) error { + return e.Delete(map[string]interface{}{ "owner": id, }) } + +func (e *EmailsTable) CountOwnedBy(id string) (int, error) { + return e.FindByAndCount("owner", id) +} + +func (e *EmailsTable) List( + owner string, + sort []string, + offset int, + limit int, +) ([]*models.Email, error) { + // Filter by owner's ID + term := e.GetTable().Filter(map[string]interface{}{ + "owner": owner, + }) + + // If sort array has contents, parse them and add to the term + if sort != nil && len(sort) > 0 { + var conds []interface{} + for _, cond := range sort { + if cond[0] == '-' { + conds = append(conds, gorethink.Desc(cond[1:])) + } else if cond[0] == '+' || cond[0] == ' ' { + conds = append(conds, gorethink.Asc(cond[1:])) + } else { + conds = append(conds, gorethink.Asc(cond)) + } + } + + term = term.OrderBy(conds...) + } + + // Slice the result in 3 cases + if offset != 0 && limit == 0 { + term = term.Skip(offset) + } + + if offset == 0 && limit != 0 { + term = term.Limit(limit) + } + + if offset != 0 && limit != 0 { + term = term.Slice(offset, offset+limit) + } + + // Run the query + cursor, err := term.Run(e.GetSession()) + if err != nil { + return nil, err + } + + // Fetch the cursor + var resp []*models.Email + err = cursor.All(&resp) + if err != nil { + return nil, err + } + + return resp, nil +} diff --git a/routes/emails.go b/routes/emails.go index 3cce604..2d9a367 100644 --- a/routes/emails.go +++ b/routes/emails.go @@ -2,6 +2,8 @@ package routes import ( "net/http" + "strconv" + "strings" "github.com/Sirupsen/logrus" "github.com/ugorji/go/codec" @@ -18,19 +20,104 @@ var ( // EmailsListResponse contains the result of the EmailsList request. type EmailsListResponse struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - ItemsCount int `json:"items_count,omitempty"` - Emails []*models.Email `json:"emails,omitempty"` + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Emails *[]*models.Email `json:"emails,omitempty"` } // EmailsList sends a list of the emails in the inbox. -func EmailsList(w http.ResponseWriter, r *http.Request) { +func EmailsList(c web.C, w http.ResponseWriter, r *http.Request) { + // Fetch the current session from the database + session := c.Env["token"].(*models.Token) + + // Parse the query + var ( + query = r.URL.Query() + sortRaw = query.Get("sort") + offsetRaw = query.Get("offset") + limitRaw = query.Get("limit") + sort []string + offset int + limit int + ) + + if offsetRaw != "" { + o, err := strconv.Atoi(offsetRaw) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + "offset": offset, + }).Error("Invalid offset") + + utils.JSONResponse(w, 400, &EmailsListResponse{ + Success: false, + Message: "Invalid offset", + }) + return + } + offset = o + } + + if limitRaw != "" { + l, err := strconv.Atoi(limitRaw) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + "limit": limit, + }).Error("Invalid limit") + + utils.JSONResponse(w, 400, &EmailsListResponse{ + Success: false, + Message: "Invalid limit", + }) + return + } + limit = l + } + + if sortRaw != "" { + sort = strings.Split(sortRaw, ",") + } + + // Get contacts from the database + emails, err := env.Emails.List(session.Owner, sort, offset, limit) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Error("Unable to fetch emails") + + utils.JSONResponse(w, 500, &EmailsListResponse{ + Success: false, + Message: "Internal error (code EM/LI/01)", + }) + return + } + + if offsetRaw != "" || limitRaw != "" { + count, err := env.Emails.CountOwnedBy(session.Owner) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + }).Error("Unable to count emails") + + utils.JSONResponse(w, 500, &EmailsListResponse{ + Success: false, + Message: "Internal error (code EM/LI/02)", + }) + return + } + w.Header().Set("X-Total-Count", strconv.Itoa(count)) + } + utils.JSONResponse(w, 200, &EmailsListResponse{ - Success: true, - ItemsCount: 1, - Emails: []*models.Email{}, + Success: true, + Emails: &emails, }) + + // GET parameters: + // sort - split by commas, prefixes: - is desc, + is asc + // offset, limit - for pagination + // Pagination ADDS X-Total-Count to the response! } type EmailsCreateRequest struct { From b93f0837b1aab31c833b0b714a07d93979975206 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 28 Dec 2014 01:29:17 +0100 Subject: [PATCH 27/30] Removed PUT /emails/:id, implemented DELETE /emails/:id and GET /emails/:id --- routes/emails.go | 93 ++++++++++++++++++++++++++++++++++++------------ setup/setup.go | 1 - 2 files changed, 70 insertions(+), 24 deletions(-) diff --git a/routes/emails.go b/routes/emails.go index 2d9a367..4e7e521 100644 --- a/routes/emails.go +++ b/routes/emails.go @@ -227,30 +227,39 @@ func EmailsCreate(c web.C, w http.ResponseWriter, r *http.Request) { // EmailsGetResponse contains the result of the EmailsGet request. type EmailsGetResponse struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Status string `json:"status,omitempty"` + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Email *models.Email `json:"email,omitempty"` } // EmailsGet responds with a single email message -func EmailsGet(w http.ResponseWriter, r *http.Request) { - utils.JSONResponse(w, 200, &EmailsGetResponse{ - Success: true, - Status: "sending", - }) -} +func EmailsGet(c web.C, w http.ResponseWriter, r *http.Request) { + // Get the email from the database + email, err := env.Emails.GetEmail(c.URLParams["id"]) + if err != nil { + utils.JSONResponse(w, 404, &EmailsGetResponse{ + Success: false, + Message: "Email not found", + }) + return + } -// EmailsUpdateResponse contains the result of the EmailsUpdate request. -type EmailsUpdateResponse struct { - Success bool `json:"success"` - Message string `json:"message"` -} + // Fetch the current session from the middleware + session := c.Env["token"].(*models.Token) -// EmailsUpdate does *something* - TODO -func EmailsUpdate(w http.ResponseWriter, r *http.Request) { - utils.JSONResponse(w, 501, &EmailsUpdateResponse{ - Success: false, - Message: "Sorry, not implemented yet", + // Check for ownership + if email.Owner != session.Owner { + utils.JSONResponse(w, 404, &EmailsGetResponse{ + Success: false, + Message: "Email not found", + }) + return + } + + // Write the email to the response + utils.JSONResponse(w, 200, &EmailsGetResponse{ + Success: true, + Email: email, }) } @@ -261,9 +270,47 @@ type EmailsDeleteResponse struct { } // EmailsDelete remvoes an email from the system -func EmailsDelete(w http.ResponseWriter, r *http.Request) { - utils.JSONResponse(w, 501, &EmailsDeleteResponse{ - Success: false, - Message: "Sorry, not implemented yet", +func EmailsDelete(c web.C, w http.ResponseWriter, r *http.Request) { + // Get the email from the database + email, err := env.Emails.GetEmail(c.URLParams["id"]) + if err != nil { + utils.JSONResponse(w, 404, &EmailsDeleteResponse{ + Success: false, + Message: "Email not found", + }) + return + } + + // Fetch the current session from the middleware + session := c.Env["token"].(*models.Token) + + // Check for ownership + if email.Owner != session.Owner { + utils.JSONResponse(w, 404, &EmailsDeleteResponse{ + Success: false, + Message: "Email not found", + }) + return + } + + // Perform the deletion + err = env.Emails.DeleteID(c.URLParams["id"]) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err, + "id": c.URLParams["id"], + }).Error("Unable to delete a email") + + utils.JSONResponse(w, 500, &EmailsDeleteResponse{ + Success: false, + Message: "Internal error (code EM/DE/01)", + }) + return + } + + // Write the email to the response + utils.JSONResponse(w, 200, &EmailsDeleteResponse{ + Success: true, + Message: "Email successfully removed", }) } diff --git a/setup/setup.go b/setup/setup.go index c169fa5..3d31cde 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -218,7 +218,6 @@ func PrepareMux(flags *env.Flags) *web.Mux { auth.Get("/emails", routes.EmailsList) auth.Post("/emails", routes.EmailsCreate) auth.Get("/emails/:id", routes.EmailsGet) - auth.Put("/emails/:id", routes.EmailsUpdate) auth.Delete("/emails/:id", routes.EmailsDelete) // Labels From c2fff71700f7e2deee450b6af77640099bd61384 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 28 Dec 2014 15:09:06 +0100 Subject: [PATCH 28/30] setup.go, the 700-line wonder --- setup/setup.go | 367 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 336 insertions(+), 31 deletions(-) diff --git a/setup/setup.go b/setup/setup.go index 3d31cde..5549c6c 100644 --- a/setup/setup.go +++ b/setup/setup.go @@ -3,11 +3,11 @@ package setup import ( "bufio" "encoding/json" - "fmt" "io/ioutil" "net/http" "net/http/httptest" "strings" + "sync" "time" "github.com/Sirupsen/logrus" @@ -27,6 +27,12 @@ import ( "github.com/lavab/glogrus" ) +// sessions contains all "subscribing" WebSockets sessions +var ( + sessions = map[string][]sockjs.Session{} + sessionsLock sync.Mutex +) + // PrepareMux sets up the API func PrepareMux(flags *env.Flags) *web.Mux { // Set up a new logger @@ -147,12 +153,84 @@ func PrepareMux(flags *env.Flags) *web.Mux { }).Fatal("Unable to initialize a JSON NATS connection") } - c.Subscribe("delivery", func(s string) { - fmt.Printf("Received a message: %s\n", s) + c.Subscribe("delivery", func(msg *struct { + ID string `json:"id"` + Owner string `json:"owner"` + }) { + // Check if we are handling owner's session + if _, ok := sessions[msg.Owner]; !ok { + return + } + + if len(sessions[msg.Owner]) == 0 { + return + } + + // Resolve the email + email, err := env.Emails.GetEmail(msg.ID) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err.Error(), + "id": msg.ID, + }).Error("Unable to resolve an email from queue") + return + } + + // Send notifications to subscribers + for _, session := range sessions[msg.Owner] { + result, _ := json.Marshal(map[string]interface{}{ + "type": "delivery", + "id": msg.ID, + "name": email.Name, + }) + err = session.Send(string(result)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + } + } }) - c.Subscribe("receipt", func(s string) { - fmt.Printf("Received a message: %s\n", s) + c.Subscribe("receipt", func(msg *struct { + ID string `json:"id"` + Owner string `json:"owner"` + }) { + // Check if we are handling owner's session + if _, ok := sessions[msg.Owner]; !ok { + return + } + + if len(sessions[msg.Owner]) == 0 { + return + } + + // Resolve the email + email, err := env.Emails.GetEmail(msg.ID) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "error": err.Error(), + "id": msg.ID, + }).Error("Unable to resolve an email from queue") + return + } + + // Send notifications to subscribers + for _, session := range sessions[msg.Owner] { + result, _ := json.Marshal(map[string]interface{}{ + "type": "receipt", + "id": msg.ID, + "name": email.Name, + }) + err = session.Send(string(result)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + } + } }) env.NATS = c @@ -283,6 +361,8 @@ func PrepareMux(flags *env.Flags) *web.Mux { }) mux.Handle("/ws/*", sockjs.NewHandler("/ws", sockjs.DefaultOptions, func(session sockjs.Session) { + var subscribed string + // A new goroutine seems to be spawned for each new session for { // Read a message from the input @@ -297,6 +377,8 @@ func PrepareMux(flags *env.Flags) *web.Mux { // Decode the message var input struct { + Type string `json:"type"` + Token string `json:"token"` ID string `json:"id"` Method string `json:"method"` Path string `json:"path"` @@ -307,6 +389,7 @@ func PrepareMux(flags *env.Flags) *web.Mux { if err != nil { // Return an error response resp, _ := json.Marshal(map[string]interface{}{ + "type": "error", "error": err, }) err := session.Send(string(resp)) @@ -320,13 +403,172 @@ func PrepareMux(flags *env.Flags) *web.Mux { continue } - // Perform the request - w := httptest.NewRecorder() - r, err := http.NewRequest(input.Method, "http://api.lavaboom.io"+input.Path, strings.NewReader(input.Body)) - if err != nil { - // Return an error response + // Check message's type + if input.Type == "subscribe" { + // Listen to user's events + + // Check if token is empty + if input.Token == "" { + // Return an error response + resp, _ := json.Marshal(map[string]interface{}{ + "type": "error", + "error": "Invalid token", + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + continue + } + + // Check the token in database + token, err := env.Tokens.GetToken(input.Token) + if err != nil { + // Return an error response + resp, _ := json.Marshal(map[string]interface{}{ + "type": "error", + "error": "Invalid token", + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + continue + } + + // Do the actual subscription + subscribed = token.Owner + sessionsLock.Lock() + + // Sessions map already contains this owner + if _, ok := sessions[token.Owner]; ok { + sessions[token.Owner] = append(sessions[token.Owner], session) + } else { + // We have to allocate a new slice + sessions[token.Owner] = []sockjs.Session{session} + } + + // Unlock the map write + sessionsLock.Unlock() + + // Return a response resp, _ := json.Marshal(map[string]interface{}{ - "error": err.Error(), + "type": "subscribed", + }) + err = session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + } else if input.Type == "unsubscribe" { + if subscribed == "" { + resp, _ := json.Marshal(map[string]interface{}{ + "type": "error", + "error": "Not subscribed", + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + } + + sessionsLock.Lock() + + if _, ok := sessions[subscribed]; !ok { + // Return a response + resp, _ := json.Marshal(map[string]interface{}{ + "type": "unsubscribed", + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + sessionsLock.Unlock() + subscribed = "" + break + } + sessionsLock.Unlock() + subscribed = "" + continue + } + + if len(sessions[subscribed]) == 1 { + delete(sessions, subscribed) + + // Return a response + resp, _ := json.Marshal(map[string]interface{}{ + "type": "unsubscribed", + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + sessionsLock.Unlock() + subscribed = "" + break + } + sessionsLock.Unlock() + subscribed = "" + continue + } + + // Find the session + index := -1 + for i, session2 := range sessions[subscribed] { + if session == session2 { + index = i + break + } + } + + // We didn't find anything + if index == -1 { + // Return a response + resp, _ := json.Marshal(map[string]interface{}{ + "type": "unsubscribed", + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + sessionsLock.Unlock() + subscribed = "" + break + } + sessionsLock.Unlock() + subscribed = "" + continue + } + + // We found it, so we are supposed to slice it + sessions[subscribed][index] = sessions[subscribed][len(sessions[subscribed])-1] + sessions[subscribed][len(sessions[subscribed])-1] = nil + sessions[subscribed] = sessions[subscribed][:len(sessions[subscribed])-1] + + // Return a response + resp, _ := json.Marshal(map[string]interface{}{ + "type": "unsubscribed", }) err := session.Send(string(resp)) if err != nil { @@ -334,36 +576,99 @@ func PrepareMux(flags *env.Flags) *web.Mux { "id": session.ID(), "error": err.Error(), }).Warn("Error while writing to a WebSocket") + sessionsLock.Unlock() + subscribed = "" break } - continue - } + sessionsLock.Unlock() + subscribed = "" + } else if input.Type == "request" { + // Perform the request + w := httptest.NewRecorder() + r, err := http.NewRequest(input.Method, "http://api.lavaboom.io"+input.Path, strings.NewReader(input.Body)) + if err != nil { + // Return an error response + resp, _ := json.Marshal(map[string]interface{}{ + "error": err.Error(), + }) + err := session.Send(string(resp)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } + continue + } - r.RequestURI = input.Path + r.RequestURI = input.Path - for key, value := range input.Headers { - r.Header.Set(key, value) + for key, value := range input.Headers { + r.Header.Set(key, value) + } + + mux.ServeHTTP(w, r) + + // Return the final response + result, _ := json.Marshal(map[string]interface{}{ + "type": "response", + "id": input.ID, + "status": w.Code, + "header": w.HeaderMap, + "body": w.Body.String(), + }) + err = session.Send(string(result)) + if err != nil { + env.Log.WithFields(logrus.Fields{ + "id": session.ID(), + "error": err.Error(), + }).Warn("Error while writing to a WebSocket") + break + } } + } - mux.ServeHTTP(w, r) + // We have to clear the subscription here too. TODO: make the code shorter + if subscribed == "" { + return + } - // Return the final response - result, _ := json.Marshal(map[string]interface{}{ - "type": "response", - "id": input.ID, - "status": w.Code, - "header": w.HeaderMap, - "body": w.Body.String(), - }) - err = session.Send(string(result)) - if err != nil { - env.Log.WithFields(logrus.Fields{ - "id": session.ID(), - "error": err.Error(), - }).Warn("Error while writing to a WebSocket") + sessionsLock.Lock() + + if _, ok := sessions[subscribed]; !ok { + sessionsLock.Unlock() + return + } + + if len(sessions[subscribed]) == 1 { + delete(sessions, subscribed) + sessionsLock.Unlock() + return + } + + // Find the session + index := -1 + for i, session2 := range sessions[subscribed] { + if session == session2 { + index = i break } } + + // We didn't find anything + if index == -1 { + sessionsLock.Unlock() + return + } + + // We found it, so we are supposed to slice it + sessions[subscribed][index] = sessions[subscribed][len(sessions[subscribed])-1] + sessions[subscribed][len(sessions[subscribed])-1] = nil + sessions[subscribed] = sessions[subscribed][:len(sessions[subscribed])-1] + + // Unlock the mutex + sessionsLock.Unlock() })) // Merge the muxes From 9406d944400bd8334dbc85eee63229a2ada2fc48 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 28 Dec 2014 16:19:38 +0100 Subject: [PATCH 29/30] Weak password test --- routes/accounts_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/routes/accounts_test.go b/routes/accounts_test.go index 6b2f61c..240d34a 100644 --- a/routes/accounts_test.go +++ b/routes/accounts_test.go @@ -149,6 +149,45 @@ func TestAccountsCreateInvitedExisting(t *testing.T) { require.Equal(t, "Username already exists", response.Message) } +func TestAccountsCreateInvitedWeakPassword(t *testing.T) { + const ( + username = "jeremy" + password = "c0067d4af4e87f00dbac63b6156828237059172d1bbeac67427345d6a9fda484" + ) + + // Prepare a token + inviteToken := models.Token{ + Resource: models.MakeResource("", "test invite token"), + Type: "invite", + } + inviteToken.ExpireSoon() + + err := env.Tokens.Insert(inviteToken) + require.Nil(t, err) + + // POST /accounts - invited + result, err := goreq.Request{ + Method: "POST", + Uri: server.URL + "/accounts", + ContentType: "application/json", + Body: routes.AccountsCreateRequest{ + Username: username, + Password: password, + Token: inviteToken.ID, + }, + }.Do() + require.Nil(t, err) + + // Unmarshal the response + var response routes.AccountsCreateResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + // Check the result's contents + require.False(t, response.Success) + require.Equal(t, "Weak password", response.Message) +} + func TestAccountsCreateInvitedExpired(t *testing.T) { const ( username = "jeremy2" From 0953b23d7114c123417d15f1e4505bb29d8410e4 Mon Sep 17 00:00:00 2001 From: Piotr Zduniak Date: Sun, 28 Dec 2014 17:09:52 +0100 Subject: [PATCH 30/30] Added email tests --- models/email.go | 6 ++ routes/emails.go | 8 +- routes/emails_test.go | 245 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 routes/emails_test.go diff --git a/models/email.go b/models/email.go index 10aaa5e..03bd9e3 100644 --- a/models/email.go +++ b/models/email.go @@ -5,6 +5,12 @@ package models type Email struct { Resource + // Kind of the email. Value is either sent or received. + Kind string `json:"kind" gorethink:"kind"` + + // Who is supposed to receive the email / what email received it. + To []string `json:"to" gorethink:"to"` + // AttachmentsIDs is a slice of the FileIDs associated with this email // For uploading attachments see `POST /upload` AttachmentIDs []string `json:"attachments" gorethink:"attachments"` diff --git a/routes/emails.go b/routes/emails.go index 4e7e521..612324b 100644 --- a/routes/emails.go +++ b/routes/emails.go @@ -125,7 +125,7 @@ type EmailsCreateRequest struct { BCC []string `json:"bcc"` ReplyTo string `json:"reply_to"` ThreadID string `json:"thread_id"` - Title string `json:"title"` + Subject string `json:"title"` Body string `json:"body"` Preview string `json:"preview"` Attachments []string `json:"attachments"` @@ -160,7 +160,7 @@ func EmailsCreate(c web.C, w http.ResponseWriter, r *http.Request) { session := c.Env["token"].(*models.Token) // Ensure that the input data isn't empty - if len(input.To) == 0 || input.Title == "" || input.Body == "" { + if len(input.To) == 0 || input.Subject == "" || input.Body == "" { utils.JSONResponse(w, 400, &EmailsCreateResponse{ Success: false, Message: "Invalid request", @@ -170,7 +170,9 @@ func EmailsCreate(c web.C, w http.ResponseWriter, r *http.Request) { // Create a new email struct email := &models.Email{ - Resource: models.MakeResource(session.Owner, input.Title), + Kind: "sent", + To: input.To, + Resource: models.MakeResource(session.Owner, input.Subject), AttachmentIDs: input.Attachments, Body: models.Encrypted{ Encoding: "json", diff --git a/routes/emails_test.go b/routes/emails_test.go new file mode 100644 index 0000000..7d4310a --- /dev/null +++ b/routes/emails_test.go @@ -0,0 +1,245 @@ +package routes_test + +import ( + "testing" + + "github.com/franela/goreq" + "github.com/stretchr/testify/require" + + "github.com/lavab/api/env" + "github.com/lavab/api/models" + "github.com/lavab/api/routes" +) + +var ( + emailID string + notOwnedEmailID string +) + +func TestEmailsCreate(t *testing.T) { + request := goreq.Request{ + Method: "POST", + Uri: server.URL + "/emails", + ContentType: "application/json", + Body: routes.EmailsCreateRequest{ + To: []string{"piotr@zduniak.net"}, + Subject: "hello world", + Body: "raw meaty email", + }, + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsCreateResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.Equal(t, len(response.Created), 1) + require.True(t, response.Success) + + emailID = response.Created[0] +} + +func TestEmailsCreateInvalidBody(t *testing.T) { + request := goreq.Request{ + Method: "POST", + Uri: server.URL + "/emails", + ContentType: "application/json", + Body: "!@#!@#!@#", + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsCreateResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.Equal(t, response.Message, "Invalid input format") + require.False(t, response.Success) +} + +func TestEmailsCreateMissingFields(t *testing.T) { + request := goreq.Request{ + Method: "POST", + Uri: server.URL + "/emails", + ContentType: "application/json", + Body: routes.EmailsCreateRequest{ + To: []string{"piotr@zduniak.net"}, + Subject: "hello world", + }, + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsCreateResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.Equal(t, response.Message, "Invalid request") + require.False(t, response.Success) +} + +func TestEmailsGet(t *testing.T) { + request := goreq.Request{ + Method: "GET", + Uri: server.URL + "/emails/" + emailID, + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsGetResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.True(t, response.Success) + require.Equal(t, "hello world", response.Email.Name) +} + +func TestEmailsGetInvalidID(t *testing.T) { + request := goreq.Request{ + Method: "GET", + Uri: server.URL + "/emails/nonexisting", + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsGetResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.Equal(t, "Email not found", response.Message) + require.False(t, response.Success) +} + +func TestEmailsGetNotOwned(t *testing.T) { + email := &models.Email{ + Resource: models.MakeResource("not", "Carpeus Caesar"), + } + + err := env.Emails.Insert(email) + require.Nil(t, err) + + notOwnedEmailID = email.ID + + request := goreq.Request{ + Method: "GET", + Uri: server.URL + "/emails/" + email.ID, + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsGetResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.Equal(t, "Email not found", response.Message) + require.False(t, response.Success) +} + +func TestEmailsList(t *testing.T) { + request := goreq.Request{ + Method: "GET", + Uri: server.URL + "/emails?offset=0&limit=1&sort=+date", + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsListResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.True(t, response.Success) + require.Equal(t, "hello world", (*response.Emails)[0].Name) +} + +func TestEmailsListInvalidOffset(t *testing.T) { + request := goreq.Request{ + Method: "GET", + Uri: server.URL + "/emails?offset=pppp&limit=1&sort=+date", + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsListResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.False(t, response.Success) + require.Equal(t, "Invalid offset", response.Message) +} + +func TestEmailsListInvalidLimit(t *testing.T) { + request := goreq.Request{ + Method: "GET", + Uri: server.URL + "/emails?offset=0&limit=pppp&sort=+date", + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsListResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.False(t, response.Success) + require.Equal(t, "Invalid limit", response.Message) +} + +func TestEmailsDelete(t *testing.T) { + request := goreq.Request{ + Method: "DELETE", + Uri: server.URL + "/emails/" + emailID, + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsDeleteResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.True(t, response.Success) + require.Equal(t, "Email successfully removed", response.Message) +} + +func TestEmailsDeleteNotExisting(t *testing.T) { + request := goreq.Request{ + Method: "DELETE", + Uri: server.URL + "/emails/nonexisting", + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsDeleteResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.False(t, response.Success) + require.Equal(t, "Email not found", response.Message) +} + +func TestEmailsDeleteNotOwned(t *testing.T) { + request := goreq.Request{ + Method: "DELETE", + Uri: server.URL + "/emails/" + notOwnedEmailID, + } + request.AddHeader("Authorization", "Bearer "+authToken) + result, err := request.Do() + require.Nil(t, err) + + var response routes.EmailsDeleteResponse + err = result.Body.FromJsonTo(&response) + require.Nil(t, err) + + require.False(t, response.Success) + require.Equal(t, "Email not found", response.Message) +}