From 0d1e2709004a6445e39759fa3b14bd67da3d507a Mon Sep 17 00:00:00 2001 From: John Refior Date: Wed, 7 Nov 2018 21:32:49 -0500 Subject: [PATCH 1/4] Dependency name change Like Sirupsen -> sirupsen, here garyburd -> gomodule. Uses of the old name are giving us headaches elsewhere, it's time to remove them. --- README.md | 4 +- benches/bench_goworker/main.go | 123 ---------------------------- benches/bench_goworkers/main.go | 131 ------------------------------ benches/bench_jobs/main.go | 140 -------------------------------- benches/bench_work/main.go | 130 ----------------------------- client.go | 2 +- client_test.go | 2 +- cmd/workenqueue/main.go | 2 +- cmd/workfakedata/main.go | 2 +- cmd/workwebui/main.go | 2 +- dead_pool_reaper.go | 2 +- dead_pool_reaper_test.go | 2 +- enqueue.go | 2 +- go.mod | 26 ++++++ go.sum | 47 +++++++++++ heartbeater.go | 2 +- heartbeater_test.go | 2 +- observer.go | 2 +- observer_test.go | 2 +- periodic_enqueuer.go | 2 +- periodic_enqueuer_test.go | 2 +- requeuer.go | 2 +- requeuer_test.go | 2 +- webui/webui.go | 2 +- webui/webui_test.go | 2 +- worker.go | 2 +- worker_pool.go | 2 +- worker_test.go | 2 +- 28 files changed, 96 insertions(+), 547 deletions(-) delete mode 100644 benches/bench_goworker/main.go delete mode 100644 benches/bench_goworkers/main.go delete mode 100644 benches/bench_jobs/main.go delete mode 100644 benches/bench_work/main.go create mode 100644 go.mod create mode 100644 go.sum diff --git a/README.md b/README.md index abd6aa16..46779058 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ To enqueue jobs, you need to make an Enqueuer with a redis namespace and a redig package main import ( - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/DispatchMe/go-work" ) @@ -63,7 +63,7 @@ In order to process jobs, you'll need to make a WorkerPool. Add middleware and j package main import ( - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/DispatchMe/go-work" "os" "os/signal" diff --git a/benches/bench_goworker/main.go b/benches/bench_goworker/main.go deleted file mode 100644 index cfc8243a..00000000 --- a/benches/bench_goworker/main.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "fmt" - "github.com/benmanns/goworker" - "github.com/garyburd/redigo/redis" - "github.com/gocraft/health" - "os" - "sync/atomic" - "time" -) - -func myJob(queue string, args ...interface{}) error { - atomic.AddInt64(&totcount, 1) - //fmt.Println("job! ", queue) - return nil -} - -var namespace = "bench_test" -var pool = newPool(":6379") - -// go run *.go -queues="myqueue,myqueue2,myqueue3,myqueue4,myqueue5" -namespace="bench_test:" -concurrency=50 -use-nuber -func main() { - - stream := health.NewStream().AddSink(&health.WriterSink{os.Stdout}) - stream.Event("wat") - cleanKeyspace() - - queues := []string{"myqueue", "myqueue2", "myqueue3", "myqueue4", "myqueue5"} - numJobs := 100000 / len(queues) - - job := stream.NewJob("enqueue_all") - for _, q := range queues { - enqueueJobs(q, numJobs) - } - job.Complete(health.Success) - - goworker.Register("MyClass", myJob) - - go monitor() - - // Blocks until process is told to exit via unix signal - goworker.Work() -} - -var totcount int64 - -func monitor() { - t := time.Tick(1 * time.Second) - - curT := 0 - c1 := int64(0) - c2 := int64(0) - prev := int64(0) - -DALOOP: - for { - select { - case <-t: - curT++ - v := atomic.AddInt64(&totcount, 0) - fmt.Printf("after %d seconds, count is %d\n", curT, v) - if curT == 1 { - c1 = v - } else if curT == 3 { - c2 = v - } - if v == prev { - break DALOOP - } - prev = v - } - } - fmt.Println("Jobs/sec: ", float64(c2-c1)/2.0) - os.Exit(0) -} - -func enqueueJobs(queue string, count int) { - conn := pool.Get() - defer conn.Close() - - for i := 0; i < count; i++ { - //workers.Enqueue(queue, "Foo", []int{i}) - conn.Do("RPUSH", "bench_test:queue:"+queue, `{"class":"MyClass","args":[]}`) - } -} - -func cleanKeyspace() { - conn := pool.Get() - defer conn.Close() - - keys, err := redis.Strings(conn.Do("KEYS", namespace+"*")) - if err != nil { - panic("could not get keys: " + err.Error()) - } - for _, k := range keys { - //fmt.Println("deleting ", k) - if _, err := conn.Do("DEL", k); err != nil { - panic("could not del: " + err.Error()) - } - } -} - -func newPool(addr string) *redis.Pool { - return &redis.Pool{ - MaxActive: 3, - MaxIdle: 3, - IdleTimeout: 240 * time.Second, - Dial: func() (redis.Conn, error) { - c, err := redis.Dial("tcp", addr) - if err != nil { - return nil, err - } - return c, nil - //return redis.NewLoggingConn(c, log.New(os.Stdout, "", 0), "redis"), err - }, - Wait: true, - //TestOnBorrow: func(c redis.Conn, t time.Time) error { - // _, err := c.Do("PING") - // return err - //}, - } -} diff --git a/benches/bench_goworkers/main.go b/benches/bench_goworkers/main.go deleted file mode 100644 index bd67b7d5..00000000 --- a/benches/bench_goworkers/main.go +++ /dev/null @@ -1,131 +0,0 @@ -package main - -import ( - "fmt" - "github.com/garyburd/redigo/redis" - "github.com/gocraft/health" - "github.com/jrallison/go-workers" - "os" - "sync/atomic" - "time" -) - -func myJob(m *workers.Msg) { - atomic.AddInt64(&totcount, 1) -} - -var namespace = "bench_test" -var pool = newPool(":6379") - -func main() { - - stream := health.NewStream().AddSink(&health.WriterSink{os.Stdout}) - stream.Event("wat") - cleanKeyspace() - - workers.Configure(map[string]string{ - // location of redis instance - "server": "localhost:6379", - // instance of the database - "database": "0", - // number of connections to keep open with redis - "pool": "10", - // unique process id for this instance of workers (for proper recovery of inprogress jobs on crash) - "process": "1", - "namespace": namespace, - }) - workers.Middleware = &workers.Middlewares{} - - queues := []string{"myqueue", "myqueue2", "myqueue3", "myqueue4", "myqueue5"} - numJobs := 100000 / len(queues) - - job := stream.NewJob("enqueue_all") - for _, q := range queues { - enqueueJobs(q, numJobs) - } - job.Complete(health.Success) - - for _, q := range queues { - workers.Process(q, myJob, 10) - } - - go monitor() - - // Blocks until process is told to exit via unix signal - workers.Run() -} - -var totcount int64 - -func monitor() { - t := time.Tick(1 * time.Second) - - curT := 0 - c1 := int64(0) - c2 := int64(0) - prev := int64(0) - -DALOOP: - for { - select { - case <-t: - curT++ - v := atomic.AddInt64(&totcount, 0) - fmt.Printf("after %d seconds, count is %d\n", curT, v) - if curT == 1 { - c1 = v - } else if curT == 3 { - c2 = v - } - if v == prev { - break DALOOP - } - prev = v - } - } - fmt.Println("Jobs/sec: ", float64(c2-c1)/2.0) - os.Exit(0) -} - -func enqueueJobs(queue string, count int) { - for i := 0; i < count; i++ { - workers.Enqueue(queue, "Foo", []int{i}) - } -} - -func cleanKeyspace() { - conn := pool.Get() - defer conn.Close() - - keys, err := redis.Strings(conn.Do("KEYS", namespace+"*")) - if err != nil { - panic("could not get keys: " + err.Error()) - } - for _, k := range keys { - //fmt.Println("deleting ", k) - if _, err := conn.Do("DEL", k); err != nil { - panic("could not del: " + err.Error()) - } - } -} - -func newPool(addr string) *redis.Pool { - return &redis.Pool{ - MaxActive: 3, - MaxIdle: 3, - IdleTimeout: 240 * time.Second, - Dial: func() (redis.Conn, error) { - c, err := redis.Dial("tcp", addr) - if err != nil { - return nil, err - } - return c, nil - //return redis.NewLoggingConn(c, log.New(os.Stdout, "", 0), "redis"), err - }, - Wait: true, - //TestOnBorrow: func(c redis.Conn, t time.Time) error { - // _, err := c.Do("PING") - // return err - //}, - } -} diff --git a/benches/bench_jobs/main.go b/benches/bench_jobs/main.go deleted file mode 100644 index cf68f5fc..00000000 --- a/benches/bench_jobs/main.go +++ /dev/null @@ -1,140 +0,0 @@ -package main - -import ( - "fmt" - "github.com/albrow/jobs" - "github.com/garyburd/redigo/redis" - "github.com/gocraft/health" - "os" - "sync/atomic" - "time" -) - -var namespace = "jobs" -var pool = newPool(":6379") - -func epsilonHandler(i int) error { - atomic.AddInt64(&totcount, 1) - return nil -} - -func main() { - stream := health.NewStream().AddSink(&health.WriterSink{os.Stdout}) - cleanKeyspace() - - queueNames := []string{"myqueue", "myqueue2", "myqueue3", "myqueue4", "myqueue5"} - queues := []*jobs.Type{} - - for _, qn := range queueNames { - q, err := jobs.RegisterType(qn, 3, epsilonHandler) - if err != nil { - panic(err) - } - queues = append(queues, q) - } - - job := stream.NewJob("enqueue_all") - - numJobs := 40000 / len(queues) - for _, q := range queues { - for i := 0; i < numJobs; i++ { - _, err := q.Schedule(100, time.Now(), i) - if err != nil { - panic(err) - } - } - } - - job.Complete(health.Success) - - go monitor() - - job = stream.NewJob("run_all") - pool, err := jobs.NewPool(&jobs.PoolConfig{ - // NumWorkers: 1000, - // BatchSize: 3000, - }) - if err != nil { - panic(err) - } - defer func() { - pool.Close() - if err := pool.Wait(); err != nil { - panic(err) - } - }() - if err := pool.Start(); err != nil { - panic(err) - } - job.Complete(health.Success) - select {} -} - -var totcount int64 - -func monitor() { - t := time.Tick(1 * time.Second) - - curT := 0 - c1 := int64(0) - c2 := int64(0) - prev := int64(0) - -DALOOP: - for { - select { - case <-t: - curT++ - v := atomic.AddInt64(&totcount, 0) - fmt.Printf("after %d seconds, count is %d\n", curT, v) - if curT == 1 { - c1 = v - } else if curT == 3 { - c2 = v - } - if v == prev { - break DALOOP - } - prev = v - } - } - fmt.Println("Jobs/sec: ", float64(c2-c1)/2.0) - os.Exit(0) -} - -func cleanKeyspace() { - conn := pool.Get() - defer conn.Close() - - keys, err := redis.Strings(conn.Do("KEYS", namespace+"*")) - if err != nil { - panic("could not get keys: " + err.Error()) - } - for _, k := range keys { - //fmt.Println("deleting ", k) - if _, err := conn.Do("DEL", k); err != nil { - panic("could not del: " + err.Error()) - } - } -} - -func newPool(addr string) *redis.Pool { - return &redis.Pool{ - MaxActive: 20, - MaxIdle: 20, - IdleTimeout: 240 * time.Second, - Dial: func() (redis.Conn, error) { - c, err := redis.Dial("tcp", addr) - if err != nil { - return nil, err - } - return c, nil - //return redis.NewLoggingConn(c, log.New(os.Stdout, "", 0), "redis"), err - }, - Wait: true, - //TestOnBorrow: func(c redis.Conn, t time.Time) error { - // _, err := c.Do("PING") - // return err - //}, - } -} diff --git a/benches/bench_work/main.go b/benches/bench_work/main.go deleted file mode 100644 index 9184b61b..00000000 --- a/benches/bench_work/main.go +++ /dev/null @@ -1,130 +0,0 @@ -package main - -import ( - "fmt" - "github.com/garyburd/redigo/redis" - "github.com/gocraft/health" - "github.com/gocraft/work" - "os" - "sync/atomic" - "time" -) - -var namespace = "bench_test" -var pool = newPool(":6379") - -type context struct{} - -func epsilonHandler(job *work.Job) error { - //fmt.Println("hi") - //a := job.Args[0] - //fmt.Printf("job: %s arg: %v\n", job.Name, a) - atomic.AddInt64(&totcount, 1) - return nil -} - -func main() { - stream := health.NewStream().AddSink(&health.WriterSink{os.Stdout}) - cleanKeyspace() - - numJobs := 10 - jobNames := []string{} - - for i := 0; i < numJobs; i++ { - jobNames = append(jobNames, fmt.Sprintf("job%d", i)) - } - - job := stream.NewJob("enqueue_all") - enqueueJobs(jobNames, 10000) - job.Complete(health.Success) - - workerPool := work.NewWorkerPool(context{}, 20, namespace, pool) - for _, jobName := range jobNames { - workerPool.Job(jobName, epsilonHandler) - } - go monitor() - - job = stream.NewJob("run_all") - workerPool.Start() - workerPool.Drain() - job.Complete(health.Success) - select {} -} - -var totcount int64 - -func monitor() { - t := time.Tick(1 * time.Second) - - curT := 0 - c1 := int64(0) - c2 := int64(0) - prev := int64(0) - -DALOOP: - for { - select { - case <-t: - curT++ - v := atomic.AddInt64(&totcount, 0) - fmt.Printf("after %d seconds, count is %d\n", curT, v) - if curT == 1 { - c1 = v - } else if curT == 3 { - c2 = v - } - if v == prev { - break DALOOP - } - prev = v - } - } - fmt.Println("Jobs/sec: ", float64(c2-c1)/2.0) - os.Exit(0) -} - -func enqueueJobs(jobs []string, count int) { - enq := work.NewEnqueuer(namespace, pool) - for _, jobName := range jobs { - for i := 0; i < count; i++ { - enq.Enqueue(jobName, work.Q{"i": i}) - } - } -} - -func cleanKeyspace() { - conn := pool.Get() - defer conn.Close() - - keys, err := redis.Strings(conn.Do("KEYS", namespace+"*")) - if err != nil { - panic("could not get keys: " + err.Error()) - } - for _, k := range keys { - //fmt.Println("deleting ", k) - if _, err := conn.Do("DEL", k); err != nil { - panic("could not del: " + err.Error()) - } - } -} - -func newPool(addr string) *redis.Pool { - return &redis.Pool{ - MaxActive: 20, - MaxIdle: 20, - IdleTimeout: 240 * time.Second, - Dial: func() (redis.Conn, error) { - c, err := redis.Dial("tcp", addr) - if err != nil { - return nil, err - } - return c, nil - //return redis.NewLoggingConn(c, log.New(os.Stdout, "", 0), "redis"), err - }, - Wait: true, - //TestOnBorrow: func(c redis.Conn, t time.Time) error { - // _, err := c.Do("PING") - // return err - //}, - } -} diff --git a/client.go b/client.go index cae61afe..3e614a37 100644 --- a/client.go +++ b/client.go @@ -2,7 +2,7 @@ package work import ( "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "sort" "strconv" "strings" diff --git a/client_test.go b/client_test.go index 88956e69..a9a8a6cb 100644 --- a/client_test.go +++ b/client_test.go @@ -2,7 +2,7 @@ package work import ( "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "testing" "time" diff --git a/cmd/workenqueue/main.go b/cmd/workenqueue/main.go index e281196c..9e5ac026 100644 --- a/cmd/workenqueue/main.go +++ b/cmd/workenqueue/main.go @@ -4,7 +4,7 @@ import ( "encoding/json" "flag" "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/gocraft/work" "os" "time" diff --git a/cmd/workfakedata/main.go b/cmd/workfakedata/main.go index 357fbf0d..0cb6e577 100644 --- a/cmd/workfakedata/main.go +++ b/cmd/workfakedata/main.go @@ -3,7 +3,7 @@ package main import ( "flag" "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/gocraft/work" "math/rand" "time" diff --git a/cmd/workwebui/main.go b/cmd/workwebui/main.go index a596f760..9bb21731 100644 --- a/cmd/workwebui/main.go +++ b/cmd/workwebui/main.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/gocraft/work/webui" ) diff --git a/dead_pool_reaper.go b/dead_pool_reaper.go index f8caa2e6..26f2e234 100644 --- a/dead_pool_reaper.go +++ b/dead_pool_reaper.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) const ( diff --git a/dead_pool_reaper_test.go b/dead_pool_reaper_test.go index d23ba4fb..d1f0b046 100644 --- a/dead_pool_reaper_test.go +++ b/dead_pool_reaper_test.go @@ -1,7 +1,7 @@ package work import ( - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "testing" "time" diff --git a/enqueue.go b/enqueue.go index 2b1d98a0..4ecd4148 100644 --- a/enqueue.go +++ b/enqueue.go @@ -1,7 +1,7 @@ package work import ( - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "sync" "time" ) diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..3398bf9d --- /dev/null +++ b/go.mod @@ -0,0 +1,26 @@ +module github.com/DispatchMe/go-work + +require ( + github.com/albrow/jobs v0.4.2 + github.com/bitly/go-simplejson v0.5.0 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/braintree/manners v0.0.0-20160418043613-82a8879fc5fd + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect + github.com/customerio/gospec v0.0.0-20130710230057-a5cc0e48aa39 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 + github.com/gocraft/web v0.0.0-20170925135945-d8611de039df + github.com/gocraft/work v0.5.1 + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/gomodule/redigo v2.0.0+incompatible + github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb + github.com/kr/pretty v0.1.0 // indirect + github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 + github.com/stretchr/testify v1.2.2 + github.com/youtube/vitess v2.1.1+incompatible // indirect + golang.org/x/net v0.0.0-20181107234226-1c5f79cfb164 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..721b1a29 --- /dev/null +++ b/go.sum @@ -0,0 +1,47 @@ +github.com/albrow/jobs v0.4.2 h1:AhhNNgtnOz3h+Grt6uuRJP+uj/AVq+ZhIBY8Mzkf4TM= +github.com/albrow/jobs v0.4.2/go.mod h1:e4sWh7D1DxPbpxrzJhNo/cMARAljpTYF/osgh2j3+r8= +github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/braintree/manners v0.0.0-20160418043613-82a8879fc5fd h1:ePesaBzdTmoMQjwqRCLP2jY+jjWMBpwws/LEQdt1fMM= +github.com/braintree/manners v0.0.0-20160418043613-82a8879fc5fd/go.mod h1:TNehV1AhBwtT7Bd+rh8G6MoGDbBLNs/sKdk3nvr4Yzg= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/customerio/gospec v0.0.0-20130710230057-a5cc0e48aa39 h1:O0YTztXI3XeJXlFhSo4wNb0VBVqSgT+hi/CjNWKvMnY= +github.com/customerio/gospec v0.0.0-20130710230057-a5cc0e48aa39/go.mod h1:OzYUFhPuL2JbjwFwrv6CZs23uBawekc6OZs+g19F0mY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 h1:pKjeDsx7HGGbjr7VGI1HksxDJqSjaGED3cSw9GeSI98= +github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0/go.mod h1:rWibcVfwbUxi/QXW84U7vNTcIcZFd6miwbt8ritxh/Y= +github.com/gocraft/web v0.0.0-20170925135945-d8611de039df h1:3gd2HoQ1d5bN0/U4aTmrB4hRKUnycYeYYBsNGY/A9fU= +github.com/gocraft/web v0.0.0-20170925135945-d8611de039df/go.mod h1:Ag7UMbZNGrnHwaXPJOUKJIVgx4QOWMOWZngrvsN6qak= +github.com/gocraft/work v0.5.1 h1:3bRjMiOo6N4zcRgZWV3Y7uX7R22SF+A9bPTk4xRXr34= +github.com/gocraft/work v0.5.1/go.mod h1:pc3n9Pb5FAESPPGfM0nL+7Q1xtgtRnF8rr/azzhQVlM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb h1:y9LFhCM3gwK94Xz9/h7GcSVLteky9pFHEkP04AqQupA= +github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb/go.mod h1:ziQRRNHCWZe0wVNzF8y8kCWpso0VMpqHJjB19DSenbE= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 h1:yOXfzNV7qkZ3nf2NPqy4BMzlCmnQzIEbI1vuqKb2FkQ= +github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701/go.mod h1:VtBIF1XX0c1nKkeAPk8i4aXkYopqQgfDqolHUIHPwNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 h1:x7xEyJDP7Hv3LVgvWhzioQqbC/KtuUhTigKlH/8ehhE= +github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/youtube/vitess v2.1.1+incompatible h1:iqObhq5grB5AR11E6ANuYyUHjI4ggQ/AlkUyontLQGw= +github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= +golang.org/x/net v0.0.0-20181107234226-1c5f79cfb164 h1:3/Nh+s1BnSj7XfWoKG7UhweBRwji2boAbiy293mqsHQ= +golang.org/x/net v0.0.0-20181107234226-1c5f79cfb164/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/heartbeater.go b/heartbeater.go index 14714e6a..55c214d4 100644 --- a/heartbeater.go +++ b/heartbeater.go @@ -2,7 +2,7 @@ package work import ( // "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "os" "sort" "strings" diff --git a/heartbeater_test.go b/heartbeater_test.go index dbc436b1..0247416a 100644 --- a/heartbeater_test.go +++ b/heartbeater_test.go @@ -2,7 +2,7 @@ package work import ( // "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "testing" "time" diff --git a/observer.go b/observer.go index e4f8d750..51f556e9 100644 --- a/observer.go +++ b/observer.go @@ -2,7 +2,7 @@ package work import ( "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "time" ) diff --git a/observer_test.go b/observer_test.go index 75be7894..633f481f 100644 --- a/observer_test.go +++ b/observer_test.go @@ -3,7 +3,7 @@ package work import ( "encoding/json" "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "testing" // "time" diff --git a/periodic_enqueuer.go b/periodic_enqueuer.go index 3f2d64f0..bc00df72 100644 --- a/periodic_enqueuer.go +++ b/periodic_enqueuer.go @@ -2,7 +2,7 @@ package work import ( "fmt" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/robfig/cron" "math/rand" "time" diff --git a/periodic_enqueuer_test.go b/periodic_enqueuer_test.go index c3e2f2d8..ac6ba67c 100644 --- a/periodic_enqueuer_test.go +++ b/periodic_enqueuer_test.go @@ -1,7 +1,7 @@ package work import ( - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/robfig/cron" "github.com/stretchr/testify/assert" "testing" diff --git a/requeuer.go b/requeuer.go index cd79de3f..26da9140 100644 --- a/requeuer.go +++ b/requeuer.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) type requeuer struct { diff --git a/requeuer_test.go b/requeuer_test.go index 3d1a7f40..7a7a8d13 100644 --- a/requeuer_test.go +++ b/requeuer_test.go @@ -1,7 +1,7 @@ package work import ( - // "github.com/garyburd/redigo/redis" + // "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" "testing" // "fmt" diff --git a/webui/webui.go b/webui/webui.go index 51ceb48e..8e24e52d 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -10,7 +10,7 @@ import ( "github.com/DispatchMe/go-work" "github.com/DispatchMe/go-work/webui/internal/assets" "github.com/braintree/manners" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/gocraft/web" ) diff --git a/webui/webui_test.go b/webui/webui_test.go index 3dda02dd..68884b59 100644 --- a/webui/webui_test.go +++ b/webui/webui_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/gocraft/work" "github.com/stretchr/testify/assert" ) diff --git a/worker.go b/worker.go index 839b63fd..6ccf2501 100644 --- a/worker.go +++ b/worker.go @@ -5,7 +5,7 @@ import ( "math/rand" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) type worker struct { diff --git a/worker_pool.go b/worker_pool.go index 051f2d95..f95ff18b 100644 --- a/worker_pool.go +++ b/worker_pool.go @@ -1,7 +1,7 @@ package work import ( - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/robfig/cron" "sort" "sync" diff --git a/worker_test.go b/worker_test.go index 42a00e27..e5cea8b9 100644 --- a/worker_test.go +++ b/worker_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" ) From 6662f02dd1fc9912c1027a398ee85a896eeaf38a Mon Sep 17 00:00:00 2001 From: John Refior Date: Wed, 7 Nov 2018 21:38:17 -0500 Subject: [PATCH 2/4] Remove benchmark dependencies --- go.mod | 14 -------------- go.sum | 31 ------------------------------- 2 files changed, 45 deletions(-) diff --git a/go.mod b/go.mod index 3398bf9d..a74e1998 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,12 @@ module github.com/DispatchMe/go-work require ( - github.com/albrow/jobs v0.4.2 - github.com/bitly/go-simplejson v0.5.0 // indirect - github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/braintree/manners v0.0.0-20160418043613-82a8879fc5fd - github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect - github.com/customerio/gospec v0.0.0-20130710230057-a5cc0e48aa39 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 github.com/gocraft/web v0.0.0-20170925135945-d8611de039df github.com/gocraft/work v0.5.1 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/gomodule/redigo v2.0.0+incompatible - github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb - github.com/kr/pretty v0.1.0 // indirect - github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 github.com/stretchr/testify v1.2.2 - github.com/youtube/vitess v2.1.1+incompatible // indirect - golang.org/x/net v0.0.0-20181107234226-1c5f79cfb164 // indirect ) diff --git a/go.sum b/go.sum index 721b1a29..ec406cb1 100644 --- a/go.sum +++ b/go.sum @@ -1,47 +1,16 @@ -github.com/albrow/jobs v0.4.2 h1:AhhNNgtnOz3h+Grt6uuRJP+uj/AVq+ZhIBY8Mzkf4TM= -github.com/albrow/jobs v0.4.2/go.mod h1:e4sWh7D1DxPbpxrzJhNo/cMARAljpTYF/osgh2j3+r8= -github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/braintree/manners v0.0.0-20160418043613-82a8879fc5fd h1:ePesaBzdTmoMQjwqRCLP2jY+jjWMBpwws/LEQdt1fMM= github.com/braintree/manners v0.0.0-20160418043613-82a8879fc5fd/go.mod h1:TNehV1AhBwtT7Bd+rh8G6MoGDbBLNs/sKdk3nvr4Yzg= -github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= -github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= -github.com/customerio/gospec v0.0.0-20130710230057-a5cc0e48aa39 h1:O0YTztXI3XeJXlFhSo4wNb0VBVqSgT+hi/CjNWKvMnY= -github.com/customerio/gospec v0.0.0-20130710230057-a5cc0e48aa39/go.mod h1:OzYUFhPuL2JbjwFwrv6CZs23uBawekc6OZs+g19F0mY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= -github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0 h1:pKjeDsx7HGGbjr7VGI1HksxDJqSjaGED3cSw9GeSI98= -github.com/gocraft/health v0.0.0-20170925182251-8675af27fef0/go.mod h1:rWibcVfwbUxi/QXW84U7vNTcIcZFd6miwbt8ritxh/Y= github.com/gocraft/web v0.0.0-20170925135945-d8611de039df h1:3gd2HoQ1d5bN0/U4aTmrB4hRKUnycYeYYBsNGY/A9fU= github.com/gocraft/web v0.0.0-20170925135945-d8611de039df/go.mod h1:Ag7UMbZNGrnHwaXPJOUKJIVgx4QOWMOWZngrvsN6qak= github.com/gocraft/work v0.5.1 h1:3bRjMiOo6N4zcRgZWV3Y7uX7R22SF+A9bPTk4xRXr34= github.com/gocraft/work v0.5.1/go.mod h1:pc3n9Pb5FAESPPGfM0nL+7Q1xtgtRnF8rr/azzhQVlM= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb h1:y9LFhCM3gwK94Xz9/h7GcSVLteky9pFHEkP04AqQupA= -github.com/jrallison/go-workers v0.0.0-20180112190529-dbf81d0b75bb/go.mod h1:ziQRRNHCWZe0wVNzF8y8kCWpso0VMpqHJjB19DSenbE= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701 h1:yOXfzNV7qkZ3nf2NPqy4BMzlCmnQzIEbI1vuqKb2FkQ= -github.com/orfjackal/nanospec.go v0.0.0-20120727230329-de4694c1d701/go.mod h1:VtBIF1XX0c1nKkeAPk8i4aXkYopqQgfDqolHUIHPwNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 h1:x7xEyJDP7Hv3LVgvWhzioQqbC/KtuUhTigKlH/8ehhE= github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/youtube/vitess v2.1.1+incompatible h1:iqObhq5grB5AR11E6ANuYyUHjI4ggQ/AlkUyontLQGw= -github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= -golang.org/x/net v0.0.0-20181107234226-1c5f79cfb164 h1:3/Nh+s1BnSj7XfWoKG7UhweBRwji2boAbiy293mqsHQ= -golang.org/x/net v0.0.0-20181107234226-1c5f79cfb164/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From e36f29766b87a4a0d4628ee466765cd24f396c7e Mon Sep 17 00:00:00 2001 From: John Refior Date: Thu, 8 Nov 2018 09:36:37 -0500 Subject: [PATCH 3/4] Update README. I also installed redis and verified that all tests pass. --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 46779058..d54d3ad2 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ DispatchMe/go-work lets you enqueue and processes background jobs in Go. Jobs ar * Web UI to manage failed jobs and observe the system. * Periodically enqueue jobs on a cron-like schedule. +## Run tests + +Redis must be installed to avoid a panic when running tests. + ## Enqueue new jobs To enqueue jobs, you need to make an Enqueuer with a redis namespace and a redigo pool. Each enqueued job has a name and can take optional arguments. Arguments are k/v pairs (serialized as JSON internally). @@ -292,7 +296,7 @@ You'll see a view that looks like this: ## Benchmarks -The benches folder contains various benchmark code. In each case, we enqueue 100k jobs across 5 queues. The jobs are almost no-op jobs: they simply increment an atomic counter. We then measure the rate of change of the counter to obtain our measurement. +The benches folder used to contain various benchmark code. In each case, we enqueued 100k jobs across 5 queues. The jobs were almost no-op jobs: they simply incremented an atomic counter. We then measured the rate of change of the counter to obtain our measurement. These were some test results: | Library | Speed | | --- | --- | @@ -301,6 +305,10 @@ The benches folder contains various benchmark code. In each case, we enqueue 100 | [benmanns/goworker](https://www.github.com/benmanns/goworker) | 10328.5 jobs/s | | [albrow/jobs](https://www.github.com/albrow/jobs) | 40 jobs/s | +The comparison benchmarks were run against repositories that were stale and unmaintained by fall of 2018. Invalid +import paths were causing tests to fail in go-lib, which has background and indexer packages that rely on this +repository. As the benchmarks were no longer needed, they were removed. + ## Authors * Jonathan Novak -- [https://github.com/cypriss](https://github.com/cypriss) From 29230b4ff50d68a2aace1ab768abf4a5ca2f1320 Mon Sep 17 00:00:00 2001 From: John Refior Date: Thu, 8 Nov 2018 09:58:47 -0500 Subject: [PATCH 4/4] Update readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d54d3ad2..ece15f73 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,11 @@ func ExportHandler(ctx *work.Context) error { Then in the web UI, you'll see the status of the worker: +``` | Name | Arguments | Started At | Check-in At | Check-in | | --- | --- | --- | --- | --- | | export | {"account_id": 123} | 2016/07/09 04:16:51 | 2016/07/09 05:03:13 | i=335000 | +``` ### Scheduled Jobs @@ -286,7 +288,7 @@ You'll see a view that looks like this: * "worker observation" - A snapshot made by an observer of what a worker is working on. * "periodic enqueuer" - A process that runs with a worker pool that periodically enqueues new jobs based on cron schedules. * "job" - the actual bundle of data that constitutes one job -* "job name" - each job has a name, like "create_watch" +* "job name" - each job has a name, like `create_watch` * "job type" - backend/private nomenclature for the handler+options for processing a job * "queue" - each job creates a queue with the same name as the job. only jobs named X go into the X queue. * "retry jobs" - If a job fails and needs to be retried, it will be put on this queue. @@ -307,7 +309,7 @@ The benches folder used to contain various benchmark code. In each case, we enqu The comparison benchmarks were run against repositories that were stale and unmaintained by fall of 2018. Invalid import paths were causing tests to fail in go-lib, which has background and indexer packages that rely on this -repository. As the benchmarks were no longer needed, they were removed. +repository. As the benchmarks were not currently needed, they were removed. ## Authors