List endpoints with repository transaction

cmd/ghsync/main.go

@@ -17,6 +17,7 @@ var (
 var app = cli.New("ghsync", version, build, "GitHub metadata sync")
 
 func main() {
+	app.AddCommand(&subcmd.ShallowCommand{})
 	app.AddCommand(&subcmd.SyncCommand{})
 	app.AddCommand(&subcmd.MigrateCommand{})
 

cmd/ghsync/subcmd/common.go

@@ -1,13 +1,24 @@
 package subcmd
 
 import (
+	"context"
+	"database/sql"
 	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
 
 	"github.com/src-d/ghsync/models/migrations"
+	"github.com/src-d/ghsync/utils"
+	"gopkg.in/src-d/go-log.v1"
 
 	"github.com/golang-migrate/migrate/v4"
 	_ "github.com/golang-migrate/migrate/v4/database/postgres"
 	bindata "github.com/golang-migrate/migrate/v4/source/go_bindata"
+	"github.com/google/go-github/github"
+	"github.com/gregjones/httpcache"
+	"github.com/gregjones/httpcache/diskcache"
+	"golang.org/x/oauth2"
 )
 
 const maxVersion uint = 1560510971
@@ -25,6 +36,45 @@ func (o PostgresOpt) URL() string {
 		o.User, o.Password, o.Host, o.Port, o.DB)
 }
 
+func (o PostgresOpt) initDB() (db *sql.DB, err error) {
+	db, err = sql.Open("postgres", o.URL())
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			db.Close()
+			db = nil
+		}
+	}()
+
+	if err = db.Ping(); err != nil {
+		return db, err
+	}
+
+	m, err := newMigrate(o.URL())
+	if err != nil {
+		return db, err
+	}
+
+	dbVersion, _, err := m.Version()
+
+	if err != nil && err != migrate.ErrNilVersion {
+		return db, err
+	}
+
+	if dbVersion != maxVersion {
+		return db, fmt.Errorf(
+			"database version mismatch. Current version is %v, but this binary needs version %v. "+
+				"Use the 'migrate' subcommand to upgrade your database", dbVersion, maxVersion)
+	}
+
+	log.With(log.Fields{"db-version": dbVersion}).Debugf("the DB version is up to date")
+	log.Infof("connection with the DB established")
+	return db, nil
+}
+
 func newMigrate(url string) (*migrate.Migrate, error) {
 	// wrap assets into Resource
 	s := bindata.Resource(migrations.AssetNames(),
@@ -38,3 +88,32 @@ func newMigrate(url string) (*migrate.Migrate, error) {
 	}
 	return migrate.NewWithSourceInstance("go-bindata", d, url)
 }
+
+func newClient(token string) (*github.Client, error) {
+	http := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: token},
+	))
+
+	dirPath := filepath.Join(os.TempDir(), "ghsync")
+	err := os.MkdirAll(dirPath, os.ModePerm)
+	if err != nil {
+		return nil, fmt.Errorf("error while creating directory %s: %v", dirPath, err)
+	}
+
+	t := httpcache.NewTransport(diskcache.New(dirPath))
+	t.Transport = &RemoveHeaderTransport{utils.NewRateLimitTransport(http.Transport)}
+	http.Transport = t
+
+	return github.NewClient(http), nil
+}
+
+type RemoveHeaderTransport struct {
+	T http.RoundTripper
+}
+
+func (t *RemoveHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Del("X-Ratelimit-Limit")
+	req.Header.Del("X-Ratelimit-Remaining")
+	req.Header.Del("X-Ratelimit-Reset")
+	return t.T.RoundTrip(req)
+}

cmd/ghsync/subcmd/shallow.go

@@ -0,0 +1,32 @@
+package subcmd
+
+import (
+	"github.com/src-d/ghsync/shallow"
+
+	"gopkg.in/src-d/go-cli.v0"
+)
+
+type ShallowCommand struct {
+	cli.Command `name:"shallow" short-description:"Shallow sync of GitHub data" long-description:"Shallow sync of GitHub data"`
+
+	Token string `long:"token" env:"GHSYNC_TOKEN" description:"GitHub personal access token" required:"true"`
+	Org   string `long:"org" env:"GHSYNC_ORG" description:"Name of the GitHub organization" required:"true"`
+
+	Postgres PostgresOpt `group:"PostgreSQL connection options"`
+}
+
+func (c *ShallowCommand) Execute(args []string) error {
+	db, err := c.Postgres.initDB()
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	client, err := newClient(c.Token)
+	if err != nil {
+		return err
+	}
+
+	orgSyncer := shallow.NewOrganizationSyncer(db, client)
+	return orgSyncer.Sync(c.Org)
+}

cmd/ghsync/subcmd/sync.go

@@ -1,19 +1,8 @@
 package subcmd
 
 import (
-	"context"
-	"database/sql"
-	"fmt"
-	"net/http"
+	"github.com/src-d/ghsync/deep"
 
-	"github.com/src-d/ghsync"
-	"github.com/src-d/ghsync/utils"
-
-	"github.com/golang-migrate/migrate/v4"
-	"github.com/google/go-github/github"
-	"github.com/gregjones/httpcache"
-	"github.com/gregjones/httpcache/diskcache"
-	"golang.org/x/oauth2"
 	"gopkg.in/src-d/go-cli.v0"
 	"gopkg.in/src-d/go-log.v1"
 	"gopkg.in/src-d/go-queue.v1"
@@ -22,7 +11,7 @@ import (
 )
 
 type SyncCommand struct {
-	cli.Command `name:"sync"`
+	cli.Command `name:"deep" short-description:"Deep sync of GitHub data" long-description:"Deep sync of GitHub data"`
 
 	Token string `long:"token" env:"GHSYNC_TOKEN" description:"GitHub personal access token" required:"true"`
 	Org   string `long:"org" env:"GHSYNC_ORG" description:"Name of the GitHub organization" required:"true"`
@@ -36,21 +25,16 @@ type SyncCommand struct {
 }
 
 func (c *SyncCommand) Execute(args []string) error {
-	db, err := c.initDB()
+	db, err := c.Postgres.initDB()
 	if err != nil {
 		return err
 	}
 	defer db.Close()
 
-	http := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: c.Token},
-	))
-
-	t := httpcache.NewTransport(diskcache.New("cache/" + c.Org))
-	t.Transport = &RemoveHeaderTransport{utils.NewRateLimitTransport(http.Transport)}
-	http.Transport = t
-
-	client := github.NewClient(http)
+	client, err := newClient(c.Token)
+	if err != nil {
+		return err
+	}
 
 	broker, err := queue.NewBroker(c.QueueOpt.Broker)
 	if err != nil {
@@ -66,7 +50,7 @@ func (c *SyncCommand) Execute(args []string) error {
 		return err
 	}
 
-	syncer := ghsync.NewSyncer(db, client, queue)
+	syncer := deep.NewSyncer(db, client, queue)
 
 	go func() {
 		err := syncer.DoOrganization(c.Org)
@@ -77,53 +61,3 @@ func (c *SyncCommand) Execute(args []string) error {
 
 	return syncer.Wait()
 }
-
-func (c *SyncCommand) initDB() (db *sql.DB, err error) {
-	db, err = sql.Open("postgres", c.Postgres.URL())
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if err != nil {
-			db.Close()
-			db = nil
-		}
-	}()
-
-	if err = db.Ping(); err != nil {
-		return db, err
-	}
-
-	m, err := newMigrate(c.Postgres.URL())
-	if err != nil {
-		return db, err
-	}
-
-	dbVersion, _, err := m.Version()
-
-	if err != nil && err != migrate.ErrNilVersion {
-		return db, err
-	}
-
-	if dbVersion != maxVersion {
-		return db, fmt.Errorf(
-			"database version mismatch. Current version is %v, but this binary needs version %v. "+
-				"Use the 'migrate' subcommand to upgrade your database", dbVersion, maxVersion)
-	}
-
-	log.With(log.Fields{"db-version": dbVersion}).Debugf("the DB version is up to date")
-	log.Infof("connection with the DB established")
-	return db, nil
-}
-
-type RemoveHeaderTransport struct {
-	T http.RoundTripper
-}
-
-func (t *RemoveHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.Header.Del("X-Ratelimit-Limit")
-	req.Header.Del("X-Ratelimit-Remaining")
-	req.Header.Del("X-Ratelimit-Reset")
-	return t.T.RoundTrip(req)
-}

common.go → deep/common.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"strings"

issue.go → deep/issue.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

issue_comment.go → deep/issue_comment.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

organization.go → deep/organization.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

pull_request.go → deep/pull_request.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

pull_request_comment.go → deep/pull_request_comment.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

pull_request_review.go → deep/pull_request_review.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

repository.go → deep/repository.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

syncer.go → deep/syncer.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"database/sql"

user.go → deep/user.go

@@ -1,4 +1,4 @@
-package ghsync
+package deep
 
 import (
 	"context"

shallow/common.go

@@ -0,0 +1,3 @@
+package shallow
+
+const listOptionsPerPage = 100