Use contribution statistics endpoint

steward/backend/mysql/mysql_storage.go

@@ -2,7 +2,6 @@ package mysql
 
 import (
 	"database/sql"
-	"time"
 
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/localhots/steward/steward"
@@ -10,26 +9,20 @@ import (
 
 type (
 	MysqlStorage struct {
-		db    *sql.DB
+		db         *sql.DB
-		State map[string]*steward.State
+		importStmt *sql.Stmt
-
-		importStmt    *sql.Stmt
-		saveStateStmt *sql.Stmt
-		loadStateStmt *sql.Stmt
 	}
 )
 
 const (
-	importQuery    = "replace into commits (sha1, author, repo, ts) values (?, ?, ?, ?)"
+	importQuery = "" +
-	saveStateQuery = "replace into state (repo, sha1, ts) values (?, ?, ?)"
+		"replace into contributions (author, repo, week, commits, additions, deletions) " +
-	loadStateQuery = "select repo, sha1, ts from state"
+		"values (?, ?, ?, ?, ?, ?)"
 )
 
 func New(host, user, pass, db string) *MysqlStorage {
 	var (
-		s = &MysqlStorage{
+		s           = &MysqlStorage{}
-			State: map[string]*steward.State{},
-		}
 		err         error
 		databaseURI = makeDatabaseURI(host, user, pass, db)
 	)
@@ -40,68 +33,22 @@ func New(host, user, pass, db string) *MysqlStorage {
 	if s.importStmt, err = s.db.Prepare(importQuery); err != nil {
 		panic(err)
 	}
-	if s.saveStateStmt, err = s.db.Prepare(saveStateQuery); err != nil {
-		panic(err)
-	}
-
-	s.loadGlobalState()
 
 	return s
 }
 
-func (ms *MysqlStorage) Import(repo string, hist map[string]*steward.Commit) {
+func (ms *MysqlStorage) ImportContributions(contrib []*steward.Contribution) {
-	var (
+	for _, c := range contrib {
-		lastTimestamp *time.Time
+		if _, err := ms.importStmt.Exec(
-		lastSha1      string
+			c.Author,
-	)
+			c.Repo,
-
+			c.Week,
-	// fmt.Println("saving", len(hist), "commits")
+			c.Commits,
-	// pretty.Println(hist)
+			c.Additions,
-
+			c.Deletions,
-	for sha1, c := range hist {
+		); err != nil {
-		if _, err := ms.importStmt.Exec(sha1, c.Author, repo, c.Timestamp); err != nil {
 			panic(err)
 		}
-		if lastTimestamp == nil || lastTimestamp.After(c.Timestamp) {
-			lastTimestamp = &c.Timestamp
-			lastSha1 = sha1
-		}
-	}
-
-	if len(hist) > 0 {
-		ms.saveRepoState(repo, lastSha1, *lastTimestamp)
-	}
-}
-
-func (ms *MysqlStorage) saveRepoState(repo string, sha1 string, ts time.Time) {
-	ms.State[repo] = &steward.State{
-		Sha1:      sha1,
-		Timestamp: ts,
-	}
-	if _, err := ms.saveStateStmt.Exec(repo, sha1, ts); err != nil {
-		panic(err)
-	}
-}
-
-func (ms *MysqlStorage) loadGlobalState() {
-	var (
-		repo string
-		sha1 string
-		ts   time.Time
-	)
-
-	rows, err := ms.db.Query(loadStateQuery)
-	if err != nil {
-		panic(err)
-	}
-	for rows.Next() {
-		if err := rows.Scan(&repo, &sha1, &ts); err != nil {
-			panic(err)
-		}
-		ms.State[repo] = &steward.State{
-			Sha1:      sha1,
-			Timestamp: ts,
-		}
 	}
 }
 

steward/commit.go

@@ -1,14 +0,0 @@
-package steward
-
-import (
-	"time"
-)
-
-type (
-	Commit struct {
-		Repo      string
-		Sha1      string
-		Author    string
-		Timestamp time.Time
-	}
-)

steward/github/github.go

@@ -6,7 +6,6 @@ import (
 
 	"code.google.com/p/goauth2/oauth"
 	gh "github.com/google/go-github/github"
-	"github.com/kr/pretty"
 	"github.com/localhots/steward/steward"
 )
 
@@ -16,8 +15,11 @@ const (
 
 type (
 	GithubClient struct {
-		owner  string
+		owner     string
-		client *gh.Client
+		client    *gh.Client
+		limit     int
+		remaining int
+		limitEnds time.Time
 	}
 )
 
@@ -46,9 +48,7 @@ func (c *GithubClient) ListRepos() []string {
 		}
 	)
 
-	fmt.Print("Loading repositories ")
 	for {
-		fmt.Print(".")
 		opt.Page++
 		repos, _, err := c.client.Repositories.ListByOrg(c.owner, opt)
 		if err != nil {
@@ -63,71 +63,50 @@ func (c *GithubClient) ListRepos() []string {
 			break
 		}
 	}
-	fmt.Print("\n")
 
 	return names
 }
 
-func (c *GithubClient) ListCommits(repo string, until *time.Time) (hist map[string]*steward.Commit, hasMore bool) {
+func (c *GithubClient) ListContributors(repo string) []*steward.Contribution {
-	hist = map[string]*steward.Commit{}
+	var (
+		contrib = []*steward.Contribution{}
+	)
 
-	opt := &gh.CommitsListOptions{}
+	cslist, resp, err := c.client.Repositories.ListContributorsStats(c.owner, repo)
-	if until != nil {
+	c.saveResponseMeta(resp)
-		opt.Until = *until
-	}
-
-	commits, _, err := c.client.Repositories.ListCommits(c.owner, repo, opt)
 	if err != nil {
-		fmt.Println("Error fetching commits: ", err.Error())
+		if err.Error() == "EOF" {
-		return
+			// Empty repository, not an actual error
-	}
+			return contrib
-
-	// fmt.Println("Fetched", len(commits), "commits until", opt.Until)
-	hasMore = (len(commits) == DefaultPerPage)
-	for _, c := range commits {
-		commit, err := makeCommit(&c)
-		if err != nil {
-			fmt.Println("Error:", err.Error())
-			continue
 		}
-		hist[commit.Sha1] = commit
+
+		fmt.Println("Error loading contributors stats for repo", repo)
+		fmt.Println(err.Error())
+		return contrib
 	}
 
-	return
+	for _, cs := range cslist {
+		for _, week := range cs.Weeks {
+			if *week.Commits == 0 {
+				continue
+			}
+
+			contrib = append(contrib, &steward.Contribution{
+				Author:    *cs.Author.Login,
+				Repo:      repo,
+				Week:      week.Week.Time.Unix(),
+				Commits:   *week.Commits,
+				Additions: *week.Additions,
+				Deletions: *week.Deletions,
+			})
+		}
+	}
+
+	return contrib
 }
 
-func makeCommit(c *gh.RepositoryCommit) (commit *steward.Commit, err error) {
+func (c *GithubClient) saveResponseMeta(res *gh.Response) {
-	defer func() {
+	c.limit = res.Limit
-		if err := recover(); err != nil {
+	c.remaining = res.Remaining
-			fmt.Print("\n\n\nTroubles with commit:")
+	c.limitEnds = res.Reset.Time
-			pretty.Println(c)
-			fmt.Println("")
-			panic(err)
-		}
-	}()
-
-	commit = &steward.Commit{}
-	if c.SHA != nil {
-		commit.Sha1 = *c.SHA
-	} else {
-		return nil, fmt.Errorf("Missing commit SHA1 field")
-	}
-
-	if c.Author != nil {
-		commit.Author = *c.Author.Login
-	} else {
-		return nil, fmt.Errorf("Missing author field")
-	}
-
-	if c.Commit != nil {
-		if c.Commit.Author != nil {
-			commit.Timestamp = *c.Commit.Author.Date
-		} else {
-			return nil, fmt.Errorf("Missing commit author field")
-		}
-	} else {
-		return nil, fmt.Errorf("Missing commit field")
-	}
-
-	return
 }

steward/state.go

@@ -1,12 +0,0 @@
-package steward
-
-import (
-	"time"
-)
-
-type (
-	State struct {
-		Sha1      string
-		Timestamp time.Time
-	}
-)