diff --git a/steward/backend/mysql/mysql_storage.go b/steward/backend/mysql/mysql_storage.go index 11e3e42..bbcf7ab 100644 --- a/steward/backend/mysql/mysql_storage.go +++ b/steward/backend/mysql/mysql_storage.go @@ -2,7 +2,6 @@ package mysql import ( "database/sql" - "time" _ "github.com/go-sql-driver/mysql" "github.com/localhots/steward/steward" @@ -10,26 +9,20 @@ import ( type ( MysqlStorage struct { - db *sql.DB - State map[string]*steward.State - - importStmt *sql.Stmt - saveStateStmt *sql.Stmt - loadStateStmt *sql.Stmt + db *sql.DB + importStmt *sql.Stmt } ) const ( - importQuery = "replace into commits (sha1, author, repo, ts) values (?, ?, ?, ?)" - saveStateQuery = "replace into state (repo, sha1, ts) values (?, ?, ?)" - loadStateQuery = "select repo, sha1, ts from state" + importQuery = "" + + "replace into contributions (author, repo, week, commits, additions, deletions) " + + "values (?, ?, ?, ?, ?, ?)" ) func New(host, user, pass, db string) *MysqlStorage { var ( - s = &MysqlStorage{ - State: map[string]*steward.State{}, - } + s = &MysqlStorage{} err error databaseURI = makeDatabaseURI(host, user, pass, db) ) @@ -40,68 +33,22 @@ func New(host, user, pass, db string) *MysqlStorage { if s.importStmt, err = s.db.Prepare(importQuery); err != nil { panic(err) } - if s.saveStateStmt, err = s.db.Prepare(saveStateQuery); err != nil { - panic(err) - } - - s.loadGlobalState() return s } -func (ms *MysqlStorage) Import(repo string, hist map[string]*steward.Commit) { - var ( - lastTimestamp *time.Time - lastSha1 string - ) - - // fmt.Println("saving", len(hist), "commits") - // pretty.Println(hist) - - for sha1, c := range hist { - if _, err := ms.importStmt.Exec(sha1, c.Author, repo, c.Timestamp); err != nil { +func (ms *MysqlStorage) ImportContributions(contrib []*steward.Contribution) { + for _, c := range contrib { + if _, err := ms.importStmt.Exec( + c.Author, + c.Repo, + c.Week, + c.Commits, + c.Additions, + c.Deletions, + ); err != nil { panic(err) } - if lastTimestamp == nil || lastTimestamp.After(c.Timestamp) { - lastTimestamp = &c.Timestamp - lastSha1 = sha1 - } - } - - if len(hist) > 0 { - ms.saveRepoState(repo, lastSha1, *lastTimestamp) - } -} - -func (ms *MysqlStorage) saveRepoState(repo string, sha1 string, ts time.Time) { - ms.State[repo] = &steward.State{ - Sha1: sha1, - Timestamp: ts, - } - if _, err := ms.saveStateStmt.Exec(repo, sha1, ts); err != nil { - panic(err) - } -} - -func (ms *MysqlStorage) loadGlobalState() { - var ( - repo string - sha1 string - ts time.Time - ) - - rows, err := ms.db.Query(loadStateQuery) - if err != nil { - panic(err) - } - for rows.Next() { - if err := rows.Scan(&repo, &sha1, &ts); err != nil { - panic(err) - } - ms.State[repo] = &steward.State{ - Sha1: sha1, - Timestamp: ts, - } } } diff --git a/steward/commit.go b/steward/commit.go deleted file mode 100644 index a99f0da..0000000 --- a/steward/commit.go +++ /dev/null @@ -1,14 +0,0 @@ -package steward - -import ( - "time" -) - -type ( - Commit struct { - Repo string - Sha1 string - Author string - Timestamp time.Time - } -) diff --git a/steward/github/github.go b/steward/github/github.go index 1b76b25..2f9be4c 100644 --- a/steward/github/github.go +++ b/steward/github/github.go @@ -6,7 +6,6 @@ import ( "code.google.com/p/goauth2/oauth" gh "github.com/google/go-github/github" - "github.com/kr/pretty" "github.com/localhots/steward/steward" ) @@ -16,8 +15,11 @@ const ( type ( GithubClient struct { - owner string - client *gh.Client + owner string + client *gh.Client + limit int + remaining int + limitEnds time.Time } ) @@ -46,9 +48,7 @@ func (c *GithubClient) ListRepos() []string { } ) - fmt.Print("Loading repositories ") for { - fmt.Print(".") opt.Page++ repos, _, err := c.client.Repositories.ListByOrg(c.owner, opt) if err != nil { @@ -63,71 +63,50 @@ func (c *GithubClient) ListRepos() []string { break } } - fmt.Print("\n") return names } -func (c *GithubClient) ListCommits(repo string, until *time.Time) (hist map[string]*steward.Commit, hasMore bool) { - hist = map[string]*steward.Commit{} +func (c *GithubClient) ListContributors(repo string) []*steward.Contribution { + var ( + contrib = []*steward.Contribution{} + ) - opt := &gh.CommitsListOptions{} - if until != nil { - opt.Until = *until - } - - commits, _, err := c.client.Repositories.ListCommits(c.owner, repo, opt) + cslist, resp, err := c.client.Repositories.ListContributorsStats(c.owner, repo) + c.saveResponseMeta(resp) if err != nil { - fmt.Println("Error fetching commits: ", err.Error()) - return - } - - // fmt.Println("Fetched", len(commits), "commits until", opt.Until) - hasMore = (len(commits) == DefaultPerPage) - for _, c := range commits { - commit, err := makeCommit(&c) - if err != nil { - fmt.Println("Error:", err.Error()) - continue + if err.Error() == "EOF" { + // Empty repository, not an actual error + return contrib } - hist[commit.Sha1] = commit + + fmt.Println("Error loading contributors stats for repo", repo) + fmt.Println(err.Error()) + return contrib } - return + for _, cs := range cslist { + for _, week := range cs.Weeks { + if *week.Commits == 0 { + continue + } + + contrib = append(contrib, &steward.Contribution{ + Author: *cs.Author.Login, + Repo: repo, + Week: week.Week.Time.Unix(), + Commits: *week.Commits, + Additions: *week.Additions, + Deletions: *week.Deletions, + }) + } + } + + return contrib } -func makeCommit(c *gh.RepositoryCommit) (commit *steward.Commit, err error) { - defer func() { - if err := recover(); err != nil { - fmt.Print("\n\n\nTroubles with commit:") - pretty.Println(c) - fmt.Println("") - panic(err) - } - }() - - commit = &steward.Commit{} - if c.SHA != nil { - commit.Sha1 = *c.SHA - } else { - return nil, fmt.Errorf("Missing commit SHA1 field") - } - - if c.Author != nil { - commit.Author = *c.Author.Login - } else { - return nil, fmt.Errorf("Missing author field") - } - - if c.Commit != nil { - if c.Commit.Author != nil { - commit.Timestamp = *c.Commit.Author.Date - } else { - return nil, fmt.Errorf("Missing commit author field") - } - } else { - return nil, fmt.Errorf("Missing commit field") - } - - return +func (c *GithubClient) saveResponseMeta(res *gh.Response) { + c.limit = res.Limit + c.remaining = res.Remaining + c.limitEnds = res.Reset.Time } diff --git a/steward/state.go b/steward/state.go deleted file mode 100644 index 21b16f8..0000000 --- a/steward/state.go +++ /dev/null @@ -1,12 +0,0 @@ -package steward - -import ( - "time" -) - -type ( - State struct { - Sha1 string - Timestamp time.Time - } -)