diff --git a/steward/backend/mysql/mysql_storage.go b/steward/backend/mysql/mysql_storage.go index 855f154..11e3e42 100644 --- a/steward/backend/mysql/mysql_storage.go +++ b/steward/backend/mysql/mysql_storage.go @@ -11,7 +11,7 @@ import ( type ( MysqlStorage struct { db *sql.DB - state map[string]*steward.State + State map[string]*steward.State importStmt *sql.Stmt saveStateStmt *sql.Stmt @@ -28,7 +28,7 @@ const ( func New(host, user, pass, db string) *MysqlStorage { var ( s = &MysqlStorage{ - state: map[string]*steward.State{}, + State: map[string]*steward.State{}, } err error databaseURI = makeDatabaseURI(host, user, pass, db) @@ -55,6 +55,9 @@ func (ms *MysqlStorage) Import(repo string, hist map[string]*steward.Commit) { lastSha1 string ) + // fmt.Println("saving", len(hist), "commits") + // pretty.Println(hist) + for sha1, c := range hist { if _, err := ms.importStmt.Exec(sha1, c.Author, repo, c.Timestamp); err != nil { panic(err) @@ -65,10 +68,16 @@ func (ms *MysqlStorage) Import(repo string, hist map[string]*steward.Commit) { } } - ms.saveRepoState(repo, lastSha1, *lastTimestamp) + if len(hist) > 0 { + ms.saveRepoState(repo, lastSha1, *lastTimestamp) + } } func (ms *MysqlStorage) saveRepoState(repo string, sha1 string, ts time.Time) { + ms.State[repo] = &steward.State{ + Sha1: sha1, + Timestamp: ts, + } if _, err := ms.saveStateStmt.Exec(repo, sha1, ts); err != nil { panic(err) } @@ -89,7 +98,7 @@ func (ms *MysqlStorage) loadGlobalState() { if err := rows.Scan(&repo, &sha1, &ts); err != nil { panic(err) } - ms.state[repo] = &steward.State{ + ms.State[repo] = &steward.State{ Sha1: sha1, Timestamp: ts, } diff --git a/steward/github/github.go b/steward/github/github.go index 6bd56ca..1b76b25 100644 --- a/steward/github/github.go +++ b/steward/github/github.go @@ -2,14 +2,16 @@ package github import ( "fmt" + "time" "code.google.com/p/goauth2/oauth" gh "github.com/google/go-github/github" + "github.com/kr/pretty" "github.com/localhots/steward/steward" ) const ( - DEFAULT_PER_PAGE = 30 + DefaultPerPage = 30 ) type ( @@ -57,7 +59,7 @@ func (c *GithubClient) ListRepos() []string { names = append(names, *repo.Name) } - if len(repos) < DEFAULT_PER_PAGE { + if len(repos) < DefaultPerPage { break } } @@ -66,34 +68,66 @@ func (c *GithubClient) ListRepos() []string { return names } -func (c *GithubClient) ListCommits(repo string) map[string]*steward.Commit { - var ( - history = map[string]*steward.Commit{} - opt = &gh.CommitsListOptions{} - ) +func (c *GithubClient) ListCommits(repo string, until *time.Time) (hist map[string]*steward.Commit, hasMore bool) { + hist = map[string]*steward.Commit{} - fmt.Print(repo, " ") - for { - fmt.Print(".") - commits, _, err := c.client.Repositories.ListCommits(c.owner, repo, opt) + opt := &gh.CommitsListOptions{} + if until != nil { + opt.Until = *until + } + + commits, _, err := c.client.Repositories.ListCommits(c.owner, repo, opt) + if err != nil { + fmt.Println("Error fetching commits: ", err.Error()) + return + } + + // fmt.Println("Fetched", len(commits), "commits until", opt.Until) + hasMore = (len(commits) == DefaultPerPage) + for _, c := range commits { + commit, err := makeCommit(&c) if err != nil { + fmt.Println("Error:", err.Error()) + continue + } + hist[commit.Sha1] = commit + } + + return +} + +func makeCommit(c *gh.RepositoryCommit) (commit *steward.Commit, err error) { + defer func() { + if err := recover(); err != nil { + fmt.Print("\n\n\nTroubles with commit:") + pretty.Println(c) + fmt.Println("") panic(err) } + }() - for _, c := range commits { - history[*c.SHA] = &steward.Commit{ - Repo: repo, - Author: *c.Author.Login, - Timestamp: *c.Commit.Author.Date, - } - opt.Until = *c.Commit.Author.Date - } - - if len(commits) < DEFAULT_PER_PAGE { - break - } + commit = &steward.Commit{} + if c.SHA != nil { + commit.Sha1 = *c.SHA + } else { + return nil, fmt.Errorf("Missing commit SHA1 field") } - fmt.Print("\n") - return history + if c.Author != nil { + commit.Author = *c.Author.Login + } else { + return nil, fmt.Errorf("Missing author field") + } + + if c.Commit != nil { + if c.Commit.Author != nil { + commit.Timestamp = *c.Commit.Author.Date + } else { + return nil, fmt.Errorf("Missing commit author field") + } + } else { + return nil, fmt.Errorf("Missing commit field") + } + + return }