diff --git a/common/interfaces.go b/common/interfaces.go index 3f58e68c..bd6cafe5 100644 --- a/common/interfaces.go +++ b/common/interfaces.go @@ -3,5 +3,6 @@ package common type IClient interface { GetUserOrOrganization(login string) (*Owner, error) GetRepositoriesFromOwner(target Owner) ([]*Repository, error) + GetRepositoriesFromOrganization(target Owner) ([]*Repository, error) GetOrganizationMembers(target Owner) ([]*Owner, error) } diff --git a/core/analysis.go b/core/analysis.go index f45b48f9..1607fefb 100644 --- a/core/analysis.go +++ b/core/analysis.go @@ -64,12 +64,18 @@ func GatherRepositories(sess *Session) { for i := 0; i < threadNum; i++ { go func() { for { + var repos []*common.Repository + var err error target, ok := <-ch if !ok { wg.Done() return } - repos, err := sess.Client.GetRepositoriesFromOwner(*target) + if *target.Type == "Organization" { + repos, err = sess.Client.GetRepositoriesFromOrganization(*target) + } else { + repos, err = sess.Client.GetRepositoriesFromOwner(*target) + } if err != nil { sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) } @@ -216,12 +222,14 @@ func findSecrets(sess *Session, repo *common.Repository, commit *object.Commit, func cloneRepository(sess *Session, repo *common.Repository, threadId int) (*git.Repository, string, error) { sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", threadId, *repo.CloneURL) + userName := "oauth2" + cloneConfig := common.CloneConfiguration{ Url: repo.CloneURL, Branch: repo.DefaultBranch, Depth: sess.Options.CommitDepth, - Token: &sess.GitLab.AccessToken, InMemClone: sess.Options.InMemClone, + Username: &userName, } var clone *git.Repository @@ -229,10 +237,10 @@ func cloneRepository(sess *Session, repo *common.Repository, threadId int) (*git var err error if sess.IsGithubSession { + cloneConfig.Token = &sess.Github.AccessToken clone, path, err = github.CloneRepository(&cloneConfig) } else { - userName := "oauth2" - cloneConfig.Username = &userName + cloneConfig.Token = &sess.GitLab.AccessToken clone, path, err = gitlab.CloneRepository(&cloneConfig) } if err != nil { diff --git a/github/apiClient.go b/github/apiClient.go index 46d1fbb2..11bb2bab 100644 --- a/github/apiClient.go +++ b/github/apiClient.go @@ -81,6 +81,43 @@ func (c Client) GetRepositoriesFromOwner(target common.Owner) ([]*common.Reposit return allRepos, nil } +func (c Client) GetRepositoriesFromOrganization(target common.Owner) ([]*common.Repository, error) { + var allRepos []*common.Repository + ctx := context.Background() + opt := &github.RepositoryListByOrgOptions{ + Type: "sources", + } + + for { + repos, resp, err := c.apiClient.Repositories.ListByOrg(ctx, *target.Login, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + r := common.Repository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.SSHURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + allRepos = append(allRepos, &r) + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + return allRepos, nil +} + func (c Client) GetOrganizationMembers(target common.Owner) ([]*common.Owner, error) { var allMembers []*common.Owner ctx := context.Background() diff --git a/github/git.go b/github/git.go index 04b13586..578aa6e7 100644 --- a/github/git.go +++ b/github/git.go @@ -2,6 +2,7 @@ package github import ( "fmt" + "strings" "gopkg.in/src-d/go-git.v4/storage/memory" "io/ioutil" @@ -9,6 +10,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) func CloneRepository(cloneConfig *common.CloneConfiguration) (*git.Repository, string, error) { @@ -19,8 +21,15 @@ func CloneRepository(cloneConfig *common.CloneConfiguration) (*git.Repository, s ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", *cloneConfig.Branch)), SingleBranch: true, Tags: git.NoTags, + Auth: &http.BasicAuth{ + Username: *cloneConfig.Username, + Password: *cloneConfig.Token, + }, } + cloneOptions.URL = strings.Replace(cloneOptions.URL, "git@github.com:", "https://github.com/", -1) + + var repository *git.Repository var err error var dir string diff --git a/gitlab/apiClient.go b/gitlab/apiClient.go index 9497ed3b..c15470a4 100644 --- a/gitlab/apiClient.go +++ b/gitlab/apiClient.go @@ -112,6 +112,29 @@ func (c Client) GetRepositoriesFromOwner(target common.Owner) ([]*common.Reposit return allProjects, nil } +func (c Client) GetRepositoriesFromOrganization(target common.Owner) ([]*common.Repository, error) { + var allProjects []*common.Repository + id := int(*target.ID) + if *target.Type == common.TargetTypeUser { + userProjects, err := c.getUserProjects(id) + if err != nil { + return nil, err + } + for _, project := range userProjects { + allProjects = append(allProjects, project) + } + } else { + groupProjects, err := c.getGroupProjects(target) + if err != nil { + return nil, err + } + for _, project := range groupProjects { + allProjects = append(allProjects, project) + } + } + return allProjects, nil +} + func (c Client) getUser(login string) (*gitlab.User, error) { users, _, err := c.apiClient.Users.ListUsers(&gitlab.ListUsersOptions{Username: gitlab.String(login)}) if err != nil {