<commit_before>package gddoexp

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

// httpClient contains all used methods to perform requests in Github API.
// This is useful for mocking and building tests.
type httpClient interface {
	Get(string) (*http.Response, error)
}

// HTTPClient is going to be used to perform the Github HTTP requests. We use
// a global variable, as it is safe for concurrent use by multiple goroutines
var HTTPClient httpClient

// IsCacheResponse detects if a HTTP response was retrieved from cache or
// not.
var IsCacheResponse func(*http.Response) bool

func init() {
	HTTPClient = new(http.Client)
}

// GithubAuth store the authentication information to allow a less
// restrictive rate limit in Github API. Authenticated requests can make up
// to 5000 requests per hour, otherwise you will be limited in 60 requests
// per hour (https://developer.github.com/v3/#rate-limiting).
type GithubAuth struct {
	ID     string
	Secret string
}

// String build the Github authentication in the request query string format.
func (g GithubAuth) String() string {
	return fmt.Sprintf("client_id=%s&client_secret=%s", g.ID, g.Secret)
}

// githubRepository stores the information of a repository. For more
// information check: https://developer.github.com/v3/repos/#get
type githubRepository struct {
	CreatedAt       time.Time `json:"created_at"`
	Fork            bool      `json:"fork"`
	ForksCount      int       `json:"forks_count"`
	NetworkCount    int       `json:"network_count"`
	StargazersCount int       `json:"stargazers_count"`
	UpdatedAt       time.Time `json:"updated_at"`
}

// getGithubRepository will retrieve the path project information. For a
// better rate limit the requests must be authenticated, for more information
// check: https://developer.github.com/v3/search/#rate-limit. This function
// also returns if the response was retrieved from a local cache.
func getGithubRepository(path string, auth *GithubAuth) (repository githubRepository, cache bool, err error) {
	if !strings.HasPrefix(path, "github.com/") {
		return repository, false, NewError(path, ErrorCodeNonGithub, nil)
	}

	normalizedPath := strings.TrimPrefix(path, "github.com/")
	normalizedPath = strings.Join(strings.Split(normalizedPath, "/")[:2], "/")

	url := "https://api.github.com/repos/" + normalizedPath
	if auth != nil {
		url += "?" + auth.String()
	}

	cache, err = doGithub(path, url, &repository)
	return repository, cache, err
}

// githubCommits stores the information of all commits from a repository. For more
// information check:
// https://developer.github.com/v3/repos/commits/#list-commits-on-a-repository.
// Note that there's a difference between the author and the committer:
// http://stackoverflow.com/questions/6755824/what-is-the-difference-between-author-and-committer-in-git
type githubCommits []struct {
	Commit struct {
		Author struct {
			Date time.Time `json:"date"`
		} `json:"author"`
	} `json:"commit"`
}

// getCommits will retrieve the commits from a Github repository. For a
// better rate limit the requests must be authenticated, for more information
// check: https://developer.github.com/v3/search/#rate-limit. This function
// also returns if the response was retrieved from a local cache.
func getCommits(path string, auth *GithubAuth) (commits githubCommits, cache bool, err error) {
	if !strings.HasPrefix(path, "github.com/") {
		return commits, false, NewError(path, ErrorCodeNonGithub, nil)
	}

	normalizedPath := strings.TrimPrefix(path, "github.com/")
	normalizedPath = strings.Join(strings.Split(normalizedPath, "/")[:2], "/")

	url := fmt.Sprintf("https://api.github.com/repos/%s/commits", normalizedPath)
	if auth != nil {
		url += "?" + auth.String()
	}

	cache, err = doGithub(path, url, &commits)
	return commits, cache, err
}

// doGithub is the low level function that do actually the work of querying
// Github API and parsing the response. It is also responsible for verifying if
// the response was a cache hit or not.
func doGithub(path, url string, obj interface{}) (cache bool, err error) {
	rsp, err := HTTPClient.Get(url)
	if err != nil {
		return false, NewError(path, ErrorCodeGithubFetch, err)
	}
	defer func() {
		if rsp.Body != nil {
			// for now we aren't checking the error
			rsp.Body.Close()
		}
	}()
<commit_msg>Avoid duplicated code<commit_after>package gddoexp

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

// httpClient contains all used methods to perform requests in Github API.
// This is useful for mocking and building tests.
type httpClient interface {
	Get(string) (*http.Response, error)
}

// HTTPClient is going to be used to perform the Github HTTP requests. We use
// a global variable, as it is safe for concurrent use by multiple goroutines
var HTTPClient httpClient

// IsCacheResponse detects if a HTTP response was retrieved from cache or
// not.
var IsCacheResponse func(*http.Response) bool

func init() {
	HTTPClient = new(http.Client)
}

// GithubAuth store the authentication information to allow a less
// restrictive rate limit in Github API. Authenticated requests can make up
// to 5000 requests per hour, otherwise you will be limited in 60 requests
// per hour (https://developer.github.com/v3/#rate-limiting).
type GithubAuth struct {
	ID     string
	Secret string
}

// String build the Github authentication in the request query string format.
func (g GithubAuth) String() string {
	return fmt.Sprintf("client_id=%s&client_secret=%s", g.ID, g.Secret)
}

// githubRepository stores the information of a repository. For more
// information check: https://developer.github.com/v3/repos/#get
type githubRepository struct {
	CreatedAt       time.Time `json:"created_at"`
	Fork            bool      `json:"fork"`
	ForksCount      int       `json:"forks_count"`
	NetworkCount    int       `json:"network_count"`
	StargazersCount int       `json:"stargazers_count"`
	UpdatedAt       time.Time `json:"updated_at"`
}

// getGithubRepository will retrieve the path project information. For a
// better rate limit the requests must be authenticated, for more information
// check: https://developer.github.com/v3/search/#rate-limit. This function
// also returns if the response was retrieved from a local cache.
func getGithubRepository(path string, auth *GithubAuth) (repository githubRepository, cache bool, err error) {
	normalizedPath, err := normalizePath(path)
	if err != nil {
		return repository, false, err
	}

	url := "https://api.github.com/repos/" + normalizedPath
	if auth != nil {
		url += "?" + auth.String()
	}

	cache, err = doGithub(path, url, &repository)
	return repository, cache, err
}

// githubCommits stores the information of all commits from a repository. For more
// information check:
// https://developer.github.com/v3/repos/commits/#list-commits-on-a-repository.
// Note that there's a difference between the author and the committer:
// http://stackoverflow.com/questions/6755824/what-is-the-difference-between-author-and-committer-in-git
type githubCommits []struct {
	Commit struct {
		Author struct {
			Date time.Time `json:"date"`
		} `json:"author"`
	} `json:"commit"`
}

// getCommits will retrieve the commits from a Github repository. For a
// better rate limit the requests must be authenticated, for more information
// check: https://developer.github.com/v3/search/#rate-limit. This function
// also returns if the response was retrieved from a local cache.
func getCommits(path string, auth *GithubAuth) (commits githubCommits, cache bool, err error) {
	normalizedPath, err := normalizePath(path)
	if err != nil {
		return commits, false, err
	}

	url := fmt.Sprintf("https://api.github.com/repos/%s/commits", normalizedPath)
	if auth != nil {
		url += "?" + auth.String()
	}

	cache, err = doGithub(path, url, &commits)
	return commits, cache, err
}

// normalizePath identify if the path is from Github and normalize it for the
// Github API request.
func normalizePath(path string) (string, error) {
	if !strings.HasPrefix(path, "github.com/") {
		return "", NewError(path, ErrorCodeNonGithub, nil)
	}

	normalizedPath := strings.TrimPrefix(path, "github.com/")
	normalizedPath = strings.Join(strings.Split(normalizedPath, "/")[:2], "/")
	return normalizedPath, nil
}

// doGithub is the low level function that do actually the work of querying
// Github API and parsing the response. It is also responsible for verifying if
// the response was a cache hit or not.
func doGithub(path, url string, obj interface{}) (cache bool, err error) {
	rsp, err := HTTPClient.Get(url)
	if err != nil {
		return false, NewError(path, ErrorCodeGithubFetch, err)
	}
	defer func() {
		if rsp.Body != nil {
			// for now we aren't checking the error
			rsp.Body.Close()
		}
	}()

{'fraction_non_alphanumeric': 0.11297309621523027, 'fraction_numerical': 0.004445964432284542, 'mean_word_length': 3.9677236693091733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3242930', 'n_tokens_mistral': 2725, 'n_tokens_neox': 2581, 'n_words': 917}