<commit_before>package gothub

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

// The HTTP host that we will hit to use the GitHub API.
const (
	GitHubUrl           string = "https://api.github.com"
)

var (
	ErrRateLimitReached        = errors.New("Rate limit reached")
)	

// The GitHub struct represents an active session to the GitHub API.
type GitHub struct {
	httpClient         *http.Client
	Authorization      string
	RateLimit          int
	RateLimitRemaining int
}

func hashAuth(u, p string) string {
	var a = fmt.Sprintf("%s:%s", u, p)
	return base64.StdEncoding.EncodeToString([]byte(a))
}

// Log in to GitHub using basic, username/password authentication.
func BasicLogin(username, password string) (*GitHub, error) {
	// Format and Base64-encode the provided username and password, in preparation for basic
	// HTTP auth.
	authorization := fmt.Sprintf("Basic %s", hashAuth(username, password))
	request, err := http.NewRequest("GET", GitHubUrl, nil)
	if err != nil {
		return nil, err
	}

	// Set the Authorization header.
	request.Header.Set("Authorization", authorization)

	// Create a new HTTP client (which we will eventually provide to the GitHub struct), for
	// issuing the above HTTP request, and future requests.
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == http.StatusOK {
		// Yaaaaaaay!
		ratelimit, err := strconv.Atoi(response.Header.Get("X-RateLimit-Limit"))
		if err != nil {
			return nil, err
		}
		
		remaining, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining"))
		if err != nil {
			return nil, err
		}

		return &GitHub{httpClient: client, Authorization: authorization,
			RateLimit: ratelimit, RateLimitRemaining: remaining}, nil
	}

	// Should we get here, the basic authentication request failed.
	e := "Authorization failed with HTTP code: %d"
	return nil, errors.New(fmt.Sprintf(e, response.StatusCode))
}

func (g *GitHub) callGithubApi(method, uri string, rs interface{}) error {
	if g.RateLimitRemaining == 0 {
		return ErrRateLimitReached
	}

	url := fmt.Sprintf("%s%s", GitHubUrl, uri)
	request, err := http.NewRequest(method, url, nil)
	if err != nil {
		return err
	}

	request.Header.Set("Authorization", g.Authorization)

	// Fire off the request.
	response, err := g.httpClient.Do(request)
	if err != nil {
		return err
	}

	// Update the rate limits
	limit, err := strconv.Atoi(response.Header.Get("X-RateLimit-Limit"))
	if err != nil {
		return err
	}
	g.RateLimit = limit

	remaining, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining"))
	if err != nil {
		return err
	}
	g.RateLimitRemaining = remaining

	// Now, marshal the HTTP response (if it was successful) onto the
	// struct provided by `rs`
	if response.StatusCode != http.StatusOK {
		e := "GitHub API responded with HTTP %d"
		err := errors.New(fmt.Sprintf(e, response.StatusCode))
		return err
	}

	// Check to make sure we actually got JSON back.
	switch response.Header.Get("Content-Type") {
	case "application/json":
		fallthrough
	case "application/json; charset=utf-8":
		var js []byte
		js, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		} else {
			err = json.Unmarshal(js, rs)
		}
	}

	return err
}<commit_msg>Added a new error gothub.ErrNoJSON<commit_after>package gothub

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

// The HTTP host that we will hit to use the GitHub API.
const (
	GitHubUrl           string = "https://api.github.com"
)

var (
	ErrRateLimitReached        = errors.New("Rate limit reached")
	ErrNoJSON = errors.New("GitHub did not return a JSON response")
)	

// The GitHub struct represents an active session to the GitHub API.
type GitHub struct {
	httpClient         *http.Client
	Authorization      string
	RateLimit          int
	RateLimitRemaining int
}

func hashAuth(u, p string) string {
	var a = fmt.Sprintf("%s:%s", u, p)
	return base64.StdEncoding.EncodeToString([]byte(a))
}

// Log in to GitHub using basic, username/password authentication.
func BasicLogin(username, password string) (*GitHub, error) {
	// Format and Base64-encode the provided username and password, in preparation for basic
	// HTTP auth.
	authorization := fmt.Sprintf("Basic %s", hashAuth(username, password))
	request, err := http.NewRequest("GET", GitHubUrl, nil)
	if err != nil {
		return nil, err
	}

	// Set the Authorization header.
	request.Header.Set("Authorization", authorization)

	// Create a new HTTP client (which we will eventually provide to the GitHub struct), for
	// issuing the above HTTP request, and future requests.
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == http.StatusOK {
		// Yaaaaaaay!
		ratelimit, err := strconv.Atoi(response.Header.Get("X-RateLimit-Limit"))
		if err != nil {
			return nil, err
		}
		
		remaining, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining"))
		if err != nil {
			return nil, err
		}

		return &GitHub{httpClient: client, Authorization: authorization,
			RateLimit: ratelimit, RateLimitRemaining: remaining}, nil
	}

	// Should we get here, the basic authentication request failed.
	e := "Authorization failed with HTTP code: %d"
	return nil, errors.New(fmt.Sprintf(e, response.StatusCode))
}

func (g *GitHub) callGithubApi(method, uri string, rs interface{}) error {
	if g.RateLimitRemaining == 0 {
		return ErrRateLimitReached
	}

	url := fmt.Sprintf("%s%s", GitHubUrl, uri)
	request, err := http.NewRequest(method, url, nil)
	if err != nil {
		return err
	}

	request.Header.Set("Authorization", g.Authorization)

	// Fire off the request.
	response, err := g.httpClient.Do(request)
	if err != nil {
		return err
	}

	// Update the rate limits
	limit, err := strconv.Atoi(response.Header.Get("X-RateLimit-Limit"))
	if err != nil {
		return err
	}
	g.RateLimit = limit

	remaining, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining"))
	if err != nil {
		return err
	}
	g.RateLimitRemaining = remaining

	// Now, marshal the HTTP response (if it was successful) onto the
	// struct provided by `rs`
	if response.StatusCode != http.StatusOK {
		e := "GitHub API responded with HTTP %d"
		err := errors.New(fmt.Sprintf(e, response.StatusCode))
		return err
	}

	// Check to make sure we actually got JSON back.
	switch response.Header.Get("Content-Type") {
	case "application/json":
		fallthrough
	case "application/json; charset=utf-8":
		var js []byte
		js, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		} else {
			err = json.Unmarshal(js, rs)
		}
	default:
		err = ErrNoJSON
	}

	return err
}
{'fraction_non_alphanumeric': 0.12311557788944724, 'fraction_numerical': 0.0023647650014779783, 'mean_word_length': 3.6285909712722297, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6284813', 'n_tokens_mistral': 2321, 'n_tokens_neox': 2191, 'n_words': 677}