<commit_before>package fetcheddit

import (
	"errors"
	"io"
	"net/http"
	//	"net/url"
)

const (
	DefaultFetch = HttpFetch("http://www.reddit.com")
)

/* A Fetcher that uses http. String represents the domain appended to
fetched paths. */
type HttpFetch string

/* Create a HTTP fetch object. "http://" is automatically prepended. 
TODO: Error handling? handling of http:// being provided? */
func CreateHttpFetch(domain string) HttpFetch {
	return HttpFetch("http://" + domain)
}

/* Fetches the path using an HTTP get request. The string value is used as the
domain (prepended to the path). The ReadCloser returns is to the body. An error
is returned if the page could not be retrieved, or if a non 200 status code is
returned. */
func (httpFetch HttpFetch) Fetch(relativePath string) (io.ReadCloser, error) {

	getUrl := string(httpFetch) + relativePath
	response, error := http.Get(getUrl)

	if error != nil {
		return nil, errors.New("Unable to retreive listing [" + getUrl + "]: " + error.Error())
	}

	if response.StatusCode != 200 {
		return nil, errors.New("Unable to retrieve listing [" + getUrl + "]: Expected response code 200")
	}

	return response.Body, nil
}<commit_msg>Removed unused function and include in HttpFetch<commit_after>package fetcheddit

import (
	"errors"
	"io"
	"net/http"
)

const (
	DefaultFetch = HttpFetch("http://www.reddit.com")
)

/* A Fetcher that uses http. String represents the domain appended to
fetched paths. */
type HttpFetch string

/* Fetches the path using an HTTP get request. The string value is used as the
domain (prepended to the path). The ReadCloser returns is to the body. An error
is returned if the page could not be retrieved, or if a non 200 status code is
returned. */
func (httpFetch HttpFetch) Fetch(relativePath string) (io.ReadCloser, error) {

	getUrl := string(httpFetch) + relativePath
	response, error := http.Get(getUrl)

	if error != nil {
		return nil, errors.New("Unable to retreive listing [" + getUrl + "]: " + error.Error())
	}

	if response.StatusCode != 200 {
		return nil, errors.New("Unable to retrieve listing [" + getUrl + "]: Expected response code 200")
	}

	return response.Body, nil
}
{'fraction_non_alphanumeric': 0.11054813450023031, 'fraction_numerical': 0.008291110087517274, 'mean_word_length': 4.062937062937063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7113925', 'n_tokens_mistral': 695, 'n_tokens_neox': 656, 'n_words': 257}