<commit_before>package data

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
)

const (
	// DiscoveryQueryParam is the query parameter that is appended to URLs to signal the request is looking for
	// repository information.
	DiscoveryQueryParam = "spread-get=1"

	// DiscoveryMetaName is the the 'name' of the <meta> tag that contains Spread package information.
	DiscoveryMetaName = "spread-ref"
)

// httpClient is a copy of DefaultClient for testing purposes.
var httpClient = http.DefaultClient

// packageInfo contains the data retrieved in the discovery process.
type packageInfo struct {
	// prefix is the package contained in the repo. It should be an exact match or prefix to the requested package name.
	prefix string
	// repoURL is the location of the repository where package data is stored.
	repoURL string
}

// DiscoverPackage uses the package name to fetch a Spread URL to the repository. Set insecure when HTTP is allowed.
// Verbose will print information to STDOUT.
func DiscoverPackage(packageName string, insecure, verbose bool) (packageInfo, error) {
	// first try HTTPS
	urlStr, res, err := fetch("https", packageName, verbose)
	if err != nil || res.StatusCode != 200 {
		if verbose {
			if err != nil {
				fmt.Fprint(Out, "https fetch failed")
			} else {
				fmt.Fprintf(Out, "ignoring https fetch with status code %d", res.StatusCode)
			}
		}
		// fallback to HTTP if insecure is allowed
		if insecure {
			urlStr, res, err = fetch("http", packageName, verbose)
		}
	}
	if err != nil {
		return packageInfo{}, err
	}

	// close body when done
	if res != nil {
		defer res.Body.Close()
	}

	if verbose {
		fmt.Fprintf(Out, "Parsing meta information from '%s' (status code %d)", urlStr, res.StatusCode)<commit_msg>close body of failed HTTPS discovery requests<commit_after>package data

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
)

const (
	// DiscoveryQueryParam is the query parameter that is appended to URLs to signal the request is looking for
	// repository information.
	DiscoveryQueryParam = "spread-get=1"

	// DiscoveryMetaName is the the 'name' of the <meta> tag that contains Spread package information.
	DiscoveryMetaName = "spread-ref"
)

// httpClient is a copy of DefaultClient for testing purposes.
var httpClient = http.DefaultClient

// packageInfo contains the data retrieved in the discovery process.
type packageInfo struct {
	// prefix is the package contained in the repo. It should be an exact match or prefix to the requested package name.
	prefix string
	// repoURL is the location of the repository where package data is stored.
	repoURL string
}

// DiscoverPackage uses the package name to fetch a Spread URL to the repository. Set insecure when HTTP is allowed.
// Verbose will print information to STDOUT.
func DiscoverPackage(packageName string, insecure, verbose bool) (packageInfo, error) {
	// first try HTTPS
	urlStr, res, err := fetch("https", packageName, verbose)
	if err != nil || res.StatusCode != 200 {
		if verbose {
			if err != nil {
				fmt.Fprint(Out, "https fetch failed")
			} else {
				fmt.Fprintf(Out, "ignoring https fetch with status code %d", res.StatusCode)
			}
		}
		// fallback to HTTP if insecure is allowed
		if insecure {
			if res != nil {
				res.Body.Close()
			}
			urlStr, res, err = fetch("http", packageName, verbose)
		}
	}
	if err != nil {
		return packageInfo{}, err
	}

	// close body when done
	if res != nil {
		defer res.Body.Close()
	}

	if verbose {
		fmt.Fprintf(Out, "Parsing meta information from '%s' (status code %d)", urlStr, res.StatusCode)
{'fraction_non_alphanumeric': 0.0937066814527308, 'fraction_numerical': 0.0022179096201829776, 'mean_word_length': 3.6138107416879794, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1640797', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1052, 'n_words': 443}