<commit_before>package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// store is a Repository that talks to the juju charm server (in ../store).
type store struct {
	baseURL   string
	cachePath string
}

const (
	storeURL  = "https://store.juju.ubuntu.com"
	cachePath = "$HOME/.juju/cache"
)

// Store returns a Repository that provides access to the juju charm store.
func Store() Repository {
	return &store{storeURL, os.ExpandEnv(cachePath)}
}

// info returns the revision and SHA256 digest of the charm referenced by curl.
func (s *store) info(curl *URL) (rev int, digest string, err error) {
	key := curl.String()
	resp, err := http.Get(s.baseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return
	}
	info, found := infos[key]
	if !found {
		err = fmt.Errorf("charm: charm store returned response without charm %q", key)
		return
	}
	for _, w := range info.Warnings {
		log.Printf("charm: WARNING: charm store reports for %q: %s", key, w)
	}
	if info.Errors != nil {
		err = fmt.Errorf(
			"charm info errors for %q: %s", key, strings.Join(info.Errors, "; "),
		)
		return
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *store) Latest(curl *URL) (int, error) {
	rev, _, err := s.info(curl.WithRevision(-1))
	return rev, err
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	h := sha256.New()
	h.Write(b)
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}<commit_msg>charm: stream SHA256 of charm<commit_after>package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// store is a Repository that talks to the juju charm server (in ../store).
type store struct {
	baseURL   string
	cachePath string
}

const (
	storeURL  = "https://store.juju.ubuntu.com"
	cachePath = "$HOME/.juju/cache"
)

// Store returns a Repository that provides access to the juju charm store.
func Store() Repository {
	return &store{storeURL, os.ExpandEnv(cachePath)}
}

// info returns the revision and SHA256 digest of the charm referenced by curl.
func (s *store) info(curl *URL) (rev int, digest string, err error) {
	key := curl.String()
	resp, err := http.Get(s.baseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return
	}
	info, found := infos[key]
	if !found {
		err = fmt.Errorf("charm: charm store returned response without charm %q", key)
		return
	}
	for _, w := range info.Warnings {
		log.Printf("charm: WARNING: charm store reports for %q: %s", key, w)
	}
	if info.Errors != nil {
		err = fmt.Errorf(
			"charm info errors for %q: %s", key, strings.Join(info.Errors, "; "),
		)
		return
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *store) Latest(curl *URL) (int, error) {
	rev, _, err := s.info(curl.WithRevision(-1))
	return rev, err
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}
{'fraction_non_alphanumeric': 0.1384498190820796, 'fraction_numerical': 0.010093315558941154, 'mean_word_length': 3.723021582733813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2676938', 'n_tokens_mistral': 1936, 'n_tokens_neox': 1835, 'n_words': 585}