<commit_before>// Package httpsource provides a HTTP source Image Server.
package httpsource

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"

	"github.com/pierrre/imageserver"
)

var contentTypeRegexp = regexp.MustCompile("^image/(.+)$")

// Server is a HTTP source Image Server.
type Server struct{}

// Get returns an Image for a HTTP source.
//
// If the source is not an url, the string representation of the source will be used to create one.
//
// Returns an error if the HTTP status code is not 200 (OK).
//
// The image type is determined by the "Content-Type" header.
func (server *Server) Get(params imageserver.Params) (*imageserver.Image, error) {
	sourceURL, err := getSourceURL(params)
	if err != nil {
		return nil, err
	}
	response, err := doRequest(sourceURL)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	image, err := parseResponse(response)
	if err != nil {
		return nil, err
	}
	return image, nil
}

func getSourceURL(params imageserver.Params) (*url.URL, error) {
	source, err := params.GetString(imageserver.SourceParam)
	if err != nil {
		return nil, err
	}
	sourceURL, err := url.ParseRequestURI(source)
	if err != nil {
		return nil, &imageserver.ParamError{
			Param:   imageserver.SourceParam,
			Message: fmt.Sprintf("parse url error: %s", err),
		}
	}
	if sourceURL.Scheme != "http" && sourceURL.Scheme != "https" {
		return nil, &imageserver.ParamError{
			Param:   imageserver.SourceParam,
			Message: "url scheme must be http(s)",
		}
	}
	return sourceURL, nil
}

func doRequest(sourceURL *url.URL) (*http.Response, error) {
	//TODO optional http client
	response, err := http.Get(sourceURL.String())
	if err != nil {
		return nil, &imageserver.ParamError{Param: imageserver.SourceParam, Message: err.Error()}
	}
	return response, nil
}

func parseResponse(response *http.Response) (*imageserver.Image, error) {
	if response.StatusCode != http.StatusOK {
		return nil, &imageserver.ParamError{
			Param:   imageserver.SourceParam,
			Message: fmt.Sprintf("http status code %d while downloading", response.StatusCode),
		}
	}
	im := new(imageserver.Image)
	contentType := response.Header.Get("Content-Type")
	if contentType != "" {
		matches := contentTypeRegexp.FindStringSubmatch(contentType)
		if matches != nil && len(matches) == 2 {
			im.Format = matches[1]
		}<commit_msg>httpsource.Server: add optional http.Client<commit_after>// Package httpsource provides a HTTP source Image Server.
package httpsource

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"

	"github.com/pierrre/imageserver"
)

var contentTypeRegexp = regexp.MustCompile("^image/(.+)$")

// Server is a HTTP source Image Server.
//
// It parses the "source" param as URL, then do a GET request.
// It returns an error if the HTTP status code is not 200 (OK).
//
// The image type is determined by the "Content-Type" header.
type Server struct {
	// Client is an optional HTTP client.
	// http.DefaultClient is used by default.
	Client *http.Client
}

// Get implements Server.
func (srv *Server) Get(params imageserver.Params) (*imageserver.Image, error) {
	sourceURL, err := getSourceURL(params)
	if err != nil {
		return nil, err
	}
	response, err := srv.doRequest(sourceURL)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	image, err := parseResponse(response)
	if err != nil {
		return nil, err
	}
	return image, nil
}

func getSourceURL(params imageserver.Params) (*url.URL, error) {
	source, err := params.GetString(imageserver.SourceParam)
	if err != nil {
		return nil, err
	}
	sourceURL, err := url.ParseRequestURI(source)
	if err != nil {
		return nil, &imageserver.ParamError{
			Param:   imageserver.SourceParam,
			Message: fmt.Sprintf("parse url error: %s", err),
		}
	}
	if sourceURL.Scheme != "http" && sourceURL.Scheme != "https" {
		return nil, &imageserver.ParamError{
			Param:   imageserver.SourceParam,
			Message: "url scheme must be http(s)",
		}
	}
	return sourceURL, nil
}

func (srv *Server) doRequest(sourceURL *url.URL) (*http.Response, error) {
	c := srv.Client
	if c == nil {
		c = http.DefaultClient
	}
	response, err := c.Get(sourceURL.String())
	if err != nil {
		return nil, &imageserver.ParamError{Param: imageserver.SourceParam, Message: err.Error()}
	}
	return response, nil
}

func parseResponse(response *http.Response) (*imageserver.Image, error) {
	if response.StatusCode != http.StatusOK {
		return nil, &imageserver.ParamError{
			Param:   imageserver.SourceParam,
			Message: fmt.Sprintf("http status code %d while downloading", response.StatusCode),
		}
	}
	im := new(imageserver.Image)
	contentType := response.Header.Get("Content-Type")
	if contentType != "" {
		matches := contentTypeRegexp.FindStringSubmatch(contentType)
		if matches != nil && len(matches) == 2 {
			im.Format = matches[1]
		}
{'fraction_non_alphanumeric': 0.13174273858921162, 'fraction_numerical': 0.002074688796680498, 'mean_word_length': 4.006230529595015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2609483', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1569, 'n_words': 466}