<commit_before>package image

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/cenkalti/backoff"
)

type APISelector struct {
	baseURL *url.URL
}

func NewAPISelector(u *url.URL) *APISelector {
	return &APISelector{baseURL: u}
}

func (as *APISelector) Select(params *Params) (string, error) {
	imageName := "default"

	fullTagSet := []string{}

	if params.Language != "" {
		fullTagSet = append(fullTagSet, "language:"+params.Language)
	}

	if params.Dist != "" {
		fullTagSet = append(fullTagSet, "dist:"+params.Dist)
	}

	if params.Group != "" {
		fullTagSet = append(fullTagSet, "group:"+params.Group)
	}

	if params.OS != "" {
		fullTagSet = append(fullTagSet, "os:"+params.OS)
	}

	tagAttempts := [][]string{}

	if params.OS == "osx" && params.OsxImage != "" {
		fullTagSet = append(fullTagSet, "osx_image:"+params.OsxImage)
		tagAttempts = append(tagAttempts,
			[]string{"osx_image:" + params.OsxImage, "language:" + params.Language},
			[]string{"osx_image:" + params.OsxImage})
	}

	tagAttempts = append(tagAttempts,
		fullTagSet,
		[]string{"language:" + params.Language},
		[]string{"dist:" + params.Dist, "language:" + params.Language},
		[]string{"dist:" + params.Dist},
		[]string{"group:" + params.Group, "language:" + params.Language},
		[]string{"group:" + params.Group},
		[]string{"os:" + params.OsxImage, "language:" + params.Language},
		[]string{"os:" + params.OsxImage})

	for _, tags := range tagAttempts {
		u, _ := url.Parse(as.baseURL.String())
		qs := url.Values{}
		qs.Set("infra", params.Infra)
		qs.Set("limit", "1")
		qs.Set("tags", strings.Join(tags, ","))

		u.RawQuery = qs.Encode()

		var body []byte

		b := backoff.NewExponentialBackOff()
		b.MaxInterval = 10 * time.Second
		b.MaxElapsedTime = time.Minute

		err := backoff.Retry(func() (err error) {
			resp, err := http.Get(u.String())
			if err != nil {<commit_msg>Query for images via API with multiple language tags<commit_after>package image

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/cenkalti/backoff"
)

type APISelector struct {
	baseURL *url.URL
}

func NewAPISelector(u *url.URL) *APISelector {
	return &APISelector{baseURL: u}
}

func (as *APISelector) Select(params *Params) (string, error) {
	imageName := "default"

	fullTagSet := []string{}

	if params.Language != "" {
		fullTagSet = append(fullTagSet, "language:"+params.Language)
	}

	if params.Dist != "" {
		fullTagSet = append(fullTagSet, "dist:"+params.Dist)
	}

	if params.Group != "" {
		fullTagSet = append(fullTagSet, "group:"+params.Group)
	}

	if params.OS != "" {
		fullTagSet = append(fullTagSet, "os:"+params.OS)
	}

	tagAttempts := [][]string{}

	if params.OS == "osx" && params.OsxImage != "" {
		fullTagSet = append(fullTagSet, "osx_image:"+params.OsxImage)
		tagAttempts = append(tagAttempts,
			[]string{"osx_image:" + params.OsxImage, "language:" + params.Language},
			[]string{"osx_image:" + params.OsxImage})
	}

	tagAttempts = append(tagAttempts,
		fullTagSet,
		[]string{"language:" + params.Language},
		[]string{"language_" + params.Language + ":true"},
		[]string{"dist:" + params.Dist, "language:" + params.Language},
		[]string{"dist:" + params.Dist, "language_" + params.Language + ":true"},
		[]string{"dist:" + params.Dist},
		[]string{"group:" + params.Group, "language:" + params.Language},
		[]string{"group:" + params.Group, "language_" + params.Language + ":true"},
		[]string{"group:" + params.Group},
		[]string{"os:" + params.OsxImage, "language:" + params.Language},
		[]string{"os:" + params.OsxImage, "language_" + params.Language + ":true"},
		[]string{"os:" + params.OsxImage})

	for _, tags := range tagAttempts {
		u, _ := url.Parse(as.baseURL.String())
		qs := url.Values{}
		qs.Set("infra", params.Infra)
		qs.Set("limit", "1")
		qs.Set("tags", strings.Join(tags, ","))

		u.RawQuery = qs.Encode()

		var body []byte

		b := backoff.NewExponentialBackOff()
		b.MaxInterval = 10 * time.Second
		b.MaxElapsedTime = time.Minute

		err := backoff.Retry(func() (err error) {
			resp, err := http.Get(u.String())
			if err != nil {
{'fraction_non_alphanumeric': 0.1931268151016457, 'fraction_numerical': 0.001452081316553727, 'mean_word_length': 4.44532279314888, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2270695', 'n_tokens_mistral': 1627, 'n_tokens_neox': 1508, 'n_words': 284}