<commit_before>package pinterest

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/url"
	"strings"
)

var (
	// callback can't be empty
	endPoint           = "http://api.pinterest.com/v1/urls/count.json?callback=cb&url="
	errInvalidResponse = errors.New("invalid JSON response")
)

type LinkStats struct {
	Normalized string
	Count      int
}

func GetLinkStats(u string) (*LinkStats, error) {
	if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
		u = "http://" + u
	}
	resp, err := Client.Get(endPoint + url.QueryEscape(u))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	// Pinterest will return the results wrapped for a callback. Unfortunately, it
	// seems there's no way to remove that.
	d, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if len(d) < 2 {
		return nil, errInvalidResponse
	}
	d = d[3 : len(d)-1]
	var res map[string]interface{}
	if err := json.Unmarshal(d, &res); err != nil {
		return nil, err
	}
	normalized, ok := res["url"].(string)
	if !ok {
		return nil, errInvalidResponse
	}
	count, ok := res["count"].(float64)
	if !ok {
		return nil, errInvalidResponse
	}
	return &LinkStats{
		Normalized: normalized,
		Count:      int(count),
	}, nil
}

type result struct {
	url   string
	stats *LinkStats
	err   error
}

func GetLinksStats(urls []string) (map[string]*LinkStats, error) {
	count := len(urls)
	ch := make(chan *result, count)
	for _, v := range urls {
		go func(u string) {
			stats, err := GetLinkStats(u)
			ch <- &result{
				url:   u,
				stats: stats,
				err:   err,
			}
		}(v)
	}
	results := make(map[string]*LinkStats, count)
	for ii := 0; ii < len(urls); ii++ {
		res := <-ch
		if res.err != nil {
			return nil, res.err
		}
		results[res.url] = res.stats
	}
	close(ch)
	return results, nil
}<commit_msg>If there's an error, return the results we managed to fetch<commit_after>package pinterest

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/url"
	"strings"
)

var (
	// callback can't be empty
	endPoint           = "http://api.pinterest.com/v1/urls/count.json?callback=cb&url="
	errInvalidResponse = errors.New("invalid JSON response")
)

type LinkStats struct {
	Normalized string
	Count      int
}

func GetLinkStats(u string) (*LinkStats, error) {
	if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
		u = "http://" + u
	}
	resp, err := Client.Get(endPoint + url.QueryEscape(u))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	// Pinterest will return the results wrapped for a callback. Unfortunately, it
	// seems there's no way to remove that.
	d, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if len(d) < 2 {
		return nil, errInvalidResponse
	}
	d = d[3 : len(d)-1]
	var res map[string]interface{}
	if err := json.Unmarshal(d, &res); err != nil {
		return nil, err
	}
	normalized, ok := res["url"].(string)
	if !ok {
		return nil, errInvalidResponse
	}
	count, ok := res["count"].(float64)
	if !ok {
		return nil, errInvalidResponse
	}
	return &LinkStats{
		Normalized: normalized,
		Count:      int(count),
	}, nil
}

type result struct {
	url   string
	stats *LinkStats
	err   error
}

func GetLinksStats(urls []string) (map[string]*LinkStats, error) {
	count := len(urls)
	ch := make(chan *result, count)
	for _, v := range urls {
		go func(u string) {
			stats, err := GetLinkStats(u)
			ch <- &result{
				url:   u,
				stats: stats,
				err:   err,
			}
		}(v)
	}
	results := make(map[string]*LinkStats, count)
	var err error
	for ii := 0; ii < len(urls); ii++ {
		res := <-ch
		if res.err != nil {
			if err == nil {
				err = res.err
			}
			continue
		}
		results[res.url] = res.stats
	}
	close(ch)
	return results, err
}
{'fraction_non_alphanumeric': 0.15343347639484978, 'fraction_numerical': 0.0037553648068669528, 'mean_word_length': 3.189887640449438, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '208465', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1392, 'n_words': 384}