<commit_before>package main

import (
	"fmt"
	"github.com/CenturyLinkLabs/docker-reg-client/registry"
	"github.com/docker/docker/pkg/units"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
	"sync"
)

var (
	client      *registry.Client
	registryURL string
)

type imageNode struct {
	id       string
	size     int64
	tags     []string
	children []*imageNode
}

func init() {
	client = registry.NewClient()
}

func getRepos() *registry.SearchResults {
	log.Print("Fetching repos...")
	results, _ := client.Search.Query("", 0, 0)
	log.Printf("%v repo(s) fetched", results.NumResults)
	return results
}

func getTags(name string) registry.TagMap {
	log.Printf("Fetching tags for %s ...", name)
	tags, _ := client.Repository.ListTags(name, registry.NilAuth{})
	log.Printf("%v tags fetched for repo %s", len(tags), name)
	fqTags := make(registry.TagMap)
	for tag, id := range tags {
		fqTags[fqTag(name, tag)] = id
	}
	return fqTags
}

func getAncestry(id string) []string {
	log.Printf("Fetching ancestry for %s ...", id)
	ancestry, _ := client.Image.GetAncestry(id, registry.NilAuth{})
	log.Printf("%v ancestors fetched for tag %s", len(ancestry), id)
	return ancestry
}

func getMetadata(id string) *registry.ImageMetadata {
	log.Printf("Fetching metadata for %s ...", id)
	metadata, _ := client.Image.GetMetadata(id, registry.NilAuth{})
	log.Printf("Metadata fetched for tag %s", id)
	return metadata
}

func fqTag(name string, t string) string {
	canonicalName := strings.TrimPrefix(name, "library/")
	return canonicalName + ":" + t<commit_msg>retry 3 times on http error<commit_after>package main

import (
	"fmt"
	"github.com/CenturyLinkLabs/docker-reg-client/registry"
	"github.com/docker/docker/pkg/units"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
	"sync"
)

var (
	client      *registry.Client
	registryURL string
)

type imageNode struct {
	id       string
	size     int64
	tags     []string
	children []*imageNode
}

func init() {
	client = registry.NewClient()
}

func getRepos() *registry.SearchResults {
	var (
		results *registry.SearchResults
		err     error
	)
	err = retry(3, func() error {
		log.Print("Fetching repos...")
		results, err = client.Search.Query("", 0, 0)
		return err
	})
	if err != nil {
		panic(err)
	}
	log.Printf("%v repo(s) fetched", results.NumResults)
	return results
}

func getTags(name string) registry.TagMap {
	var (
		tags registry.TagMap
		err  error
	)
	err = retry(3, func() error {
		log.Printf("Fetching tags for %s ...", name)
		tags, err = client.Repository.ListTags(name, registry.NilAuth{})
		return err
	})
	if err != nil {
		panic(err)
	}
	log.Printf("%v tags fetched for repo %s", len(tags), name)
	fqTags := make(registry.TagMap)
	for tag, id := range tags {
		fqTags[fqTag(name, tag)] = id
	}
	return fqTags
}

func getAncestry(id string) []string {
	var (
		ancestry []string
		err  error
	)
	err = retry(3, func() error {
		log.Printf("Fetching ancestry for %s ...", id)
		ancestry, err = client.Image.GetAncestry(id, registry.NilAuth{})
		return err
	})
	if err != nil {
		panic(err)
	}
	log.Printf("%v ancestors fetched for tag %s", len(ancestry), id)
	return ancestry
}

func getMetadata(id string) *registry.ImageMetadata {
	var (
		metadata *registry.ImageMetadata
		err  error
	)
	err = retry(3, func() error {
		log.Printf("Fetching metadata for %s ...", id)
		metadata, err = client.Image.GetMetadata(id, registry.NilAuth{})
		return err
	})
	if err != nil {
		panic(err)
	}
	log.Printf("Metadata fetched for tag %s", id)
	return metadata
}

func retry(max int, fn func() error) error {
	var err error
	attempt := 1
	for {
		err = fn()
		attempt++
		if err == nil || attempt > max {
			break
		}
	}
	return err
}

func fqTag(name string, t string) string {
	canonicalName := strings.TrimPrefix(name, "library/")
	return canonicalName + ":" + t
{'fraction_non_alphanumeric': 0.14574049803407602, 'fraction_numerical': 0.003669724770642202, 'mean_word_length': 3.746268656716418, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '267278', 'n_tokens_mistral': 1443, 'n_tokens_neox': 1398, 'n_words': 358}