<commit_before>package main

import (
	"encoding/xml"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"regexp"
	"sync"
)

type svnIndex struct {
	XMLName xml.Name   `xml:"svn"`
	Entries []svnEntry `xml:"index>file"`
}

type svnEntry struct {
	Name string `xml:"href,attr"`
}

const maxRequests = 10

// DownloadSvnDir reads an SVN XML directory index and downloads all the listed (filtered) files
func DownloadSvnDir(svnDirUrl string, filter *regexp.Regexp, outDir string) error {
	response, err := http.Get(svnDirUrl)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	var index svnIndex
	if err := xml.NewDecoder(response.Body).Decode(&index); err != nil {
		return err
	}

	var downloadErr error = nil

	wg := new(sync.WaitGroup)
	c := make(chan int, maxRequests)
	for _, e := range index.Entries {
		if filter.MatchString(e.Name) {
			url := svnDirUrl + "/" + e.Name
			file := filepath.Join(outDir, e.Name)
			wg.Add(1)
			go func() {
				defer wg.Done()
				c <- 1
				if err := downloadFile(url, file); err != nil && downloadErr == nil {
					downloadErr = err
				}
				<-c
			}()
		}
	}
	wg.Wait()

	return downloadErr
}

func downloadFile(url, file string) error {
	log.Println("Downloading", url)

	response, err := http.Get(url)
	if err != nil {<commit_msg>Avoid starting a goroutine until we want to start a download.<commit_after>package main

import (
	"encoding/xml"
	"io/ioutil"
	"log"
	"net/http"
	"path/filepath"
	"regexp"
	"sync"
)

type svnIndex struct {
	XMLName xml.Name   `xml:"svn"`
	Entries []svnEntry `xml:"index>file"`
}

type svnEntry struct {
	Name string `xml:"href,attr"`
}

const maxRequests = 10

// DownloadSvnDir reads an SVN XML directory index and downloads all the listed (filtered) files
func DownloadSvnDir(svnDirUrl string, filter *regexp.Regexp, outDir string) error {
	response, err := http.Get(svnDirUrl)
	if err != nil {
		return err
	}
	defer response.Body.Close()

	var index svnIndex
	if err := xml.NewDecoder(response.Body).Decode(&index); err != nil {
		return err
	}

	var downloadErr error = nil

	wg := new(sync.WaitGroup)
	c := make(chan int, maxRequests)
	for _, e := range index.Entries {
		if filter.MatchString(e.Name) {
			c <- 1
			wg.Add(1)
			url := svnDirUrl + "/" + e.Name
			file := filepath.Join(outDir, e.Name)
			go func(url, file string) {
				defer wg.Done()
				if err := downloadFile(url, file); err != nil && downloadErr == nil {
					downloadErr = err
				}
				<-c
			}(url, file)
		}
	}
	wg.Wait()

	return downloadErr
}

func downloadFile(url, file string) error {
	log.Println("Downloading", url)

	response, err := http.Get(url)
	if err != nil {
{'fraction_non_alphanumeric': 0.1375, 'fraction_numerical': 0.0030303030303030303, 'mean_word_length': 3.476271186440678, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2517051', 'n_tokens_mistral': 1051, 'n_tokens_neox': 959, 'n_words': 266}