<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"

func progress(done *bool) {
	for !*done {
		fmt.Print(".")
		time.Sleep(200 * time.Millisecond)
	}
	fmt.Println()
}

func getUcdFile(fileName string) {
	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	done := false
	go progress(&done)
	defer func() {
		done = true
	}()
	response, err := http.Get(url)
	if err != nil {
		panic(err)<commit_msg>renamed function progress to progressDisplay<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"

func progressDisplay(done *bool) {
	for !*done {
		fmt.Print(".")
		time.Sleep(150 * time.Millisecond)
	}
	fmt.Println()
}

func getUcdFile(fileName string) {
	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	done := false
	go progressDisplay(&done)
	defer func() {
		done = true
	}()
	response, err := http.Get(url)
	if err != nil {
		panic(err)
{'fraction_non_alphanumeric': 0.16216216216216217, 'fraction_numerical': 0.004633204633204633, 'mean_word_length': 4.042801556420233, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916861', 'n_tokens_mistral': 541, 'n_tokens_neox': 529, 'n_words': 113}