<commit_before>/* Copyright (c) 2015, <NAME> <<EMAIL>> */
/* See LICENSE for licensing information */

package main

import (
	"bufio"
	"log"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strings"
	"text/template"
)

const path = "tlds.go"

var tldsTmpl = template.Must(template.New("tlds").Parse(`// Generated by tldsgen

package xurls

// TLDs is a sorted list of all public top-level domains
//
// Sources:
//  * https://data.iana.org/TLD/tlds-alpha-by-domain.txt
//  * https://publicsuffix.org/list/effective_tld_names.dat
var TLDs = []string{
{{range $_, $value := .TLDs}}` + "\t`" + `{{$value}}` + "`" + `,
{{end}}}
`))

func addFromIana(addTld func(tld string)) error {
	url := "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
	log.Printf("Fetching %s", url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	re := regexp.MustCompile(`^[^#]+$`)
	for scanner.Scan() {
		line := scanner.Text()
		tld := re.FindString(line)
		addTld(tld)
	}
	return nil
}

func addFromPublicSuffix(addTld func(tld string)) error {
	url := "https://publicsuffix.org/list/effective_tld_names.dat"
	log.Printf("Fetching %s", url)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	re := regexp.MustCompile(`^[^/.]+$`)
	for scanner.Scan() {
		line := scanner.Text()
		tld := re.FindString(line)
		addTld(tld)
	}
	return nil
}

func tldList() ([]string, error) {
	tlds := make(map[string]struct{})
	addTld := func(tld string) {
		if tld == "" {
			return
		}
		tld = strings.ToLower(tld)
		if strings.HasPrefix(tld, "xn--") {
			return
		}
		tlds[tld] = struct{}{}
	}
	if err := addFromIana(addTld); err != nil {
		return nil, err
	}
	if err := addFromPublicSuffix(addTld); err != nil {
		return nil, err
	}
	list := make([]string, 0, len(tlds))
	for tld := range tlds {
		list = append(list, tld)
	}
	sort.Strings(list)
	return list, nil
}

func writeTlds(tlds []string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	return tldsTmpl.Execute(f, struct {
		TLDs []string
	}{
		TLDs: tlds,<commit_msg>Make TLD lists fetching concurrent<commit_after>/* Copyright (c) 2015, <NAME> <<EMAIL>> */
/* See LICENSE for licensing information */

package main

import (
	"bufio"
	"log"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"text/template"
)

const path = "tlds.go"

var tldsTmpl = template.Must(template.New("tlds").Parse(`// Generated by tldsgen

package xurls

// TLDs is a sorted list of all public top-level domains
//
// Sources:
//  * https://data.iana.org/TLD/tlds-alpha-by-domain.txt
//  * https://publicsuffix.org/list/effective_tld_names.dat
var TLDs = []string{
{{range $_, $value := .TLDs}}` + "\t`" + `{{$value}}` + "`" + `,
{{end}}}
`))

func cleanTld(tld string) string {
	tld = strings.ToLower(tld)
	if strings.HasPrefix(tld, "xn--") {
		return ""
	}
	return tld
}

func fromIana(tldChan chan string, wg *sync.WaitGroup) {
	url := "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
	log.Printf("Fetching %s", url)
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error fetcihng %s: %v", url, err)
		return
	}
	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	re := regexp.MustCompile(`^[^#]+$`)
	for scanner.Scan() {
		line := scanner.Text()
		tld := re.FindString(line)
		tld = cleanTld(tld)
		if tld == "" {
			continue
		}
		tldChan <- tld
	}
	wg.Done()
}

func fromPublicSuffix(tldChan chan string, wg *sync.WaitGroup) {
	url := "https://publicsuffix.org/list/effective_tld_names.dat"
	log.Printf("Fetching %s", url)
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("Error fetcihng %s: %v", url, err)
		return
	}
	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	re := regexp.MustCompile(`^[^/.]+$`)
	for scanner.Scan() {
		line := scanner.Text()
		tld := re.FindString(line)
		tld = cleanTld(tld)
		if tld == "" {
			continue
		}
		tldChan <- tld
	}
	wg.Done()
}

func tldList() ([]string, error) {

	tldChan := make(chan string)
	var wg sync.WaitGroup
	wg.Add(2)

	go fromIana(tldChan, &wg)
	go fromPublicSuffix(tldChan, &wg)

	tlds := make(map[string]struct{})
	go func() {
		for tld := range tldChan {
			tlds[tld] = struct{}{}
		}
	}()
	wg.Wait()

	list := make([]string, 0, len(tlds))
	for tld := range tlds {
		list = append(list, tld)
	}

	sort.Strings(list)
	return list, nil
}

func writeTlds(tlds []string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	return tldsTmpl.Execute(f, struct {
		TLDs []string
	}{
		TLDs: tlds,
{'fraction_non_alphanumeric': 0.18714596949891069, 'fraction_numerical': 0.0023965141612200436, 'mean_word_length': 3.5864135864135864, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2811118', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1926, 'n_words': 412}