<commit_before>/* Copyright (c) 2015, <NAME> <<EMAIL>> */
/* See LICENSE for licensing information */

package main

import (
	"log"
	"os"
	"sort"
	"strings"
	"text/template"

	"golang.org/x/net/idna"

	"github.com/mvdan/xurls"
)

const path = "regex.go"

var regexTmpl = template.Must(template.New("regex").Parse(`// Generated by regexgen

package xurls

const ({{ range $key, $value := . }}
	{{$key}} = ` + "`" + `{{$value}}` + "`" + `{{end}}
)
`))

// These schemes may be followed by just ":" instead of "://", so instead of
// allowing for arbitrary schemes we're limiting the regex to just a few
// well-known ones.
var schemes = []string{
	`bitcoin`,
	`magnet`,
	`mailto`,
	`sms`,
	`tel`,
	`xmpp`,
}

func writeRegex(tlds []string) error {
	allTldsSet := make(map[string]struct{})
	for _, tldlist := range [...][]string{tlds, xurls.PseudoTLDs} {
		for _, tld := range tldlist {
			allTldsSet[tld] = struct{}{}
			asciiTld, err := idna.ToASCII(tld)
			if err != nil {
				return err
			}
			allTldsSet[asciiTld] = struct{}{}
		}
	}
	var allTlds []string
	for tld := range allTldsSet {
		allTlds = append(allTlds, tld)
	}
	sort.Strings(allTlds)
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	return regexTmpl.Execute(f, map[string]string{
		"gtld       ": `(?i)(` + strings.Join(allTlds, `|`) + `)(?-i)`,
		"otherScheme": `(?i)(` + strings.Join(schemes, `|`) + `)(?-i):`,
	})
}

func main() {
	log.Printf("Generating %s...", path)
	if err := writeRegex(xurls.TLDs); err != nil {
		log.Fatalf("Could not write %s: %s", path, err)
	}<commit_msg>Detect duplicate TLDs<commit_after>/* Copyright (c) 2015, <NAME> <<EMAIL>> */
/* See LICENSE for licensing information */

package main

import (
	"log"
	"os"
	"sort"
	"strings"
	"text/template"

	"golang.org/x/net/idna"

	"github.com/mvdan/xurls"
)

const path = "regex.go"

var regexTmpl = template.Must(template.New("regex").Parse(`// Generated by regexgen

package xurls

const ({{ range $key, $value := . }}
	{{$key}} = ` + "`" + `{{$value}}` + "`" + `{{end}}
)
`))

// These schemes may be followed by just ":" instead of "://", so instead of
// allowing for arbitrary schemes we're limiting the regex to just a few
// well-known ones.
var schemes = []string{
	`bitcoin`,
	`magnet`,
	`mailto`,
	`sms`,
	`tel`,
	`xmpp`,
}

func writeRegex(tlds []string) error {
	allTldsSet := make(map[string]struct{})
	add := func(tld string) {
		if _, e := allTldsSet[tld]; e {
			log.Fatalf("Duplicate TLD: %s", tld)
		}
		allTldsSet[tld] = struct{}{}
	}
	for _, tldlist := range [...][]string{tlds, xurls.PseudoTLDs} {
		for _, tld := range tldlist {
			add(tld)
			asciiTld, err := idna.ToASCII(tld)
			if err != nil {
				return err
			}
			if asciiTld != tld {
				add(asciiTld)
			}
		}
	}
	var allTlds []string
	for tld := range allTldsSet {
		allTlds = append(allTlds, tld)
	}
	sort.Strings(allTlds)
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	return regexTmpl.Execute(f, map[string]string{
		"gtld       ": `(?i)(` + strings.Join(allTlds, `|`) + `)(?-i)`,
		"otherScheme": `(?i)(` + strings.Join(schemes, `|`) + `)(?-i):`,
	})
}

func main() {
	log.Printf("Generating %s...", path)
	if err := writeRegex(xurls.TLDs); err != nil {
		log.Fatalf("Could not write %s: %s", path, err)
	}
{'fraction_non_alphanumeric': 0.20453848512726158, 'fraction_numerical': 0.0024532352039251764, 'mean_word_length': 3.511756569847856, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492350', 'n_tokens_mistral': 1421, 'n_tokens_neox': 1340, 'n_words': 312}