<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"exp/norm"
	"exp/regexp"
	"flag"
	"fmt"
	"http"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
	"strconv"
	"time"
	"utf8"
)

func main() {
	flag.Parse()
	loadTestData()
	CharacterByCharacterTests()
	StandardTests()
	PerformanceTest()
	if errorCount == 0 {
		fmt.Println("PASS")
	}
}

const file = "NormalizationTest.txt"

var url = flag.String("url",
	"http://www.unicode.org/Public/6.0.0/ucd/"+file,
	"URL of Unicode database directory")
var localFiles = flag.Bool("local",
	false,
	"data files have been copied to the current directory; for debugging only")

var logger = log.New(os.Stderr, "", log.Lshortfile)

// This regression test runs the test set in NormalizationTest.txt
// (taken from http://www.unicode.org/Public/6.0.0/ucd/).
//
// NormalizationTest.txt has form:
// @Part0 # Specific cases
// #
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where 
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:
// 1. The following invariants must be true for all conformant implementations
//
//    NFC
//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
//      c4 ==  NFC(c4) ==  NFC(c5)
//
//    NFD
//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
//      c5 ==  NFD(c4) ==  NFD(c5)
//
//    NFKC
//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
//
//    NFKD
//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
//
// 2. For every code point X assigned in this version of Unicode that is not
//    specifically listed in Part 1, the following invariants must be true
//    for all conformant implementations:
//
//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
//

// Column types.
const (
	cRaw = iota
	cNFC
	cNFD
	cNFKC
	cNFKD
	cMaxColumns
)

// Holds data from NormalizationTest.txt
var part []Part

type Part struct {
	name   string
	number int
	tests  []Test
}

type Test struct {
	name   string
	partnr int
	number int
	rune   int                 // used for character by character test
	cols   [cMaxColumns]string // Each has 5 entries, see below.
}

func (t Test) Name() string {
	if t.number < 0 {
		return part[t.partnr].name
	}
	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}

var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n`) // TODO: using $ iso \n does not work
// TODO: the following regexp does not work: `^(?:([\dA-F ]+);){5} # (.*)\n`
var testRe = regexp.MustCompile(`^(?:([\dA-F ]+);)(?:([\dA-F ]+);)(?:([\dA-F ]+);)(?:([\dA-F ]+);)(?:([\dA-F ]+);) # (.*)\n`)

var counter int

// Load the data form NormalizationTest.txt
func loadTestData() {
	if *localFiles {
		pwd, _ := os.Getwd()
		*url = "file://" + path.Join(pwd, file)
	}
	t := &http.Transport{}
	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	c := &http.Client{Transport: t}
	resp, err := c.Get(*url)
	if err != nil {
		logger.Fatal(err)
	}
	if resp.StatusCode != 200 {<commit_msg>exp/norm: reverting to using strings.Repeat, as it doesn't look like exp/regexp is going to support returning multiple matches for a single repeated group.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"exp/norm"
	"exp/regexp"
	"flag"
	"fmt"
	"http"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
	"strconv"
	"time"
	"utf8"
)

func main() {
	flag.Parse()
	loadTestData()
	CharacterByCharacterTests()
	StandardTests()
	PerformanceTest()
	if errorCount == 0 {
		fmt.Println("PASS")
	}
}

const file = "NormalizationTest.txt"

var url = flag.String("url",
	"http://www.unicode.org/Public/6.0.0/ucd/"+file,
	"URL of Unicode database directory")
var localFiles = flag.Bool("local",
	false,
	"data files have been copied to the current directory; for debugging only")

var logger = log.New(os.Stderr, "", log.Lshortfile)

// This regression test runs the test set in NormalizationTest.txt
// (taken from http://www.unicode.org/Public/6.0.0/ucd/).
//
// NormalizationTest.txt has form:
// @Part0 # Specific cases
// #
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where 
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:
// 1. The following invariants must be true for all conformant implementations
//
//    NFC
//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
//      c4 ==  NFC(c4) ==  NFC(c5)
//
//    NFD
//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
//      c5 ==  NFD(c4) ==  NFD(c5)
//
//    NFKC
//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
//
//    NFKD
//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
//
// 2. For every code point X assigned in this version of Unicode that is not
//    specifically listed in Part 1, the following invariants must be true
//    for all conformant implementations:
//
//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
//

// Column types.
const (
	cRaw = iota
	cNFC
	cNFD
	cNFKC
	cNFKD
	cMaxColumns
)

// Holds data from NormalizationTest.txt
var part []Part

type Part struct {
	name   string
	number int
	tests  []Test
}

type Test struct {
	name   string
	partnr int
	number int
	rune   int                 // used for character by character test
	cols   [cMaxColumns]string // Each has 5 entries, see below.
}

func (t Test) Name() string {
	if t.number < 0 {
		return part[t.partnr].name
	}
	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}

var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n`) // TODO: using $ iso \n does not work
var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n`)

var counter int

// Load the data form NormalizationTest.txt
func loadTestData() {
	if *localFiles {
		pwd, _ := os.Getwd()
		*url = "file://" + path.Join(pwd, file)
	}
	t := &http.Transport{}
	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	c := &http.Client{Transport: t}
	resp, err := c.Get(*url)
	if err != nil {
		logger.Fatal(err)
	}
	if resp.StatusCode != 200 {
{'fraction_non_alphanumeric': 0.1784959226819692, 'fraction_numerical': 0.03261854424645122, 'mean_word_length': 3.0066545674531158, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4869272', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2681, 'n_words': 738}