<commit_before>package parse

import (
	"fmt"
	"reflect"
	"strconv"
	"testing"

	"golang.org/x/text/unicode/norm"
)

type equalityCheck struct {
	id                  int
	fieldNum            int
	expectedFieldExists bool
	t                   *testing.T

	parsedValue        reflect.Value
	parsedFieldName    string
	parsedFieldValue   reflect.Value
	expectedValue      reflect.Value
	expectedFieldName  string
	expectedFieldValue reflect.Value
}

func newEqualityCheck(t *testing.T, parsedVal reflect.Value, expectedVal reflect.Value) *equalityCheck {
	return &equalityCheck{
		id:            int(parsedVal.FieldByName("ID").Interface().(ID)),
		parsedValue:   parsedVal,
		expectedValue: expectedVal,
	}
}

func (e *equalityCheck) error() {
	var got, exp string
	switch r := e.parsedValue.Interface().(type) {
	case ID:
		got = e.parsedValue.Interface().(ID).String()
		exp = e.expectedValue.Interface().(ID).String()
	case itemElement:
		got = e.parsedValue.Interface().(itemElement).String()
		exp = e.expectedValue.Interface().(itemElement).String()
	case Line:
		got = e.parsedValue.Interface().(Line).String()
		exp = e.expectedValue.Interface().(Line).String()
	case StartPosition:
		got = e.parsedValue.Interface().(StartPosition).String()
		exp = e.expectedValue.Interface().(StartPosition).String()
	case int:
		got = strconv.Itoa(e.parsedValue.Interface().(int))
		exp = strconv.Itoa(e.expectedValue.Interface().(int))
	case string:
		got = e.parsedValue.Interface().(string)
		exp = e.expectedValue.Interface().(string)
	default:
		panic(fmt.Sprintf("%T is not implemented!", r))
	}
	e.t.Errorf("\n(ID: %d) Got: %s = %q, Expect: %s = %q", e.id, e.parsedFieldName, got, e.expectedFieldName, exp)
}

func (e *equalityCheck) check(fieldNum int) error {
	e.fieldNum = fieldNum
	e.expectedFieldValue = e.expectedValue.Field(e.fieldNum)
	e.expectedFieldName = e.expectedValue.Type().Field(e.fieldNum).Name

	var parsedValueStruct reflect.StructField
	parsedValueStruct, e.expectedFieldExists = e.parsedValue.Type().FieldByName(e.expectedFieldName)
	e.parsedFieldValue = e.parsedValue.FieldByName(e.expectedFieldName)
	e.parsedFieldName = parsedValueStruct.Name

	if !e.expectedFieldExists {
		return fmt.Errorf("ID: %d does not contain field %q", e.id, e.expectedFieldName)
	}

	// Handle special cases when comparing fields
	switch e.expectedFieldName {
	case "Text":
		e.expectedFieldValue = reflect.ValueOf(norm.NFC.String(e.expectedFieldValue.Interface().(string)))
	case "StartPosition":
		if e.parsedFieldValue.Interface().(StartPosition) == 1 {
			// Ignore StartPositions that begin at 1 from the parsed output items. This allows
			// startPosition to be excluded from the expected items tests (*_items.json).
			return nil
		}
	}

	if e.expectedFieldValue.Interface() != e.parsedFieldValue.Interface() {
		e.error()
	}

	return nil
}<commit_msg>lexer_test: check doesn't need to return error<commit_after>package parse

import (
	"fmt"
	"reflect"
	"strconv"
	"testing"

	"golang.org/x/text/unicode/norm"
)

type equalityCheck struct {
	id                  int
	fieldNum            int
	expectedFieldExists bool
	t                   *testing.T

	parsedValue        reflect.Value
	parsedFieldName    string
	parsedFieldValue   reflect.Value
	expectedValue      reflect.Value
	expectedFieldName  string
	expectedFieldValue reflect.Value
}

func newEqualityCheck(t *testing.T, parsedVal reflect.Value, expectedVal reflect.Value) *equalityCheck {
	return &equalityCheck{
		id:            int(parsedVal.FieldByName("ID").Interface().(ID)),
		parsedValue:   parsedVal,
		expectedValue: expectedVal,
	}
}

func (e *equalityCheck) error() {
	var got, exp string
	switch r := e.parsedValue.Interface().(type) {
	case ID:
		got = e.parsedValue.Interface().(ID).String()
		exp = e.expectedValue.Interface().(ID).String()
	case itemElement:
		got = e.parsedValue.Interface().(itemElement).String()
		exp = e.expectedValue.Interface().(itemElement).String()
	case Line:
		got = e.parsedValue.Interface().(Line).String()
		exp = e.expectedValue.Interface().(Line).String()
	case StartPosition:
		got = e.parsedValue.Interface().(StartPosition).String()
		exp = e.expectedValue.Interface().(StartPosition).String()
	case int:
		got = strconv.Itoa(e.parsedValue.Interface().(int))
		exp = strconv.Itoa(e.expectedValue.Interface().(int))
	case string:
		got = e.parsedValue.Interface().(string)
		exp = e.expectedValue.Interface().(string)
	default:
		panic(fmt.Sprintf("%T is not implemented!", r))
	}
	e.t.Errorf("\n(ID: %d) Got: %s = %q, Expect: %s = %q", e.id, e.parsedFieldName, got, e.expectedFieldName, exp)
}

func (e *equalityCheck) check(fieldNum int) {
	e.fieldNum = fieldNum
	e.expectedFieldValue = e.expectedValue.Field(e.fieldNum)
	e.expectedFieldName = e.expectedValue.Type().Field(e.fieldNum).Name

	var parsedValueStruct reflect.StructField
	parsedValueStruct, e.expectedFieldExists = e.parsedValue.Type().FieldByName(e.expectedFieldName)
	e.parsedFieldValue = e.parsedValue.FieldByName(e.expectedFieldName)
	e.parsedFieldName = parsedValueStruct.Name

	if !e.expectedFieldExists {
		e.t.Errorf("ID: %d does not contain field %q", e.id, e.expectedFieldName)
		return
	}

	// Handle special cases when comparing fields
	switch e.expectedFieldName {
	case "Text":
		e.expectedFieldValue = reflect.ValueOf(norm.NFC.String(e.expectedFieldValue.Interface().(string)))
	case "StartPosition":
		if e.parsedFieldValue.Interface().(StartPosition) == 1 {
			// Ignore StartPositions that begin at 1 from the parsed output items. This allows
			// startPosition to be excluded from the expected items tests (*_items.json).
			return
		}
	}

	if e.expectedFieldValue.Interface() != e.parsedFieldValue.Interface() {
		e.error()
	}
}
{'fraction_non_alphanumeric': 0.1327171259155912, 'fraction_numerical': 0.0006975933031042902, 'mean_word_length': 4.185352622061483, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '641190', 'n_tokens_mistral': 1911, 'n_tokens_neox': 1874, 'n_words': 392}