<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package collate contains types for comparing and sorting Unicode strings
// according to a given collation order.  Package locale provides a high-level
// interface to collation. Users should typically use that package instead.
package collate

import (
	"bytes"
	"exp/norm"
)

// Level identifies the collation comparison level.
// The primary level corresponds to the basic sorting of text.
// The secondary level corresponds to accents and related linguistic elements.
// The tertiary level corresponds to casing and related concepts.
// The quaternary level is derived from the other levels by the
// various algorithms for handling variable elements.
type Level int

const (
	Primary Level = iota
	Secondary
	Tertiary
	Quaternary
	Identity
)

// AlternateHandling identifies the various ways in which variables are handled.
// A rune with a primary weight lower than the variable top is considered a
// variable. 
// See http://www.unicode.org/reports/tr10/#Variable_Weighting for details.
type AlternateHandling int

const (
	// AltNonIgnorable turns off special handling of variables.
	AltNonIgnorable AlternateHandling = iota

	// AltBlanked sets variables and all subsequent primary ignorables to be
	// ignorable at all levels. This is identical to removing all variables
	// and subsequent primary ignorables from the input.
	AltBlanked

	// AltShifted sets variables to be ignorable for levels one through three and
	// adds a fourth level based on the values of the ignored levels.
	AltShifted

	// AltShiftTrimmed is a slight variant of AltShifted that is used to
	// emulate POSIX.
	AltShiftTrimmed
)

// Collator provides functionality for comparing strings for a given
// collation order.
type Collator struct {
	// Strength sets the maximum level to use in comparison.
	Strength Level

	// Alternate specifies an alternative handling of variables.
	Alternate AlternateHandling

	// Backwards specifies the order of sorting at the secondary level.
	// This option exists predominantly to support reverse sorting of accents in French.
	Backwards bool

	// TODO: implement:
	// With HiraganaQuaternary enabled, Hiragana codepoints will get lower values
	// than all the other non-variable code points. Strength must be greater or
	// equal to Quaternary for this to take effect.
	HiraganaQuaternary bool

	// If CaseLevel is true, a level consisting only of case characteristics will
	// be inserted in front of the tertiary level.  To ignore accents but take
	// cases into account, set Strength to Primary and CaseLevel to true.
	CaseLevel bool

	// If Numeric is true, any sequence of decimal digits (category is Nd) is sorted
	// at a primary level with its numeric value.  For example, "A-21" < "A-123".
	Numeric bool

	f norm.Form

	t *table
}

// Locales returns the list of locales for which collating differs from its parent locale.
func Locales() []string {
	return availableLocales
}

// New returns a new Collator initialized for the given locale.
func New(loc string) *Collator {
	// TODO: handle locale selection according to spec.
	t := &mainTable
	if loc != "" {
		if idx, ok := locales[loc]; ok {
			t = mainTable.indexedTable(idx)
		}
	}
	return &Collator{
		Strength: Quaternary,
		f:        norm.NFD,
		t:        t,
	}<commit_msg>exp/locale/collate: clarification in comments on use of returned value.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package collate contains types for comparing and sorting Unicode strings
// according to a given collation order.  Package locale provides a high-level
// interface to collation. Users should typically use that package instead.
package collate

import (
	"bytes"
	"exp/norm"
)

// Level identifies the collation comparison level.
// The primary level corresponds to the basic sorting of text.
// The secondary level corresponds to accents and related linguistic elements.
// The tertiary level corresponds to casing and related concepts.
// The quaternary level is derived from the other levels by the
// various algorithms for handling variable elements.
type Level int

const (
	Primary Level = iota
	Secondary
	Tertiary
	Quaternary
	Identity
)

// AlternateHandling identifies the various ways in which variables are handled.
// A rune with a primary weight lower than the variable top is considered a
// variable. 
// See http://www.unicode.org/reports/tr10/#Variable_Weighting for details.
type AlternateHandling int

const (
	// AltNonIgnorable turns off special handling of variables.
	AltNonIgnorable AlternateHandling = iota

	// AltBlanked sets variables and all subsequent primary ignorables to be
	// ignorable at all levels. This is identical to removing all variables
	// and subsequent primary ignorables from the input.
	AltBlanked

	// AltShifted sets variables to be ignorable for levels one through three and
	// adds a fourth level based on the values of the ignored levels.
	AltShifted

	// AltShiftTrimmed is a slight variant of AltShifted that is used to
	// emulate POSIX.
	AltShiftTrimmed
)

// Collator provides functionality for comparing strings for a given
// collation order.
type Collator struct {
	// Strength sets the maximum level to use in comparison.
	Strength Level

	// Alternate specifies an alternative handling of variables.
	Alternate AlternateHandling

	// Backwards specifies the order of sorting at the secondary level.
	// This option exists predominantly to support reverse sorting of accents in French.
	Backwards bool

	// TODO: implement:
	// With HiraganaQuaternary enabled, Hiragana codepoints will get lower values
	// than all the other non-variable code points. Strength must be greater or
	// equal to Quaternary for this to take effect.
	HiraganaQuaternary bool

	// If CaseLevel is true, a level consisting only of case characteristics will
	// be inserted in front of the tertiary level.  To ignore accents but take
	// cases into account, set Strength to Primary and CaseLevel to true.
	CaseLevel bool

	// If Numeric is true, any sequence of decimal digits (category is Nd) is sorted
	// at a primary level with its numeric value.  For example, "A-21" < "A-123".
	Numeric bool

	f norm.Form

	t *table
}

// Locales returns the list of locales for which collating differs from its parent locale.
// The returned value should not be modified.
func Locales() []string {
	return availableLocales
}

// New returns a new Collator initialized for the given locale.
func New(loc string) *Collator {
	// TODO: handle locale selection according to spec.
	t := &mainTable
	if loc != "" {
		if idx, ok := locales[loc]; ok {
			t = mainTable.indexedTable(idx)
		}
	}
	return &Collator{
		Strength: Quaternary,
		f:        norm.NFD,
		t:        t,
	}
{'fraction_non_alphanumeric': 0.06022889842632332, 'fraction_numerical': 0.0031473533619456365, 'mean_word_length': 3.5782580222658806, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3208524', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1825, 'n_words': 931}