<commit_before>package plethora

import (
	"io"
	"reflect"
	"strings"
)

// seperator is used as seperator in various string utilities
// Note: changing this will break existing databases
const seperator = ":"

var nameToProvider map[string]DataProvider
var providerToName map[string]string

// DataType is a kind of data
//
// Each DataType is backed by a DataProvider that stores data in a
// place the provider sees fit. It is possible for multiple providers
// to exist for a single DataType.
type DataType string

// DataProvider returns the Data associated with the identifier passed.
type DataProvider func(identifier string) (Data, error)

// Data is the interface used to support arbitrary kinds
// of data in plethora. All data types need to support the
// interface to be able to register with plethora.
type Data interface {
	// Type returns the type of data this is
	Type() DataType
	// Provider returns the DataProvider of this data
	Provider() DataProvider
	// Identifier is called to get an unique identifier to this
	// data. The identifier only has to be unique to the
	// DataProvider returned by Provider.
	Identifier() string
	// Render should write a html representation of the data to
	// the writer given.
	Render(w io.Writer) error
}

type identifier string

// Identifier returns an identifier unique to this Data, this is
// different from Data.Identifier in that the former is only unique
// to the provider associated with the Data. The identifier returned
// by Identifier is unique in the whole system.
func Identifier(d Data) identifier {
	p := d.Provider()
	if p == nil {
		panic("illegal: data returned nil provider")
	}

	return identifier(providerName(p) + seperator + d.Identifier())
}

// RegisterProvider registers a DataProvider with the given name.
//
// A DataProvider can be registered under multiple names
//
// Note: changing the name of a data provider after any data has
// entered the system will result in data becoming unreachable.
// Therefore if you want to change the name of your data provider
// you should register a backwards-compatible version under the old
// name to be used with the existing data.
func RegisterProvider(name string, provider DataProvider) {
	if nameToProvider[name] != nil {
		panic("illegal: double register for single name")
	}

	if strings.Index(name, seperator) > 0 {
		panic("illegal: seperator contained in provider name")
	}

	nameToProvider[name] = provider
	providerToName[providerName(provider)] = name
}

// providerName returns a name unique to this DataProvider
func providerName(p DataProvider) string {
	return reflect.TypeOf(p).String()
}<commit_msg>Seperator -> Separator<commit_after>package plethora

import (
	"io"
	"reflect"
	"strings"
)

// separator is used as separator in various string utilities
// Note: changing this will break existing databases
const separator = ":"

var nameToProvider map[string]DataProvider
var providerToName map[string]string

// DataType is a kind of data
//
// Each DataType is backed by a DataProvider that stores data in a
// place the provider sees fit. It is possible for multiple providers
// to exist for a single DataType.
type DataType string

// DataProvider returns the Data associated with the identifier passed.
type DataProvider func(identifier string) (Data, error)

// Data is the interface used to support arbitrary kinds
// of data in plethora. All data types need to support the
// interface to be able to register with plethora.
type Data interface {
	// Type returns the type of data this is
	Type() DataType
	// Provider returns the DataProvider of this data
	Provider() DataProvider
	// Identifier is called to get an unique identifier to this
	// data. The identifier only has to be unique to the
	// DataProvider returned by Provider.
	Identifier() string
	// Render should write a html representation of the data to
	// the writer given.
	Render(w io.Writer) error
}

type identifier string

// Identifier returns an identifier unique to this Data, this is
// different from Data.Identifier in that the former is only unique
// to the provider associated with the Data. The identifier returned
// by Identifier is unique in the whole system.
func Identifier(d Data) identifier {
	p := d.Provider()
	if p == nil {
		panic("illegal: data returned nil provider")
	}

	return identifier(providerName(p) + separator + d.Identifier())
}

// RegisterProvider registers a DataProvider with the given name.
//
// A DataProvider can be registered under multiple names
//
// Note: changing the name of a data provider after any data has
// entered the system will result in data becoming unreachable.
// Therefore if you want to change the name of your data provider
// you should register a backwards-compatible version under the old
// name to be used with the existing data.
func RegisterProvider(name string, provider DataProvider) {
	if nameToProvider[name] != nil {
		panic("illegal: double register for single name")
	}

	if strings.Index(name, separator) > 0 {
		panic("illegal: separator contained in provider name")
	}

	nameToProvider[name] = provider
	providerToName[providerName(provider)] = name
}

// providerName returns a name unique to this DataProvider
func providerName(p DataProvider) string {
	return reflect.TypeOf(p).String()
}
{'fraction_non_alphanumeric': 0.07211629224089107, 'fraction_numerical': 0.00037757221068529355, 'mean_word_length': 3.9981132075471697, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7166690', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1347, 'n_words': 685}