<commit_before>package schema

import (
	"fmt"
	"log"
	"reflect"
	"strings"

	"golang.org/x/net/context"
)

// Schema defines fields for a document
type Schema struct {
	// Description of the object described by this schema
	Description string
	// Fields defines the schema's allowed fields
	Fields Fields
}

// Validator is an interface used to validate schema against actual data
type Validator interface {
	GetField(name string) *Field
	Prepare(ctx context.Context, payload map[string]interface{}, original *map[string]interface{}, replace bool) (changes map[string]interface{}, base map[string]interface{})
	Validate(changes map[string]interface{}, base map[string]interface{}) (doc map[string]interface{}, errs map[string][]interface{})
}

// Compiler is an interface defining a validator that can be compiled at run time in order
// to check validator configuration validity and/or prepare some data for a faster execution.
type Compiler interface {
	Compile() error
}

type internal struct{}

// Tombstone is used to mark a field for removal
var Tombstone = internal{}

func addFieldError(errs map[string][]interface{}, field string, err interface{}) {
	errs[field] = append(errs[field], err)
}

func mergeFieldErrors(errs map[string][]interface{}, mergeErrs map[string][]interface{}) {
	// TODO recursive merge
	for field, values := range mergeErrs {
		if dest, found := errs[field]; found {
			for _, value := range values {
				dest = append(dest, value)
			}
		} else {
			errs[field] = values
		}
	}
}

// Compile implements Compiler interface and call the same function on each field
func (s Schema) Compile() error {
	// Search for all Dependecy on fields, and compile then
	if err := compileDependencies(s, s); err != nil {
		return err<commit_msg>Update: standalone use case for Compile<commit_after>package schema

import (
	"fmt"
	"log"
	"reflect"
	"strings"

	"golang.org/x/net/context"
)

// Schema defines fields for a document
type Schema struct {
	// Description of the object described by this schema
	Description string
	// Fields defines the schema's allowed fields
	Fields Fields
}

// Validator is an interface used to validate schema against actual data
type Validator interface {
	GetField(name string) *Field
	Prepare(ctx context.Context, payload map[string]interface{}, original *map[string]interface{}, replace bool) (changes map[string]interface{}, base map[string]interface{})
	Validate(changes map[string]interface{}, base map[string]interface{}) (doc map[string]interface{}, errs map[string][]interface{})
}

// Compiler is an interface defining a validator that can be compiled at run time in order
// to check validator configuration validity and/or prepare some data for a faster execution.
type Compiler interface {
	Compile() error
}

type internal struct{}

// Tombstone is used to mark a field for removal
var Tombstone = internal{}

func addFieldError(errs map[string][]interface{}, field string, err interface{}) {
	errs[field] = append(errs[field], err)
}

func mergeFieldErrors(errs map[string][]interface{}, mergeErrs map[string][]interface{}) {
	// TODO recursive merge
	for field, values := range mergeErrs {
		if dest, found := errs[field]; found {
			for _, value := range values {
				dest = append(dest, value)
			}
		} else {
			errs[field] = values
		}
	}
}

// Compile implements Compiler interface and call the same function on each field.
// Note: if you use schema as a standalone library, it is the *caller's* responsibility
// to invoke the Compile method before using Prepare or Validate on a Schema instance,
// otherwise FieldValidator instances may not be initialized correctly.
func (s Schema) Compile() error {
	// Search for all Dependecy on fields, and compile then
	if err := compileDependencies(s, s); err != nil {
		return err
{'fraction_non_alphanumeric': 0.10342105263157894, 'fraction_numerical': 0.0, 'mean_word_length': 4.631111111111111, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4776772', 'n_tokens_mistral': 1101, 'n_tokens_neox': 1052, 'n_words': 426}