<commit_before>package pass

import (
	"fmt"
)

type Blob struct {
	// a map of boundary names to their indices, tuples of (inclusive, exclusive)
	boundaries map[string][2]int
	data       []byte
}

// given a bunch of (name, size)... parameter tuples, constructs a blob from
// them. the internal field boundaries are calculated from the given names and
// sizes. if the final parameter is a byte array, it's used to initalize the
// blob.
func NewBlob(params ...interface{}) Blob {
	// require at _least_ one field
	if len(params) < 2 {
		panic("At least one field is required")
	}

	// used as the final boundaries map, but used initially to check for duplicate
	// field names.
	boundaries := make(map[string][2]int)
	fieldNamePlaceholder := [2]int{-1, -1}

	// verify the parameters separately to ensure that they alternate (string,
	// int) and are valid. this is slower than doing it as we go, but much easier
	// to read.
	lastWasString := false
	for i, param := range params {
		switch t := param.(type) {
		default:
			// only accept the given types
			panic(fmt.Sprintf("Unexpected paramter type %T", t))
		case string:
			// make sure the field names are non-null and unique
			if lastWasString {
				panic("Parameter list must have the pattern (string, int)...")
			} else if len(t) == 0 {
				panic("Field names must not be empty")
			} else if _, ok := boundaries[t]; ok {
				panic(fmt.Sprintf("Duplicate field name: %s", t))
			}

			// store the marker value in this field name
			boundaries[t] = fieldNamePlaceholder

			lastWasString = true
		case int:
			// make sure field sizes are at least 1
			if !lastWasString {
				panic("Parameter list must have the pattern (string, int)...")
			} else if t < 1 {
				panic(fmt.Sprintf("Field sizes must be >= 1 (got: %d)", t))
			}
			lastWasString = false
		case []byte:
			// make sure that the initialization bytes, if any, only come at the very
			// end of all the other paramters.
			if i != len(params)-1 {
				panic("Any initialization byte array must come at the end, if at all")
			}
		}
	}

	// build the internal boundaries map from the given paramters
	fieldName := ""
	boundaryStart := 0
	var initBytes []byte = nil
	for _, param := range params {
		switch t := param.(type) {
		case string:
			// store the name until we get a boundary size
			fieldName = t
		case int:
			// store a new boundary pair
			end := boundaryStart + t
			boundaries[fieldName] = [2]int{boundaryStart, end}

			// move the boundary marker to the new start position if the field was of
			// non-zero length. otherwise, we should start at the same place as last<commit_msg>Fix comment typo<commit_after>package pass

import (
	"fmt"
)

type Blob struct {
	// a map of boundary names to their indices, tuples of (inclusive, exclusive)
	boundaries map[string][2]int
	data       []byte
}

// given a bunch of (name, size)... parameter tuples, constructs a blob from
// them. the internal field boundaries are calculated from the given names and
// sizes. if the final parameter is a byte array, it's used to initalize the
// blob.
func NewBlob(params ...interface{}) Blob {
	// require at _least_ one field
	if len(params) < 2 {
		panic("At least one field is required")
	}

	// used as the final boundaries map, but used initially to check for duplicate
	// field names.
	boundaries := make(map[string][2]int)
	fieldNamePlaceholder := [2]int{-1, -1}

	// verify the parameters separately to ensure that they alternate (string,
	// int) and are valid. this is slower than doing it as we go, but much easier
	// to read.
	lastWasString := false
	for i, param := range params {
		switch t := param.(type) {
		default:
			// only accept the given types
			panic(fmt.Sprintf("Unexpected paramter type %T", t))
		case string:
			// make sure the field names are non-null and unique
			if lastWasString {
				panic("Parameter list must have the pattern (string, int)...")
			} else if len(t) == 0 {
				panic("Field names must not be empty")
			} else if _, ok := boundaries[t]; ok {
				panic(fmt.Sprintf("Duplicate field name: %s", t))
			}

			// store the marker value in this field name
			boundaries[t] = fieldNamePlaceholder

			lastWasString = true
		case int:
			// make sure field sizes are at least 1
			if !lastWasString {
				panic("Parameter list must have the pattern (string, int)...")
			} else if t < 1 {
				panic(fmt.Sprintf("Field sizes must be >= 1 (got: %d)", t))
			}
			lastWasString = false
		case []byte:
			// make sure that the initialization bytes, if any, only come at the very
			// end of all the other paramters.
			if i != len(params)-1 {
				panic("Any initialization byte array must come at the end, if at all")
			}
		}
	}

	// build the internal boundaries map from the given parameters
	fieldName := ""
	boundaryStart := 0
	var initBytes []byte = nil
	for _, param := range params {
		switch t := param.(type) {
		case string:
			// store the name until we get a boundary size
			fieldName = t
		case int:
			// store a new boundary pair
			end := boundaryStart + t
			boundaries[fieldName] = [2]int{boundaryStart, end}

			// move the boundary marker to the new start position if the field was of
			// non-zero length. otherwise, we should start at the same place as last
{'fraction_non_alphanumeric': 0.10384176492963104, 'fraction_numerical': 0.00494484594903005, 'mean_word_length': 3.02371843917368, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1471590', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1534, 'n_words': 695}