<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"text/template"
)

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func groupByMulti(entries []*RuntimeContainer, key, sep string) map[string][]*RuntimeContainer {
	groups := make(map[string][]*RuntimeContainer)
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			items := strings.Split(value.(string), sep)
			for _, item := range items {
				groups[item] = append(groups[item], v)
			}

		}
	}
	return groups
}

// groupBy groups a list of *RuntimeContainers by the path property key
func groupBy(entries []*RuntimeContainer, key string) map[string][]*RuntimeContainer {
	groups := make(map[string][]*RuntimeContainer)
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			groups[value.(string)] = append(groups[value.(string)], v)
		}
	}
	return groups
}

// groupByKeys is the same as groupBy but only returns a list of keys
func groupByKeys(entries []*RuntimeContainer, key string) []string {
	groups := groupBy(entries, key)
	ret := []string{}
	for k, _ := range groups {
		ret = append(ret, k)
	}
	return ret
}

// selects entries based on key
func where(entries interface{}, key string, cmp interface{}) (interface{}, error) {
	entriesVal := reflect.ValueOf(entries)

	switch entriesVal.Kind() {
	case reflect.Array, reflect.Slice:
		break
	default:
		return nil, fmt.Errorf("Must pass an array or slice to 'where'; received %v", entries)
	}

	selection := make([]interface{}, 0)
	for i := 0; i < entriesVal.Len(); i++ {
		v := reflect.Indirect(entriesVal.Index(i)).Interface()

		value := deepGet(v, key)
		if reflect.DeepEqual(value, cmp) {
			selection = append(selection, v)
		}
	}

	return selection, nil
}

// selects entries where a key exists
func whereExist(entries []*RuntimeContainer, key string) []*RuntimeContainer {
	selection := []*RuntimeContainer{}
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			selection = append(selection, v)
		}
	}
	return selection
}

// selects entries where a key does not exist
func whereNotExist(entries []*RuntimeContainer, key string) []*RuntimeContainer {
	selection := []*RuntimeContainer{}
	for _, v := range entries {
		value := deepGet(*v, key)
		if value == nil {
			selection = append(selection, v)
		}
	}
	return selection
}

// selects entries based on key.  Assumes key is delimited and breaks it apart before comparing
func whereAny(entries []*RuntimeContainer, key, sep string, cmp []string) []*RuntimeContainer {
	selection := []*RuntimeContainer{}
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			items := strings.Split(value.(string), sep)
			if len(intersect(cmp, items)) > 0 {
				selection = append(selection, v)
			}
		}
	}
	return selection
}

// selects entries based on key.  Assumes key is delimited and breaks it apart before comparing
func whereAll(entries []*RuntimeContainer, key, sep string, cmp []string) []*RuntimeContainer {
	selection := []*RuntimeContainer{}
	req_count := len(cmp)
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			items := strings.Split(value.(string), sep)
			if len(intersect(cmp, items)) == req_count {
				selection = append(selection, v)
			}
		}
	}
	return selection
}

// hasPrefix returns whether a given string is a prefix of another string
func hasPrefix(prefix, s string) bool {
	return strings.HasPrefix(s, prefix)
}<commit_msg>Use generalizedWhere for all where* functions<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"text/template"
)

func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func groupByMulti(entries []*RuntimeContainer, key, sep string) map[string][]*RuntimeContainer {
	groups := make(map[string][]*RuntimeContainer)
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			items := strings.Split(value.(string), sep)
			for _, item := range items {
				groups[item] = append(groups[item], v)
			}

		}
	}
	return groups
}

// groupBy groups a list of *RuntimeContainers by the path property key
func groupBy(entries []*RuntimeContainer, key string) map[string][]*RuntimeContainer {
	groups := make(map[string][]*RuntimeContainer)
	for _, v := range entries {
		value := deepGet(*v, key)
		if value != nil {
			groups[value.(string)] = append(groups[value.(string)], v)
		}
	}
	return groups
}

// groupByKeys is the same as groupBy but only returns a list of keys
func groupByKeys(entries []*RuntimeContainer, key string) []string {
	groups := groupBy(entries, key)
	ret := []string{}
	for k, _ := range groups {
		ret = append(ret, k)
	}
	return ret
}

// Generalized where function
func generalizedWhere(entries interface{}, key string, test func(interface{}) bool) (interface{}, error) {
	entriesVal := reflect.ValueOf(entries)

	switch entriesVal.Kind() {
	case reflect.Array, reflect.Slice:
		break
	default:
		return nil, fmt.Errorf("Must pass an array or slice to 'where'; received %v", entries)
	}

	selection := make([]interface{}, 0)
	for i := 0; i < entriesVal.Len(); i++ {
		v := reflect.Indirect(entriesVal.Index(i)).Interface()

		value := deepGet(v, key)
		if test(value) {
			selection = append(selection, v)
		}
	}

	return selection, nil
}

// selects entries based on key
func where(entries interface{}, key string, cmp interface{}) (interface{}, error) {
	return generalizedWhere(entries, key, func(value interface{}) bool {
		return reflect.DeepEqual(value, cmp)
	})
}

// selects entries where a key exists
func whereExist(entries interface{}, key string) (interface{}, error) {
	return generalizedWhere(entries, key, func(value interface{}) bool {
		return value != nil
	})
}

// selects entries where a key does not exist
func whereNotExist(entries interface{}, key string) (interface{}, error) {
	return generalizedWhere(entries, key, func(value interface{}) bool {
		return value == nil
	})
}

// selects entries based on key.  Assumes key is delimited and breaks it apart before comparing
func whereAny(entries interface{}, key, sep string, cmp []string) (interface{}, error) {
	return generalizedWhere(entries, key, func(value interface{}) bool {
		if value == nil {
			return false
		} else {
			items := strings.Split(value.(string), sep)
			return len(intersect(cmp, items)) > 0
		}
	})
}

// selects entries based on key.  Assumes key is delimited and breaks it apart before comparing
func whereAll(entries interface{}, key, sep string, cmp []string) (interface{}, error) {
	req_count := len(cmp)
	return generalizedWhere(entries, key, func(value interface{}) bool {
		if value == nil {
			return false
		} else {
			items := strings.Split(value.(string), sep)
			return len(intersect(cmp, items)) == req_count
		}
	})
}

// hasPrefix returns whether a given string is a prefix of another string
func hasPrefix(prefix, s string) bool {
	return strings.HasPrefix(s, prefix)
}
{'fraction_non_alphanumeric': 0.13820148749154834, 'fraction_numerical': 0.0010818120351588911, 'mean_word_length': 4.1756473058082575, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3566317', 'n_tokens_mistral': 2500, 'n_tokens_neox': 2344, 'n_words': 752}