<commit_before>/*
  Package jq provides functions to resolve paths of the form fld1/fld2/42/fld/4 to be resolved
  in nested structures as returned by e.g. json.Unmarshal.

  TODO the decision of returning nil (not found) or an error is not entirely consistent yet.
*/
package jq

import (
	"encoding/json"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"
)

type quantifier int

const (
	ALL quantifier = iota
)

func (q quantifier) String() string {
	switch q {
	case ALL:
		return "ALL"
	}
	return fmt.Sprintf("<quantifier %d>", int(q))
}

func isSigned(k reflect.Kind) bool {
	switch k {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return true
	}
	return false
}

// Q recursively queries the root object with the path composed of the indices.
//
// If index has no elements, it returns root.
//
// If root is a struct or a map with a string key type
// and the first element of index is a string,
// it will return Q applied to the corresponding field or map value
// with the remainder of the index.
//
// If root is an array or slice, or a map with integer key type,
// and the first element of index is an integer type or a string that parses as an integer type
// it will return Q applied to the corresponding value with the
// remainder of the index.
//
// If the first element of index is the special value ALL, Q returns multiple values as follows:
// If root is a struct or a map Q returns a map of the field names or keys
// of root to the result of Q applied to the corresponding values with the remainder of the index,
// but any errors are omitted from the result set.
// If root is a slice or array, Q returns a slice of the results of Q
// applied to the elements of root with the remainder of the index.
//
// If the value is not present, Q returns nil, but if the
// index has the wrong type for the root element it will return an error.
func Q(root interface{}, index ...interface{}) interface{} {
	if len(index) == 0 {
		return root
	}

	if i, ok := index[0].(quantifier); ok {
		if i == ALL {
			switch v := reflect.ValueOf(root); v.Kind() {
			case reflect.Struct:
				m := make(map[string]interface{})
				for ii := 0; ii < v.NumField(); ii++ {
					f := v.Type().Field(ii)
					if f.PkgPath != "" { // not exported
						continue
					}
					r := v.Field(ii)
					if !r.IsValid() {
						continue
					}
					rr := Q(r.Interface(), index[1:]...)
					// Fields will typically vary in type, and many of them may not be indexable
					// like the rest of the query requires.  It seems more convenient for the user
					// to just filter these elements out here.
					if _, ok := rr.(error); ok {
						continue
					}
					m[f.Name] = rr
				}
				return m

			case reflect.Map:
				k := v.Type().Key()
				var dum []interface{} // dont know how else to make typeof interface.
				m := reflect.MakeMap(reflect.MapOf(k, reflect.TypeOf(dum).Elem()))
				for _, kk := range v.MapKeys() {
					vv := v.MapIndex(kk)
					rr := Q(vv.Interface(), index[1:]...)
					if rr == nil {
						continue
					}
					// see above
					if _, ok := rr.(error); ok {
						continue
					}
					m.SetMapIndex(kk, reflect.ValueOf(rr))
				}
				return m.Interface()

			case reflect.Array, reflect.Slice:
				var a []interface{}
				for ii := 0; ii < v.Len(); ii++ {
					r := v.Index(ii)
					if r.IsValid() {
						a = append(a, Q(r.Interface(), index[1:]...))
					}
				}
				return a
			}
			return fmt.Errorf("type %T does not support retrieving ALL", root)
		}
		panic(fmt.Errorf("unsupported quantifier %d", i))
	}

	switch v := reflect.ValueOf(root); v.Kind() {
	case reflect.Struct:
		switch i := reflect.ValueOf(index[0]); i.Kind() {
		case reflect.String:
			r := v.FieldByName(strings.Title(i.String())) // get the corresponding exported field only
			if r.IsValid() {
				return Q(r.Interface(), index[1:]...)
			}
			return nil
		}
		return fmt.Errorf("cannot use %v (type %T) as struct field name", index[0], index[0])
<commit_msg>simplified quantifier test<commit_after>/*
  Package jq provides functions to resolve paths of the form fld1/fld2/42/fld/4 to be resolved
  in nested structures as returned by e.g. json.Unmarshal.

  TODO the decision of returning nil (not found) or an error is not entirely consistent yet.
*/
package jq

import (
	"encoding/json"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"
)

type quantifier int

const (
	ALL quantifier = iota
)

func (q quantifier) String() string {
	switch q {
	case ALL:
		return "ALL"
	}
	return fmt.Sprintf("<quantifier %d>", int(q))
}

func isSigned(k reflect.Kind) bool {
	switch k {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return true
	}
	return false
}

// Q recursively queries the root object with the path composed of the indices.
//
// If index has no elements, it returns root.
//
// If root is a struct or a map with a string key type
// and the first element of index is a string,
// it will return Q applied to the corresponding field or map value
// with the remainder of the index.
//
// If root is an array or slice, or a map with integer key type,
// and the first element of index is an integer type or a string that parses as an integer type
// it will return Q applied to the corresponding value with the
// remainder of the index.
//
// If the first element of index is the special value ALL, Q returns multiple values as follows:
// If root is a struct or a map Q returns a map of the field names or keys
// of root to the result of Q applied to the corresponding values with the remainder of the index,
// but any errors are omitted from the result set.
// If root is a slice or array, Q returns a slice of the results of Q
// applied to the elements of root with the remainder of the index.
//
// If the value is not present, Q returns nil, but if the
// index has the wrong type for the root element it will return an error.
func Q(root interface{}, index ...interface{}) interface{} {
	if len(index) == 0 {
		return root
	}

	if i, ok := index[0].(quantifier); ok && i == ALL {
		switch v := reflect.ValueOf(root); v.Kind() {
		case reflect.Struct:
			m := make(map[string]interface{})
			for ii := 0; ii < v.NumField(); ii++ {
				f := v.Type().Field(ii)
				if f.PkgPath != "" { // not exported
					continue
				}
				r := v.Field(ii)
				if !r.IsValid() {
					continue
				}
				rr := Q(r.Interface(), index[1:]...)
				// Fields will typically vary in type, and many of them may not be indexable
				// like the rest of the query requires.  It seems more convenient for the user
				// to just filter these elements out here.
				if _, ok := rr.(error); ok {
					continue
				}
				m[f.Name] = rr
			}
			return m

		case reflect.Map:
			k := v.Type().Key()
			var dum []interface{} // dont know how else to make typeof interface.
			m := reflect.MakeMap(reflect.MapOf(k, reflect.TypeOf(dum).Elem()))
			for _, kk := range v.MapKeys() {
				vv := v.MapIndex(kk)
				rr := Q(vv.Interface(), index[1:]...)
				if rr == nil {
					continue
				}
				// see above
				if _, ok := rr.(error); ok {
					continue
				}
				m.SetMapIndex(kk, reflect.ValueOf(rr))
			}
			return m.Interface()

		case reflect.Array, reflect.Slice:
			var a []interface{}
			for ii := 0; ii < v.Len(); ii++ {
				r := v.Index(ii)
				if r.IsValid() {
					a = append(a, Q(r.Interface(), index[1:]...))
				}
			}
			return a
		}
		return fmt.Errorf("type %T does not support retrieving ALL", root)
	}

	if v, ok := index[0].(quantifier); ok {
		panic(fmt.Errorf("unsupported %s", v))
	}

	switch v := reflect.ValueOf(root); v.Kind() {
	case reflect.Struct:
		switch i := reflect.ValueOf(index[0]); i.Kind() {
		case reflect.String:
			r := v.FieldByName(strings.Title(i.String())) // get the corresponding exported field only
			if r.IsValid() {
				return Q(r.Interface(), index[1:]...)
			}
			return nil
		}
		return fmt.Errorf("cannot use %v (type %T) as struct field name", index[0], index[0])

{'fraction_non_alphanumeric': 0.12372817485240548, 'fraction_numerical': 0.005903780932043713, 'mean_word_length': 2.7521206409048067, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7494202', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2492, 'n_words': 1043}