<commit_before>package secondly

import (
	"log"
	"reflect"
)

type field struct {
	Path  string      `json:"path"`
	Name  string      `json:"name"`
	Kind  string      `json:"kind"`
	Value interface{} `json:"val"`
}

func extractFields(st interface{}, path string) []field {
	var res []field

	val := reflect.ValueOf(st)
	if val.Kind() == reflect.Ptr {
		val = reflect.Indirect(val)
	}
	typ := val.Type()

	for i := 0; i < val.NumField(); i++ {
		ftyp := typ.Field(i)
		fval := val.Field(i)

		switch kind := fval.Kind(); kind {
		case reflect.Struct:
			sub := extractFields(fval.Interface(), ftyp.Name+".")
			res = append(res, sub...)
		case reflect.Bool,
			reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64,
			reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64,
			reflect.Float32,
			reflect.Float64,
			reflect.String:
			res = append(res, field{
				Path:  path + ftyp.Name,
				Name:  ftyp.Name,
				Kind:  kind.String(),
				Value: fval.Interface(),
			})
		default:
			log.Printf("Field type %q not supported for field %q\n", kind, path+ftyp.Name)
		}
	}

	return res
}

func diff(a, b interface{}) map[string][]interface{} {
	af := indexFields(extractFields(a, ""))
	bf := indexFields(extractFields(b, ""))<commit_msg>Use json keys as field path tokens<commit_after>package secondly

import (
	"log"
	"reflect"
)

type field struct {
	Path  string      `json:"path"`
	Name  string      `json:"name"`
	Kind  string      `json:"kind"`
	Value interface{} `json:"value"`
}

func extractFields(st interface{}, path string) []field {
	var res []field

	val := reflect.ValueOf(st)
	if val.Kind() == reflect.Ptr {
		val = reflect.Indirect(val)
	}
	typ := val.Type()

	for i := 0; i < val.NumField(); i++ {
		ftyp := typ.Field(i)
		fval := val.Field(i)
		tag := ftyp.Tag.Get("json")

		switch kind := fval.Kind(); kind {
		case reflect.Struct:
			sub := extractFields(fval.Interface(), path+tag+".")
			res = append(res, sub...)
		case reflect.Bool,
			reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64,
			reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64,
			reflect.Float32,
			reflect.Float64,
			reflect.String:
			res = append(res, field{
				Path:  path + tag,
				Name:  ftyp.Name,
				Kind:  kind.String(),
				Value: fval.Interface(),
			})
		default:
			log.Printf("Field type %q not supported for field %q\n", kind, path+tag)
		}
	}

	return res
}

func diff(a, b interface{}) map[string][]interface{} {
	af := indexFields(extractFields(a, ""))
	bf := indexFields(extractFields(b, ""))
{'fraction_non_alphanumeric': 0.1627554882664648, 'fraction_numerical': 0.014383043149129448, 'mean_word_length': 3.2288, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2379489', 'n_tokens_mistral': 1088, 'n_tokens_neox': 950, 'n_words': 226}