<commit_before>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	str := js.Global.Get("JSON").Call("stringify", js.InternalObject(jsonify).Invoke(js.InternalObject(v)), nil, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	boolKind       = 1
	intKind        = 2
	int8Kind       = 3
	int16Kind      = 4
	int32Kind      = 5
	int64Kind      = 6
	uintKind       = 7
	uint8Kind      = 8
	uint16Kind     = 9
	uint32Kind     = 10
	uint64Kind     = 11
	uintptrKind    = 12
	float32Kind    = 13
	float64Kind    = 14
	complex64Kind  = 15
	complex128Kind = 16
	arrayKind      = 17
	interfaceKind  = 20
	mapKind        = 21
	ptrKind        = 22
	sliceKind      = 23
	stringKind     = 24
	structKind     = 25
)

func jsonify(v *js.Object) *js.Object {
	t := v.Get("constructor")
	for t.Get("kind").Int() == ptrKind {
		if v.Get("$val") != t.Get("nil") {
			v = v.Call("$get")
			t = t.Get("elem")
		}
	}
	switch t.Get("kind").Int() {
	case boolKind, intKind, int8Kind, int16Kind, int32Kind, uintKind, uint8Kind, uint16Kind, uint32Kind, uintptrKind, float32Kind, float64Kind, int64Kind, uint64Kind, complex64Kind, complex128Kind, ptrKind, stringKind:
		return js.Global.Call("$externalize", v, t)
	case arrayKind, sliceKind:
		l := v.Get("$length").Int()
		a := js.Global.Get("Array").New(l)
		for i := 0; i < l; i++ {
			a.SetIndex(i, js.InternalObject(jsonify).Invoke(v.Get("$array").Index(i)))
		}
		return a
	case interfaceKind:
		return js.InternalObject(jsonify).Invoke(v.Get("$val"))
	case mapKind:
		m := js.Global.Get("Object").New()
		keys := js.Global.Get("Object").Call("keys", v)
		len := keys.Length()
		for i := 0; i < len; i++ {
			k := keys.Index(i).String()
			m.Set(k, js.InternalObject(jsonify).Invoke(v.Get(k)))
		}
		return m
	case structKind:
		s := js.Global.Get("Object").New()
		todo := [][2]*js.Object{{v, t}}
		fieldsTodo := make(map[string]*js.Object)
		for _, val := range todo {
			v := val[0]
			t := val[1]
			fields := t.Get("fields")
			fieldsLen := fields.Length()
			for i := 0; i < fieldsLen; i++ {
				f := fields.Index(i)
				if f.Get("pkg").Length() > 0 {
					continue
				}
				fName := f.Get("prop").String()
				name := f.Get("name").String()
				anon := false
				if name == "" {
					name = fName // filter $x?
					anon = true
				}
				n, o := parseTag(getJSONTag(f.Get("tag")))
				if n == "-" || (o.Contains("omitempty") && isEmpty(v.Get(fName), f.Get("typ"))) {
					continue
				}
				jName := name
				if n != "" {
					jName = n
				}
				if _, ok := fieldsTodo[jName]; ok {
					fieldsTodo[jName] = nil
				} else if anon {
					todo = append(todo, [2]*js.Object{v.Get(fName), f.Get("typ")})
				} else if s.Get(jName) == js.Undefined {
					val := v.Get(fName)
					if val.Get("constructor").Get("kind") == js.Undefined {
						t := js.Global.Get("Object").New()
						t.Set("kind", f.Get("kind"))
						val.Set("constructor", t)
					}
					fieldsTodo[jName] = val
				}
			}
			for name, f := range fieldsTodo {
				if f != nil {
					s.Set(name, js.InternalObject(jsonify).Invoke(f))
					delete(fieldsTodo, name)<commit_msg>Continuing rewrite<commit_after>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	str := js.Global.Get("JSON").Call("stringify", js.InternalObject(jsonify).Invoke(js.InternalObject(v)), nil, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	boolKind       = 1
	intKind        = 2
	int8Kind       = 3
	int16Kind      = 4
	int32Kind      = 5
	int64Kind      = 6
	uintKind       = 7
	uint8Kind      = 8
	uint16Kind     = 9
	uint32Kind     = 10
	uint64Kind     = 11
	uintptrKind    = 12
	float32Kind    = 13
	float64Kind    = 14
	complex64Kind  = 15
	complex128Kind = 16
	arrayKind      = 17
	interfaceKind  = 20
	mapKind        = 21
	ptrKind        = 22
	sliceKind      = 23
	stringKind     = 24
	structKind     = 25
)

func jsonify(v *js.Object) *js.Object {
	t := v.Get("constructor")
	for t.Get("kind").Int() == ptrKind {
		if v.Get("$val") != t.Get("nil") {
			v = v.Call("$get")
			t = t.Get("elem")
		}
	}
	switch t.Get("kind").Int() {
	case boolKind, intKind, int8Kind, int16Kind, int32Kind, uintKind, uint8Kind, uint16Kind, uint32Kind, uintptrKind, float32Kind, float64Kind, int64Kind, uint64Kind, complex64Kind, complex128Kind, ptrKind, stringKind:
		return js.Global.Call("$externalize", v, t)
	case arrayKind, sliceKind:
		l := v.Get("$length").Int()
		a := js.Global.Get("Array").New(l)
		for i := 0; i < l; i++ {
			a.SetIndex(i, js.InternalObject(jsonify).Invoke(v.Get("$array").Index(i)))
		}
		return a
	case interfaceKind:
		return js.InternalObject(jsonify).Invoke(v.Get("$val"))
	case mapKind:
		m := js.Global.Get("Object").New()
		keys := js.Global.Get("Object").Call("keys", v)
		len := keys.Length()
		for i := 0; i < len; i++ {
			val := v.Get(keys.Index(i).String())
			m.Set(val.Get("k").String(), js.InternalObject(jsonify).Invoke(val.Get("v")))
		}
		return m
	case structKind:
		s := js.Global.Get("Object").New()
		todo := [][2]*js.Object{{v, t}}
		fieldsTodo := make(map[string]*js.Object)
		for _, val := range todo {
			v := val[0]
			t := val[1]
			fields := t.Get("fields")
			fieldsLen := fields.Length()
			for i := 0; i < fieldsLen; i++ {
				f := fields.Index(i)
				if f.Get("pkg").Length() > 0 {
					continue
				}
				fName := f.Get("prop").String()
				name := f.Get("name").String()
				anon := false
				if name == "" {
					name = fName // filter $x?
					anon = true
				}
				n, o := parseTag(getJSONTag(f.Get("tag")))
				if n == "-" || (o.Contains("omitempty") && isEmpty(v.Get(fName), f.Get("typ"))) {
					continue
				}
				jName := name
				if n != "" {
					jName = n
				}
				if _, ok := fieldsTodo[jName]; ok {
					fieldsTodo[jName] = nil
				} else if anon {
					todo = append(todo, [2]*js.Object{v.Get(fName), f.Get("typ")})
				} else if s.Get(jName) == js.Undefined {
					val := v.Get(fName)
					if o.Contains("string") && stringable(f.Get("typ")) {
						s := js.Global.Get("Object").New()
						s.Set("kind", stringKind)
						t := js.Global.Get("Object").New()
						t.Set("elem", s)
						t.Set("kind", ptrKind)
						v := js.Global.Get("Object").New()
						if f.Get("typ").Get("kind").Int() == stringKind {
							v.Set("$val", js.Global.Get("JSON").Call("stringify", val))
						} else {
							v.Set("$val", val.Call("toString"))
						}
						v.Set("constructor", t)
						v.Set("$get", js.MakeFunc(func(this *js.Object, arguments []*js.Object) interface{} {
							return this.Get("$val")
						}))
						val = v

					} else if val.Get("constructor").Get("kind") == js.Undefined {
						t := js.Global.Get("Object").New()
						t.Set("elem", f.Get("typ"))
						t.Set("kind", ptrKind)
						v := js.Global.Get("Object").New()
						v.Set("$val", val)
						v.Set("constructor", t)
						v.Set("$get", js.MakeFunc(func(this *js.Object, arguments []*js.Object) interface{} {
							return this.Get("$val")
						}))
						val = v
					} else if f.Get("typ").Get("kind").Int() == interfaceKind {
						val = val.Get("$val")
					}
					fieldsTodo[jName] = val
				}
			}
			for name, f := range fieldsTodo {
				if f != nil {
					s.Set(name, js.InternalObject(jsonify).Invoke(f))
					//delete(fieldsTodo, name)
{'fraction_non_alphanumeric': 0.1912087912087912, 'fraction_numerical': 0.022466422466422466, 'mean_word_length': 2.9266538830297217, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1146929', 'n_tokens_mistral': 3614, 'n_tokens_neox': 3012, 'n_words': 643}