<commit_before>package rfc6902

import (
	"strconv"
	"errors"
	"fmt"
)

var ErrorInvalidJSONPath = errors.New("Invalid JSON Path")

/*
Patcher to maniplate a json doc.
*/
type patcher struct {
	pointer jsonptr
	jsonObject interface{}
}

func (p *patcher) parent() *patcher {
	if len(p.pointer) == 1 {
		return nil
	}
	return &patcher{jsonptr(p.pointer[:len(p.pointer)-1]), p.jsonObject}
}

func (p *patcher) value() (interface{}, error) {
	v, err := value(p.pointer, &p.jsonObject)
	if err != nil {
		return nil, err
	}
	return *v, nil
}

func (p *patcher) existing() bool {
	_, err := value(p.pointer, &p.jsonObject)
	return err == nil
}

func(p *patcher) setExistingValue(v interface{}) error {
	ref, err := p.parentValue()
	if err != nil {
		return err
	}

	switch t := ref.(type) {
	case map[string]interface{}:
		t[p.pointer[len(p.pointer)-1].token()] = v
	case []interface{}:
		i, err := strconv.Atoi(p.pointer[len(p.pointer)-1].token())
		if err != nil {
			return err
		}
		parent, err := p.parentValue()	
		if err != nil {
			return err
		}
		pa, ok := parent.([]interface{})
		if !ok {
			panic("unable to convert to []interface{}")
		}
		na := append(pa[:i], append([]interface{}{v}, pa[i:]...)...)

		parentRef := p.parent()
		if parentRef == nil {
			p.setParentValue(na)
			return nil
		} 

		parentRef.setExistingValue(na)

	default:
		panic("Unknown type.")
	}
	return nil
}

func (p *patcher) parentValue() (interface{}, error) {
	v, err := value(p.pointer[:len(p.pointer)-1], &p.jsonObject)
	return *v, err
}

func (p *patcher) setParentValue(v interface{}) {
	ref, _ := value(p.pointer[:len(p.pointer)-1], &p.jsonObject)
	*ref = v
}

func (p *patcher) remove() error {
	ref, err := p.parentValue()
	if err != nil {
		return err
	}

	switch t := ref.(type) {
	case map[string]interface{}:
		delete(t, p.pointer[len(p.pointer)-1].token())
	}
	return nil
}

func value(fields jsonptr, ref *interface{}) (value *interface{}, err error) {

	value = ref
	for _, field := range fields {
		el := field.token()
		switch t := (*ref).(type) {
		case map[string]interface{}:
			vv, ok := t[el]
			if !ok {
				return nil, ErrorInvalidJSONPath
			}
			value = &vv
			ref = &vv
		case []interface{}:
			idx, err := strconv.Atoi(string(el))
			if err != nil {
				panic("unable to convert to integer")
			}
			if idx >= len(t) {
				return nil, ErrorInvalidJSONPath
			}
			vv := t[idx]
			value = &vv
			ref = &vv
		default:
			panic(fmt.Sprintf("unknown type %T\n", t))
		}
	}
	return
}
<commit_msg>Support for removing array element<commit_after>package rfc6902

import (
	"strconv"
	"errors"
	"fmt"
)

var ErrorInvalidJSONPath = errors.New("Invalid JSON Path")

/*
Patcher to maniplate a json doc.
*/
type patcher struct {
	pointer jsonptr
	jsonObject interface{}
}

func (p *patcher) parent() *patcher {
	if len(p.pointer) == 1 {
		return nil
	}
	return &patcher{jsonptr(p.pointer[:len(p.pointer)-1]), p.jsonObject}
}

func (p *patcher) value() (interface{}, error) {
	v, err := value(p.pointer, &p.jsonObject)
	if err != nil {
		return nil, err
	}
	return *v, nil
}

func (p *patcher) existing() bool {
	_, err := value(p.pointer, &p.jsonObject)
	return err == nil
}

func(p *patcher) setExistingValue(v interface{}) error {
	ref, err := p.parentValue()
	if err != nil {
		return err
	}

	switch t := ref.(type) {
	case map[string]interface{}:
		t[p.pointer[len(p.pointer)-1].token()] = v
	case []interface{}:
		i, err := strconv.Atoi(p.pointer[len(p.pointer)-1].token())
		if err != nil {
			return err
		}
		parent, err := p.parentValue()
		if err != nil {
			return err
		}
		pa, ok := parent.([]interface{})
		if !ok {
			panic("unable to convert to []interface{}")
		}
		na := append(pa[:i], append([]interface{}{v}, pa[i:]...)...)

		parentRef := p.parent()
		if parentRef == nil {
			p.setParentValue(na)
			return nil
		}

		parentRef.setExistingValue(na)

	default:
		panic("Unknown type.")
	}
	return nil
}

func (p *patcher) parentValue() (interface{}, error) {
	v, err := value(p.pointer[:len(p.pointer)-1], &p.jsonObject)
	return *v, err
}

func (p *patcher) setParentValue(v interface{}) {
	ref, _ := value(p.pointer[:len(p.pointer)-1], &p.jsonObject)
	*ref = v
}

func (p *patcher) remove() error {
	ref, err := p.parentValue()
	if err != nil {
		return err
	}

	parent := p.parent()
	switch t := ref.(type) {
	case map[string]interface{}:
		delete(t, p.pointer[len(p.pointer) - 1].token())
	case []interface{}:
		i, err := strconv.Atoi(p.pointer[len(p.pointer)-1].token())
		if err != nil {
			return err
		}
		parentValue, err := parent.value()
		if err != nil {
			return err
		}
		container, ok := parentValue.([]interface{})
		if !ok {
			panic("Unable to convert parent to []interface{}")
		}

		newArray := make([]interface{}, len(container) - 2)
		copy(newArray, container[:i])
		newArray = append(newArray, container[i+1:]...)
		parent.setExistingValue(newArray)
	default:
		panic("Unknown type for removal")
	}
	return nil
}

func value(fields jsonptr, ref *interface{}) (value *interface{}, err error) {

	value = ref
	for _, field := range fields {
		el := field.token()
		switch t := (*ref).(type) {
		case map[string]interface{}:
			vv, ok := t[el]
			if !ok {
				return nil, ErrorInvalidJSONPath
			}
			value = &vv
			ref = &vv
		case []interface{}:
			idx, err := strconv.Atoi(string(el))
			if err != nil {
				panic("unable to convert to integer")
			}
			if idx >= len(t) {
				return nil, ErrorInvalidJSONPath
			}
			vv := t[idx]
			value = &vv
			ref = &vv
		default:
			panic(fmt.Sprintf("unknown type %T\n", t))
		}
	}
	return
}

{'fraction_non_alphanumeric': 0.17813620071684588, 'fraction_numerical': 0.004480286738351254, 'mean_word_length': 3.429365079365079, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5745014', 'n_tokens_mistral': 2270, 'n_tokens_neox': 2061, 'n_words': 537}