<commit_before>package core

import (
	"encoding/json"
	"testing"

	"github.com/coreos/etcd/client"
	"github.com/stretchr/testify/suite"
)

func TestObject(t *testing.T) {
	suite.Run(t, new(objectTestSuite))
}

type objectTestSuite struct {
	suite.Suite
}

func (s *objectTestSuite) TestMarshal() {
	v := 123
	str, err := marshaller(v)
	s.NoError(err)
	s.Equal(str, "123")
}

func (s *objectTestSuite) TestUnmarshal() {
	str := `"123"`
	var v interface{}
	err := unmarshaller(str, &v)
	s.NoError(err)

	var exp interface{} = "123"
	s.Equal(v, exp)
}

func (s *objectTestSuite) TestSubkey() {
	skey, err := subkey("/aaa/bbb", "/aaa/bbb/ccc")
	s.NoError(err)
	s.Equal(skey, "ccc")

	skey, err = subkey("/aaa/bbb", "/aaa/bbb/ccc/ddd")
	s.NoError(err)
	s.Equal(skey, "ccc/ddd")

	skey, err = subkey("/aaa/bbb", "/aaa/bbb")
	s.NoError(err)
	s.Equal(skey, "")

	skey, err = subkey("/aaa/bbb", "/aaa/bbb/")
	s.NoError(err)
	s.Equal(skey, "")

	_, err = subkey("/xxx/bbb", "/aaa/bbb/ccc")
	s.Error(err)
}

func (s *objectTestSuite) TestToValue() {
	rawJSON := `{
		"key": <KEY>",
		"dir": true,
		"modifiedIndex": 123,
		"nodes": [
			{
				"key": <KEY>",
				"dir": false,
				"modifiedIndex": 123,
				"value": "111"
			},
			{
				"key": <KEY>",
				"dir": false,
				"modifiedIndex": 456,
				"value": "222"
			},
			{
				"key": <KEY>",
				"dir": true,
				"modifiedIndex": 333,
				"nodes": [
					{
						"key": "/aaa/bbb/eee/fff",
						"dir": false,
						"modifiedIndex": 373,
						"value": "333"
					}				
				]
			}
		]
	}`

	var root *client.Node
	s.NoError(json.Unmarshal([]byte(rawJSON), &root))

	result, version, err := toValue(root, unmarshaller)
	s.NoError(err)
	s.Equal(version, uint64(456))
	s.Equal(result, map[string]interface{}{
		"ccc": float64(111),
		"ddd": float64(222),
		"eee": map[string]interface{}{
			"fff": float64(333),
		},
	})
}

func (s *objectTestSuite) TestToValueError() {
	rawJSON := `{
		"key": <KEY>",
		"dir": true,
		"modifiedIndex": 123,
		"nodes": [
			{
				"key": <KEY>",
				"dir": false,
				"modifiedIndex": 123,
				"value": "111"
			}
		]
	}`

	var root *client.Node
	s.NoError(json.Unmarshal([]byte(rawJSON), &root))

	_, _, err := toValue(root, unmarshaller)
	s.Error(err)
}

func (s *objectTestSuite) TestSetByPath() {
	var err error
	var v interface{} = map[string]interface{}{
		"ccc": 111,
		"ddd": 222,
		"eee": map[string]interface{}{
			"fff": 333,
		},
	}
	var exp interface{} = map[string]interface{}{
		"aaa": map[string]interface{}{
			"bbb": 10,
		},
		"ccc": 111,
		"ddd": 222,
		"eee": map[string]interface{}{
			"fff": 333,
			"ggg": 15,
		},
		"uuu": 33,
	}

	v, err = setByPath(v, "uuu", 33)
	s.NoError(err)
	v, err = setByPath(v, "aaa/bbb", 10)
	s.NoError(err)
	v, err = setByPath(v, "eee/ggg", 15)
	s.NoError(err)
	s.Equal(v, exp)

	// cannot convert int to map
	v, err = setByPath(v, "ddd/kkk", 15)
	s.Error(err)
	// v still ok
	s.Equal(v, exp)

	// replace root
	v, err = setByPath(v, "", 60)
	s.Equal(v, 60)
}<commit_msg>address comment<commit_after>package core

import (
	"encoding/json"
	"testing"

	"github.com/coreos/etcd/client"
	"github.com/stretchr/testify/suite"
)

func TestObject(t *testing.T) {
	suite.Run(t, new(objectTestSuite))
}

type objectTestSuite struct {
	suite.Suite
}

func (s *objectTestSuite) TestMarshal() {
	v := 123
	str, err := marshaller(v)
	s.NoError(err)
	s.Equal(str, "123")
}

func (s *objectTestSuite) TestUnmarshal() {
	str := `"123"`
	var v interface{}
	err := unmarshaller(str, &v)
	s.NoError(err)

	var exp interface{} = "123"
	s.Equal(v, exp)
}

func (s *objectTestSuite) TestSubkey() {
	skey, err := subkey("/aaa/bbb", "/aaa/bbb/ccc")
	s.NoError(err)
	s.Equal(skey, "ccc")

	skey, err = subkey("/aaa/bbb", "/aaa/bbb/ccc/ddd")
	s.NoError(err)
	s.Equal(skey, "ccc/ddd")

	skey, err = subkey("/aaa/bbb", "/aaa/bbb")
	s.NoError(err)
	s.Equal(skey, "")

	skey, err = subkey("/aaa/bbb", "/aaa/bbb/")
	s.NoError(err)
	s.Equal(skey, "")

	_, err = subkey("/xxx/bbb", "/aaa/bbb/ccc")
	s.Error(err)
}

func (s *objectTestSuite) TestToValue() {
	rawJSON := `{
		"key": <KEY>",
		"dir": true,
		"modifiedIndex": 123,
		"nodes": [
			{
				"key": "/<KEY>",
				"dir": false,
				"modifiedIndex": 123,
				"value": "111"
			},
			{
				"key": "/<KEY>",
				"dir": false,
				"modifiedIndex": 456,
				"value": "222"
			},
			{
				"key": "/<KEY>",
				"dir": true,
				"modifiedIndex": 333,
				"nodes": [
					{
						"key": "/aaa/bbb/eee/fff",
						"dir": false,
						"modifiedIndex": 373,
						"value": "333"
					}				
				]
			}
		]
	}`

	var root *client.Node
	s.NoError(json.Unmarshal([]byte(rawJSON), &root))

	result, version, err := toValue(root, unmarshaller)
	s.NoError(err)
	s.Equal(version, uint64(456))
	s.Equal(result, map[string]interface{}{
		"ccc": float64(111),
		"ddd": float64(222),
		"eee": map[string]interface{}{
			"fff": float64(333),
		},
	})
}

func (s *objectTestSuite) TestToValueError() {
	rawJSON := `{
		"key": <KEY>",
		"dir": true,
		"modifiedIndex": 123,
		"nodes": [
			{
				"key": <KEY>",
				"dir": false,
				"modifiedIndex": 123,
				"value": "111"
			}
		]
	}`

	var root *client.Node
	s.NoError(json.Unmarshal([]byte(rawJSON), &root))

	_, _, err := toValue(root, unmarshaller)
	s.Error(err)
}

func (s *objectTestSuite) TestSetByPath() {
	var err error
	var v interface{} = map[string]interface{}{
		"ccc": 111,
		"ddd": 222,
		"eee": map[string]interface{}{
			"fff": 333,
		},
	}
	var exp interface{} = map[string]interface{}{
		"aaa": map[string]interface{}{
			"bbb": 10,
		},
		"ccc": 111,
		"ddd": 222,
		"eee": map[string]interface{}{
			"fff": 333,
			"ggg": 15,
		},
		"uuu": 33,
	}

	v, err = setByPath(v, "uuu", 33)
	s.NoError(err)
	v, err = setByPath(v, "aaa/bbb", 10)
	s.NoError(err)
	v, err = setByPath(v, "eee/ggg", 15)
	s.NoError(err)
	s.Equal(v, exp)

	// cannot convert int to map
	v, err = setByPath(v, "ddd/kkk", 15)
	s.Error(err)
	// v still ok
	s.Equal(v, exp)

	// replace root
	v, err = setByPath(v, "", 60)
	s.NoError(err)
	s.Equal(v, 60)
}
{'fraction_non_alphanumeric': 0.2169215425531915, 'fraction_numerical': 0.03357712765957447, 'mean_word_length': 3.297857142857143, 'pattern_counts': {'":': 88, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '119129', 'n_tokens_mistral': 3096, 'n_tokens_neox': 2592, 'n_words': 522}