<commit_before>package tree

import (
	"errors"
	"fmt"
	"sort"
)

var (
	ErrBadPath = errors.New("invalid path")
)

// superset of Marker
type Node interface {
	Start() (int64, int64) // line, col
	End()   (int64, int64)
	Get(path []interface{}) (Node, error)
	GetMarker() Marker
	pos() []*Pos // just used for iterating through the markers to fill in line and column from index
}

func FixLineColumn(n Node, source []byte) {
	p := n.pos()
	sort.Slice(p, func(i, j int) bool {
		return p[i].Index < p[j].Index
	})
	pi := 0
	line, col := int64(1), int64(1)
	for i, c := range source {
		if int64(i) == p[pi].Index {
			p[pi].Line = line
			p[pi].Column = col
			pi++
		}
		col++
		if c == '\n' {
			line++
			col = 1
		}
	}
}

// Key is used to differentiate leaves describing the start and end of where
// a key starts and where a value starts.
type Key string

type Pos struct {
	Index  int64
	Line   int64
	Column int64
}

func posString(p *Pos) string {
	if p == nil {
		return ""
	}
	return fmt.Sprintf("line %d col %d", p.Line, p.Column)
}

func posLC(p *Pos) (int64, int64) {
	if p == nil {
		return 0, 0
	}
	return p.Line, p.Column
}

// IndexMarkers are composed of information regarding the start and
// end of where a Node exists in its source. Depending on where the marker
// came from (e.g. json or yaml) the *Idx or End* fields may not be populated.
type Marker struct {
	StartP *Pos
	EndP   *Pos
}

func (m Marker) Start() (int64, int64) {
	return posLC(m.StartP)
}

func (m Marker) End() (int64, int64) {
	return posLC(m.EndP)
}

func (m Marker) String() string {
	// Just do start for now, figure out end later
	return posString(m.StartP)
}

func (marker Marker) GetMarker() Marker {
	return marker
}

func MarkerFromIndices(start, end int64) Marker {
	return Marker{
		StartP: &Pos{Index: start},
		EndP:   &Pos{Index: end},
	}
}

func appendPos(l []*Pos, p *Pos) []*Pos {
	if p != nil {
		return append(l, p)
	}
	return l
}

type MapNode struct {
	Marker
	Children map[string]Node
	Keys     map[string]Leaf
}

func (m MapNode) Get(path []interface{}) (Node, error) {
	if len(path) == 0 {
		return m, nil
	}
	switch p := path[0].(type) {
	case string:
		if r, ok := m.Children[p]; ok {
			return r.Get(path[1:])
		} else {
			return nil, ErrBadPath
		}
	case Key:
		if r, ok := m.Keys[string(p)]; ok {
			return r.Get(path[1:])
		} else {
			return nil, ErrBadPath<commit_msg>tree: don't add negative offsets, fix index oob<commit_after>package tree

import (
	"errors"
	"fmt"
	"sort"
)

var (
	ErrBadPath = errors.New("invalid path")
)

// superset of Marker
type Node interface {
	Start() (int64, int64) // line, col
	End()   (int64, int64)
	Get(path []interface{}) (Node, error)
	GetMarker() Marker
	pos() []*Pos // just used for iterating through the markers to fill in line and column from index
}

func FixLineColumn(n Node, source []byte) {
	p := n.pos()
	sort.Slice(p, func(i, j int) bool {
		return p[i].Index < p[j].Index
	})
	pi := 0
	line, col := int64(1), int64(1)
	for i, c := range source {
		if pi == len(p) {
			return
		}
		if int64(i) == p[pi].Index {
			p[pi].Line = line
			p[pi].Column = col
			pi++
		}
		col++
		if c == '\n' {
			line++
			col = 1
		}
	}
}

// Key is used to differentiate leaves describing the start and end of where
// a key starts and where a value starts.
type Key string

type Pos struct {
	Index  int64
	Line   int64
	Column int64
}

func posString(p *Pos) string {
	if p == nil {
		return ""
	}
	return fmt.Sprintf("line %d col %d", p.Line, p.Column)
}

func posLC(p *Pos) (int64, int64) {
	if p == nil {
		return 0, 0
	}
	return p.Line, p.Column
}

// IndexMarkers are composed of information regarding the start and
// end of where a Node exists in its source. Depending on where the marker
// came from (e.g. json or yaml) the *Idx or End* fields may not be populated.
type Marker struct {
	StartP *Pos
	EndP   *Pos
}

func (m Marker) Start() (int64, int64) {
	return posLC(m.StartP)
}

func (m Marker) End() (int64, int64) {
	return posLC(m.EndP)
}

func (m Marker) String() string {
	// Just do start for now, figure out end later
	return posString(m.StartP)
}

func (marker Marker) GetMarker() Marker {
	return marker
}

func MarkerFromIndices(start, end int64) Marker {
	m := Marker{}
	if start >= 0 {
		m.StartP = &Pos{Index: start}
	}
	if end >= 0 {
		m.EndP = &Pos{Index: end}
	}
	return m
}

func appendPos(l []*Pos, p *Pos) []*Pos {
	if p != nil {
		return append(l, p)
	}
	return l
}

type MapNode struct {
	Marker
	Children map[string]Node
	Keys     map[string]Leaf
}

func (m MapNode) Get(path []interface{}) (Node, error) {
	if len(path) == 0 {
		return m, nil
	}
	switch p := path[0].(type) {
	case string:
		if r, ok := m.Children[p]; ok {
			return r.Get(path[1:])
		} else {
			return nil, ErrBadPath
		}
	case Key:
		if r, ok := m.Keys[string(p)]; ok {
			return r.Get(path[1:])
		} else {
			return nil, ErrBadPath
{'fraction_non_alphanumeric': 0.1438188494492044, 'fraction_numerical': 0.01835985312117503, 'mean_word_length': 3.113255033557047, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893567', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1888, 'n_words': 640}