<commit_before>package types

import (
	"fmt"
	"log"
	"reflect"
	"sort"
	"strings"
	"yap/alg"
	"yap/alg/graph"
	"yap/util"
)

type Morpheme struct {
	graph.BasicDirectedEdge
	Form       string
	Lemma      string
	CPOS       string
	POS        string
	Features   map[string]string
	TokenID    int
	FeatureStr string
}

type EMorpheme struct {
	Morpheme
	EForm, ELemma    int
	EFCPOS, EPOS     int
	EFeatures        int
	EMHost, EMSuffix int
}

var _ DepNode = &Morpheme{}
var _ DepNode = &EMorpheme{}

func NewRootMorpheme() *EMorpheme {
	return &EMorpheme{Morpheme: Morpheme{
		graph.BasicDirectedEdge{0, 0, 0},
		ROOT_TOKEN, ROOT_TOKEN, ROOT_TOKEN, ROOT_TOKEN,
		nil, 0, "",
	}}
}

func (m *Morpheme) ID() int {
	return m.BasicDirectedEdge.ID()
}

func (m *Morpheme) From() int {
	return m.BasicDirectedEdge.From()
}

func (m *Morpheme) To() int {
	return m.BasicDirectedEdge.To()
}

func (m *Morpheme) StringNoLemma() string {
	return fmt.Sprintf("%v-%v-%v-%s", m.Form, m.CPOS, m.POS, m.FeatureStr)
}

func (m *Morpheme) String() string {
	return fmt.Sprintf("%v-%v-%v-%v-%s-(%v->%v)", m.Form, m.Lemma, m.CPOS, m.POS, m.FeatureStr, m.From(), m.To())
}

func (m *Morpheme) Copy() *Morpheme {
	newMorph := new(Morpheme)
	*newMorph = *m
	newMorph.Features = make(map[string]string)
	for k, v := range m.Features {
		newMorph.Features[k] = v
	}
	return newMorph
}

func (m *Morpheme) EMorpheme() *EMorpheme {
	newMorph := new(Morpheme)
	*newMorph = *m
	newMorph.Features = make(map[string]string)
	for k, v := range m.Features {
		newMorph.Features[k] = v
	}
	return &EMorpheme{Morpheme: *newMorph}
}

func (m *Morpheme) Equal(otherEq util.Equaler) bool {
	other := otherEq.(*Morpheme)
	featEq := (len(m.Features) == 0 && len(other.Features) == 0) || reflect.DeepEqual(m.Features, other.Features)
	return m.Form == other.Form &&
		m.Lemma == other.Lemma &&
		m.CPOS == other.CPOS &&
		m.POS == other.POS &&
		featEq && m.FeatureStr == other.FeatureStr
}

func (m *EMorpheme) Equal(otherEq util.Equaler) bool {
	other := otherEq.(*EMorpheme)
	return (&m.Morpheme).Equal(&other.Morpheme)
	// return m.EForm == other.EForm &&
	// 	m.Lemma == other.Lemma &&
	// 	m.EPOS == other.EPOS &&
	// 	m.EFCPOS == other.EFCPOS &&
	// 	m.EFeatures == other.EFeatures
}

func (m *EMorpheme) Copy() *EMorpheme {
	newMorph := new(EMorpheme)
	*newMorph = *m
	newMorph.Features = make(map[string]string)
	for k, v := range m.Features {
		newMorph.Features[k] = v
	}
	return newMorph
}

var _ graph.DirectedEdge = &Morpheme{}
var _ graph.DirectedEdge = &EMorpheme{}

type Morphemes []*EMorpheme
type BasicMorphemes []*Morpheme

var _ alg.Index = make(Morphemes, 1)

func (m Morphemes) Len() int {
	return len(m)
}

func (m Morphemes) Less(i, j int) bool {
	return m[i].From() < m[j].From() ||
		(m[i].From() == m[j].From() && m[i].To() < m[j].To())
}

func (m Morphemes) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
	m[i].BasicDirectedEdge[0], m[j].BasicDirectedEdge[0] = m[j].BasicDirectedEdge[0], m[i].BasicDirectedEdge[0]
}

func (m Morphemes) Index(index int) (int, bool) {
	if index >= len(m) {
		return 0, false
	}
	return len(m) - 1 - index, true
}

func (m Morphemes) AsBasic() BasicMorphemes {
	retval := make([]*Morpheme, len(m))
	for i, val := range m {
		retval[i] = &val.Morpheme
	}
	return BasicMorphemes(retval)
}

func (m *Morphemes) Merge(new *EMorpheme) {
	for _, morph := range *m {
		if morph.Equal(new) && new.From() == morph.From() {
			return
		}
	}
	*m = append(*m, new)
}

func (m *BasicMorphemes) Union(others BasicMorphemes) {
	if len(others) != 1 {
		panic("Can't Union with another morpheme set with size other than 1")
	}
	other := others[0]
	for _, cur := range *m {
		if cur.Equal(other) {
			return
		}
	}
	other.BasicDirectedEdge[0] = len(*m)
	*m = append(*m, other)
}

func (m *BasicMorphemes) Equal(others BasicMorphemes) bool {
	for i, cur := range *m {
		if !cur.Equal(others[i]) {
			return false
		}
	}
	return true
}

func (m Morphemes) Standalone() BasicMorphemes {
	// TODO: think of a better name - should mean 'retrieve the
	// raw morphemes, as if they appear by themselves'
	morphs := make(BasicMorphemes, len(m))
	for i, curMorph := range m {
		newMorph := new(Morpheme)
		*newMorph = curMorph.Morpheme
		newMorph.BasicDirectedEdge = [3]int{i, i, i + 1}
		newMorph.TokenID = 0
		morphs[i] = newMorph
	}<commit_msg>Verify array length equality to avoid out-of-range bug<commit_after>package types

import (
	"fmt"
	"log"
	"reflect"
	"sort"
	"strings"
	"yap/alg"
	"yap/alg/graph"
	"yap/util"
)

type Morpheme struct {
	graph.BasicDirectedEdge
	Form       string
	Lemma      string
	CPOS       string
	POS        string
	Features   map[string]string
	TokenID    int
	FeatureStr string
}

type EMorpheme struct {
	Morpheme
	EForm, ELemma    int
	EFCPOS, EPOS     int
	EFeatures        int
	EMHost, EMSuffix int
}

var _ DepNode = &Morpheme{}
var _ DepNode = &EMorpheme{}

func NewRootMorpheme() *EMorpheme {
	return &EMorpheme{Morpheme: Morpheme{
		graph.BasicDirectedEdge{0, 0, 0},
		ROOT_TOKEN, ROOT_TOKEN, ROOT_TOKEN, ROOT_TOKEN,
		nil, 0, "",
	}}
}

func (m *Morpheme) ID() int {
	return m.BasicDirectedEdge.ID()
}

func (m *Morpheme) From() int {
	return m.BasicDirectedEdge.From()
}

func (m *Morpheme) To() int {
	return m.BasicDirectedEdge.To()
}

func (m *Morpheme) StringNoLemma() string {
	return fmt.Sprintf("%v-%v-%v-%s", m.Form, m.CPOS, m.POS, m.FeatureStr)
}

func (m *Morpheme) String() string {
	return fmt.Sprintf("%v-%v-%v-%v-%s-(%v->%v)", m.Form, m.Lemma, m.CPOS, m.POS, m.FeatureStr, m.From(), m.To())
}

func (m *Morpheme) Copy() *Morpheme {
	newMorph := new(Morpheme)
	*newMorph = *m
	newMorph.Features = make(map[string]string)
	for k, v := range m.Features {
		newMorph.Features[k] = v
	}
	return newMorph
}

func (m *Morpheme) EMorpheme() *EMorpheme {
	newMorph := new(Morpheme)
	*newMorph = *m
	newMorph.Features = make(map[string]string)
	for k, v := range m.Features {
		newMorph.Features[k] = v
	}
	return &EMorpheme{Morpheme: *newMorph}
}

func (m *Morpheme) Equal(otherEq util.Equaler) bool {
	other := otherEq.(*Morpheme)
	featEq := (len(m.Features) == 0 && len(other.Features) == 0) || reflect.DeepEqual(m.Features, other.Features)
	return m.Form == other.Form &&
		m.Lemma == other.Lemma &&
		m.CPOS == other.CPOS &&
		m.POS == other.POS &&
		featEq && m.FeatureStr == other.FeatureStr
}

func (m *EMorpheme) Equal(otherEq util.Equaler) bool {
	other := otherEq.(*EMorpheme)
	return (&m.Morpheme).Equal(&other.Morpheme)
	// return m.EForm == other.EForm &&
	// 	m.Lemma == other.Lemma &&
	// 	m.EPOS == other.EPOS &&
	// 	m.EFCPOS == other.EFCPOS &&
	// 	m.EFeatures == other.EFeatures
}

func (m *EMorpheme) Copy() *EMorpheme {
	newMorph := new(EMorpheme)
	*newMorph = *m
	newMorph.Features = make(map[string]string)
	for k, v := range m.Features {
		newMorph.Features[k] = v
	}
	return newMorph
}

var _ graph.DirectedEdge = &Morpheme{}
var _ graph.DirectedEdge = &EMorpheme{}

type Morphemes []*EMorpheme
type BasicMorphemes []*Morpheme

var _ alg.Index = make(Morphemes, 1)

func (m Morphemes) Len() int {
	return len(m)
}

func (m Morphemes) Less(i, j int) bool {
	return m[i].From() < m[j].From() ||
		(m[i].From() == m[j].From() && m[i].To() < m[j].To())
}

func (m Morphemes) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
	m[i].BasicDirectedEdge[0], m[j].BasicDirectedEdge[0] = m[j].BasicDirectedEdge[0], m[i].BasicDirectedEdge[0]
}

func (m Morphemes) Index(index int) (int, bool) {
	if index >= len(m) {
		return 0, false
	}
	return len(m) - 1 - index, true
}

func (m Morphemes) AsBasic() BasicMorphemes {
	retval := make([]*Morpheme, len(m))
	for i, val := range m {
		retval[i] = &val.Morpheme
	}
	return BasicMorphemes(retval)
}

func (m *Morphemes) Merge(new *EMorpheme) {
	for _, morph := range *m {
		if morph.Equal(new) && new.From() == morph.From() {
			return
		}
	}
	*m = append(*m, new)
}

func (m *BasicMorphemes) Union(others BasicMorphemes) {
	if len(others) != 1 {
		panic("Can't Union with another morpheme set with size other than 1")
	}
	other := others[0]
	for _, cur := range *m {
		if cur.Equal(other) {
			return
		}
	}
	other.BasicDirectedEdge[0] = len(*m)
	*m = append(*m, other)
}

func (m *BasicMorphemes) Equal(others BasicMorphemes) bool {
	if len(*m) != len(others) {
		return false
	}
	for i, cur := range *m {
		if !cur.Equal(others[i]) {
			return false
		}
	}
	return true
}

func (m Morphemes) Standalone() BasicMorphemes {
	// TODO: think of a better name - should mean 'retrieve the
	// raw morphemes, as if they appear by themselves'
	morphs := make(BasicMorphemes, len(m))
	for i, curMorph := range m {
		newMorph := new(Morpheme)
		*newMorph = curMorph.Morpheme
		newMorph.BasicDirectedEdge = [3]int{i, i, i + 1}
		newMorph.TokenID = 0
		morphs[i] = newMorph
	}
{'fraction_non_alphanumeric': 0.1649543378995434, 'fraction_numerical': 0.0045662100456621, 'mean_word_length': 3.7356756756756755, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412243', 'n_tokens_mistral': 3853, 'n_tokens_neox': 3679, 'n_words': 846}