<commit_before>// Copyright 2015 <NAME>, OMRF. All rights reserved.
// Use of this code is governed by a 3 clause BSD style license
// Full license details in LICENSE file distributed with this software

package matcher

import "math"

//A Pair is the basic thing that makes up a match
type Pair struct {
	a string
	b string
}

//NewPair creates a new pairing between to items
func NewPair(a, b string) Pair {
	return Pair{a: a, b: b}
}

//Eql returns true if the two pairs are the same
//A Pair is the same regardless of the order of the elements
func (a Pair) Eql(b Pair) bool {
	return (a.a == b.a || a.a == b.b) && (a.b == b.a || a.b == b.b)
}

type matches []string

func (m matches) IndexOf(t string) int {
	for i, v := range m {
		if t == v {
			return i
		}
	}
	return -1
}

//MatchSet represents a collection of Pairs more or less
type MatchSet struct {
	pairs map[string]matches
}

//Copy returns a new copy of the MatchSet
func (m *MatchSet) Copy() (n MatchSet) {
	n = NewMatchSet()
	for k, v := range m.pairs {
		n.pairs[k] = make([]string, len(v))
		copy(n.pairs[k], v)
	}
	return
}

//NewMatchSet creates a new MatchSet collection
func NewMatchSet() MatchSet {
	return MatchSet{make(map[string]matches)}
}

//AddPair adds a new pair of matched items to the collection
func (m *MatchSet) AddPair(p Pair) {
	m.pairs[p.a] = append(m.pairs[p.a], p.b)
	m.pairs[p.b] = append(m.pairs[p.b], p.a)
}

//RemovePair takes a pair of matched items out of the collection if its there
func (m *MatchSet) RemovePair(p Pair) {
	m.delete(p.a, p.b)
	m.delete(p.b, p.a)
}

//Purge takes a single ID, then removes any and all Pairs that contain at least half of it
func (m *MatchSet) Purge(i string) {
	if v, ok := m.pairs[i]; ok {
		for _, b := range v {
			m.RemovePair(NewPair(i, b))
		}
	}
}

func (m *MatchSet) delete(a, b string) {
	if v, ok := m.pairs[a]; ok {
		i := v.IndexOf(b)
		if i >= 0 {
			m.pairs[a] = append(v[:i], v[i+1:]...)
			if len(m.pairs[a]) <= 0 {
				delete(m.pairs, a)
			}
		}
	}
}

//NumPairs returns the number of total pairs/matches in this collection
func (m *MatchSet) NumPairs() (l int) {
	for _, v := range m.pairs {<commit_msg>Fix bug in Purge of leaving elements<commit_after>// Copyright 2015 <NAME>, OMRF. All rights reserved.
// Use of this code is governed by a 3 clause BSD style license
// Full license details in LICENSE file distributed with this software

package matcher

import "math"

//A Pair is the basic thing that makes up a match
type Pair struct {
	a string
	b string
}

//NewPair creates a new pairing between to items
func NewPair(a, b string) Pair {
	return Pair{a: a, b: b}
}

//Eql returns true if the two pairs are the same
//A Pair is the same regardless of the order of the elements
func (a Pair) Eql(b Pair) bool {
	return (a.a == b.a || a.a == b.b) && (a.b == b.a || a.b == b.b)
}

type matches []string

func (m matches) IndexOf(t string) int {
	for i, v := range m {
		if t == v {
			return i
		}
	}
	return -1
}

//MatchSet represents a collection of Pairs more or less
type MatchSet struct {
	pairs map[string]matches
}

//Copy returns a new copy of the MatchSet
func (m *MatchSet) Copy() (n MatchSet) {
	n = NewMatchSet()
	for k, v := range m.pairs {
		n.pairs[k] = make([]string, len(v))
		copy(n.pairs[k], v)
	}
	return
}

//NewMatchSet creates a new MatchSet collection
func NewMatchSet() MatchSet {
	return MatchSet{make(map[string]matches)}
}

//AddPair adds a new pair of matched items to the collection
func (m *MatchSet) AddPair(p Pair) {
	m.pairs[p.a] = append(m.pairs[p.a], p.b)
	m.pairs[p.b] = append(m.pairs[p.b], p.a)
}

//RemovePair takes a pair of matched items out of the collection if its there
func (m *MatchSet) RemovePair(p Pair) {
	m.delete(p.a, p.b)
	m.delete(p.b, p.a)
}

//Purge takes a single ID, then removes any and all Pairs that contain at least half of it
func (m *MatchSet) Purge(i string) {
	if v, ok := m.pairs[i]; ok {
		for _, b := range v {
			//defer used, since RemovePair will alter the slice v & its range
			//this can lead to items being NOT purged, with defer we make sure
			//we loop through all the items. Defer used in place of copying v for "fun"
			defer m.RemovePair(NewPair(i, b))
		}
	}
}

func (m *MatchSet) delete(a, b string) {
	if v, ok := m.pairs[a]; ok {
		i := v.IndexOf(b)
		if i >= 0 {
			m.pairs[a] = append(v[:i], v[i+1:]...)
			if len(m.pairs[a]) <= 0 {
				delete(m.pairs, a)
			}
		}
	}
}

//NumPairs returns the number of total pairs/matches in this collection
func (m *MatchSet) NumPairs() (l int) {
	for _, v := range m.pairs {
{'fraction_non_alphanumeric': 0.1338012722088177, 'fraction_numerical': 0.0039482342618995395, 'mean_word_length': 3.18348623853211, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3992543', 'n_tokens_mistral': 1784, 'n_tokens_neox': 1691, 'n_words': 654}