<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

type RawPoint struct {
	String             string
	Pattern            string
	Stance             string
	OriginalStanceText string
	OriginalTopic      string
	Post               string
	Content            string
}

type Point struct {
	String             string
	Verb               string
	Stance             string
	OriginalStanceText string
	OriginalTopic      string
	Post               string
	Content            string `json:"-"`
	Components         []string
	Words              []string
	Relations          []string
}

func (p *Point) print() {
	fmt.Printf("(%v) : %v\n", strings.Join(p.Components, " "), p.String)
}

func (p *Point) componentString() string {
	return strings.Join(p.Components, " ")
}

func (p *Point) matches(p2 Point) bool {
	if p.Verb != p2.Verb {
		return false
	}
	return p.matchesWords(p2) || p.componentString() == p2.componentString()
}

func (p *Point) matchesWords(p2 Point) bool {
	return compare(p.Words, p2.Words) && compare(p2.Words, p.Words)
}

func compare(X, Y []string) bool {
	m := make(map[string]int)

	for _, y := range Y {
		m[y]++
	}

	var ret []string
	for _, x := range X {
		if m[x] > 0 {
			m[x]--
			continue
		}
		ret = append(ret, x)
	}

	return len(ret) == 0
}

func containsStr(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
func containsInt(s []int, e int) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func removeDuplicates(elements []int) []int {
	encountered := map[int]bool{}
	result := []int{}

	for v := range elements {
		if encountered[elements[v]] == true {
		} else {
			encountered[elements[v]] = true
			result = append(result, elements[v])
		}
	}
	return result
}

func pairs(arr []int) [][]int {
	var pairs [][]int
	for i := 0; i < len(arr); i++ {
		for j := i + 1; j < len(arr); j++ {
			pairs = append(pairs, []int{arr[i], arr[j]})
		}
	}
	return pairs
}

func upgradePoint(p RawPoint) Point {
	pattern := regexp.MustCompile("cop|pass").ReplaceAllString(p.Pattern, "")
	components := strings.Split(pattern, " ")
	var verb string
	var words []string
	var relations []string
	for _, v := range components {
		parts := strings.Split(v, ".")
		if parts[1] == "verb" {
			verb = parts[0]
		}
		words = append(words, parts[0])
		relations = append(relations, parts[1])
	}
	return Point{
		String:             p.String,
		Stance:             p.Stance,
		OriginalStanceText: p.OriginalStanceText,
		OriginalTopic:      p.OriginalTopic,
		Post:               p.Post,
		Content:            p.Content,
		Verb:               verb,
		Components:         components,
		Relations:          relations,
		Words:              words,
	}
}

type ByLen [][]Point

func (a ByLen) Len() int           { return len(a) }
func (a ByLen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }

func main() {
	b, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		panic(err)
	}

	points := []Point{}
	contents := strings.Split(string(b), "\n")
	for i, v := range contents {
		if i == 0 {
			fmt.Println(v)
			continue
		} else if len(v) < 4 {
			continue
		}
		var rawPoint = RawPoint{}
		json.Unmarshal([]byte(v[0:len(v)-1]), &rawPoint)
		points = append(points, upgradePoint(rawPoint))
	}

	bannedList := strings.Split("it.nsubj that.nsubj this.nsubj which.nsubj what.nsubj", " ")
	bannedPersonList := strings.Split("object continue come go sit open close begin end believe happen leave understand realize debate speak show stand call refer believe lose change care hear write disagree read tell start talk explain come live take support guess feel follow make go get move agree find fail feel ask argue try", " ")
	for i, v := range bannedPersonList {
		bannedPersonList[i] = fmt.Sprintf("%v.verb", v)
	}
	bannedComponentList := []string{
		"PERSON.nsubj be.verb correct.dobj",
		"PERSON.nsubj be.verb able.dobj",
		"PERSON.nsubj be.verb good.dobj",
		"PERSON.nsubj be.verb likely.dobj",
		"PERSON.nsubj be.verb sorry.dobj",
		"PERSON.nsubj be.verb say.dobj",
		"PERSON.nsubj be.verb aware.dobj",
		"PERSON.nsubj be.verb one.dobj",
		"PERSON.nsubj be.verb sure.dobj",
		"PERSON.nsubj be.verb wrong.dobj",
		"PERSON.nsubj be.verb glad.dobj",
		"PERSON.nsubj be.verb here.dobj",
		"PERSON.nsubj be.verb willing.dobj",
		"PERSON.nsubj be.verb right.dobj",
		"PERSON.nsubj be.verb true.dobj",
		"PERSON.nsubj be.verb false.dobj",
		"PERSON.nsubj be.verb favor.dobj",
		"PERSON.nsubj be.verb interested.dobj",
		"PERSON.nsubj want.verb have.xcomp",<commit_msg>Commenters are no longer allowed to think + wonder<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

type RawPoint struct {
	String             string
	Pattern            string
	Stance             string
	OriginalStanceText string
	OriginalTopic      string
	Post               string
	Content            string
}

type Point struct {
	String             string
	Verb               string
	Stance             string
	OriginalStanceText string
	OriginalTopic      string
	Post               string
	Content            string `json:"-"`
	Components         []string
	Words              []string
	Relations          []string
}

func (p *Point) print() {
	fmt.Printf("(%v) : %v\n", strings.Join(p.Components, " "), p.String)
}

func (p *Point) componentString() string {
	return strings.Join(p.Components, " ")
}

func (p *Point) matches(p2 Point) bool {
	if p.Verb != p2.Verb {
		return false
	}
	return p.matchesWords(p2) || p.componentString() == p2.componentString()
}

func (p *Point) matchesWords(p2 Point) bool {
	return compare(p.Words, p2.Words) && compare(p2.Words, p.Words)
}

func compare(X, Y []string) bool {
	m := make(map[string]int)

	for _, y := range Y {
		m[y]++
	}

	var ret []string
	for _, x := range X {
		if m[x] > 0 {
			m[x]--
			continue
		}
		ret = append(ret, x)
	}

	return len(ret) == 0
}

func containsStr(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}
func containsInt(s []int, e int) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}
	return false
}

func removeDuplicates(elements []int) []int {
	encountered := map[int]bool{}
	result := []int{}

	for v := range elements {
		if encountered[elements[v]] == true {
		} else {
			encountered[elements[v]] = true
			result = append(result, elements[v])
		}
	}
	return result
}

func pairs(arr []int) [][]int {
	var pairs [][]int
	for i := 0; i < len(arr); i++ {
		for j := i + 1; j < len(arr); j++ {
			pairs = append(pairs, []int{arr[i], arr[j]})
		}
	}
	return pairs
}

func upgradePoint(p RawPoint) Point {
	pattern := regexp.MustCompile("cop|pass").ReplaceAllString(p.Pattern, "")
	components := strings.Split(pattern, " ")
	var verb string
	var words []string
	var relations []string
	for _, v := range components {
		parts := strings.Split(v, ".")
		if parts[1] == "verb" {
			verb = parts[0]
		}
		words = append(words, parts[0])
		relations = append(relations, parts[1])
	}
	return Point{
		String:             p.String,
		Stance:             p.Stance,
		OriginalStanceText: p.OriginalStanceText,
		OriginalTopic:      p.OriginalTopic,
		Post:               p.Post,
		Content:            p.Content,
		Verb:               verb,
		Components:         components,
		Relations:          relations,
		Words:              words,
	}
}

type ByLen [][]Point

func (a ByLen) Len() int           { return len(a) }
func (a ByLen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }

func main() {
	b, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		panic(err)
	}

	points := []Point{}
	contents := strings.Split(string(b), "\n")
	for i, v := range contents {
		if i == 0 {
			fmt.Println(v)
			continue
		} else if len(v) < 4 {
			continue
		}
		var rawPoint = RawPoint{}
		json.Unmarshal([]byte(v[0:len(v)-1]), &rawPoint)
		points = append(points, upgradePoint(rawPoint))
	}

	bannedList := strings.Split("it.nsubj that.nsubj this.nsubj which.nsubj what.nsubj", " ")
	bannedPersonList := strings.Split("object continue come go sit open close begin end believe happen leave understand realize debate speak show stand call refer believe lose change care hear write disagree read tell start talk explain come live take support guess feel follow make go get move agree find fail think wonder feel ask argue try", " ")
	for i, v := range bannedPersonList {
		bannedPersonList[i] = fmt.Sprintf("%v.verb", v)
	}
	bannedComponentList := []string{
		"PERSON.nsubj be.verb correct.dobj",
		"PERSON.nsubj be.verb able.dobj",
		"PERSON.nsubj be.verb good.dobj",
		"PERSON.nsubj be.verb likely.dobj",
		"PERSON.nsubj be.verb sorry.dobj",
		"PERSON.nsubj be.verb say.dobj",
		"PERSON.nsubj be.verb aware.dobj",
		"PERSON.nsubj be.verb one.dobj",
		"PERSON.nsubj be.verb sure.dobj",
		"PERSON.nsubj be.verb wrong.dobj",
		"PERSON.nsubj be.verb glad.dobj",
		"PERSON.nsubj be.verb here.dobj",
		"PERSON.nsubj be.verb willing.dobj",
		"PERSON.nsubj be.verb right.dobj",
		"PERSON.nsubj be.verb true.dobj",
		"PERSON.nsubj be.verb false.dobj",
		"PERSON.nsubj be.verb favor.dobj",
		"PERSON.nsubj be.verb interested.dobj",
		"PERSON.nsubj want.verb have.xcomp",
{'fraction_non_alphanumeric': 0.14354219948849106, 'fraction_numerical': 0.004262574595055414, 'mean_word_length': 2.8542094455852154, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '65262', 'n_tokens_mistral': 3602, 'n_tokens_neox': 3393, 'n_words': 910}