<commit_before>package filter

import (
	"github.com/jamesmcminn/ir-tools"
	"log"
)

type StopWordFilter struct {
	stopwords map[string]bool
}

func NewStopWordFilter(stopwordFile string) *StopWordFilter {
	terms, err := irtools.ReadLines(stopwordFile)

	if err != nil {
		log.Fatal(err)
	}
	stopwords := map[string]bool{}

	for _, term := range terms {
		stopwords[term] = true
	}

	return &StopWordFilter{stopwords}
}

func (filter *StopWordFilter) Filter(input []string) (output []string) {
	for i := range input {
		if _, found := filter.stopwords[input[i]]; found == true {<commit_msg>Updated import path due to move to mirgit<commit_after>package filter

import (
	"log"
	"mirgit.dcs.gla.ac.uk/JamesMcMinn/ir-tools"
)

type StopWordFilter struct {
	stopwords map[string]bool
}

func NewStopWordFilter(stopwordFile string) *StopWordFilter {
	terms, err := irtools.ReadLines(stopwordFile)

	if err != nil {
		log.Fatal(err)
	}
	stopwords := map[string]bool{}

	for _, term := range terms {
		stopwords[term] = true
	}

	return &StopWordFilter{stopwords}
}

func (filter *StopWordFilter) Filter(input []string) (output []string) {
	for i := range input {
		if _, found := filter.stopwords[input[i]]; found == true {
{'fraction_non_alphanumeric': 0.13218390804597702, 'fraction_numerical': 0.0, 'mean_word_length': 4.777251184834123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1039850', 'n_tokens_mistral': 442, 'n_tokens_neox': 421, 'n_words': 101}