<commit_before>// Copyright 2017 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package felix

import "unicode"
import "strings"
import (
	"net/url"
	"regexp"

	"github.com/pkg/errors"
)

type ItemFilter interface {
	Filter(item Item, next func(Item))
}

// ItemFilterFunc is an adapter to allow the use of ordinary functions as filters.
// If f is a function with the appropriate signature, ItemFilterFunc(f) is a ItemFilter that calls f.
type ItemFilterFunc func(Item, func(Item))

// Filter calls the underlying ItemFilterFunc
func (f ItemFilterFunc) Filter(item Item, next func(Item)) {
	f(item, next)
}

// FilterItems should just filter until in-Channel is closed? Or is quit channel needed?
func FilterItems(in <-chan Item, out chan<- Item, filters ...ItemFilter) {

	// final filter chain output
	var final = func(item Item) {
		out <- item
	}

	chain := buildItemFilterChain(filters...)

	for item := range in {
		chain.Filter(item, final)
	}

	close(out)
}

func buildItemFilterChain(filters ...ItemFilter) ItemFilter {
	if len(filters) > 1 {
		return ItemFilterFunc(func(item Item, next func(Item)) {
			filters[0].Filter(item, func(item Item) {
				buildItemFilterChain(filters[1:]...).Filter(item, next)
			})
		})
	} else if len(filters) == 1 {
		return filters[0]
	} else {
		return ItemFilterFunc(func(item Item, next func(Item)) {
			next(item)
		})
	}
}

// ItemTitleFilter only accepts items where the title matches
// at least one of the given titles. (After conversion to lower case and
// stripping of all non-alphanumeric characters)
func ItemTitleFilter(titles ...string) ItemFilter {

	validTitles := make([][]string, 0, len(titles))
	for _, t := range titles {
		validTitles = append(validTitles, strings.Split(sanitizeTitle(t), " "))
	}

	return ItemFilterFunc(func(item Item, next func(Item)) {
		itemTitle := sanitizeTitle(item.Title)
		for _, title := range validTitles {
			found := true
			for _, tf := range title {
				if !strings.Contains(itemTitle, tf) {
					found = false
				}
			}
			if found {
				next(item)
				return
			}
		}
	})
}

// sanitizeTitle strips all non-alphanumeric characters from a string
// and converts it to lower case for easier comparison.
func sanitizeTitle(title string) string {

	t := make([]rune, 0, len(title))
	emitted := false
	skipped := false

	for _, r := range title {
		if unicode.IsDigit(r) || unicode.IsLetter(r) {
			if skipped && emitted {
				t = append(t, ' ')
			}
			t = append(t, unicode.ToLower(r))
			emitted = true
			skipped = false<commit_msg>Added optional Stringer interface for filters<commit_after>// Copyright 2017 <NAME>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package felix

import (
	"bytes"
	"fmt"
	"net/url"
	"regexp"
	"strings"
	"unicode"

	"github.com/pkg/errors"
)

// Stringer is an optional interface for all filters to provide a 'native' textual representation.
type Stringer interface {
	String() string
}

// FilterString return the concatenated string output of all passed filters that implement the Stringer interface.
func FilterString(itemFilters []ItemFilter, linkFilters []LinkFilter) string {
	var b bytes.Buffer

	for _, f := range itemFilters {
		if s, ok := f.(Stringer); ok {
			b.WriteString(s.String())
		}
	}

	for _, f := range linkFilters {
		if s, ok := f.(Stringer); ok {
			b.WriteString(s.String())
		}
	}

	return b.String()
}

type ItemFilter interface {
	Filter(item Item, next func(Item))
}

// ItemFilterFunc is an adapter to allow the use of ordinary functions as filters.
// If f is a function with the appropriate signature, ItemFilterFunc(f) is a ItemFilter that calls f.
type ItemFilterFunc func(Item, func(Item))

// Filter calls the underlying ItemFilterFunc
func (f ItemFilterFunc) Filter(item Item, next func(Item)) {
	f(item, next)
}

// internal helper type to provide an additional Stringer implementation
type itemFilter struct {
	ItemFilter
	s string
}

func (f itemFilter) String() string {
	return f.s
}

// FilterItems should just filter until in-Channel is closed? Or is quit channel needed?
func FilterItems(in <-chan Item, out chan<- Item, filters ...ItemFilter) {

	// final filter chain output
	var final = func(item Item) {
		out <- item
	}

	chain := buildItemFilterChain(filters...)

	for item := range in {
		chain.Filter(item, final)
	}

	close(out)
}

func buildItemFilterChain(filters ...ItemFilter) ItemFilter {
	if len(filters) > 1 {
		return ItemFilterFunc(func(item Item, next func(Item)) {
			filters[0].Filter(item, func(item Item) {
				buildItemFilterChain(filters[1:]...).Filter(item, next)
			})
		})
	} else if len(filters) == 1 {
		return filters[0]
	} else {
		return ItemFilterFunc(func(item Item, next func(Item)) {
			next(item)
		})
	}
}

// ItemTitleFilter only accepts items where the title matches
// at least one of the given titles. (After conversion to lower case and
// stripping of all non-alphanumeric characters)
func ItemTitleFilter(titles ...string) ItemFilter {

	validTitles := make([][]string, 0, len(titles))
	var b bytes.Buffer
	for _, t := range titles {
		validTitles = append(validTitles, strings.Split(sanitizeTitle(t), " "))
		fmt.Fprintf(&b, "ITEM_TITLE:%s\n", t)
	}

	filter := ItemFilterFunc(func(item Item, next func(Item)) {
		itemTitle := sanitizeTitle(item.Title)
		for _, title := range validTitles {
			found := true
			for _, tf := range title {
				if !strings.Contains(itemTitle, tf) {
					found = false
				}
			}
			if found {
				next(item)
				return
			}
		}
	})

	return itemFilter{filter, b.String()}
}

// sanitizeTitle strips all non-alphanumeric characters from a string
// and converts it to lower case for easier comparison.
func sanitizeTitle(title string) string {

	t := make([]rune, 0, len(title))
	emitted := false
	skipped := false

	for _, r := range title {
		if unicode.IsDigit(r) || unicode.IsLetter(r) {
			if skipped && emitted {
				t = append(t, ' ')
			}
			t = append(t, unicode.ToLower(r))
			emitted = true
			skipped = false
{'fraction_non_alphanumeric': 0.11818329806283574, 'fraction_numerical': 0.0035813120625101742, 'mean_word_length': 3.6439909297052155, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6925713', 'n_tokens_mistral': 2120, 'n_tokens_neox': 1930, 'n_words': 652}