<commit_before>// This file defines all filters and convenience functions for filters.
// Should be functionally equivalent to stable gobgpdump
// Current Filter options:
// -Source AS number (NewASFilter("1,2,3", true))
// -Destination AS number (NewASFilter("1,2,3", false))

// TODO
// -Maybe a function to return an array of filters based on config
//	options?
// -Time filter
// -Filters based on other attributes (peers, prefixes seen, etc.)
package gobgpdump

import (
	"fmt"
	mrt "github.com/CSUNetSec/protoparse/protocol/mrt"
	radix "github.com/armon/go-radix"
	"net"
	"strconv"
	"strings"
)

type Filter func(mbs *mrt.MrtBufferStack) bool

type PrefixFilter struct {
	prefixes []string
	rt       *radix.Tree
}

func NewPrefixFilter(raw string) Filter {
	pf := PrefixFilter{}
	pf.rt = radix.New()
	prefstrings := strings.Split(raw, ",")
	for _, p := range prefstrings {
		parts := strings.Split(p, "/")
		if len(parts) != 2 {
			panic("malformed prefix string")
		}
		mask, err := maskstr2uint8(parts[1])
		if err != nil {
			panic(fmt.Sprintf("error parsing mask:%s err:%s", parts[1], err))
		}
		pf.rt.Insert(IpToRadixkey(net.ParseIP(parts[0]).To4(), mask), true)
	}
	pf.prefixes = prefstrings
	return pf.filterBySeen
}

func (pf PrefixFilter) filterBySeen(mbs *mrt.MrtBufferStack) bool {
	advPrefs, err := getAdvertizedPrefixes(mbs)
	if err == nil {
		for _, pref := range advPrefs {
			if _, _, found := pf.rt.LongestPrefix(IpToRadixkey(pref.IP, pref.Mask)); found {
				return true
			}
		}
	}

	wdnPrefs, err := getWithdrawnPrefixes(mbs)
	if err == nil {
		for _, pref := range wdnPrefs {
			if _, _, found := pf.rt.LongestPrefix(IpToRadixkey(pref.IP, pref.Mask)); found {
				return true
			}
		}
	}
	return false
}

func (pf PrefixFilter) matchesOne(pref string) bool {
	for _, comp := range pf.prefixes {
		if pref == comp {
			return true
		}
	}
	return false
}

type ASFilter struct {
	asList []uint32
}

// Returns an AS filter with the list of AS's in the form "1,2,3,4"
// If src is true, filters messages by source AS number
// otherwise filters by destination AS number
func NewASFilter(list string, src bool) (Filter, error) {
	aslist, err := parseASList(list)
	if err != nil {
		return nil, err
	}

	asf := ASFilter{aslist}
	if src {
		return asf.FilterBySource, nil
	} else {
		return asf.FilterByDest, nil
	}
}

func (asf ASFilter) FilterBySource(mbs *mrt.MrtBufferStack) bool {
	path, err := getASPath(mbs)
	if err != nil || len(path) < 1 {
		return false
	}

	return asf.matchesOne(path[len(path)-1])
}

func (asf ASFilter) FilterByDest(mbs *mrt.MrtBufferStack) bool {
	path, err := getASPath(mbs)
	if err != nil || len(path) < 1 {
		return false
	}

	return asf.matchesOne(path[0])
}

// Convenience function used by both FilterBySrc/Dest
func (asf ASFilter) matchesOne(comp uint32) bool {
	for _, asnum := range asf.asList {
		if asnum == comp {
			return true
		}
	}
	return false
}

func parseASList(str string) ([]uint32, error) {
	list := strings.Split(str, ",")
	aslist := make([]uint32, len(list))

	for i := 0; i < len(aslist); i++ {
		as, err := strconv.ParseUint(list[i], 10, 32)
		if err != nil {
			return nil, err
		}
		aslist[i] = uint32(as)
	}

	return aslist, nil
}

func filterAll(filters []Filter, mbs *mrt.MrtBufferStack) bool {
	for _, fil := range filters {
		if fil != nil && !fil(mbs) {
			return false
		}
	}
	return true
}
<commit_msg>removing unused prefix matching funtion<commit_after>// This file defines all filters and convenience functions for filters.
// Should be functionally equivalent to stable gobgpdump
// Current Filter options:
// -Source AS number (NewASFilter("1,2,3", true))
// -Destination AS number (NewASFilter("1,2,3", false))

// TODO
// -Maybe a function to return an array of filters based on config
//	options?
// -Time filter
// -Filters based on other attributes (peers, prefixes seen, etc.)
package gobgpdump

import (
	"fmt"
	mrt "github.com/CSUNetSec/protoparse/protocol/mrt"
	radix "github.com/armon/go-radix"
	"net"
	"strconv"
	"strings"
)

type Filter func(mbs *mrt.MrtBufferStack) bool

type PrefixFilter struct {
	prefixes []string
	rt       *radix.Tree
}

func NewPrefixFilter(raw string) Filter {
	pf := PrefixFilter{}
	pf.rt = radix.New()
	prefstrings := strings.Split(raw, ",")
	for _, p := range prefstrings {
		parts := strings.Split(p, "/")
		if len(parts) != 2 {
			panic("malformed prefix string")
		}
		mask, err := maskstr2uint8(parts[1])
		if err != nil {
			panic(fmt.Sprintf("error parsing mask:%s err:%s", parts[1], err))
		}
		pf.rt.Insert(IpToRadixkey(net.ParseIP(parts[0]).To4(), mask), true)
	}
	pf.prefixes = prefstrings
	return pf.filterBySeen
}

func (pf PrefixFilter) filterBySeen(mbs *mrt.MrtBufferStack) bool {
	advPrefs, err := getAdvertizedPrefixes(mbs)
	if err == nil {
		for _, pref := range advPrefs {
			if _, _, found := pf.rt.LongestPrefix(IpToRadixkey(pref.IP, pref.Mask)); found {
				return true
			}
		}
	}

	wdnPrefs, err := getWithdrawnPrefixes(mbs)
	if err == nil {
		for _, pref := range wdnPrefs {
			if _, _, found := pf.rt.LongestPrefix(IpToRadixkey(pref.IP, pref.Mask)); found {
				return true
			}
		}
	}
	return false
}

type ASFilter struct {
	asList []uint32
}

// Returns an AS filter with the list of AS's in the form "1,2,3,4"
// If src is true, filters messages by source AS number
// otherwise filters by destination AS number
func NewASFilter(list string, src bool) (Filter, error) {
	aslist, err := parseASList(list)
	if err != nil {
		return nil, err
	}

	asf := ASFilter{aslist}
	if src {
		return asf.FilterBySource, nil
	} else {
		return asf.FilterByDest, nil
	}
}

func (asf ASFilter) FilterBySource(mbs *mrt.MrtBufferStack) bool {
	path, err := getASPath(mbs)
	if err != nil || len(path) < 1 {
		return false
	}

	return asf.matchesOne(path[len(path)-1])
}

func (asf ASFilter) FilterByDest(mbs *mrt.MrtBufferStack) bool {
	path, err := getASPath(mbs)
	if err != nil || len(path) < 1 {
		return false
	}

	return asf.matchesOne(path[0])
}

// Convenience function used by both FilterBySrc/Dest
func (asf ASFilter) matchesOne(comp uint32) bool {
	for _, asnum := range asf.asList {
		if asnum == comp {
			return true
		}
	}
	return false
}

func parseASList(str string) ([]uint32, error) {
	list := strings.Split(str, ",")
	aslist := make([]uint32, len(list))

	for i := 0; i < len(aslist); i++ {
		as, err := strconv.ParseUint(list[i], 10, 32)
		if err != nil {
			return nil, err
		}
		aslist[i] = uint32(as)
	}

	return aslist, nil
}

func filterAll(filters []Filter, mbs *mrt.MrtBufferStack) bool {
	for _, fil := range filters {
		if fil != nil && !fil(mbs) {
			return false
		}
	}
	return true
}

{'fraction_non_alphanumeric': 0.13781714457288696, 'fraction_numerical': 0.010809187809638193, 'mean_word_length': 3.648988136775994, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3647376', 'n_tokens_mistral': 2587, 'n_tokens_neox': 2460, 'n_words': 685}