<commit_before>package server

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	"crypto/sha256"
	"net/url"

	"regexp"

	"github.com/PuerkitoBio/goquery"
	"path"
	"strconv"
)

func filter(action Action, req *http.Request) bool {
	if matched, _ := regexp.MatchString(action.Path, req.URL.Path); matched {
	} else {
		return false
	}

	CheckMethod := func(req *http.Request, methods []string) bool {
		if len(methods) == 0 {
			return true
		}

		for _, method := range methods {
			if method == req.Method {
				return true
			}
		}
		return false
	}

	if !CheckMethod(req, action.Method) {
		return false
	}

	CheckRemoteAddr := func(req *http.Request, addrs []string) bool {
		if len(addrs) == 0 {
			return true
		}

		remoteHost, _, _ := net.SplitHostPort(req.RemoteAddr)
		for _, remoteAddr := range addrs {
			if remoteAddr == remoteHost {
				return true<commit_msg>match on query string as well<commit_after>package server

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	"crypto/sha256"
	"net/url"

	"regexp"

	"github.com/PuerkitoBio/goquery"
	"path"
	"strconv"
)

func filter(action Action, req *http.Request) bool {
	if matched, _ := regexp.MatchString(action.Path, req.URL.RequestURI()); matched {
	} else {
		return false
	}

	CheckMethod := func(req *http.Request, methods []string) bool {
		if len(methods) == 0 {
			return true
		}

		for _, method := range methods {
			if method == req.Method {
				return true
			}
		}
		return false
	}

	if !CheckMethod(req, action.Method) {
		return false
	}

	CheckRemoteAddr := func(req *http.Request, addrs []string) bool {
		if len(addrs) == 0 {
			return true
		}

		remoteHost, _, _ := net.SplitHostPort(req.RemoteAddr)
		for _, remoteAddr := range addrs {
			if remoteAddr == remoteHost {
				return true
{'fraction_non_alphanumeric': 0.1422680412371134, 'fraction_numerical': 0.005154639175257732, 'mean_word_length': 3.3133333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1833075', 'n_tokens_mistral': 821, 'n_tokens_neox': 736, 'n_words': 172}