<commit_before># Copyright (C) 2010  <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
    The following operators are understood:

        ~q          Request
        ~s          Response

    Headers:

        Patterns are matched against "name: value" strings. Field names are
        all-lowercase.

        ~h rex      Header line in either request or response
        ~hq rex     Header in request
        ~hs rex     Header in response

        ~b rex      Expression in the body of either request or response
        ~bq rex     Expression in the body of request
        ~bq rex     Expression in the body of response
        ~t rex      Shortcut for content-type header.

        ~m rex      Method
        ~u rex      URL
        ~c CODE     Response code.
        rex         Equivalent to ~u rex
"""
import re, sys
import contrib.pyparsing as pp
import flow


class _Token:
    def dump(self, indent=0, fp=sys.stdout):
        print >> fp, "\t"*indent, self.__class__.__name__,
        if hasattr(self, "expr"):
            print >> fp, "(%s)"%self.expr,
        print >> fp


class _Action(_Token):
    @classmethod
    def make(klass, s, loc, toks):
        return klass(*toks[1:])


class FErr(_Action):
    code = "e"
    help = "Match error"
    def __call__(self, conn):
        return isinstance(conn, flow.Error)


class FReq(_Action):
    code = "q"
    help = "Match request"
    def __call__(self, conn):
        return isinstance(conn, flow.Request)


class FResp(_Action):
    code = "s"
    help = "Match response"
    def __call__(self, conn):
        return isinstance(conn, flow.Response)


class _Rex(_Action):
    def __init__(self, expr):
        self.expr = expr
        try:
            self.re = re.compile(self.expr)
        except:
            raise ValueError, "Cannot compile expression."

def _check_content_type(expr, o):
    val = o.headers["content-type"]
    if val and re.search(expr, val[0]):
        return True
    return False


class FContentType(_Rex):
    code = "t"
    help = "Content-type header"
    def __call__(self, o):
        if _check_content_type(self.expr, o):
            return True
        elif isinstance(o, flow.Response) and _check_content_type(self.expr, o.request):
            return True
        else:
            return False


class FRequestContentType(_Rex):
    code = "tq"
    help = "Request Content-Type header"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            return _check_content_type(self.expr, o.request)
        else:
            return _check_content_type(self.expr, o)


class FResponseContentType(_Rex):
    code = "ts"
    help = "Request Content-Type header"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            return _check_content_type(self.expr, o)
        else:
            return False


class FHead(_Rex):
    code = "h"
    help = "Header"
    def __call__(self, o):
        val = o.headers.match_re(self.expr)
        if not val and isinstance(o, flow.Response):
            val = o.request.headers.match_re(self.expr)
        return val


class FHeadRequest(_Rex):
    code = "hq"
    help = "Request header"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            h = o.request.headers
        else:
            h = o.headers
        return h.match_re(self.expr)


class FHeadResponse(_Rex):
    code = "hs"
    help = "Response header"
    def __call__(self, o):
        if not isinstance(o, flow.Response):
            return False
        return o.headers.match_re(self.expr)


class FBod(_Rex):
    code = "b"
    help = "Body"
    def __call__(self, o):
        if o.content and re.search(self.expr, o.content):
            return True
        elif isinstance(o, flow.Response) and o.request.content and re.search(self.expr, o.request.content):
            return True
        return False


class FBodRequest(_Rex):
    code = "bq"
    help = "Request body"
    def __call__(self, o):
        if isinstance(o, flow.Response) and o.request.content and re.search(self.expr, o.request.content):
            return True
        elif not isinstance(o, flow.Response) and o.content and re.search(self.expr, o.content):
            return True
        return False


class FBodResponse(_Rex):
    code = "bs"
    help = "Response body"
    def __call__(self, o):
        if not isinstance(o, flow.Response):
            return False
        elif o.content and re.search(self.expr, o.content):
            return True
        return False


class FMethod(_Rex):
    code = "m"
    help = "Method"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            return False
        elif o.method:
            return re.search(self.expr, o.method, re.IGNORECASE)
        return False


class FUrl(_Rex):
    code = "u"
    help = "URL"
    # FUrl is special, because it can be "naked".
    @classmethod
    def make(klass, s, loc, toks):
        if len(toks) > 1:
            toks = toks[1:]
        return klass(*toks)

    def __call__(self, o):
        if isinstance(o, flow.Response):
            c = o.request
        elif isinstance(o, flow.Request):
            c = o
        else:
            return False<commit_msg>Fix bug in method filter matching.<commit_after># Copyright (C) 2010  <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
    The following operators are understood:

        ~q          Request
        ~s          Response

    Headers:

        Patterns are matched against "name: value" strings. Field names are
        all-lowercase.

        ~h rex      Header line in either request or response
        ~hq rex     Header in request
        ~hs rex     Header in response

        ~b rex      Expression in the body of either request or response
        ~bq rex     Expression in the body of request
        ~bq rex     Expression in the body of response
        ~t rex      Shortcut for content-type header.

        ~m rex      Method
        ~u rex      URL
        ~c CODE     Response code.
        rex         Equivalent to ~u rex
"""
import re, sys
import contrib.pyparsing as pp
import flow


class _Token:
    def dump(self, indent=0, fp=sys.stdout):
        print >> fp, "\t"*indent, self.__class__.__name__,
        if hasattr(self, "expr"):
            print >> fp, "(%s)"%self.expr,
        print >> fp


class _Action(_Token):
    @classmethod
    def make(klass, s, loc, toks):
        return klass(*toks[1:])


class FErr(_Action):
    code = "e"
    help = "Match error"
    def __call__(self, conn):
        return isinstance(conn, flow.Error)


class FReq(_Action):
    code = "q"
    help = "Match request"
    def __call__(self, conn):
        return isinstance(conn, flow.Request)


class FResp(_Action):
    code = "s"
    help = "Match response"
    def __call__(self, conn):
        return isinstance(conn, flow.Response)


class _Rex(_Action):
    def __init__(self, expr):
        self.expr = expr
        try:
            self.re = re.compile(self.expr)
        except:
            raise ValueError, "Cannot compile expression."

def _check_content_type(expr, o):
    val = o.headers["content-type"]
    if val and re.search(expr, val[0]):
        return True
    return False


class FContentType(_Rex):
    code = "t"
    help = "Content-type header"
    def __call__(self, o):
        if _check_content_type(self.expr, o):
            return True
        elif isinstance(o, flow.Response) and _check_content_type(self.expr, o.request):
            return True
        else:
            return False


class FRequestContentType(_Rex):
    code = "tq"
    help = "Request Content-Type header"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            return _check_content_type(self.expr, o.request)
        else:
            return _check_content_type(self.expr, o)


class FResponseContentType(_Rex):
    code = "ts"
    help = "Request Content-Type header"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            return _check_content_type(self.expr, o)
        else:
            return False


class FHead(_Rex):
    code = "h"
    help = "Header"
    def __call__(self, o):
        val = o.headers.match_re(self.expr)
        if not val and isinstance(o, flow.Response):
            val = o.request.headers.match_re(self.expr)
        return val


class FHeadRequest(_Rex):
    code = "hq"
    help = "Request header"
    def __call__(self, o):
        if isinstance(o, flow.Response):
            h = o.request.headers
        else:
            h = o.headers
        return h.match_re(self.expr)


class FHeadResponse(_Rex):
    code = "hs"
    help = "Response header"
    def __call__(self, o):
        if not isinstance(o, flow.Response):
            return False
        return o.headers.match_re(self.expr)


class FBod(_Rex):
    code = "b"
    help = "Body"
    def __call__(self, o):
        if o.content and re.search(self.expr, o.content):
            return True
        elif isinstance(o, flow.Response) and o.request.content and re.search(self.expr, o.request.content):
            return True
        return False


class FBodRequest(_Rex):
    code = "bq"
    help = "Request body"
    def __call__(self, o):
        if isinstance(o, flow.Response) and o.request.content and re.search(self.expr, o.request.content):
            return True
        elif not isinstance(o, flow.Response) and o.content and re.search(self.expr, o.content):
            return True
        return False


class FBodResponse(_Rex):
    code = "bs"
    help = "Response body"
    def __call__(self, o):
        if not isinstance(o, flow.Response):
            return False
        elif o.content and re.search(self.expr, o.content):
            return True
        return False


class FMethod(_Rex):
    code = "m"
    help = "Method"
    def __call__(self, o):
        if isinstance(o, flow.Request):
            return re.search(self.expr, o.method, re.IGNORECASE)
        return False


class FUrl(_Rex):
    code = "u"
    help = "URL"
    # FUrl is special, because it can be "naked".
    @classmethod
    def make(klass, s, loc, toks):
        if len(toks) > 1:
            toks = toks[1:]
        return klass(*toks)

    def __call__(self, o):
        if isinstance(o, flow.Response):
            c = o.request
        elif isinstance(o, flow.Request):
            c = o
        else:
            return False
{'fraction_non_alphanumeric': 0.09703805751679008, 'fraction_numerical': 0.001722059583261581, 'mean_word_length': 2.1631263616557734, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7613566', 'n_tokens_mistral': 3503, 'n_tokens_neox': 3432, 'n_words': 1152}