<commit_before>#!/usr/bin/env python3
# Copyright (c) 2015 <NAME>, <NAME>
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import re
import sys
import requests
import argparse
import tempfile
import mimetypes
import subprocess
import configparser
import logging as log
import magic
from xdg import BaseDirectory
from enum import Enum
from functools import reduce
from urllib.parse import urlparse

from mario.parser import make_parser, parse_rules_file
from mario.util import ElasticDict

class Kind(Enum):
    raw = 1
    url = 2

def lookup_content_type(url):
    headers = {'User-agent' : 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) '
               'Gecko/20100101 Firefox/36.0'}

    try:
        request = requests.head(url, headers=headers)
        response = request.headers['content-type']
    except (requests.RequestException, KeyError):
        return None, None

    if ';' in response:
        content_type, encoding = response.split(';', maxsplit=1)
        return content_type, encoding.strip()

    return response, None


def escape_match_group_references(action):
    return re.sub(r"{(\d*)}", r"{\\\1}", action)


def get_var_references(s):
    start = 0
    while True:
        p1 = s.find("{", start)
        p2 = s.find("}", p1+1)
        if p1 != -1 and p2 != -1:
            start = p2+1
            yield s[p1:p2+1]
        else:
            raise StopIteration


def kind_is_func(msg, arguments, cache):
    try:
        return msg['kind'] == Kind[arguments[0]], msg, cache
    except KeyError:
        return False, msg, cache


def arg_is_func(msg, arguments, cache):
    arg, checks = arguments

    ret = arg.format(**msg) in checks
    return ret, msg, cache


def arg_matches_func(msg, arguments, cache):
    arg, patterns = arguments
    arg = arg.format(**msg)
<commit_msg>core: Fix for PEP 479.<commit_after>#!/usr/bin/env python3
# Copyright (c) 2015 <NAME>, <NAME>
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import re
import sys
import requests
import argparse
import tempfile
import mimetypes
import subprocess
import configparser
import logging as log
import magic
from xdg import BaseDirectory
from enum import Enum
from functools import reduce
from urllib.parse import urlparse

from mario.parser import make_parser, parse_rules_file
from mario.util import ElasticDict

class Kind(Enum):
    raw = 1
    url = 2

def lookup_content_type(url):
    headers = {'User-agent' : 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) '
               'Gecko/20100101 Firefox/36.0'}

    try:
        request = requests.head(url, headers=headers)
        response = request.headers['content-type']
    except (requests.RequestException, KeyError):
        return None, None

    if ';' in response:
        content_type, encoding = response.split(';', maxsplit=1)
        return content_type, encoding.strip()

    return response, None


def escape_match_group_references(action):
    return re.sub(r"{(\d*)}", r"{\\\1}", action)


def get_var_references(s):
    start = 0
    while True:
        p1 = s.find("{", start)
        p2 = s.find("}", p1+1)
        if p1 != -1 and p2 != -1:
            start = p2+1
            yield s[p1:p2+1]
        else:
            return


def kind_is_func(msg, arguments, cache):
    try:
        return msg['kind'] == Kind[arguments[0]], msg, cache
    except KeyError:
        return False, msg, cache


def arg_is_func(msg, arguments, cache):
    arg, checks = arguments

    ret = arg.format(**msg) in checks
    return ret, msg, cache


def arg_matches_func(msg, arguments, cache):
    arg, patterns = arguments
    arg = arg.format(**msg)

{'fraction_non_alphanumeric': 0.10831099195710456, 'fraction_numerical': 0.023324396782841823, 'mean_word_length': 2.9565217391304346, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3251411', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1163, 'n_words': 379}