<commit_before># Copyright (c) 2015. Mount Sinai School of Medicine
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import collections
import re
import json
from future.utils import raise_

import typechecks
import pandas
import numpy

try:
    # Python 2
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

try:
    # Python 2
    from urlparse import parse_qsl
except ImportError:
    from urllib.parse import parse_qsl

STANDARD_EVALUATION_ENVIRONMENT = {
    "os": os,
    "sys": sys,
    "collections": collections,
    "re": re,
    "json": json,
    "pandas": pandas,
    "numpy": numpy,
}

def parse_url_fragment(url):
    parsed = urlparse(url)
    try:
        if parsed.fragment:
            # If our fragment begins with an '&' symbol, we ignore it. 
            unparsed_fragment = parsed.fragment
            if unparsed_fragment.startswith("&"):
                unparsed_fragment = unparsed_fragment[1:]
            fragment = parse_qsl(unparsed_fragment, strict_parsing=True)
        else:
            fragment = []
    except ValueError as e:
        raise ValueError("Couldn't parse fragment '%s': %s" % (
            parsed.fragment, e))

    return (parsed._replace(fragment='').geturl(), fragment)

def string_to_boolean(s):
    value = str(s).lower()
    if value in ("true", "1"):
        return True
    elif value in ("false", 0):
        return False
    raise ValueError("Not a boolean string: %s" % s)

class EvaluationEnvironment(object):
    def __init__(self, wrapped_list, extra={}):
        self._wrapped_list = wrapped_list
        self._extra = extra

    def __getitem__(self, key):
        for wrapped in self._wrapped_list:
            if hasattr(wrapped, key):
                return getattr(wrapped, key)
        if key in self._extra:
            return self._extra[key]
        raise KeyError("No key: %s" % key)

    def __str__(self):
        all_bindings = []
        for wrapped in self._wrapped_list:
            all_bindings.extend(dir(wrapped))
        all_bindings.extend(self._extra)
        display_bindings = [
            binding for binding in all_bindings
            if not binding.startswith("_")
        ]
        return ("<Environment with bindings: %s>"
            % " ".join(sorted(display_bindings)))

RAISE = object()

def evaluate_expression(expression, bindings, error_value=RAISE):
    typechecks.require_string(expression)

    # Since Python 2 doesn't have a nonlocal keyword, we have to box up the
    # error_value, so we can reassign to it in the ``on_error`` function
    # below.<commit_msg>Fix bug in evaluation.py for certain python versions where hasattr does not detect properties<commit_after># Copyright (c) 2015. Mount Sinai School of Medicine
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import collections
import re
import json
from future.utils import raise_

import typechecks
import pandas
import numpy

try:
    # Python 2
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

try:
    # Python 2
    from urlparse import parse_qsl
except ImportError:
    from urllib.parse import parse_qsl

STANDARD_EVALUATION_ENVIRONMENT = {
    "os": os,
    "sys": sys,
    "collections": collections,
    "re": re,
    "json": json,
    "pandas": pandas,
    "numpy": numpy,
}

def parse_url_fragment(url):
    parsed = urlparse(url)
    try:
        if parsed.fragment:
            # If our fragment begins with an '&' symbol, we ignore it. 
            unparsed_fragment = parsed.fragment
            if unparsed_fragment.startswith("&"):
                unparsed_fragment = unparsed_fragment[1:]
            fragment = parse_qsl(unparsed_fragment, strict_parsing=True)
        else:
            fragment = []
    except ValueError as e:
        raise ValueError("Couldn't parse fragment '%s': %s" % (
            parsed.fragment, e))

    return (parsed._replace(fragment='').geturl(), fragment)

def string_to_boolean(s):
    value = str(s).lower()
    if value in ("true", "1"):
        return True
    elif value in ("false", 0):
        return False
    raise ValueError("Not a boolean string: %s" % s)

class EvaluationEnvironment(object):
    def __init__(self, wrapped_list, extra={}):
        self._wrapped_list = wrapped_list
        self._extra = extra

    def __getitem__(self, key):
        for wrapped in self._wrapped_list:
            try:
                return getattr(wrapped, key)
            except AttributeError:
                pass
        if key in self._extra:
            return self._extra[key]
        raise KeyError("No key: %s" % key)

    def __str__(self):
        all_bindings = []
        for wrapped in self._wrapped_list:
            all_bindings.extend(dir(wrapped))
        all_bindings.extend(self._extra)
        display_bindings = [
            binding for binding in all_bindings
            if not binding.startswith("_")
        ]
        return ("<Environment with bindings: %s>"
            % " ".join(sorted(display_bindings)))

RAISE = object()

def evaluate_expression(expression, bindings, error_value=RAISE):
    typechecks.require_string(expression)

    # Since Python 2 doesn't have a nonlocal keyword, we have to box up the
    # error_value, so we can reassign to it in the ``on_error`` function
    # below.
{'fraction_non_alphanumeric': 0.08839692259381378, 'fraction_numerical': 0.004396294551734966, 'mean_word_length': 2.778173190984579, 'pattern_counts': {'":': 14, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '882294', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1801, 'n_words': 627}