<commit_before>#!/usr/bin/env python2
# coding=utf-8
"""
Utility belt
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from functools import wraps

__author__ = "<NAME>"
__copyright__ = "Copyright 2015, summer.ai"
__date__ = "2015-11-09"
__email__ = "<EMAIL>"

import hashlib
import re
from unidecode import unidecode
import threading
import traceback
import functools32 as functools
import logging
import unicodecsv as csv
import datetime
from urlparse import urlparse
from collections import Mapping, MutableSequence
import subprocess

log = logging.getLogger('serapis.search')


class AttrDict(object):
    """
    Read-only dynamic attribute class for dictionaries.

        >>> b = AttrDict({"x": {"foo": 2, "bar": [3, 4, 5]}})
        >>> b.x
        {"foo": 2, "bar": [3, 4, 5]}
        >>> b.x.foo
        2
        >>> b.x.bar[-1]
        5
    """
    def __init__(self, mapping):
        self.__data = dict(mapping)
    
    def __getattr__(self, name):
        if name == "__data":
            return self.__data
        if hasattr(self.__data, name):
            return getattr(self.__data, name)
        else:
            return AttrDict.build(self.__data[name])

    @classmethod
    def build(cls, obj):
        if isinstance(obj, Mapping):
            return cls(obj)
        elif isinstance(obj, MutableSequence):
            return [cls.build(item) for item in obj]
        else:
            return obj

    def __repr__(self):
        return self.__data.__repr__()


class AsynchronousRequest(object):
    """Class for threaded function calls. Usage:

    >>> urls = ("http://summer.ai", "http://google.com")
    >>> jobs = [AsynchronousRequest(crawl, url) for url in urls]
    >>> while not all(jobs):
    >>>     time.sleep(.5)
    >>> results = [job.value for job in jobs]

    Note that AsynchronousRequest will be falsey until the function
    returns or raises an exception. You can explicitly check whether
    the request is completed with AsynchronousRequest.done

    Properties:
        value -- return value of the function (or None)
        error -- Exception raised, if any
        done -- True if the function has returned or raised an Exception
    """
    def __init__(self, function, *args, **kwargs):
        """
        Args:
            function: func -- function to call asynchronously
            args, kwargs -- arguments to the function
        Returns:
            AsynchronousRequest
        """
        self.value = None
        self.error = None
        self._function = function
        self._thread = threading.Thread(target=self._fetch, args=(function,) + args, kwargs=kwargs)
        self._thread.start()

    def _fetch(self, function, *args, **kwargs):
        try:
            self.value = function(*args, **kwargs)
        except Exception as e:
            log.error("Exception in AsynchronousRequest '{}' -- {}".format(function.__name__, traceback.format_exc()))
            self.error = e

    @property
    def done(self):
        return not self._thread.isAlive()

    def __nonzero__(self):
        return self.done


def now():
    """Returns the current date and time in ISO8601 format.

    Returns:
        str -- e.g. '2015-12-17T20:21:10'
    """
    return datetime.datetime.now().replace(microsecond=0,).isoformat()


def merge_dict(target, *to_merge):
    """Merges dictionaries into a target. If keys already exist,
    only merges target if they are not falsey.

    Args:
        target: dict -- Will be modified
        *to_merge: dicts -- Will be merged into target
    Returns:
        dict -- modified target
    """
    for d in to_merge:
        if d:
            for k, v in d.items():
                if k not in target or d[k]:
                    target[k] = d[k]
    return target


@functools.lru_cache()
def squashed(text, keep=''):<commit_msg>collapse whitespace preprocessing<commit_after>#!/usr/bin/env python2
# coding=utf-8
"""
Utility belt
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from functools import wraps

__author__ = "<NAME>"
__copyright__ = "Copyright 2015, summer.ai"
__date__ = "2015-11-09"
__email__ = "<EMAIL>"

import hashlib
import re
from unidecode import unidecode
import threading
import traceback
import functools32 as functools
import logging
import unicodecsv as csv
import datetime
from urlparse import urlparse
from collections import Mapping, MutableSequence
import subprocess

log = logging.getLogger('serapis.search')


class AttrDict(object):
    """
    Read-only dynamic attribute class for dictionaries.

        >>> b = AttrDict({"x": {"foo": 2, "bar": [3, 4, 5]}})
        >>> b.x
        {"foo": 2, "bar": [3, 4, 5]}
        >>> b.x.foo
        2
        >>> b.x.bar[-1]
        5
    """
    def __init__(self, mapping):
        self.__data = dict(mapping)
    
    def __getattr__(self, name):
        if name == "__data":
            return self.__data
        if hasattr(self.__data, name):
            return getattr(self.__data, name)
        else:
            return AttrDict.build(self.__data[name])

    @classmethod
    def build(cls, obj):
        if isinstance(obj, Mapping):
            return cls(obj)
        elif isinstance(obj, MutableSequence):
            return [cls.build(item) for item in obj]
        else:
            return obj

    def __repr__(self):
        return self.__data.__repr__()


class AsynchronousRequest(object):
    """Class for threaded function calls. Usage:

    >>> urls = ("http://summer.ai", "http://google.com")
    >>> jobs = [AsynchronousRequest(crawl, url) for url in urls]
    >>> while not all(jobs):
    >>>     time.sleep(.5)
    >>> results = [job.value for job in jobs]

    Note that AsynchronousRequest will be falsey until the function
    returns or raises an exception. You can explicitly check whether
    the request is completed with AsynchronousRequest.done

    Properties:
        value -- return value of the function (or None)
        error -- Exception raised, if any
        done -- True if the function has returned or raised an Exception
    """
    def __init__(self, function, *args, **kwargs):
        """
        Args:
            function: func -- function to call asynchronously
            args, kwargs -- arguments to the function
        Returns:
            AsynchronousRequest
        """
        self.value = None
        self.error = None
        self._function = function
        self._thread = threading.Thread(target=self._fetch, args=(function,) + args, kwargs=kwargs)
        self._thread.start()

    def _fetch(self, function, *args, **kwargs):
        try:
            self.value = function(*args, **kwargs)
        except Exception as e:
            log.error("Exception in AsynchronousRequest '{}' -- {}".format(function.__name__, traceback.format_exc()))
            self.error = e

    @property
    def done(self):
        return not self._thread.isAlive()

    def __nonzero__(self):
        return self.done


def clean_sentence(sentence, term, replacement='_TERM_'):
    """Replaces all variants of term with a replacement.

        >>> s_clean, variants = clean_sentence("I've had a Déjà Vu!", "deja-vu")
        >> s_clean
        "I've had a _TERM_"
        >> variants
        ["Déjà Vu"]

    Args:
        sentence: str
        term: str
        replacement: str
    Returns:
        tuple -- Contains the cleaned sentence and all variants found.
    """
    variants = collect_variants(sentence, term)
    s_clean = multiple_replace(sentence, {v: replacement for v in variants}) if variants else sentence
    return ' '.join(s_clean.split()), variants


def now():
    """Returns the current date and time in ISO8601 format.

    Returns:
        str -- e.g. '2015-12-17T20:21:10'
    """
    return datetime.datetime.now().replace(microsecond=0,).isoformat()


def merge_dict(target, *to_merge):
    """Merges dictionaries into a target. If keys already exist,
    only merges target if they are not falsey.

    Args:
        target: dict -- Will be modified
        *to_merge: dicts -- Will be merged into target
    Returns:
        dict -- modified target
    """
    for d in to_merge:
        if d:
            for k, v in d.items():
                if k not in target or d[k]:
                    target[k] = d[k]
    return target


@functools.lru_cache()
def squashed(text, keep=''):
{'fraction_non_alphanumeric': 0.11030461684911946, 'fraction_numerical': 0.011185149928605425, 'mean_word_length': 2.3891129032258065, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5346944', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2391, 'n_words': 780}