<commit_before>from __future__ import division

from utils import check_type

import logging

import numpy as np
import pandas as pd

import itertools

class StandardSeries(pd.Series):
    """ A pandas Series like object with additional methods for standardising data. For example, functions to clean string data and numerical information. 
    
    """

    def __init__(self, *args, **kwargs):
        super(self.__class__, self).__init__(*args, **kwargs)

    @check_type
    def clean(self, lower=True, replace_by_none='[^ \-\_A-Za-z0-9]+', replace_by_whitespace='[\-\_]', remove_brackets=True, inplace=True):
        """
        clean(lower=True, replace_by_none='[^ \-\_A-Za-z0-9]+', replace_by_whitespace='[\-\_]', remove_brackets=True, inplace=True)
        
        Remove special tokens from a series of strings. 
    
        :param lower: Convert the strings in lowercase characters.
        :param replace_by_none: A regular expression that is replaced by ''.
        :param replace_by_whitespace: A regular expression that is replaced by a whitespace.
        :param inplace: If True, replace the current strings by their cleaned variant.
        :param remove_brackets: Remove all content between brackets and the brackets themselves. 

        :return: A cleaned Series of strings.
        :rtype: standardise.StandardSeries

        For example:

        >>> s = recordlinkage.StandardSeries(['Mary-ann', 'Bob :)', 'Angel', 'Bob (alias Billy)'])
        >>> print s.clean()
        <NAME>
        bob
        angel
        bob

        """

        result = self if inplace else self.copy()

        # Lower the result if lower is True
        result = result.str.lower() if lower else result 

        # Remove all content between brackets
        if remove_brackets:
            result = result.str.replace(r'(\[.*?\]|\(.*?\)|\{.*?\})', '')

        # Remove the special characters
        result = result.str.replace(replace_by_none, '')
        result = result.str.replace(replace_by_whitespace, ' ')

        # Remove multiple whitespaces
        result = result.str.replace(r'\s\s+', ' ')

        # Strip result
        result = result.str.lstrip().str.rstrip()

        if inplace:
            self._update_inplace(result._data)
            return

        return result

    @check_type
    def phonenumbers(self, replace_by_none='[^0-9]+', inplace=True):
        """ Clean string formatted phonenumbers into string of intergers. 

        :return: A StandardSeries with cleaned phonenumbers.
        :rtype: standardise.StandardSeries
        """
        string = self if inplace else self.copy()
        string = string.astype(str)

        # Remove all special tokens
        string = string.str.replace(replace_by_none, '')

        # Convert string into integer
        return string.astype(str)

    @check_type
    def count_values(self):
        """DEPRECEATED SOON: Count the number of times a value occurs. The difference with pandas.value_counts is that this function returns the values for each row. 
    
        :return: A StandardSeries with value counts.
        :rtype: standardise.StandardSeries
        """

        logging.warning('This function is depreceated soon. Use value_occurence instead.')

        value_count = self.fillna(0)

        return StandardSeries(value_count.groupby(by=value_count.values).transform('count'))

    @check_type
    def value_occurence(self):
        """
        Count the number of times a value occurs. The difference with pandas.value_counts is that this function returns the values for each row. 

        :return: A StandardSeries with value counts.
        :rtype: standardise.StandardSeries
        """

        value_count = self.copy()
        value_count.fillna('NAN', inplace=True)

        return value_count.groupby(by=value_count.values).transform('count')

    # @check_type
    # def names(self, encode_method=None):
 
    #     if encode:
    #         return encode(encode_method)
    #     else:
    #         return self

    @check_type
    def encode(self, encode_method, inplace=True):

        try:
            import jellyfish
        except ImportError:
            print "Install jellyfish to use string encoding."

        if inplace:
            string = self
        else:
            string = self.copy()

        number_of_unique_values = len(string.unique())

        if encode_method == 'soundex':
            return string.str.upper().str.decode('utf-8').apply(lambda x: jellyfish.soundex(x) if pd.notnull(x) else np.nan)

        elif encode_method == 'nysiis':
            return string.str.upper().str.decode('utf-8').apply(lambda x: jellyfish.nysiis(x) if pd.notnull(x) else np.nan)

        elif encode_method == 'unicode':
            return  string.str.decode('utf-8')

        else:
            raise Exception("encoding method not found")

        print "Reduction ratio is %s" % (1-number_of_unique_values/len(self.unique()))

        if not inplace:
            return string

    @check_type
    def similar_values(self, threshold=0.8, inplace=True):
        """
        similar_values(threshold=0.8, inplace=True)

        Group strings with high similarities. 
    
        :param threshold: Two strings with similarity above this threshold are considered to be the same string. The threshold is a value equal or between 0 and 1. Default 0.8. 
        :param inplace: If True, replace the current strings by their cleaned variant. Default: True.

        :return: A Series of strings.
        :rtype: standardise.StandardSeries

        """
        try:<commit_msg>Add inplace to other functions<commit_after>from __future__ import division

from utils import check_type

import logging

import numpy as np
import pandas as pd

import itertools

class StandardSeries(pd.Series):
    """ A pandas Series like object with additional methods for standardising data. For example, functions to clean string data and numerical information. 
    
    """

    def __init__(self, *args, **kwargs):
        super(self.__class__, self).__init__(*args, **kwargs)

    @check_type
    def clean(self, lower=True, replace_by_none='[^ \-\_A-Za-z0-9]+', replace_by_whitespace='[\-\_]', remove_brackets=True, inplace=True):
        """
        clean(lower=True, replace_by_none='[^ \-\_A-Za-z0-9]+', replace_by_whitespace='[\-\_]', remove_brackets=True, inplace=True)
        
        Remove special tokens from a series of strings. 
    
        :param lower: Convert the strings in lowercase characters.
        :param replace_by_none: A regular expression that is replaced by ''.
        :param replace_by_whitespace: A regular expression that is replaced by a whitespace.
        :param inplace: If True, replace the current strings by their cleaned variant.
        :param remove_brackets: Remove all content between brackets and the brackets themselves. 

        :return: A cleaned Series of strings.
        :rtype: standardise.StandardSeries

        For example:

        >>> s = recordlinkage.StandardSeries(['Mary-ann', 'Bob :)', 'Angel', 'Bob (alias Billy)'])
        >>> print s.clean()
        <NAME>
        bob
        angel
        bob

        """

        result = self if inplace else self.copy()

        # Lower the result if lower is True
        result = result.str.lower() if lower else result 

        # Remove all content between brackets
        if remove_brackets:
            result = result.str.replace(r'(\[.*?\]|\(.*?\)|\{.*?\})', '')

        # Remove the special characters
        result = result.str.replace(replace_by_none, '')
        result = result.str.replace(replace_by_whitespace, ' ')

        # Remove multiple whitespaces
        result = result.str.replace(r'\s\s+', ' ')

        # Strip result
        result = result.str.lstrip().str.rstrip()

        if inplace:
            self._update_inplace(result._data)
            return

        return result

    @check_type
    def phonenumbers(self, country='USA' ,inplace=True):
        """ Clean string formatted phonenumbers into string of intergers. 

        :return: A StandardSeries with cleaned phonenumbers.
        :rtype: standardise.StandardSeries
        """
        result = self if inplace else self.copy()

        result = result.astype(str)

        # Remove all special tokens
        result = result.str.replace('[^0-9]+', '')

        # Convert result into integer
        result = result.astype(str)

        if inplace:
            self._update_inplace(result._data)
            return

        return result

    @check_type
    def count_values(self):
        """DEPRECEATED SOON: Count the number of times a value occurs. The difference with pandas.value_counts is that this function returns the values for each row. 
    
        :return: A StandardSeries with value counts.
        :rtype: standardise.StandardSeries
        """

        logging.warning('This function is depreceated soon. Use value_occurence instead.')

        value_count = self.fillna(0)

        return StandardSeries(value_count.groupby(by=value_count.values).transform('count'))

    @check_type
    def value_occurence(self):
        """
        Count the number of times a value occurs. The difference with pandas.value_counts is that this function returns the values for each row. 

        :return: A StandardSeries with value counts.
        :rtype: standardise.StandardSeries
        """

        value_count = self.copy()
        value_count.fillna('NAN', inplace=True)

        return value_count.groupby(by=value_count.values).transform('count')

    @check_type
    def encode(self, encode_method, inplace=True):

        try:
            import jellyfish
        except ImportError:
            print "Install jellyfish to use string encoding."

        result = self if inplace else self.copy()

        number_of_unique_values = len(result.unique())

        if encode_method == 'soundex':
            return result.str.upper().str.decode('utf-8').apply(lambda x: jellyfish.soundex(x) if pd.notnull(x) else np.nan)

        elif encode_method == 'nysiis':
            return result.str.upper().str.decode('utf-8').apply(lambda x: jellyfish.nysiis(x) if pd.notnull(x) else np.nan)

        elif encode_method == 'unicode':
            return  result.str.decode('utf-8')

        else:
            raise Exception("encoding method not found")

        if inplace:
            self._update_inplace(result._data)
            return

        return result

    @check_type
    def similar_values(self, threshold=0.8, inplace=True):
        """
        similar_values(threshold=0.8, inplace=True)

        Group strings with high similarities. 
    
        :param threshold: Two strings with similarity above this threshold are considered to be the same string. The threshold is a value equal or between 0 and 1. Default 0.8. 
        :param inplace: If True, replace the current strings by their cleaned variant. Default: True.

        :return: A Series of strings.
        :rtype: standardise.StandardSeries

        """
        try:
{'fraction_non_alphanumeric': 0.09855334538878843, 'fraction_numerical': 0.003345388788426763, 'mean_word_length': 2.463055729492799, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497878', 'n_tokens_mistral': 3155, 'n_tokens_neox': 2974, 'n_words': 930}