<commit_before>import pandas as pd
import numpy as np
import json
import re
import copy
import itertools
import math
import re, string
import sqlite3

from collections import OrderedDict
from quantipy.core.helpers.constants import DTYPE_MAP
from quantipy.core.helpers.constants import MAPPED_PATTERN
from itertools import product
from quantipy.core.view import View
from quantipy.core.view_generators.view_mapper import ViewMapper
from quantipy.core.helpers import functions

from quantipy.core.tools.dp.dimensions.reader import quantipy_from_dimensions
from quantipy.core.tools.dp.decipher.reader import quantipy_from_decipher
from quantipy.core.tools.dp.spss.reader import parse_sav_file
from quantipy.core.tools.dp.spss.writer import save_sav
from quantipy.core.tools.dp.ascribe.reader import quantipy_from_ascribe

def unicoder(obj, decoder='UTF-8'):
    """
    Decodes all the text (keys and strings) in obj.
    
    Recursively mines obj for any str objects, whether keys
    or values, converting any str objects to unicode.
    
    Parameters
    ----------
    obj : object
        The object to be mined.
        
    Returns
    -------
    obj : object
        The recursively decoded object. 
    """
    
    if isinstance(obj, list):
        obj = [
            unicoder(item)
            for item in obj
        ]
    if isinstance(obj, tuple):
        obj = tuple([
            unicoder(item)
            for item in obj
        ])
    elif isinstance(obj, (dict)):
        obj = {
            key: unicoder(value)
            for key, value in obj.iteritems()
        }
    elif isinstance(obj, str):
        obj = unicode(obj, decoder)
    
    return obj

def load_json(path_json, hook=OrderedDict):
    ''' Returns a python object from the json file located at path_json
    '''

    with open(path_json) as f:
        obj = json.load(f, object_pairs_hook=hook)

        return obj

def loads_json(json_text, hook=OrderedDict):
    ''' Returns a python object from the json string json_text
    '''

    obj = json.loads(json_text, object_pairs_hook=hook)

    return obj

def load_csv(path_csv):
    
    return pd.DataFrame.from_csv(path_csv)
<commit_msg>Added additional unicode verification courtesy of ftfy (new dependency).<commit_after>import pandas as pd
import numpy as np
import json
import re
import copy
import itertools
import math
import re, string
import sqlite3

from ftfy import fix_text

from collections import OrderedDict
from quantipy.core.helpers.constants import DTYPE_MAP
from quantipy.core.helpers.constants import MAPPED_PATTERN
from itertools import product
from quantipy.core.view import View
from quantipy.core.view_generators.view_mapper import ViewMapper
from quantipy.core.helpers import functions

from quantipy.core.tools.dp.dimensions.reader import quantipy_from_dimensions
from quantipy.core.tools.dp.decipher.reader import quantipy_from_decipher
from quantipy.core.tools.dp.spss.reader import parse_sav_file
from quantipy.core.tools.dp.spss.writer import save_sav
from quantipy.core.tools.dp.ascribe.reader import quantipy_from_ascribe

def unicoder(obj, decoder='UTF-8'):
    """
    Decodes all the text (keys and strings) in obj.
    
    Recursively mines obj for any str objects, whether keys or values,
    converting any str objects to unicode and then correcting the 
    unicode (which may have been decoded incorrectly) using ftfy.
    
    Parameters
    ----------
    obj : object
        The object to be mined.
        
    Returns
    -------
    obj : object
        The recursively decoded object. 
    """
    
    if isinstance(obj, list):
        obj = [
            unicoder(item)
            for item in obj
        ]
    if isinstance(obj, tuple):
        obj = tuple([
            unicoder(item)
            for item in obj
        ])
    elif isinstance(obj, (dict)):
        obj = {
            key: unicoder(value)
            for key, value in obj.iteritems()
        }
    elif isinstance(obj, str):
        obj = fix_text(unicode(obj, decoder))
    
    return obj

def load_json(path_json, hook=OrderedDict):
    ''' Returns a python object from the json file located at path_json
    '''

    with open(path_json) as f:
        obj = json.load(f, object_pairs_hook=hook)

        return obj

def loads_json(json_text, hook=OrderedDict):
    ''' Returns a python object from the json string json_text
    '''

    obj = json.loads(json_text, object_pairs_hook=hook)

    return obj

def load_csv(path_csv):
    
    return pd.DataFrame.from_csv(path_csv)

{'fraction_non_alphanumeric': 0.07673702726473175, 'fraction_numerical': 0.0008795074758135445, 'mean_word_length': 2.7876769358867612, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1733767', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1358, 'n_words': 419}