<commit_before>"""
A Python script to convert excel files into JSON.
"""
from __future__ import print_function
import json
import re
import sys
import codecs
import os
from pyxform import constants, aliases
from pyxform.errors import PyXFormError
from pyxform.xls2json_backends import xls_to_dict, csv_to_dict
from pyxform.utils import is_valid_xml_tag, unicode, basestring


def print_pyobj_to_json(pyobj, path=None):
    """
    dump a python nested array/dict structure to the specified file
    or stdout if no file is specified
    """
    if path:
        fp = codecs.open(path, mode="w", encoding="utf-8")
        json.dump(pyobj, fp=fp, ensure_ascii=False, indent=4)
        fp.close()
    else:
        print(json.dumps(pyobj, ensure_ascii=False, indent=4))


def merge_dicts(dict_a, dict_b, default_key="default"):
    """
    Recursively merge two nested dicts into a single dict.
    When keys match their values are merged using
    a recursive call to this function,
    otherwise they are just added to the output dict.
    """
    if dict_a is None or dict_a == {}:
        return dict_b
    if dict_b is None or dict_b == {}:
        return dict_a

    if type(dict_a) is not dict:
        if default_key in dict_b:
            return dict_b
        dict_a = {default_key: dict_a}
    if type(dict_b) is not dict:
        if default_key in dict_a:
            return dict_a
        dict_b = {default_key: dict_b}

    all_keys = set(dict_a.keys()).union(set(dict_b.keys()))

    out_dict = dict()
    for key in all_keys:
        out_dict[key] = merge_dicts(
            dict_a.get(key), dict_b.get(key), default_key)
    return out_dict


def list_to_nested_dict(lst):
    """
    [1,2,3,4] -> {1:{2:{3:4}}}
    """
    if len(lst) > 1:
        return {lst[0]: list_to_nested_dict(lst[1:])}
    else:
        return lst[0]


def dealias_and_group_headers(dict_array, header_aliases, use_double_colons,
                              default_language=u"default", ignore_case=False):
    """
    For each row in the worksheet, group all keys that contain a double colon.
    So
        {"text::english": "hello", "text::french" : "bonjour"}
    becomes
        {"text": {"english": "hello", "french" : "bonjour"}.
    Dealiasing is done to the first token
    (the first term separated by the delimiter).
    default_language -- used to group labels/hints/etc
    without a language specified with localized versions.
    """
    group_delimiter = u"::"
    out_dict_array = list()
    for row in dict_array:
        out_row = dict()
        for header, val in row.items():

            if ignore_case:
                header = header.lower()

            if use_double_colons:
                tokens = header.split(group_delimiter)

            # else:
            #   We do the initial parse using single colons
            #   for backwards compatibility and
            #   only the first single is used
            #   in order to avoid nesting jr:something tokens.
            #   if len(tokens) > 1:
            #       tokens[1:] = [u":".join(tokens[1:])]
            else:
                # I think the commented out section above
                # break if there is something like media:image:english
                # so maybe a better backwards compatibility hack
                # is to join any jr token with the next token
                tokens = header.split(u":")
                if "jr" in tokens:
                    jr_idx = tokens.index("jr")
                    tokens[jr_idx] = u":".join(tokens[jr_idx: jr_idx + 2])
                    tokens.pop(jr_idx + 1)

            dealiased_first_token = header_aliases.get(tokens[0], tokens[0])
            tokens = dealiased_first_token.split(group_delimiter) + tokens[1:]
            new_key = tokens[0]
            new_value = list_to_nested_dict(tokens[1:] + [val])
            out_row = merge_dicts(
                out_row, {new_key: new_value}, default_language)

        out_dict_array.append(out_row)
    return out_dict_array


def dealias_types(dict_array):
    """
    Look at all the type values in a dict array and if any aliases are found,
    replace them with the name they map to.
    """
    for row in dict_array:
        found_type = row.get(constants.TYPE)
        if found_type in aliases._type.keys():
            row[constants.TYPE] = aliases._type[found_type]
    return dict_array


def clean_text_values(dict_array):
    """
    Go though the dict array and strips all text values.
    Also replaces multiple spaces with single spaces.
    Note that the keys don't get cleaned, which could be an issue.
    """
    for row in dict_array:
        for key, value in row.items():
            if isinstance(value, basestring):
                row[key] = re.sub(r"( )+", " ", value.strip())
    return dict_array


# This is currently unused because name uniqueness is checked in json2xform.
def check_name_uniqueness(dict_array):
    """
    Make sure all names are unique
    Raises and exception if a duplicate is found
    """
    # This set is used to validate the uniqueness of names.
    name_set = set()<commit_msg>convert directional quote characters (exported by microsoft excel, etc.) into their ascii-friendly equivalents.<commit_after>"""
A Python script to convert excel files into JSON.
"""
from __future__ import print_function, unicode_literals
import json
import re
import sys
import codecs
import os
from pyxform import constants, aliases
from pyxform.errors import PyXFormError
from pyxform.xls2json_backends import xls_to_dict, csv_to_dict
from pyxform.utils import is_valid_xml_tag, unicode, basestring

SMART_QUOTES = {
    '\u2018': "'",
    '\u2019': "'",
    '\u201c': '"',
    '\u201d': '"',
}


def print_pyobj_to_json(pyobj, path=None):
    """
    dump a python nested array/dict structure to the specified file
    or stdout if no file is specified
    """
    if path:
        fp = codecs.open(path, mode="w", encoding="utf-8")
        json.dump(pyobj, fp=fp, ensure_ascii=False, indent=4)
        fp.close()
    else:
        print(json.dumps(pyobj, ensure_ascii=False, indent=4))


def merge_dicts(dict_a, dict_b, default_key="default"):
    """
    Recursively merge two nested dicts into a single dict.
    When keys match their values are merged using
    a recursive call to this function,
    otherwise they are just added to the output dict.
    """
    if dict_a is None or dict_a == {}:
        return dict_b
    if dict_b is None or dict_b == {}:
        return dict_a

    if type(dict_a) is not dict:
        if default_key in dict_b:
            return dict_b
        dict_a = {default_key: dict_a}
    if type(dict_b) is not dict:
        if default_key in dict_a:
            return dict_a
        dict_b = {default_key: dict_b}

    all_keys = set(dict_a.keys()).union(set(dict_b.keys()))

    out_dict = dict()
    for key in all_keys:
        out_dict[key] = merge_dicts(
            dict_a.get(key), dict_b.get(key), default_key)
    return out_dict


def list_to_nested_dict(lst):
    """
    [1,2,3,4] -> {1:{2:{3:4}}}
    """
    if len(lst) > 1:
        return {lst[0]: list_to_nested_dict(lst[1:])}
    else:
        return lst[0]


def dealias_and_group_headers(dict_array, header_aliases, use_double_colons,
                              default_language=u"default", ignore_case=False):
    """
    For each row in the worksheet, group all keys that contain a double colon.
    So
        {"text::english": "hello", "text::french" : "bonjour"}
    becomes
        {"text": {"english": "hello", "french" : "bonjour"}.
    Dealiasing is done to the first token
    (the first term separated by the delimiter).
    default_language -- used to group labels/hints/etc
    without a language specified with localized versions.
    """
    group_delimiter = u"::"
    out_dict_array = list()
    for row in dict_array:
        out_row = dict()
        for header, val in row.items():

            if ignore_case:
                header = header.lower()

            if use_double_colons:
                tokens = header.split(group_delimiter)

            # else:
            #   We do the initial parse using single colons
            #   for backwards compatibility and
            #   only the first single is used
            #   in order to avoid nesting jr:something tokens.
            #   if len(tokens) > 1:
            #       tokens[1:] = [u":".join(tokens[1:])]
            else:
                # I think the commented out section above
                # break if there is something like media:image:english
                # so maybe a better backwards compatibility hack
                # is to join any jr token with the next token
                tokens = header.split(u":")
                if "jr" in tokens:
                    jr_idx = tokens.index("jr")
                    tokens[jr_idx] = u":".join(tokens[jr_idx: jr_idx + 2])
                    tokens.pop(jr_idx + 1)

            dealiased_first_token = header_aliases.get(tokens[0], tokens[0])
            tokens = dealiased_first_token.split(group_delimiter) + tokens[1:]
            new_key = tokens[0]
            new_value = list_to_nested_dict(tokens[1:] + [val])
            out_row = merge_dicts(
                out_row, {new_key: new_value}, default_language)

        out_dict_array.append(out_row)
    return out_dict_array


def dealias_types(dict_array):
    """
    Look at all the type values in a dict array and if any aliases are found,
    replace them with the name they map to.
    """
    for row in dict_array:
        found_type = row.get(constants.TYPE)
        if found_type in aliases._type.keys():
            row[constants.TYPE] = aliases._type[found_type]
    return dict_array


def clean_text_values(dict_array):
    """
    Go though the dict array and strips all text values.
    Also replaces multiple spaces with single spaces.
    Note that the keys don't get cleaned, which could be an issue.
    """
    for row in dict_array:
        for key, value in row.items():
            if isinstance(value, basestring):
                value = re.sub(r"( )+", " ", value.strip())
                for smart_quote, dumb_quote in SMART_QUOTES.items():
                    if smart_quote in value:
                        value = value.replace(smart_quote, dumb_quote)
                row[key] = value
    return dict_array


# This is currently unused because name uniqueness is checked in json2xform.
def check_name_uniqueness(dict_array):
    """
    Make sure all names are unique
    Raises and exception if a duplicate is found
    """
    # This set is used to validate the uniqueness of names.
    name_set = set()
{'fraction_non_alphanumeric': 0.09231345379646101, 'fraction_numerical': 0.006366445089411104, 'mean_word_length': 2.3215174129353233, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2315037', 'n_tokens_mistral': 3295, 'n_tokens_neox': 3179, 'n_words': 1013}