<commit_before>"""
Utility functions for parsing flatfiles,
mapping to JSON, cleaning.
"""
# see tabfile_feeder(coerce_unicode) if needed
#from __future__ import unicode_literals
import itertools
import csv
import os, os.path
import json
import collections

from .common import open_anyfile, is_str, ask, safewfile, anyfile

csv.field_size_limit(10000000)   # default is 131072, too small for some big files


# remove keys whos values are ".", "-", "", "NA", "none", " "
# and remove empty dictionaries
def dict_sweep(d, vals=[".", "-", "", "NA", "none", " ", "Not Available", "unknown"]):
    """
    @param d: a dictionary
    @param vals: a string or list of strings to sweep
    """
    for key, val in list(d.items()):
        if val in vals:
            del d[key]
        elif isinstance(val, list):
            for item in val:
                if item in vals:
                    val.remove(item)
                elif isinstance(item, dict):
                    dict_sweep(item, vals)
            if len(val) == 0:
                del d[key]
        elif isinstance(val, dict):
            dict_sweep(val, vals)
            if len(val) == 0:
                del d[key]
    return d


def to_number(val):
    """convert an input string to int/float."""
    if is_str(val):
        try:
            return int(val)
        except ValueError:
            try:
                return float(val)
            except ValueError:
                pass
    return val


def boolean_convert(d, convert_keys=[], level=0):
    """Explore document d and specified convert keys to boolean.
    Use dotfield notation for inner keys"""
    for key, val in d.items():
        if isinstance(val, dict):
            d[key] = boolean_convert(val, convert_keys)
        if key in [ak.split(".")[level] for ak in convert_keys if len(ak.split(".")) > level]:
            if isinstance(val, list) or isinstance(val, tuple):
                if val and isinstance(val[0],dict):
                    d[key] = [boolean_convert(v,convert_keys,level+1) for v in val]
                else:
                    d[key] = [to_boolean(x) for x in val]
            elif isinstance(val, dict) or isinstance(val, collections.OrderedDict):
                d[key] = boolean_convert(val, convert_keys, level+1)
            else:
                d[key] = to_boolean(val)
    return d


def to_boolean(val,true_str=['true','1', 't', 'y', 'yes', 'Y','Yes','YES',1],false_str=['false','0','f','n','N','No','no','NO',0]):
    if type(val)!=str:
        return bool(val)
    else:
        if val in true_str:
            return True
        elif val in false_str:
            return False


def merge_duplicate_rows(rows, db):
    """
    @param rows: rows to be grouped by
    @param db: database name, string
    """
    rows = list(rows)
    first_row = rows[0]
    other_rows = rows[1:]
    for row in other_rows:
        for i in first_row[db]:
            if i in row[db]:
                if row[db][i] != first_row[db][i]:
                    aa = first_row[db][i]
                    if not isinstance(aa, list):
                        aa = [aa]
                    aa.append(row[db][i])
                    first_row[db][i] = aa
            else:
                continue
    return first_row


def unique_ids(src_module):
    i = src_module.load_data()
    out = list(i)
    id_list = [a['_id'] for a in out if a]
    myset = set(id_list)
    print(len(out), "Documents produced")
    print(len(myset), "Unique IDs")
    return out


def rec_handler(infile, block_end='\n', skip=0, include_block_end=False, as_list=False):
    '''A generator to return a record (block of text)
       at once from the infile. The record is separated by
       one or more empty lines by default.<commit_msg>dict_sweep : fixed to remove [None, None] value items. merge_duplicate_rows : fixed for incomplete merge<commit_after>"""
Utility functions for parsing flatfiles,
mapping to JSON, cleaning.
"""
# see tabfile_feeder(coerce_unicode) if needed
#from __future__ import unicode_literals
import itertools
import csv
import os, os.path
import json
import collections

from .common import open_anyfile, is_str, ask, safewfile, anyfile

csv.field_size_limit(10000000)   # default is 131072, too small for some big files


# remove keys whos values are ".", "-", "", "NA", "none", " "
# and remove empty dictionaries
def dict_sweep(d, vals=[".", "-", "", "NA", "none", " ", "Not Available", "unknown"]):
    """
    @param d: a dictionary
    @param vals: a string or list of strings to sweep
    """
    for key, val in list(d.items()):
        if val in vals:
            del d[key]
        elif isinstance(val, list):
            val = [v for v in val if v not in vals]
            for item in val:
                if isinstance(item, dict):
                    dict_sweep(item, vals)
            if len(val) == 0:
                del d[key]
            else:
                d[key] = val
        elif isinstance(val, dict):
            dict_sweep(val, vals)
            if len(val) == 0:
                del d[key]
    return d


def to_number(val):
    """convert an input string to int/float."""
    if is_str(val):
        try:
            return int(val)
        except ValueError:
            try:
                return float(val)
            except ValueError:
                pass
    return val


def boolean_convert(d, convert_keys=[], level=0):
    """Explore document d and specified convert keys to boolean.
    Use dotfield notation for inner keys"""
    for key, val in d.items():
        if isinstance(val, dict):
            d[key] = boolean_convert(val, convert_keys)
        if key in [ak.split(".")[level] for ak in convert_keys if len(ak.split(".")) > level]:
            if isinstance(val, list) or isinstance(val, tuple):
                if val and isinstance(val[0],dict):
                    d[key] = [boolean_convert(v,convert_keys,level+1) for v in val]
                else:
                    d[key] = [to_boolean(x) for x in val]
            elif isinstance(val, dict) or isinstance(val, collections.OrderedDict):
                d[key] = boolean_convert(val, convert_keys, level+1)
            else:
                d[key] = to_boolean(val)
    return d


def to_boolean(val,true_str=['true','1', 't', 'y', 'yes', 'Y','Yes','YES',1],false_str=['false','0','f','n','N','No','no','NO',0]):
    if type(val)!=str:
        return bool(val)
    else:
        if val in true_str:
            return True
        elif val in false_str:
            return False


def merge_duplicate_rows(rows, db):
    """
    @param rows: rows to be grouped by
    @param db: database name, string
    """
    rows = list(rows)

    keys = set()
    for row in rows:
        for k in row[db]:
            keys.add(k)

    first_row = rows[0]
    other_rows = rows[1:]
    for row in other_rows:
        for i in keys:
            try:
                aa = first_row[db][i]
            except KeyError:
                try:
                    first_row[db][i] = row[db][i]
                except:
                    pass
                continue
            if i in row[db]:
                if row[db][i] != first_row[db][i]:
                    if not isinstance(aa, list):
                        aa = [aa]
                    aa.append(row[db][i])
                    first_row[db][i] = aa
            else:
                continue
    return first_row


def unique_ids(src_module):
    i = src_module.load_data()
    out = list(i)
    id_list = [a['_id'] for a in out if a]
    myset = set(id_list)
    print(len(out), "Documents produced")
    print(len(myset), "Unique IDs")
    return out


def rec_handler(infile, block_end='\n', skip=0, include_block_end=False, as_list=False):
    '''A generator to return a record (block of text)
       at once from the infile. The record is separated by
       one or more empty lines by default.
{'fraction_non_alphanumeric': 0.1190897597977244, 'fraction_numerical': 0.006826801517067004, 'mean_word_length': 1.977418140760256, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822000', 'n_tokens_mistral': 2488, 'n_tokens_neox': 2367, 'n_words': 754}