<commit_before>import re
import string
import random
import json

from kpi.utils.sluggify import sluggify_label, is_valid_nodeName


def _increment(name):
    return name + '_0'


def _rand_id(n):
    return ''.join(random.choice(string.ascii_uppercase + string.digits)
                   for _ in range(n))


def _has_name(row):
    return 'name' in row and row['name'] != ''


def _is_group_end(row):
    row_type = row['type']
    return isinstance(row_type, basestring) and \
        (row_type.startswith('end ') or row_type.startswith('end_'))


def autoname_fields__depr(surv_content):
    '''
    Note: this method is deprecated but kept around to link prior deployments
    which don't have any names saved.
    '''
    surv_list = surv_content.get('survey')
    kuid_names = {}
    for surv_row in surv_list:
        if not _has_name(surv_row):
            if _is_group_end(surv_row):
                continue
            if 'label' in surv_row:
                next_name = sluggify_valid_xml__depr(surv_row['label'])
            else:
                raise ValueError('Label cannot be translated: %s' %
                                 json.dumps(surv_row))
            while next_name in kuid_names.values():
                next_name = _increment(next_name)
            if 'kuid' not in surv_row:
                surv_row['kuid'] = _rand_id(8)
            if surv_row['kuid'] in kuid_names:
                raise Exception("Duplicate kuid: %s" % surv_row['kuid'])
            surv_row['name'] = next_name
            kuid_names[surv_row['kuid']] = next_name
    # kuid is unused, and can't be compared with replacement method
    for surv_row in surv_list:
        if 'kuid' in surv_row:
            del surv_row['kuid']
    return surv_list


def autoname_fields(surv_content):
    surv_list = surv_content.get('survey')
    other_names = []

    # cycle through existing names ane ensure that names are valid and unique
    rows_with_names = filter(lambda r: _has_name(r) and not _is_group_end(r),<commit_msg>closes #950<commit_after>import re
import string
import random
import json

from kpi.utils.sluggify import sluggify_label, is_valid_nodeName


def _increment(name):
    return name + '_0'


def _rand_id(n):
    return ''.join(random.choice(string.ascii_uppercase + string.digits)
                   for _ in range(n))


def _has_name(row):
    return 'name' in row and row['name'] != ''


def _is_group_end(row):
    row_type = row['type']
    return isinstance(row_type, basestring) and \
        (row_type.startswith('end ') or row_type.startswith('end_'))


def autoname_fields__depr(surv_content):
    '''
    Note: this method is deprecated but kept around to link prior deployments
    which don't have any names saved.
    '''
    surv_list = surv_content.get('survey')
    kuid_names = {}
    for surv_row in surv_list:
        if not _has_name(surv_row):
            if _is_group_end(surv_row):
                continue
            if 'label' in surv_row:
                next_name = sluggify_valid_xml__depr(surv_row['label'])
            elif surv_row.get('type') == 'group':
                next_name = sluggify_valid_xml__depr('Grp')
            else:
                raise ValueError('Label cannot be translated: %s' %
                                 json.dumps(surv_row))
            while next_name in kuid_names.values():
                next_name = _increment(next_name)
            if 'kuid' not in surv_row:
                surv_row['kuid'] = _rand_id(8)
            if surv_row['kuid'] in kuid_names:
                raise Exception("Duplicate kuid: %s" % surv_row['kuid'])
            surv_row['name'] = next_name
            kuid_names[surv_row['kuid']] = next_name
    # kuid is unused, and can't be compared with replacement method
    for surv_row in surv_list:
        if 'kuid' in surv_row:
            del surv_row['kuid']
    return surv_list


def autoname_fields(surv_content):
    surv_list = surv_content.get('survey')
    other_names = []

    # cycle through existing names ane ensure that names are valid and unique
    rows_with_names = filter(lambda r: _has_name(r) and not _is_group_end(r),
{'fraction_non_alphanumeric': 0.09581126624939817, 'fraction_numerical': 0.001685122773230621, 'mean_word_length': 2.5092905405405403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6097221', 'n_tokens_mistral': 1348, 'n_tokens_neox': 1318, 'n_words': 335}