<commit_before>"""Development script to be able to identify an entity given a variety of inputs."""

import os
import re


def _expand_entity(data, entity, seen=None):
    
    if seen is None:
        seen = set()
    if entity in seen:
        return
    seen.add(entity)
    
    _type = entity['type'].lower()
    data['type'] = entity['type']
    data['id'] = entity['id']
    
    for k, v in entity.iteritems():
        if k != 'type':
            data.setdefault(_type + '_' + k, v)
        if isinstance(v, dict):
            _expand_entity(data, v, seen)


def _data_from_path(sgfs, path, entity_types=None):
    data = {}
    entities = sgfs.entities_from_path(path, entity_type=entity_types)
    for entity in entities:
        _expand_entity(data, entity)
    return (entities[0] if entities else None, data)


def parse_spec(sgfs, spec, entity_types=None, project_from_page=False):

    # We used to accept multiple parameters, but now we just take one.
    if not isinstance(spec, str):
        if not spec:
            spec = '.'
        elif len(spec) == 1:
            spec = spec[0]
        else:
            raise TypeError('spec must be string or single-item list; got %r' % spec)

    # Accept integer IDs if we know we want a specific type.
    if spec.isdigit():
        if len(entity_types) == 1:
            return {'type': entity_types[0], 'id': int(spec)}
        else:
            raise ValueError('int-only spec with %d entity_types' % len(entity_types))

    # Paths (which must have been created via Tank).<commit_msg>Must merge all results from parse_spec<commit_after>"""Development script to be able to identify an entity given a variety of inputs."""

import os
import re


def _expand_entity(data, entity, seen=None):
    
    if seen is None:
        seen = set()
    if entity in seen:
        return
    seen.add(entity)
    
    _type = entity['type'].lower()
    data['type'] = entity['type']
    data['id'] = entity['id']
    
    for k, v in entity.iteritems():
        if k != 'type':
            data.setdefault(_type + '_' + k, v)
        if isinstance(v, dict):
            _expand_entity(data, v, seen)


def _data_from_path(sgfs, path, entity_types=None):
    data = {}
    entities = sgfs.entities_from_path(path, entity_type=entity_types)
    for entity in entities:
        _expand_entity(data, entity)
    return (entities[0] if entities else None, data)


def parse_spec(sgfs, spec, entity_types=None, project_from_page=False):

    # We used to accept multiple parameters, but now we just take one.
    if not isinstance(spec, str):
        if not spec:
            spec = '.'
        elif len(spec) == 1:
            spec = spec[0]
        else:
            raise TypeError('spec must be string or single-item list; got %r' % spec)

    # Accept integer IDs if we know we want a specific type.
    if spec.isdigit():
        if len(entity_types) == 1:
            return sgfs.session.merge({'type': entity_types[0], 'id': int(spec)})
        else:
            raise ValueError('int-only spec with %d entity_types' % len(entity_types))

    # Paths (which must have been created via Tank).
{'fraction_non_alphanumeric': 0.11255961844197138, 'fraction_numerical': 0.003179650238473768, 'mean_word_length': 2.519015659955257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3357458', 'n_tokens_mistral': 971, 'n_tokens_neox': 948, 'n_words': 318}