<commit_before>from __future__ import print_function, unicode_literals
from collections import defaultdict
from contextlib import closing, contextmanager
import glob
import os
import string
try:
    from StringIO import StringIO
except ImportError:
    # Python 3
    from io import StringIO

import yaml


def str_or_list(val):
    if isinstance(val, str):
        yield val
    else:
        assert isinstance(val, list)
        for item in val:
            yield item


basename_permitted_chars = string.ascii_lowercase + string.digits

def basename_for_name(name):
    name = ''.join((c if c in basename_permitted_chars else '_')
                   for c in name.lower())
    return '_'.join(part for part in name.split('_') if part)


all_basenames = set()
groups = {}
components = {}
group_members = defaultdict(set)


def register_element(info):
    if (info.get('display_as_group', False) or
        info.get('unmoveable', False)):
        # Elements with these attributes are used for structuring
        # MWorks XML files and aren't real components.  Ignore them.
        return

    name = info['name']
    basename = basename_for_name(name)
    assert basename not in all_basenames, ('Duplicate basename: %r' %
                                           basename)
    info['basename'] = basename
    all_basenames.add(basename)

    if info.get('is_group', False):
        groups[name] = info
    else:
        group = set(str_or_list(info.get('group', [])))
        for ancestor in str_or_list(info.get('isa', [])):
            group.update(components[ancestor]['group'])
        info['group'] = group
        assert group, 'No groups for %r' % name
        for g in group:
            if not info.get('abstract', False):
                group_members[g].add(basename)
        components[name] = info


yaml_bundle_path = '/Resources/MWComponents.yaml'
yaml_files = ['/Library/Frameworks/MWorksCore.framework' + yaml_bundle_path]
yaml_files += glob.glob('/Library/Application Support/MWorks/Plugins/Core'
                        '/*.bundle/Contents' + yaml_bundle_path)

for filename in yaml_files:
    with open(filename) as fp:
        for info in yaml.safe_load_all(fp):
            register_element(info)

os.chdir('components')


@contextmanager<commit_msg>Implemented parameter inheritance in generate_component_ref.py<commit_after>from __future__ import print_function, unicode_literals
from collections import defaultdict
from contextlib import closing, contextmanager
import glob
import os
import string
try:
    from StringIO import StringIO
except ImportError:
    # Python 3
    from io import StringIO

import yaml


def str_or_list(val):
    if isinstance(val, str):
        yield val
    else:
        assert isinstance(val, list)
        for item in val:
            yield item


basename_permitted_chars = string.ascii_lowercase + string.digits

def basename_for_name(name):
    name = ''.join((c if c in basename_permitted_chars else '_')
                   for c in name.lower())
    return '_'.join(part for part in name.split('_') if part)


all_basenames = set()
groups = {}
components = {}
group_members = defaultdict(set)


def register_element(info):
    if (info.get('display_as_group', False) or
        info.get('unmoveable', False)):
        # Elements with these attributes are used for structuring
        # MWorks XML files and aren't real components.  Ignore them.
        return

    name = info['name']
    basename = basename_for_name(name)
    assert basename not in all_basenames, ('Duplicate basename: %r' %
                                           basename)
    info['basename'] = basename
    all_basenames.add(basename)

    if info.get('is_group', False):
        groups[name] = info
    else:
        group = set(str_or_list(info.get('group', [])))
        parameters = info.get('parameters', [])
        param_names = set(p['name'] for p in parameters)
        for ancestor in str_or_list(info.get('isa', [])):
            ancestor = components[ancestor]
            group.update(ancestor['group'])
            for p in ancestor.get('parameters', []):
                if p['name'] not in param_names:
                    parameters.append(p)
                    param_names.add(p['name'])
        info['group'] = group
        if parameters:
            info['parameters'] = parameters
        assert group, 'No groups for %r' % name
        for g in group:
            if not info.get('abstract', False):
                group_members[g].add(basename)
        components[name] = info


yaml_bundle_path = '/Resources/MWComponents.yaml'
yaml_files = ['/Library/Frameworks/MWorksCore.framework' + yaml_bundle_path]
yaml_files += glob.glob('/Library/Application Support/MWorks/Plugins/Core'
                        '/*.bundle/Contents' + yaml_bundle_path)

for filename in yaml_files:
    with open(filename) as fp:
        for info in yaml.safe_load_all(fp):
            register_element(info)

os.chdir('components')


@contextmanager
{'fraction_non_alphanumeric': 0.1017017017017017, 'fraction_numerical': 0.0004004004004004004, 'mean_word_length': 2.6925351071692534, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2735497', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1417, 'n_words': 392}