<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2011, <NAME>
All rights reserved.
"""

from nibble import config
reload(config)
from nibble import spm
reload(spm)

from pdb import set_trace

if __name__ == '__main__':
    input_config = ['/Users/scottburns/Code/Nibble/config/cutting.yaml',
                    '/Users/scottburns/Code/Nibble/config/LDRC_KKI/LDRC_KKI.yaml',
                    '/Users/scottburns/Code/Nibble/config/LDRC_KKI/subjects.yaml']
    output_path = '/Users/scottburns/Code/Nibble/config/generated/LDRC_KKI_final.yaml'

    #starting with config
    cfg = config.Configurator(input_config)
    all_data = cfg.load_yaml(verbose=False)
    cfg.save_yaml(output_path)
        
    print('')
    print('')
    
    # starting with SPM
    total = config.yaml2data(output_path)
    ver = total['version']
    subjects = total['subjects']
    if ver == 1:
        project = total['project']
        tdl = project['todo']
        paradigms = project['paradigms']
        subjects = project['subjects']
        for stream in tdl: # only spm for now
            for paradigm in paradigms:
                for subj in subjects:
                    if stream == 'SPM':
                        pieces = tdl[stream]
                        spm_obj = spm.SPM(subj, paradigm, pieces, total)
                        spm_obj.resolve()
                        spm_obj.dump()<commit_msg>subjects is now a top level list<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2011, <NAME>
All rights reserved.
"""

from nibble import config
reload(config)
from nibble import spm
reload(spm)

from pdb import set_trace

if __name__ == '__main__':
    input_config = ['/Users/scottburns/Code/Nibble/config/cutting.yaml',
                    '/Users/scottburns/Code/Nibble/config/LDRC_KKI/LDRC_KKI.yaml',
                    '/Users/scottburns/Code/Nibble/config/LDRC_KKI/subjects.yaml']
    output_path = '/Users/scottburns/Code/Nibble/config/generated/LDRC_KKI_final.yaml'

    #starting with config
    cfg = config.Configurator(input_config)
    all_data = cfg.load_yaml(verbose=False)
    cfg.save_yaml(output_path)
        
    print('')
    print('')
    
    # starting with SPM
    total = config.yaml2data(output_path)
    ver = total['version']
    subjects = total['subjects']
    if ver == 1:
        project = total['project']
        tdl = project['todo']
        paradigms = project['paradigms']
        for stream in tdl: # only spm for now
            for paradigm in paradigms:
                for subj in subjects:
                    if stream == 'SPM':
                        pieces = tdl[stream]
                        spm_obj = spm.SPM(subj, paradigm, pieces, total)
                        spm_obj.resolve()
                        spm_obj.dump()
{'fraction_non_alphanumeric': 0.11179007907979871, 'fraction_numerical': 0.004313443565780014, 'mean_word_length': 2.134009009009009, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '51699', 'n_tokens_mistral': 933, 'n_tokens_neox': 891, 'n_words': 183}