<commit_before>__author__ = 'kuhn'

import os


from batchxslt import cmdiresource

corpus = "/tmp/cmdi/PF/corpus/"
event = "/tmp/cmdi/PF/events/"
speakers = "//tmp/cmdi/PF/speakers/"
transcripts = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/transcripts/"

cmdi_final = '/tmp/cmdi/cmdiPF/'

resourcetree = cmdiresource.ResourceTreeCollection(corpus, event, speakers, transcripts)

resourcetree.build_resourceproxy()

for nodename in resourcetree.nodes_iter():
    if '_E_' in nodename and (resourcetree.node.get(nodename).get('type') != 'transcript'):
        resourcetree.speaker2event(nodename)

# FIXME: build_parts is not working
resourcetree.build_parts()

for nodename in resourcetree.nodes_iter():

    if '_E_' in nodename and resourcetree.node.get(nodename).get('type') != 'transcript':
        resourcetree.write_cmdi(nodename, os.path.join(cmdi_final, nodename + '.cmdi'))
    if nodename is 'PF':
        resourcetree.write_cmdi(nodename, os.path.join(cmdi_final, nodename + '.cmdi'))<commit_msg>changed types attribute for nodes to access metadata types more efficiently<commit_after>__author__ = 'kuhn'

import os


from batchxslt import cmdiresource

corpus = "/tmp/cmdi/PF/corpus/"
event = "/tmp/cmdi/PF/events/"
speakers = "//tmp/cmdi/PF/speakers/"
transcripts = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/transcripts/"

cmdi_final = '/tmp/cmdi/cmdiPF/'

resourcetree = cmdiresource.ResourceTreeCollection(corpus, event, speakers, transcripts)

resourcetree.build_resourceproxy()

resourcetree.build_parts()

for nodename in resourcetree.nodes_iter():
    if resourcetree.node.get(nodename).get('type') != 'speaker':
        resourcetree.define_parts(nodename)

for nodename in resourcetree.nodes_iter():
    if resourcetree.node.get(nodename).get('type') == 'event':
        resourcetree.speaker2event(nodename)

# FIXME: build_parts is not working


for nodename in resourcetree.nodes_iter():

    if resourcetree.node.get(nodename).get('type') != 'event':
        resourcetree.write_cmdi(nodename, os.path.join(cmdi_final, nodename + '.cmdi'))
    if nodename is 'PF':
        resourcetree.write_cmdi(nodename, os.path.join(cmdi_final, nodename + '.cmdi'))
{'fraction_non_alphanumeric': 0.13379310344827586, 'fraction_numerical': 0.005517241379310344, 'mean_word_length': 4.818181818181818, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3902859', 'n_tokens_mistral': 816, 'n_tokens_neox': 800, 'n_words': 139}