<commit_before>__author__ = 'kuhn'

from batchxslt import processor
from batchxslt import cmdiresource
import codecs
import os

dgd_corpus = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/metadata/corpora/extern"
dgd_events = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/metadata/events/extern"
dgd_speakers = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/metadata/speakers/extern"

xsl_corpus = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/xslt/dgdCorpus2cmdi.xsl"
xsl_events = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/xslt/dgdEvent2cmdi.xsl"
xsl_speakers = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/xslt/dgdSpeaker2cmdi.xsl"


pf_corpus = os.path.join(dgd_corpus, 'PF--_extern.xml')
pf_events = os.path.join(dgd_events, 'PF')
pf_speakers = os.path.join(dgd_speakers, 'PF')



<commit_msg>changed types attribute for nodes to access metadata types more efficiently<commit_after>__author__ = 'kuhn'

from batchxslt import processor
from batchxslt import cmdiresource
import codecs
import os

dgd_corpus = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/metadata/corpora/extern"
dgd_events = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/metadata/events/extern"
dgd_speakers = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/metadata/speakers/extern"

corpus_xsl = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/xslt/dgdCorpus2cmdi.xsl"
event_xsl = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/xslt/dgdEvent2cmdi.xsl"
speaker_xsl = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/xslt/dgdSpeaker2cmdi.xsl"

saxon_jar = "/home/kuhn/Data/IDS/svn/dgd2_data/dgd2cmdi/batchxslt/saxon/saxon9he.jar"

pf_corpus = os.path.join(dgd_corpus, 'PF--_extern.xml')
pf_events = os.path.join(dgd_events, 'PF')
pf_speakers = os.path.join(dgd_speakers, 'PF')

xsl_processor = processor.XSLBatchProcessor(saxon_jar)

xsl_processor.transform(corpus_xsl, pf_corpus, "cmdi_", '/tmp/cmdi/PF/corpus/')

xsl_processor.transform(event_xsl, pf_events, "cmdi_", '/tmp/cmdi/PF/events/')

xsl_processor.transform(speaker_xsl, pf_speakers, "cmdi_", '/tmp/cmdi/PF/speakers/')

from batchxslt import cmdiresource

corpus = "/tmp/cmdi/PF/corpus/"
event = "/tmp/cmdi/PF/events/"
speakers = "//tmp/cmdi/PF/speakers/"
transcripts = "/home/kuhn/Data/IDS/svn_rev1233/dgd2_data/transcripts/"

resourcetree = cmdiresource.ResourceTreeCollection(corpus, event, speakers, transcripts)
resourcetree.build_resourceproxy()

{'fraction_non_alphanumeric': 0.13835674755214986, 'fraction_numerical': 0.023839931885908897, 'mean_word_length': 5.714285714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5168164', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1016, 'n_words': 114}