<commit_before># Sources:
open-file org.culturegraph.mf.stream.source.FileOpener
open-gzip org.culturegraph.mf.stream.source.GzipOpener
open-bzip2 org.culturegraph.mf.stream.source.Bzip2Opener
open-http org.culturegraph.mf.stream.source.HttpOpener
open-resource org.culturegraph.mf.stream.source.ResourceOpener
open-tar org.culturegraph.mf.stream.source.TarReader

read-string org.culturegraph.mf.stream.source.StringReader
read-dir	org.culturegraph.mf.stream.source.DirReader
read-triples	org.culturegraph.mf.stream.source.TripleReader

retrieve-triple-objects org.culturegraph.mf.stream.pipe.TripleObjectRetriever

sort-triples org.culturegraph.mf.stream.pipe.sort.TripleSort
count-triples org.culturegraph.mf.stream.pipe.sort.TripleCount
collect-triples org.culturegraph.mf.stream.pipe.sort.TripleCollect
stream-to-triples org.culturegraph.mf.stream.converter.StreamToTriples
filter-triples org.culturegraph.mf.stream.pipe.TripleFilter
triples-to-stream org.culturegraph.mf.stream.converter.TriplesToStream
calculate-metrics org.culturegraph.mf.stream.pipe.stat.CooccurrenceMetricCalculator

jscript org.culturegraph.mf.stream.pipe.JScriptObjectPipe

as-lines	org.culturegraph.mf.stream.converter.LineReader
as-records	org.culturegraph.mf.stream.converter.RecordReader
as-formeta-records	org.culturegraph.mf.stream.converter.FormetaRecordsReader
# Decoders:
decode-pica org.culturegraph.mf.stream.converter.bib.PicaDecoder
decode-mab org.culturegraph.mf.stream.converter.bib.MabDecoder
decode-cgentity org.culturegraph.mf.stream.converter.CGEntityDecoder
decode-cgtext org.culturegraph.mf.stream.converter.CGTextDecoder
decode-formeta org.culturegraph.mf.stream.converter.FormetaDecoder
decode-csv org.culturegraph.mf.stream.converter.CsvDecoder
decode-marc21 org.culturegraph.mf.stream.converter.bib.MarcDecoder
decode-xml org.culturegraph.mf.stream.converter.xml.XmlDecoder
decode-string org.culturegraph.mf.stream.pipe.StringDecoder
regex-decode org.culturegraph.mf.stream.converter.RegexDecoder

read-beacon org.culturegraph.mf.stream.reader.BeaconReader

# XML-Handler:
handle-cg-xml org.culturegraph.mf.stream.converter.xml.CGXmlHandler
handle-generic-xml org.culturegraph.mf.stream.converter.xml.GenericXmlHandler
handle-marcxml org.culturegraph.mf.stream.converter.xml.MarcXmlHandler

# Encoders:
encode-literals org.culturegraph.mf.stream.converter.StreamLiteralFormater
encode-cgentity org.culturegraph.mf.stream.converter.CGEntityEncoder
encode-formeta org.culturegraph.mf.stream.converter.FormetaEncoder
encode-json org.culturegraph.mf.stream.converter.JsonEncoder
encode-pica org.culturegraph.mf.stream.converter.bib.PicaEncoder
encode-marc21 org.culturegraph.mf.stream.converter.bib.Marc21Encoder

write	org.culturegraph.mf.stream.sink.ObjectWriter
write-triples	org.culturegraph.mf.stream.sink.TripleWriter
write-triple-objects org.culturegraph.mf.stream.sink.TripleObjectWriter

log-stream	org.culturegraph.mf.stream.pipe.StreamLogger
log-object	org.culturegraph.mf.stream.pipe.ObjectLogger
log-time	org.culturegraph.mf.stream.pipe.ObjectTimer
#catch-exceptions org.culturegraph.mf.stream.pipe.ObjectExceptionCatcher (duplicate entry)


change-id	org.culturegraph.mf.stream.pipe.IdChangePipe
flatten	org.culturegraph.mf.stream.pipe.StreamFlattener
<commit_msg>Resolve #237: Merge branch hbz/culturegraph/issue-235...<commit_after># Sources:
open-file org.culturegraph.mf.stream.source.FileOpener
open-gzip org.culturegraph.mf.stream.source.GzipOpener
open-bzip2 org.culturegraph.mf.stream.source.Bzip2Opener
open-http org.culturegraph.mf.stream.source.HttpOpener
open-resource org.culturegraph.mf.stream.source.ResourceOpener
open-tar org.culturegraph.mf.stream.source.TarReader

read-string org.culturegraph.mf.stream.source.StringReader
read-dir	org.culturegraph.mf.stream.source.DirReader
read-triples	org.culturegraph.mf.stream.source.TripleReader

retrieve-triple-objects org.culturegraph.mf.stream.pipe.TripleObjectRetriever

sort-triples org.culturegraph.mf.stream.pipe.sort.TripleSort
count-triples org.culturegraph.mf.stream.pipe.sort.TripleCount
collect-triples org.culturegraph.mf.stream.pipe.sort.TripleCollect
stream-to-triples org.culturegraph.mf.stream.converter.StreamToTriples
filter-triples org.culturegraph.mf.stream.pipe.TripleFilter
triples-to-stream org.culturegraph.mf.stream.converter.TriplesToStream
calculate-metrics org.culturegraph.mf.stream.pipe.stat.CooccurrenceMetricCalculator

jscript org.culturegraph.mf.stream.pipe.JScriptObjectPipe

as-lines	org.culturegraph.mf.stream.converter.LineReader
as-records	org.culturegraph.mf.stream.converter.RecordReader
as-formeta-records	org.culturegraph.mf.stream.converter.FormetaRecordsReader
# Decoders:
decode-pica org.culturegraph.mf.stream.converter.bib.PicaDecoder
decode-mab org.culturegraph.mf.stream.converter.bib.MabDecoder
decode-cgentity org.culturegraph.mf.stream.converter.CGEntityDecoder
decode-cgtext org.culturegraph.mf.stream.converter.CGTextDecoder
decode-formeta org.culturegraph.mf.stream.converter.FormetaDecoder
decode-csv org.culturegraph.mf.stream.converter.CsvDecoder
decode-marc21 org.culturegraph.mf.stream.converter.bib.MarcDecoder
decode-xml org.culturegraph.mf.stream.converter.xml.XmlDecoder
decode-string org.culturegraph.mf.stream.pipe.StringDecoder
regex-decode org.culturegraph.mf.stream.converter.RegexDecoder

read-beacon org.culturegraph.mf.stream.reader.BeaconReader

# XML-Handler:
handle-cg-xml org.culturegraph.mf.stream.converter.xml.CGXmlHandler
handle-generic-xml org.culturegraph.mf.stream.converter.xml.GenericXmlHandler
handle-marcxml org.culturegraph.mf.stream.converter.xml.MarcXmlHandler
handle-picaxml	org.culturegraph.mf.stream.converter.xml.PicaXmlHandler

# Encoders:
encode-literals org.culturegraph.mf.stream.converter.StreamLiteralFormater
encode-cgentity org.culturegraph.mf.stream.converter.CGEntityEncoder
encode-formeta org.culturegraph.mf.stream.converter.FormetaEncoder
encode-json org.culturegraph.mf.stream.converter.JsonEncoder
encode-pica org.culturegraph.mf.stream.converter.bib.PicaEncoder
encode-marc21 org.culturegraph.mf.stream.converter.bib.Marc21Encoder

write	org.culturegraph.mf.stream.sink.ObjectWriter
write-triples	org.culturegraph.mf.stream.sink.TripleWriter
write-triple-objects org.culturegraph.mf.stream.sink.TripleObjectWriter

log-stream	org.culturegraph.mf.stream.pipe.StreamLogger
log-object	org.culturegraph.mf.stream.pipe.ObjectLogger
log-time	org.culturegraph.mf.stream.pipe.ObjectTimer
#catch-exceptions org.culturegraph.mf.stream.pipe.ObjectExceptionCatcher (duplicate entry)


change-id	org.culturegraph.mf.stream.pipe.IdChangePipe
flatten	org.culturegraph.mf.stream.pipe.StreamFlattener

{'fraction_non_alphanumeric': 0.10176858513189449, 'fraction_numerical': 0.0032973621103117505, 'mean_word_length': 6.414444444444444, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3096291', 'n_tokens_mistral': 2301, 'n_tokens_neox': 2174, 'n_words': 220}