<commit_before>// default file = "/home/sebastian/temp/job1r112A001.mf.xml";
default file = "/home/sebastian/temp/marc_mfcompatible/job1r115A001.format.xml.gz";
// basedir = "/home/sebastian/workspace/mfWorkflows/src/main/resources/transformation/resource";

file|
open-gzip|
// open-file|
decode-xml|
handle-marcxml|
morph(FLUX_DIR + "resourceMorph.xml")|
encode-esbulk|
// write-esbulk(baseOutDir="<Your path here>", fileSize="10000");
index-esbulk(esNode="localhost", esPort="9300", esClustername="linked-swissbib", recordsPerUpload="10000");<commit_msg>FLUX file adapted to new workflow<commit_after>// default file = "/home/sebastian/temp/job1r112A001.mf.xml";
default file = "/home/sebastian/temp/marc_mfcompatible/job1r115A001.format.xml.gz";
// basedir = "/home/sebastian/workspace/mfWorkflows/src/main/resources/transformation/resource";

file|
open-gzip|
// open-file|
decode-xml|
handle-marcxml|
morph(FLUX_DIR + "resourceMorph.xml")|
encode-esbulk|
// write-esbulk(baseOutDir="<Your path here>", fileSize="10000");
index-esbulk(esNodes="localhost", esClustername="linked-swissbib", recordsPerUpload="10000");
{'fraction_non_alphanumeric': 0.16964285714285715, 'fraction_numerical': 0.04642857142857143, 'mean_word_length': 6.523489932885906, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3230739', 'n_tokens_mistral': 444, 'n_tokens_neox': 406, 'n_words': 53}