<commit_before>indir       = "/home/seb/temp/tempSample";
outdir      = "/home/seb/temp/tempSample";
filesize    = "10000";
esnodes     = "localhost:9300";
escluster   = "elasticsearch";
records     = "5000";
index       = "testsb3";


indir|
read-dir|
open-gzip|
decode-xml|
handle-marcxml|
stream-tee| {
    morph(FLUX_DIR + "resourceMorph.xml")|
    change-id|
    encode-esbulk-ng(escapeChars="true", header="true", index=index, type="bibliographicResource")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    morph(FLUX_DIR + "documentMorph.xml")|
    change-id|
    encode-esbulk-ng(escapeChars="true", header="true", index=index, type="document")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    morph(FLUX_DIR + "personMorph.xml")|
    split-entities|
    change-id|
    encode-esbulk-ng(escapeChars="true", header="true", index=index, type="person")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    morph(FLUX_DIR + "organisationMorph.xml")|
    split-entities|
    change-id|
    encode-esbulk-ng(escapeChars="true", header="true", index=index, type="organisation")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    filter(FLUX_DIR + "workFilter.xml")|
    morph(FLUX_DIR + "workMorph1.xml")|
    stream-to-triples(redirect="true")|
    sort-triples(by="all")|
    collect-triples|
    morph(FLUX_DIR + "workMorph2.xml")|
    encode-esbulk-ng(escapeChars="true", header="true", index=index, type="work")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
};

//TODO:
//morph(FLUX_DIR + "itemMorph.xml")|<commit_msg>Functionality for printing out JSON compliant strings (instead of Bulk API compliant)<commit_after>indir       = "/home/seb/temp/marcDataMFSample";
outdir      = "/home/seb/temp/marcDataMFSample";
file        = "/home/seb/temp/marcDataMFSample/sample.xml.gz";
filesize    = "10000";
esnodes     = "localhost:9300";
escluster   = "elasticsearch";
records     = "5000";
index       = "testsb";
bulkheader      = "false";
jsoncompliant = "true";

indir|
read-dir|
//file|
open-gzip|
decode-xml|
handle-marcxml|
stream-tee| {
    morph(FLUX_DIR + "resourceMorph.xml")|
    change-id|
    encode-esbulk-ng(escapeChars="true", header=bulkheader, index=index, type="bibliographicResource")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize, jsonCompliant=jsoncompliant)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    morph(FLUX_DIR + "documentMorph.xml")|
    change-id|
    encode-esbulk-ng(escapeChars="true", header=bulkheader, index=index, type="document")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize, jsonCompliant=jsoncompliant)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    morph(FLUX_DIR + "personMorph.xml")|
    split-entities|
    change-id|
    encode-esbulk-ng(escapeChars="true", header=bulkheader, index=index, type="person")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize, jsonCompliant=jsoncompliant)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    morph(FLUX_DIR + "organisationMorph.xml")|
    split-entities|
    change-id|
    encode-esbulk-ng(escapeChars="true", header=bulkheader, index=index, type="organisation")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize, jsonCompliant=jsoncompliant)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
}{
    filter(FLUX_DIR + "workFilter.xml")|
    morph(FLUX_DIR + "workMorph1.xml")|
    stream-to-triples(redirect="true")|
    sort-triples(by="all")|
    collect-triples|
    morph(FLUX_DIR + "workMorph2.xml")|
    encode-esbulk-ng(escapeChars="true", header=bulkheader, index=index, type="work")|
    write-esbulk(baseOutDir=outdir, fileSize=filesize, jsonCompliant=jsoncompliant)
    //index-esbulk(esNodes=esnodes, esClustername=escluster, recordsPerUpload=records)
};

//TODO:
//morph(FLUX_DIR + "itemMorph.xml")|
{'fraction_non_alphanumeric': 0.15164585698070374, 'fraction_numerical': 0.007037457434733258, 'mean_word_length': 5.179523141654979, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 21}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '255579', 'n_tokens_mistral': 1620, 'n_tokens_neox': 1587, 'n_words': 204}