<commit_before>#!/bin/bash

# The Java class deploys and runs a pipeline that converts PMC nxml files to plain text
# The plain text is cached in a separate file, as well as an annotation file containing
# document section annotations. The run catalog is updated accordingly.
export MAVEN_OPTS="-Xmx10G"

PIPELINE_WORK_DIR=/nlp_data/pipeline_work/pmcoa/xml2txt
mkdir -p "$PIPELINE_WORK_DIR"

mvn -f /nlp-pipelines.git/nlp-pipelines-runner/pom.xml exec:java \
-Dexec.mainClass="edu.ucdenver.ccp.nlp.pipelines.runner.impl.PmcNxml2TxtPipeline" \
-Dexec.args="/nlp_data/catalog $PIPELINE_WORK_DIR $1"<commit_msg>Added arguments for cas pool size and num-to-process to xml2txt.sh<commit_after>#!/bin/bash

# The Java class deploys and runs a pipeline that converts PMC nxml files to plain text
# The plain text is cached in a separate file, as well as an annotation file containing
# document section annotations. The run catalog is updated accordingly.
#
# arg1 = the broker URL
# arg2 = number of documents to process (<0 = process all)
# arg3 = CAS pool size
#
export MAVEN_OPTS="-Xmx10G"

PIPELINE_WORK_DIR=/nlp_data/pipeline_work/pmcoa/xml2txt
mkdir -p "$PIPELINE_WORK_DIR"

mvn -f /nlp-pipelines.git/nlp-pipelines-runner/pom.xml exec:java \
-Dexec.mainClass="edu.ucdenver.ccp.nlp.pipelines.runner.impl.PmcNxml2TxtPipeline" \
-Dexec.args="/nlp_data/catalog $PIPELINE_WORK_DIR $1 $2 $3"
{'fraction_non_alphanumeric': 0.09747292418772563, 'fraction_numerical': 0.01227436823104693, 'mean_word_length': 4.897872340425532, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1772489', 'n_tokens_mistral': 508, 'n_tokens_neox': 489, 'n_words': 139}