<commit_before>#!/bin/bash

SCRIPT_DIR=$(dirname "$0")
ROOT_DIR=${SCRIPT_DIR}/../..
RUN_DATE=$(date +"%Y%m%d-%k%M%S")

WORKFLOWS_DIR=${ROOT_DIR}/workflows
DATA_ROOT=${ROOT_DIR}/data
LOG_ROOT=${ROOT_DIR}/log

SCRIPT_NAME=$(basename "$0")
SCRIPT_NAME_TRIMMED="${SCRIPT_NAME%.*}"
JOB_NAME="${SCRIPT_NAME_TRIMMED}-${RUN_DATE}"
echo $JOB_NAME

OUT_DIR=${DATA_ROOT}/${JOB_NAME}/
LOG_DIR=${LOG_ROOT}/${JOB_NAME}/

mkdir -p $OUT_DIR
mkdir -p $LOG_DIR
echo "Starting on $(date)..."
cwltool \
  --debug \
  --outdir ${OUT_DIR} \
  ${WORKFLOWS_DIR}/exomeseq-01-preprocessing.cwl \
  exomeseq-01-preprocessing.json \
  > >(tee ${LOG_DIR}/${SCRIPT_NAME_TRIMMED}-out.log) \
  2> >(tee ${LOG_DIR}/${SCRIPT_NAME_TRIMMED}-err.log >&2)
echo "Finished on $(date)..."<commit_msg>Updates CWL tmpdir in exomeseq-01.sh since /tmp was filling up<commit_after>#!/bin/bash

SCRIPT_DIR=$(dirname "$0")
ROOT_DIR=${SCRIPT_DIR}/../..
RUN_DATE=$(date +"%Y%m%d-%k%M%S")

WORKFLOWS_DIR=${ROOT_DIR}/workflows
DATA_ROOT=${ROOT_DIR}/data
LOG_ROOT=${ROOT_DIR}/log

SCRIPT_NAME=$(basename "$0")
SCRIPT_NAME_TRIMMED="${SCRIPT_NAME%.*}"
JOB_NAME="${SCRIPT_NAME_TRIMMED}-${RUN_DATE}"
echo $JOB_NAME

OUT_DIR=${DATA_ROOT}/${JOB_NAME}/
LOG_DIR=${LOG_ROOT}/${JOB_NAME}/
TMP_DIR=${DATA_ROOT}/tmp/tmp

mkdir -p $OUT_DIR
mkdir -p $LOG_DIR
echo "Starting on $(date)..."
cwltool \
  --debug \
  --outdir ${OUT_DIR} \
  --tmpdir-prefix ${TMP_DIR} \
  --tmp-outdir-prefix ${TMP_DIR} \
  ${WORKFLOWS_DIR}/exomeseq-01-preprocessing.cwl \
  exomeseq-01-preprocessing.json \
  > >(tee ${LOG_DIR}/${SCRIPT_NAME_TRIMMED}-out.log) \
  2> >(tee ${LOG_DIR}/${SCRIPT_NAME_TRIMMED}-err.log >&2)
echo "Finished on $(date)..."
{'fraction_non_alphanumeric': 0.21780986762936222, 'fraction_numerical': 0.010830324909747292, 'mean_word_length': 5.275471698113208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1683390', 'n_tokens_mistral': 739, 'n_tokens_neox': 712, 'n_words': 99}