<commit_before>#!/usr/bin/env bash
# Collection of routine functions

ORIG_DATA_DIR=external
EXP_DIR=exp
TEMP_DIR=${EXP_DIR}/temp
DATA_DIR=${EXP_DIR}/datasets
MAT_DIR=${EXP_DIR}/matrices
RESULTS_DIR=${EXP_DIR}/results
#TRIALS=`seq -f "%03g" 0 99`
#TRIALS=`seq -f "%03g" 0 0`

OVERWRITE=0
QSUB=1
#EXE=sh\ -c
EXE=echo

QSUB_DIR=${EXP_DIR}/qsub
QSUB_COMMAND='qsub -ug gr20111 -q tc -A p=1:t=1:c=1:m=128M'
UserGroup=gr20111
Queue=gr20100b


exec_command(){
  comm=$1

  # if len(args)>1 and File.exists($2) and OVERWRITE==0
  if [ $# -gt 1 -a ${OVERWRITE} -eq 0 ]; then
    local tar_num=$($2)
    if [[ ${tar_num} -eq 0 ]]; then
      # skip execution
      return
    fi
  fi
  if [ ${QSUB} -eq 0 ]; then
    ${EXE} "${comm}"
    return
  fi
  if [ $# -gt 2 ]; then
    option=$3<commit_msg>purge environmental variables from routines.<commit_after>#!/usr/bin/env bash
# Collection of routine functions


source scripts/my_env.sh

ORIG_DATA_DIR=external
EXP_DIR=exp
TEMP_DIR=${EXP_DIR}/temp
DATA_DIR=${EXP_DIR}/datasets
MAT_DIR=${EXP_DIR}/matrices
RESULTS_DIR=${EXP_DIR}/results
#TRIALS=`seq -f "%03g" 0 99`
#TRIALS=`seq -f "%03g" 0 0`


QSUB_DIR=${EXP_DIR}/qsub
QSUB_COMMAND='qsub -ug gr20111 -q tc -A p=1:t=1:c=1:m=128M'
UserGroup=gr20111
Queue=gr20100b


exec_command(){
  comm=$1

  # if len(args)>1 and File.exists($2) and OVERWRITE==0
  if [ $# -gt 1 -a ${OVERWRITE} -eq 0 ]; then
    local tar_num=$($2)
    if [[ ${tar_num} -eq 0 ]]; then
      # skip execution
      return
    fi
  fi
  if [ ${QSUB} -eq 0 ]; then
    ${EXE} "${comm}"
    return
  fi
  if [ $# -gt 2 ]; then
    option=$3
{'fraction_non_alphanumeric': 0.17042606516290726, 'fraction_numerical': 0.05263157894736842, 'mean_word_length': 3.6559766763848396, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '191282', 'n_tokens_mistral': 727, 'n_tokens_neox': 664, 'n_words': 158}