<commit_before>#!/usr/bin/bash

#SBATCH --job-name=BTRT
#SBATCH --ntasks=1
#SBATCH --tasks-per-node=1
#SBATCH --mail-user=<EMAIL>
#SBATCH --mail-type=ALL
#SBATCH -p swat_plus
#SBATCH -t 48:00:00
#SBATCH --array=0-30

START_DAY=($(seq -f "%02g" 1 30))
END_DAY=($(seq -f "%02g" 1 30))

# cd to directory where job was submitted from
cd $SLURM_SUBMIT_DIR

# get the day information from the array
start_day=${START_DAY[$SLURM_ARRAY_TASK_ID]}
end_day=${END_DAY[$SLURM_ARRAY_TASK_ID]}

end_mon=06
start_mon=06
year=2014
in_dir=/condo/swatcommon/common/myrorss/$year/$year$start_mon$start_day/segmotion_maxrefl/PolygonTable/scale_0/
out_dir=/condo/swatwork/arkweather/best_track/tracks_RT/201506/segmotion/

echo $SLURM_ARRAY_TASK_ID
#echo $in_dir
#echo $(seq -f "%02g" 9 30)

python archiveBTRT.py $year$start_mon$start_day $year$end_mon$end_day $in_dir $out_dir -i xml -t seg_json






<commit_msg>Updated the .sh file with recommended settings<commit_after>#!/usr/bin/bash

#SBATCH --job-name=BTRT
#SBATCH --ntasks=1
#SBATCH --tasks-per-node=1
#SBATCH --mail-user=<EMAIL>
#SBATCH --mail-type=ALL
#SBATCH -p swat_plus
#SBATCH -t 48:00:00
#SBATCH --array=0-30

START_DAY=($(seq -f "%02g" 1 30))
END_DAY=($(seq -f "%02g" 1 30))

# cd to directory where job was submitted from
cd $SLURM_SUBMIT_DIR

# get the day information from the array
start_day=${START_DAY[$SLURM_ARRAY_TASK_ID]}
end_day=${END_DAY[$SLURM_ARRAY_TASK_ID]}

end_mon=06
start_mon=06
year=2014
in_dir=/condo/swatcommon/common/myrorss/$year/$year$start_mon$start_day/segmotion_maxrefl/PolygonTable/scale_0/
out_dir=/condo/swatwork/arkweather/best_track/tracks_RT/201506/segmotion/

echo $SLURM_ARRAY_TASK_ID
#echo $in_dir
#echo $(seq -f "%02g" 9 30)

python archiveBTRT.py $year$start_mon$start_day $year$end_mon$end_day $in_dir $out_dir -it xml -ot seg_json -bt 5 -bd 10 -ht 30







{'fraction_non_alphanumeric': 0.14525745257452574, 'fraction_numerical': 0.04715447154471545, 'mean_word_length': 5.43205574912892, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4955668', 'n_tokens_mistral': 870, 'n_tokens_neox': 784, 'n_words': 158}