<commit_before>#!/bin/bash

# Batch qsub submission script for model search over SdA layer sizes 

arr=(`ls $SCRATCH/gpu_models/SdA/pretrain_pkl_files/batch_100_size_25ch_mom_0.8_wd_0.00001_lr_0.005`)
offset=0
len=${#arr[*]}

for((i=1; i<=$len; i+=2 ))
do
    let prev=$i-1
    first=${arr[$i]}
    second=${arr[$prev]}
    qsub submit_finetune_gravity.sh -v FIRSTMODEL="$first",SECONDMODEL="$second",OFFSET="$offset"
    ((offset+=5))
    sleep 5
    
    # Each pair of jobs needs 30 data chunks, and there are 211 in total.  Reset the offset parameter if necessary.
    if (( offset > 181 )); then 
       offset=0
    fi    
done<commit_msg>oops, forgot to update the meta job submission script for finetuning.<commit_after>#!/bin/bash

# Batch qsub submission script for model search over SdA layer sizes 

arr=(`ls $SCRATCH/gpu_models/SdA/pretrain_pkl_files/40`)
offset=0
len=${#arr[*]}

for((i=1; i<=$len; i+=2 ))
do
    let prev=$i-1
    first=${arr[$i]}
    second=${arr[$prev]}
    qsub submit_finetune_gravity.sh -v FIRSTMODEL="$first",SECONDMODEL="$second",OFFSET="$offset"
    ((offset+=5))
    sleep 5
    
    # Each pair of jobs needs 30 data chunks, and there are 211 in total.  Reset the offset parameter if necessary.
    if (( offset > 181 )); then 
       offset=0
    fi    
done
{'fraction_non_alphanumeric': 0.14066102997694083, 'fraction_numerical': 0.03766333589546503, 'mean_word_length': 3.6666666666666665, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2352366', 'n_tokens_mistral': 499, 'n_tokens_neox': 455, 'n_words': 125}