<commit_before>#!/bin/bash

# Batch qsub submission script for 3,5 layer models 

# Submit 3 layer jobs

three_arr=(`ls $SCRATCH/gpu_tests/SdA_results/3_layers/pretrain_pkl_files/40/`)
offset=0
len=${#three_arr[*]}

for((i=1; i<=$len; i+=2 ))
do
    let prev=$i-1
    first=${three_arr[$i]}
    second=${three_arr[$prev]}
    qsub submit_finetune_gravity_3layers.sh -v FIRSTMODEL="$first",SECONDMODEL="$second",OFFSET="$offset"
    ((offset+=5))
    sleep 5
    
    # Each pair of jobs needs 30 data chunks, and there are 211 in total.  Reset the offset parameter if necessary.
    if (( offset > 181 )); then 
       offset=0
    fi    
done

# Submit 5 layer jobs

five_arr=(`ls $SCRATCH/gpu_tests/SdA_results/5_layers/pretrain_pkl_files/40/`)
offset=0
len=${#five_arr[*]}

for((i=1; i<=$len; i+=2 ))
do
    let prev=$i-1
    first=${five_arr[$i]}
    second=${five_arr[$prev]}
    qsub submit_finetune_gravity_5layers.sh -v FIRSTMODEL="$first",SECONDMODEL="$second",OFFSET="$offset"
    ((offset+=5))
    sleep 5
    
    # Each pair of jobs needs 30 data chunks, and there are 211 in total.  Reset the offset parameter if necessary.
    if (( offset > 181 )); then 
       offset=0
    fi    
done

<commit_msg>changed dimension for finetuning job submission<commit_after>#!/bin/bash

# Batch qsub submission script for 3,5 layer models 

# Submit 3 layer jobs

three_arr=(`ls $SCRATCH/gpu_tests/SdA_results/3_layers/pretrain_pkl_files/10/gb`)
offset=0
len=${#three_arr[*]}

for((i=1; i<=$len; i+=2 ))
do
    let prev=$i-1
    first=${three_arr[$i]}
    second=${three_arr[$prev]}
    qsub submit_finetune_gravity_3layers.sh -v FIRSTMODEL="$first",SECONDMODEL="$second",OFFSET="$offset"
    ((offset+=5))
    sleep 5
    
    # Each pair of jobs needs 30 data chunks, and there are 211 in total.  Reset the offset parameter if necessary.
    if (( offset > 181 )); then 
       offset=0
    fi    
done

# Submit 5 layer jobs

five_arr=(`ls $SCRATCH/gpu_tests/SdA_results/5_layers/pretrain_pkl_files/10/gb`)
offset=0
len=${#five_arr[*]}

for((i=1; i<=$len; i+=2 ))
do
    let prev=$i-1
    first=${five_arr[$i]}
    second=${five_arr[$prev]}
    qsub submit_finetune_gravity_5layers.sh -v FIRSTMODEL="$first",SECONDMODEL="$second",OFFSET="$offset"
    ((offset+=5))
    sleep 5
    
    # Each pair of jobs needs 30 data chunks, and there are 211 in total.  Reset the offset parameter if necessary.
    if (( offset > 181 )); then 
       offset=0
    fi    
done


{'fraction_non_alphanumeric': 0.14331983805668017, 'fraction_numerical': 0.0340080971659919, 'mean_word_length': 3.6447368421052633, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3832253', 'n_tokens_mistral': 950, 'n_tokens_neox': 880, 'n_words': 223}