<commit_before>#!/bin/bash

# base_model or sub_model_1 or sub_model_2 or so on
#model_type="$1"

model_name="video_dcc_boosting_weightclip"
MODEL_DIR="../model/${model_name}"

vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"

rm ${MODEL_DIR}/ensemble.conf
if [ ! -f $vocab_file ]; then
  cd resources
  wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
  echo "OOV" > train.video_id.vocab
  cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
  cd ..
fi

vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
  echo $vocab_file is valid
else
  echo $vocab_file is corrupted
  exit 1
fi

if [ ! -f $default_freq_file ]; then
  cat $vocab_file | awk '{print 1}' > $default_freq_file
fi

base_model_dir="${MODEL_DIR}/base_model"

# base model (4 epochs)
mkdir -p $base_model_dir
for j in {1..2}; do 
  CUDA_VISIBLE_DEVICES=0 python train.py \
    --train_dir="$base_model_dir" \
    --train_data_pattern="/Youtube-8M/data/video/train/train*" \
    --frame_features=False \
    --feature_names="mean_rgb,mean_audio" \
    --feature_sizes="1024,128" \
    --model=DeepCombineChainModel \
    --moe_num_mixtures=4 \
    --deep_chain_relu_cells=256 \
    --deep_chain_layers=4 \
    --label_loss=MultiTaskCrossEntropyLoss \
    --multitask=True \
    --support_type="label,label,label,label" \
    --num_supports=18864 \
    --support_loss_percent=0.05 \
    --reweight=True \
    --sample_vocab_file="$vocab_file" \
    --sample_freq_file="$default_freq_file" \
    --keep_checkpoint_every_n_hour=8.0 \
    --keep_checkpoint_interval=6 \
    --base_learning_rate=0.01 \
    --data_augmenter=NoiseAugmenter \
    --input_noise_level=0.2 \
    --num_readers=2 \
    --num_epochs=2 \
    --batch_size=1024
done

last_freq_file=$default_freq_file

for i in {1..8}; do
  sub_model_dir="${MODEL_DIR}/sub_model_${i}"
  cp -r $base_model_dir $sub_model_dir

  echo "training model #$i, reweighting with $last_freq_file"
  # train N models with re-weighted samples
  CUDA_VISIBLE_DEVICES=0 python train.py \
    --train_dir="$sub_model_dir" \
    --train_data_pattern="/Youtube-8M/data/video/train/train*" \
    --frame_features=False \
    --feature_names="mean_rgb,mean_audio" \
    --feature_sizes="1024,128" \
    --model=DeepCombineChainModel \
    --moe_num_mixtures=4 \<commit_msg>update video dcc boosting weightclip<commit_after>#!/bin/bash

# base_model or sub_model_1 or sub_model_2 or so on
#model_type="$1"

model_name="video_dcc_boosting_weightclip"
MODEL_DIR="../model/${model_name}"

vocab_file="resources/train.video_id.vocab"
default_freq_file="resources/train.video_id.freq"

rm ${MODEL_DIR}/ensemble.conf
if [ ! -f $vocab_file ]; then
  cd resources
  wget http://us.data.yt8m.org/1/ground_truth_labels/train_labels.csv
  echo "OOV" > train.video_id.vocab
  cat train_labels.csv | cut -d ',' -f 1 >> train.video_id.vocab
  cd ..
fi

vocab_checksum=$(md5sum $vocab_file | cut -d ' ' -f 1)
if [ "$vocab_checksum" == "b74b8f2592cad5dd21bf614d1438db98" ]; then
  echo $vocab_file is valid
else
  echo $vocab_file is corrupted
  exit 1
fi

if [ ! -f $default_freq_file ]; then
  cat $vocab_file | awk '{print 1}' > $default_freq_file
fi

base_model_dir="${MODEL_DIR}/base_model"

# base model (4 epochs)
if [ ! -d $base_model_dir ]; then
  mkdir -p $base_model_dir
  for j in {1..2}; do 
    CUDA_VISIBLE_DEVICES=0 python train.py \
      --train_dir="$base_model_dir" \
      --train_data_pattern="/Youtube-8M/data/video/train/train*" \
      --frame_features=False \
      --feature_names="mean_rgb,mean_audio" \
      --feature_sizes="1024,128" \
      --model=DeepCombineChainModel \
      --moe_num_mixtures=4 \
      --deep_chain_relu_cells=256 \
      --deep_chain_layers=4 \
      --label_loss=MultiTaskCrossEntropyLoss \
      --multitask=True \
      --support_type="label,label,label,label" \
      --num_supports=18864 \
      --support_loss_percent=0.05 \
      --reweight=True \
      --sample_vocab_file="$vocab_file" \
      --sample_freq_file="$default_freq_file" \
      --keep_checkpoint_every_n_hour=8.0 \
      --keep_checkpoint_interval=6 \
      --base_learning_rate=0.01 \
      --data_augmenter=NoiseAugmenter \
      --input_noise_level=0.2 \
      --num_readers=2 \
      --num_epochs=2 \
      --batch_size=1024
  done
fi

last_freq_file=$default_freq_file

for i in {1..8}; do
  sub_model_dir="${MODEL_DIR}/sub_model_${i}"
  cp -r $base_model_dir $sub_model_dir

  echo "training model #$i, reweighting with $last_freq_file"
  # train N models with re-weighted samples
  CUDA_VISIBLE_DEVICES=0 python train.py \
    --train_dir="$sub_model_dir" \
    --train_data_pattern="/Youtube-8M/data/video/train/train*" \
    --frame_features=False \
    --feature_names="mean_rgb,mean_audio" \
    --feature_sizes="1024,128" \
    --model=DeepCombineChainModel \
    --moe_num_mixtures=4 \
{'fraction_non_alphanumeric': 0.13928138877674606, 'fraction_numerical': 0.03229713362939039, 'mean_word_length': 3.3849557522123894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4743781', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1936, 'n_words': 299}