<commit_before>#!/bin/bash

#python sort_of_clevr_generator.py

#python -u main.py --model=RN      --epochs=20  | tee  model/training_RN.log   
# Test set after epoch  1 : Relation accuracy: 67% | Non-relation accuracy: 61%
# Test set after epoch  5 : Relation accuracy: 71% | Non-relation accuracy: 61%
# Test set after epoch 10 : Relation accuracy: 73% | Non-relation accuracy: 68%
# Test set after epoch 20 : Relation accuracy: 88% | Non-relation accuracy: 100%
# 252secs per epoch on 760 GTX


#python -u main.py --model=CNN_MLP --epochs=100 | tee  model/training_CNN_MLP.log
# Test set after epoch  1 : Relation accuracy: 53% | Non-relation accuracy: 59%
# Test set after epoch  5 : Relation accuracy: 72% | Non-relation accuracy: 59%
# Test set after epoch 10 : Relation accuracy: 73% | Non-relation accuracy: 62%
# Test set after epoch 20 : Relation accuracy: 66% | Non-relation accuracy: 61%
# Test set after epoch 40 : Relation accuracy: 68% | Non-relation accuracy: 61%
# Test set after epoch 50 : Relation accuracy: 62% | Non-relation accuracy: 62%
# Test set after epoch 70 : Relation accuracy: 69% | Non-relation accuracy: 66%
# Test set after epoch 100 : Relation accuracy: 67% | Non-relation accuracy: 67%
#  43secs per epoch on 760 GTX


#python -u main.py --model=RFS     --epochs=20  | tee  model/training_RFS.log
# No softmax on output (Do'h)
# Epoch   0 (end) :   Test set: Relation accuracy: 35% | Non-relation accuracy: 17%
# Epoch  10 (end) :   Test set: Relation accuracy:  0% | Non-relation accuracy: 34%

# hidden_size=16
# Epoch 100 (end) :   Test set: Relation accuracy: 43% | Non-relation accuracy: 55%

# hidden_size=64
#  starts to plateau at ~8 epochs of training
# Epoch  10 (end) :   Test set: Relation accuracy: 45% | Non-relation accuracy: 52%
# Epoch  20 (end) :   Test set: Relation accuracy: 45% | Non-relation accuracy: 51%


# Test set after epoch  1 : Relation accuracy: 34% | Non-relation accuracy: 16%
# Test set after epoch  2 : Relation accuracy: 35% | Non-relation accuracy: 33%
# Test set after epoch  5 : Relation accuracy: 43% | Non-relation accuracy: 50%
# Test set after epoch 10 : Relation accuracy: 43% | Non-relation accuracy: 51%
# Test set after epoch 20 : Relation accuracy: 43% | Non-relation accuracy: 51%
#  76secs per epoch on Titax X (Maxwell)


# Additional rnn layer to create stream queries :  model/training_RFS-02-2rnn-layers-in-stream.log
#  Test set after epoch  1 : Relation accuracy: 33% | Non-relation accuracy: 16%
#  Test set after epoch  2 : Relation accuracy: 34% | Non-relation accuracy: 17%
#  Test set after epoch  5 : Relation accuracy: 43% | Non-relation accuracy: 51%
#  Test set after epoch 10 : Relation accuracy: 44% | Non-relation accuracy: 54%
#  Test set after epoch 15 : Relation accuracy: 44% | Non-relation accuracy: 55%
#  Test set after epoch 16 : Relation accuracy: 45% | Non-relation accuracy: 55%
#  Test set after epoch 17 : Relation accuracy: 46% | Non-relation accuracy: 55%
#  Test set after epoch 18 : Relation accuracy: 47% | Non-relation accuracy: 57%
#  Test set after epoch 19 : Relation accuracy: 49% | Non-relation accuracy: 56%
#  Test set after epoch 20 : Relation accuracy: 50% | Non-relation accuracy: 56%

# More :: python -u main.py --model=RFS     --epochs=20 --resume epoch_RFS_20.pth | tee --append model/training_RFS-02-2rnn-layers-in-stream.log

#  Test set after epoch 25 : Relation accuracy: 58% | Non-relation accuracy: 57%
#  Test set after epoch 30 : Relation accuracy: 63% | Non-relation accuracy: 58%
#  Test set after epoch 35 : Relation accuracy: 67% | Non-relation accuracy: 58%
#  Test set after epoch 40 : Relation accuracy: 68% | Non-relation accuracy: 57%

# More :: python -u main.py --model=RFS     --epochs=10 --resume epoch_RFS_20.pth | tee --append model/training_RFS-02-2rnn-layers-in-stream.log

#  Test set after epoch 45 : Relation accuracy: 70% | Non-relation accuracy: 59%
#  Test set after epoch 50 : Relation accuracy: 71% | Non-relation accuracy: 58%


# python -u main.py --model=RFS --epochs=20  | tee  model/training_RFS-key-is-param.log



# Run with higher learning rates initially:

python -u main.py --model=RFS --epochs=20 --lr=0.001  | tee model/training_RFS-key-is-param.log
#  Test set after epoch 20 : Relation accuracy: 41% | Non-relation accuracy: 81%

python -u main.py --model=RFS --epochs=30 --lr=0.0003 --resume epoch_RFS_20.pth | tee --append  model/training_RFS-key-is-param.log
#  Test set after epoch 50 : Relation accuracy: 80% | Non-relation accuracy: 98%

python -u main.py --model=RFS --epochs=150 --lr=0.0001 --resume epoch_RFS_30.pth | tee --append  model/training_RFS-key-is-param.log
# 220mins expected: ~<4hr


python -u main.py --model=RFS --epochs=50 --lr=0.001 --resume 0 --template model/{}_2item-span_{:03d}.pth | tee --append logs/RFS_2item-span.log<commit_msg>Surprisingly good result with depth-2 RNN...<commit_after>#!/bin/bash

#python sort_of_clevr_generator.py

#python -u main.py --model=RN      --epochs=20  | tee  model/training_RN.log   
# Test set after epoch  1 : Relation accuracy: 67% | Non-relation accuracy: 61%
# Test set after epoch  5 : Relation accuracy: 71% | Non-relation accuracy: 61%
# Test set after epoch 10 : Relation accuracy: 73% | Non-relation accuracy: 68%
# Test set after epoch 20 : Relation accuracy: 88% | Non-relation accuracy: 100%
# 252secs per epoch on 760 GTX


#python -u main.py --model=CNN_MLP --epochs=100 | tee  model/training_CNN_MLP.log
# Test set after epoch  1 : Relation accuracy: 53% | Non-relation accuracy: 59%
# Test set after epoch  5 : Relation accuracy: 72% | Non-relation accuracy: 59%
# Test set after epoch 10 : Relation accuracy: 73% | Non-relation accuracy: 62%
# Test set after epoch 20 : Relation accuracy: 66% | Non-relation accuracy: 61%
# Test set after epoch 40 : Relation accuracy: 68% | Non-relation accuracy: 61%
# Test set after epoch 50 : Relation accuracy: 62% | Non-relation accuracy: 62%
# Test set after epoch 70 : Relation accuracy: 69% | Non-relation accuracy: 66%
# Test set after epoch 100 : Relation accuracy: 67% | Non-relation accuracy: 67%
#  43secs per epoch on 760 GTX


#python -u main.py --model=RFS     --epochs=20  | tee  model/training_RFS.log
# No softmax on output (Do'h)
# Epoch   0 (end) :   Test set: Relation accuracy: 35% | Non-relation accuracy: 17%
# Epoch  10 (end) :   Test set: Relation accuracy:  0% | Non-relation accuracy: 34%

# hidden_size=16
# Epoch 100 (end) :   Test set: Relation accuracy: 43% | Non-relation accuracy: 55%

# hidden_size=64
#  starts to plateau at ~8 epochs of training
# Epoch  10 (end) :   Test set: Relation accuracy: 45% | Non-relation accuracy: 52%
# Epoch  20 (end) :   Test set: Relation accuracy: 45% | Non-relation accuracy: 51%


# Test set after epoch  1 : Relation accuracy: 34% | Non-relation accuracy: 16%
# Test set after epoch  2 : Relation accuracy: 35% | Non-relation accuracy: 33%
# Test set after epoch  5 : Relation accuracy: 43% | Non-relation accuracy: 50%
# Test set after epoch 10 : Relation accuracy: 43% | Non-relation accuracy: 51%
# Test set after epoch 20 : Relation accuracy: 43% | Non-relation accuracy: 51%
#  76secs per epoch on Titax X (Maxwell)


# Additional rnn layer to create stream queries :  model/training_RFS-02-2rnn-layers-in-stream.log
#  Test set after epoch  1 : Relation accuracy: 33% | Non-relation accuracy: 16%
#  Test set after epoch  2 : Relation accuracy: 34% | Non-relation accuracy: 17%
#  Test set after epoch  5 : Relation accuracy: 43% | Non-relation accuracy: 51%
#  Test set after epoch 10 : Relation accuracy: 44% | Non-relation accuracy: 54%
#  Test set after epoch 15 : Relation accuracy: 44% | Non-relation accuracy: 55%
#  Test set after epoch 16 : Relation accuracy: 45% | Non-relation accuracy: 55%
#  Test set after epoch 17 : Relation accuracy: 46% | Non-relation accuracy: 55%
#  Test set after epoch 18 : Relation accuracy: 47% | Non-relation accuracy: 57%
#  Test set after epoch 19 : Relation accuracy: 49% | Non-relation accuracy: 56%
#  Test set after epoch 20 : Relation accuracy: 50% | Non-relation accuracy: 56%

# More :: python -u main.py --model=RFS     --epochs=20 --resume epoch_RFS_20.pth | tee --append model/training_RFS-02-2rnn-layers-in-stream.log

#  Test set after epoch 25 : Relation accuracy: 58% | Non-relation accuracy: 57%
#  Test set after epoch 30 : Relation accuracy: 63% | Non-relation accuracy: 58%
#  Test set after epoch 35 : Relation accuracy: 67% | Non-relation accuracy: 58%
#  Test set after epoch 40 : Relation accuracy: 68% | Non-relation accuracy: 57%

# More :: python -u main.py --model=RFS     --epochs=10 --resume epoch_RFS_20.pth | tee --append model/training_RFS-02-2rnn-layers-in-stream.log

#  Test set after epoch 45 : Relation accuracy: 70% | Non-relation accuracy: 59%
#  Test set after epoch 50 : Relation accuracy: 71% | Non-relation accuracy: 58%


# python -u main.py --model=RFS --epochs=20  | tee  model/training_RFS-key-is-param.log



# Run with higher learning rates initially:

python -u main.py --model=RFS --epochs=20 --lr=0.001  | tee model/training_RFS-key-is-param.log
#  Test set after epoch 20 : Relation accuracy: 41% | Non-relation accuracy: 81%

python -u main.py --model=RFS --epochs=30 --lr=0.0003 --resume epoch_RFS_20.pth | tee --append  model/training_RFS-key-is-param.log
#  Test set after epoch 50 : Relation accuracy: 80% | Non-relation accuracy: 98%

python -u main.py --model=RFS --epochs=150 --lr=0.0001 --resume epoch_RFS_30.pth | tee --append  model/training_RFS-key-is-param.log
# 220mins expected: ~<4hr


python -u main.py --model=RFS --epochs=50 --lr=0.001 --resume 0 --template model/{}_2item-span_{:03d}.pth | tee --append logs/RFS_2item-span.log
(env3) [andrewsm@square relationships-from-entity-stream]$ grep Test logs/RFS_2item-span.log 
  Test set after epoch  1 : Relation accuracy: 54% | Non-relation accuracy: 57%
  Test set after epoch  2 : Relation accuracy: 73% | Non-relation accuracy: 60%
  Test set after epoch  3 : Relation accuracy: 69% | Non-relation accuracy: 59%
  Test set after epoch  4 : Relation accuracy: 73% | Non-relation accuracy: 62%
  Test set after epoch  5 : Relation accuracy: 73% | Non-relation accuracy: 63%
  Test set after epoch  6 : Relation accuracy: 74% | Non-relation accuracy: 62%
  Test set after epoch  7 : Relation accuracy: 59% | Non-relation accuracy: 62%
  Test set after epoch  8 : Relation accuracy: 55% | Non-relation accuracy: 58%
  Test set after epoch  9 : Relation accuracy: 73% | Non-relation accuracy: 61%
  Test set after epoch 10 : Relation accuracy: 73% | Non-relation accuracy: 61%
  Test set after epoch 11 : Relation accuracy: 74% | Non-relation accuracy: 62%
  Test set after epoch 12 : Relation accuracy: 74% | Non-relation accuracy: 61%
  Test set after epoch 13 : Relation accuracy: 74% | Non-relation accuracy: 61%
  Test set after epoch 14 : Relation accuracy: 73% | Non-relation accuracy: 61%
  Test set after epoch 15 : Relation accuracy: 71% | Non-relation accuracy: 64%
  Test set after epoch 16 : Relation accuracy: 73% | Non-relation accuracy: 72%
  Test set after epoch 17 : Relation accuracy: 76% | Non-relation accuracy: 74%
  Test set after epoch 18 : Relation accuracy: 76% | Non-relation accuracy: 78%
  Test set after epoch 19 : Relation accuracy: 79% | Non-relation accuracy: 82%
  Test set after epoch 20 : Relation accuracy: 80% | Non-relation accuracy: 86%
  Test set after epoch 21 : Relation accuracy: 87% | Non-relation accuracy: 90%
  Test set after epoch 22 : Relation accuracy: 83% | Non-relation accuracy: 96%
  Test set after epoch 23 : Relation accuracy: 92% | Non-relation accuracy: 97%
  Test set after epoch 24 : Relation accuracy: 48% | Non-relation accuracy: 92%
  Test set after epoch 25 : Relation accuracy: 61% | Non-relation accuracy: 98%
  Test set after epoch 26 : Relation accuracy: 91% | Non-relation accuracy: 98%
  Test set after epoch 27 : Relation accuracy: 92% | Non-relation accuracy: 99%
  Test set after epoch 28 : Relation accuracy: 71% | Non-relation accuracy: 98%
  Test set after epoch 29 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 30 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 31 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 32 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 33 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 34 : Relation accuracy: 92% | Non-relation accuracy: 99%
  Test set after epoch 35 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 36 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 37 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 38 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 39 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 40 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 41 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 42 : Relation accuracy: 93% | Non-relation accuracy: 99%
  Test set after epoch 43 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 44 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 45 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 46 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 47 : Relation accuracy: 94% | Non-relation accuracy: 99%
  Test set after epoch 48 : Relation accuracy: 67% | Non-relation accuracy: 99%
  Test set after epoch 49 : Relation accuracy: 45% | Non-relation accuracy: 92%
  Test set after epoch 50 : Relation accuracy: 92% | Non-relation accuracy: 99%
{'fraction_non_alphanumeric': 0.10857763300760044, 'fraction_numerical': 0.06688382193268187, 'mean_word_length': 3.77731673582296, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '147732', 'n_tokens_mistral': 5128, 'n_tokens_neox': 4299, 'n_words': 1734}