pere commited on
Commit
bf9f476
1 Parent(s): 91d785f

another try with specific finetune gin file

Browse files
finetune_large_mt5_sentencefix_v4.gin CHANGED
@@ -31,7 +31,7 @@ train_script.train:
31
  # `num_decodes` is equivalent to a beam size in a beam search decoding.
32
  models.EncoderDecoderModel.predict_batch_with_aux.num_decodes = 4
33
 
34
- partitioning.ModelBasedPjitPartitioner.num_partitions = 4
35
 
36
 
37
  #from t5.models import mesh_transformer
 
31
  # `num_decodes` is equivalent to a beam size in a beam search decoding.
32
  models.EncoderDecoderModel.predict_batch_with_aux.num_decodes = 4
33
 
34
+ partitioning.ModelBasedPjitPartitioner.num_partitions = 2
35
 
36
 
37
  #from t5.models import mesh_transformer
finetune_large_mt5_sentencefix_v4_16.gin ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __gin__ import dynamic_registration
2
+ import tasks_v4
3
+
4
+ import __main__ as train_script
5
+ from t5.data import mixtures
6
+ from t5x import models
7
+ from t5x import partitioning
8
+ from t5x import utils
9
+
10
+ include "t5x/examples/t5/mt5/large.gin"
11
+ include "t5x/configs/runs/finetune.gin"
12
+
13
+ MIXTURE_OR_TASK_NAME = "sentencefix"
14
+ TASK_FEATURE_LENGTHS = {"inputs": 256, "targets": 256}
15
+ TRAIN_STEPS = 1_200_000 # 1000000 pre-trained steps + 20000 fine-tuning steps.
16
+ USE_CACHED_TASKS = False
17
+ DROPOUT_RATE = 0.0
18
+ RANDOM_SEED = 0
19
+
20
+ # `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
21
+ # using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
22
+ # set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
23
+ # `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
24
+ #LOSS_NORMALIZING_FACTOR = 234496
25
+ INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/mt5_large/checkpoint_1000000"
26
+
27
+ train_script.train:
28
+ eval_period = 500
29
+ partitioner = @partitioning.ModelBasedPjitPartitioner()
30
+
31
+ # `num_decodes` is equivalent to a beam size in a beam search decoding.
32
+ models.EncoderDecoderModel.predict_batch_with_aux.num_decodes = 4
33
+
34
+ partitioning.PjitPartitioner.num_partitions = 4
35
+
36
+ #from t5.models import mesh_transformer
37
+ #import t5.models
38
+ #mesh_transformer.learning_rate_schedules.constant_learning_rate.learning_rate = 0.0005
39
+ #run.learning_rate_schedule = @learning_rate_schedules.constant_learning_rate
40
+
small_wmt_finetune.gin ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __gin__ import dynamic_registration
2
+
3
+ import __main__ as train_script
4
+ from t5.data import mixtures
5
+ from t5x import models
6
+ from t5x import partitioning
7
+ from t5x import utils
8
+
9
+ include "t5x/examples/t5/t5_1_1/small.gin"
10
+ include "t5x/configs/runs/finetune.gin"
11
+
12
+ MIXTURE_OR_TASK_NAME = "wmt_t2t_ende_v003"
13
+ TASK_FEATURE_LENGTHS = {"inputs": 256, "targets": 256}
14
+ TRAIN_STEPS = 1_020_000 # 1000000 pre-trained steps + 20000 fine-tuning steps.
15
+ DROPOUT_RATE = 0.0
16
+ INITIAL_CHECKPOINT_PATH = "gs://t5-data/pretrained_models/t5x/t5_1_1_small/checkpoint_1000000"
17
+ # `LOSS_NORMALIZING_FACTOR`: When fine-tuning a model that was pre-trained
18
+ # using Mesh Tensorflow (e.g. the public T5 / mT5 / ByT5 models), this should be
19
+ # set to `pretraining batch_size` * `target_token_length`. For T5 and T5.1.1:
20
+ # `2048 * 114`. For mT5: `1024 * 229`. For ByT5: `1024 * 189`.
21
+ LOSS_NORMALIZING_FACTOR = 233472
train_large_v4_16.sh CHANGED
@@ -5,8 +5,7 @@ export PYTHONPATH=${PROJECT_DIR}
5
 
6
  python3 ${T5X_DIR}/t5x/train.py \
7
  --gin_search_paths=${PROJECT_DIR} \
8
- --gin.partitioning.ModelBasedPjitPartitioner.num_partitions=4 \
9
- --gin_file="finetune_large_mt5_sentencefix_v4.gin" \
10
  --gin.MODEL_DIR="'${MODEL_DIR}'" \
11
  --tfds_data_dir=${TFDS_DATA_DIR}
12
 
 
5
 
6
  python3 ${T5X_DIR}/t5x/train.py \
7
  --gin_search_paths=${PROJECT_DIR} \
8
+ --gin_file="finetune_large_mt5_sentencefix_v4_16.gin" \
 
9
  --gin.MODEL_DIR="'${MODEL_DIR}'" \
10
  --tfds_data_dir=${TFDS_DATA_DIR}
11