add modified transducer

README.md

@@ -9,15 +9,16 @@ metrics:
 ---
 ### TedLium3 Zipformer
 
-Using the codes from this PR https://github.com/k2-fsa/icefall/pull/1125.
+**`rnnt_type=regular`**
 
 The WERs are
 
 |                                    |     dev    |    test    | comment                                  |
 |------------------------------------|------------|------------|------------------------------------------|
-|          greedy search             | 6.32       | 5.83       | --epoch 50, --avg 22, --max-duration 500 |
-| modified beam search (beam size 4) | 6.16       | 5.79       | --epoch 50, --avg 22, --max-duration 500 |
-| fast beam search (set as default)  | 6.30    ß   | 5.89       | --epoch 50, --avg 22, --max-duration 500 |
+|          greedy search             | 6.74       | 6.16       | --epoch 50, --avg 22, --max-duration 500 |
+|      beam search (beam size 4)     | 6.56       | 5.95       | --epoch 50, --avg 22, --max-duration 500 |
+| modified beam search (beam size 4) | 6.54       | 6.00       | --epoch 50, --avg 22, --max-duration 500 |
+| fast beam search (set as default)  | 6.91       | 6.28       | --epoch 50, --avg 22, --max-duration 500 |
 
 The training command for reproducing is given below:
 
@@ -81,3 +82,35 @@ avg=22
   --max-contexts 4 \
   --max-states 8
 ```
+
+**`rnnt_type=modified`**
+
+Using the codes from this PR https://github.com/k2-fsa/icefall/pull/1125.
+
+The WERs are
+
+|                                    |     dev    |    test    | comment                                  |
+|------------------------------------|------------|------------|------------------------------------------|
+|          greedy search             | 6.32       | 5.83       | --epoch 50, --avg 22, --max-duration 500 |
+| modified beam search (beam size 4) | 6.16       | 5.79       | --epoch 50, --avg 22, --max-duration 500 |
+| fast beam search (set as default)  | 6.30       | 5.89       | --epoch 50, --avg 22, --max-duration 500 |
+
+The training command for reproducing is given below:
+
+```
+export CUDA_VISIBLE_DEVICES="0,1,2,3"
+
+./zipformer/train.py \
+  --use-fp16 true \
+  --world-size 4 \
+  --num-epochs 50 \
+  --start-epoch 0 \
+  --exp-dir zipformer/exp \
+  --max-duration 1000 \
+  --rnnt-type modified
+```
+
+The tensorboard training log can be found at
+https://tensorboard.dev/experiment/AKXbJha0S9aXyfmuvG4h5A/#scalars
+
+The decoding commands are same as above.

decoding_results/regular_transducer/beam_search/log-decode-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model-2023-06-15-16-05-54

@@ -0,0 +1,26 @@
+2023-06-15 16:05:54,523 INFO [decode.py:675] Decoding started
+2023-06-15 16:05:54,524 INFO [decode.py:681] Device: cuda:0
+2023-06-15 16:05:54,531 INFO [decode.py:691] {'best_train_loss': inf, 'best_valid_loss': inf, 'best_train_epoch': -1, 'best_valid_epoch': -1, 'batch_idx_train': 0, 'log_interval': 50, 'reset_interval': 200, 'valid_interval': 3000, 'feature_dim': 80, 'subsampling_factor': 4, 'warm_step': 2000, 'env_info': {'k2-version': '1.24.3', 'k2-build-type': 'Debug', 'k2-with-cuda': True, 'k2-git-sha1': '38211604d6a24b15f320578a1a38f6c12d7a711c', 'k2-git-date': 'Mon Jun 12 10:59:44 2023', 'lhotse-version': '1.15.0.dev+git.f1fd23d.clean', 'torch-version': '2.0.0+cu117', 'torch-cuda-available': True, 'torch-cuda-version': '11.7', 'python-version': '3.8', 'icefall-git-branch': 'ted/zipformer', 'icefall-git-sha1': '323a299-dirty', 'icefall-git-date': 'Tue Jun 13 04:47:15 2023', 'icefall-path': '/exp/draj/jsalt2023/icefall', 'k2-path': '/exp/draj/jsalt2023/k2/k2/python/k2/__init__.py', 'lhotse-path': '/exp/draj/jsalt2023/lhotse/lhotse/__init__.py', 'hostname': 'r2n02', 'IP address': '10.1.2.2'}, 'epoch': 50, 'iter': 0, 'avg': 22, 'use_averaged_model': True, 'exp_dir': PosixPath('zipformer/exp/v5'), 'bpe_model': 'data/lang_bpe_500/bpe.model', 'lang_dir': PosixPath('data/lang_bpe_500'), 'decoding_method': 'beam_search', 'beam_size': 4, 'beam': 20.0, 'ngram_lm_scale': 0.01, 'max_contexts': 8, 'max_states': 64, 'context_size': 2, 'max_sym_per_frame': 1, 'num_paths': 200, 'nbest_scale': 0.5, 'num_encoder_layers': '2,2,3,4,3,2', 'downsampling_factor': '1,2,4,8,4,2', 'feedforward_dim': '512,768,1024,1536,1024,768', 'num_heads': '4,4,4,8,4,4', 'encoder_dim': '192,256,384,512,384,256', 'query_head_dim': '32', 'value_head_dim': '12', 'pos_head_dim': '4', 'pos_dim': 48, 'encoder_unmasked_dim': '192,192,256,256,256,192', 'cnn_module_kernel': '31,31,15,15,15,31', 'decoder_dim': 512, 'joiner_dim': 512, 'causal': False, 'chunk_size': '16,32,64,-1', 'left_context_frames': '64,128,256,-1', 'manifest_dir': PosixPath('data/manifests'), 'max_duration': 500, 'bucketing_sampler': True, 'num_buckets': 30, 'concatenate_cuts': False, 'duration_factor': 1.0, 'gap': 1.0, 'on_the_fly_feats': False, 'shuffle': True, 'return_cuts': True, 'num_workers': 2, 'enable_spec_aug': True, 'spec_aug_time_warp_factor': 80, 'enable_musan': True, 'res_dir': PosixPath('zipformer/exp/v5/beam_search'), 'suffix': 'epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model', 'blank_id': 0, 'unk_id': 2, 'vocab_size': 500}
+2023-06-15 16:05:54,532 INFO [decode.py:693] About to create model
+2023-06-15 16:05:55,243 INFO [decode.py:760] Calculating the averaged model over epoch range from 28 (excluded) to 50
+2023-06-15 16:05:59,616 INFO [decode.py:794] Number of model parameters: 65549011
+2023-06-15 16:05:59,617 INFO [asr_datamodule.py:361] About to get dev cuts
+2023-06-15 16:05:59,620 INFO [asr_datamodule.py:366] About to get test cuts
+2023-06-15 16:07:14,020 INFO [decode.py:572] batch 0/?, cuts processed until now is 30
+2023-06-15 16:21:11,947 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/beam_search/recogs-dev-beam_size_4-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 16:21:11,979 INFO [utils.py:562] [dev-beam_size_4] %WER 6.56% [1195 / 18226, 187 ins, 400 del, 608 sub ]
+2023-06-15 16:21:12,046 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/beam_search/errs-dev-beam_size_4-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 16:21:12,047 INFO [decode.py:617] 
+For dev, WER of different settings are:
+beam_size_4	6.56	best for dev
+
+2023-06-15 16:22:30,203 INFO [decode.py:572] batch 0/?, cuts processed until now is 40
+2023-06-15 16:42:42,018 INFO [decode.py:572] batch 20/?, cuts processed until now is 1063
+2023-06-15 16:46:02,358 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/beam_search/recogs-test-beam_size_4-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 16:46:02,413 INFO [utils.py:562] [test-beam_size_4] %WER 5.95% [1693 / 28430, 212 ins, 683 del, 798 sub ]
+2023-06-15 16:46:02,509 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/beam_search/errs-test-beam_size_4-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 16:46:02,511 INFO [decode.py:617] 
+For test, WER of different settings are:
+beam_size_4	5.95	best for test
+
+2023-06-15 16:46:02,511 INFO [decode.py:825] Done!

decoding_results/regular_transducer/beam_search/wer-summary-dev-beam_size_4-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+beam_size_4	6.56

decoding_results/regular_transducer/beam_search/wer-summary-test-beam_size_4-epoch-50-avg-22-beam_search-beam-size-4-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+beam_size_4	5.95

decoding_results/regular_transducer/fast_beam_search/log-decode-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model-2023-06-15-16-02-29

@@ -0,0 +1,26 @@
+2023-06-15 16:02:29,961 INFO [decode.py:675] Decoding started
+2023-06-15 16:02:29,962 INFO [decode.py:681] Device: cuda:0
+2023-06-15 16:02:29,970 INFO [decode.py:691] {'best_train_loss': inf, 'best_valid_loss': inf, 'best_train_epoch': -1, 'best_valid_epoch': -1, 'batch_idx_train': 0, 'log_interval': 50, 'reset_interval': 200, 'valid_interval': 3000, 'feature_dim': 80, 'subsampling_factor': 4, 'warm_step': 2000, 'env_info': {'k2-version': '1.24.3', 'k2-build-type': 'Debug', 'k2-with-cuda': True, 'k2-git-sha1': '38211604d6a24b15f320578a1a38f6c12d7a711c', 'k2-git-date': 'Mon Jun 12 10:59:44 2023', 'lhotse-version': '1.15.0.dev+git.f1fd23d.clean', 'torch-version': '2.0.0+cu117', 'torch-cuda-available': True, 'torch-cuda-version': '11.7', 'python-version': '3.8', 'icefall-git-branch': 'ted/zipformer', 'icefall-git-sha1': '323a299-dirty', 'icefall-git-date': 'Tue Jun 13 04:47:15 2023', 'icefall-path': '/exp/draj/jsalt2023/icefall', 'k2-path': '/exp/draj/jsalt2023/k2/k2/python/k2/__init__.py', 'lhotse-path': '/exp/draj/jsalt2023/lhotse/lhotse/__init__.py', 'hostname': 'r2n02', 'IP address': '10.1.2.2'}, 'epoch': 50, 'iter': 0, 'avg': 22, 'use_averaged_model': True, 'exp_dir': PosixPath('zipformer/exp/v5'), 'bpe_model': 'data/lang_bpe_500/bpe.model', 'lang_dir': PosixPath('data/lang_bpe_500'), 'decoding_method': 'fast_beam_search', 'beam_size': 4, 'beam': 20.0, 'ngram_lm_scale': 0.01, 'max_contexts': 8, 'max_states': 64, 'context_size': 2, 'max_sym_per_frame': 1, 'num_paths': 200, 'nbest_scale': 0.5, 'num_encoder_layers': '2,2,3,4,3,2', 'downsampling_factor': '1,2,4,8,4,2', 'feedforward_dim': '512,768,1024,1536,1024,768', 'num_heads': '4,4,4,8,4,4', 'encoder_dim': '192,256,384,512,384,256', 'query_head_dim': '32', 'value_head_dim': '12', 'pos_head_dim': '4', 'pos_dim': 48, 'encoder_unmasked_dim': '192,192,256,256,256,192', 'cnn_module_kernel': '31,31,15,15,15,31', 'decoder_dim': 512, 'joiner_dim': 512, 'causal': False, 'chunk_size': '16,32,64,-1', 'left_context_frames': '64,128,256,-1', 'manifest_dir': PosixPath('data/manifests'), 'max_duration': 500, 'bucketing_sampler': True, 'num_buckets': 30, 'concatenate_cuts': False, 'duration_factor': 1.0, 'gap': 1.0, 'on_the_fly_feats': False, 'shuffle': True, 'return_cuts': True, 'num_workers': 2, 'enable_spec_aug': True, 'spec_aug_time_warp_factor': 80, 'enable_musan': True, 'res_dir': PosixPath('zipformer/exp/v5/fast_beam_search'), 'suffix': 'epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model', 'blank_id': 0, 'unk_id': 2, 'vocab_size': 500}
+2023-06-15 16:02:29,970 INFO [decode.py:693] About to create model
+2023-06-15 16:02:30,674 INFO [decode.py:760] Calculating the averaged model over epoch range from 28 (excluded) to 50
+2023-06-15 16:02:50,856 INFO [decode.py:794] Number of model parameters: 65549011
+2023-06-15 16:02:50,857 INFO [asr_datamodule.py:361] About to get dev cuts
+2023-06-15 16:02:50,860 INFO [asr_datamodule.py:366] About to get test cuts
+2023-06-15 16:02:56,598 INFO [decode.py:572] batch 0/?, cuts processed until now is 30
+2023-06-15 16:03:41,470 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/fast_beam_search/recogs-dev-beam_20.0_max_contexts_8_max_states_64-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model.txt
+2023-06-15 16:03:41,539 INFO [utils.py:562] [dev-beam_20.0_max_contexts_8_max_states_64] %WER 6.91% [1260 / 18226, 182 ins, 467 del, 611 sub ]
+2023-06-15 16:03:41,601 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/fast_beam_search/errs-dev-beam_20.0_max_contexts_8_max_states_64-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model.txt
+2023-06-15 16:03:41,602 INFO [decode.py:617] 
+For dev, WER of different settings are:
+beam_20.0_max_contexts_8_max_states_64	6.91	best for dev
+
+2023-06-15 16:03:44,090 INFO [decode.py:572] batch 0/?, cuts processed until now is 40
+2023-06-15 16:04:22,021 INFO [decode.py:572] batch 20/?, cuts processed until now is 1063
+2023-06-15 16:04:34,908 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/fast_beam_search/recogs-test-beam_20.0_max_contexts_8_max_states_64-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model.txt
+2023-06-15 16:04:34,954 INFO [utils.py:562] [test-beam_20.0_max_contexts_8_max_states_64] %WER 6.28% [1785 / 28430, 186 ins, 788 del, 811 sub ]
+2023-06-15 16:04:35,049 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/fast_beam_search/errs-test-beam_20.0_max_contexts_8_max_states_64-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model.txt
+2023-06-15 16:04:35,050 INFO [decode.py:617] 
+For test, WER of different settings are:
+beam_20.0_max_contexts_8_max_states_64	6.28	best for test
+
+2023-06-15 16:04:35,050 INFO [decode.py:825] Done!

decoding_results/regular_transducer/fast_beam_search/wer-summary-dev-beam_20.0_max_contexts_8_max_states_64-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+beam_20.0_max_contexts_8_max_states_64	6.91

decoding_results/regular_transducer/fast_beam_search/wer-summary-test-beam_20.0_max_contexts_8_max_states_64-epoch-50-avg-22-beam-20.0-max-contexts-8-max-states-64-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+beam_20.0_max_contexts_8_max_states_64	6.28

decoding_results/regular_transducer/greedy_search/log-decode-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model-2023-06-15-15-55-01

@@ -0,0 +1,26 @@
+2023-06-15 15:55:01,021 INFO [decode.py:675] Decoding started
+2023-06-15 15:55:01,023 INFO [decode.py:681] Device: cuda:0
+2023-06-15 15:55:01,031 INFO [decode.py:691] {'best_train_loss': inf, 'best_valid_loss': inf, 'best_train_epoch': -1, 'best_valid_epoch': -1, 'batch_idx_train': 0, 'log_interval': 50, 'reset_interval': 200, 'valid_interval': 3000, 'feature_dim': 80, 'subsampling_factor': 4, 'warm_step': 2000, 'env_info': {'k2-version': '1.24.3', 'k2-build-type': 'Debug', 'k2-with-cuda': True, 'k2-git-sha1': '38211604d6a24b15f320578a1a38f6c12d7a711c', 'k2-git-date': 'Mon Jun 12 10:59:44 2023', 'lhotse-version': '1.15.0.dev+git.f1fd23d.clean', 'torch-version': '2.0.0+cu117', 'torch-cuda-available': True, 'torch-cuda-version': '11.7', 'python-version': '3.8', 'icefall-git-branch': 'ted/zipformer', 'icefall-git-sha1': '323a299-dirty', 'icefall-git-date': 'Tue Jun 13 04:47:15 2023', 'icefall-path': '/exp/draj/jsalt2023/icefall', 'k2-path': '/exp/draj/jsalt2023/k2/k2/python/k2/__init__.py', 'lhotse-path': '/exp/draj/jsalt2023/lhotse/lhotse/__init__.py', 'hostname': 'r2n04', 'IP address': '10.1.2.4'}, 'epoch': 50, 'iter': 0, 'avg': 22, 'use_averaged_model': True, 'exp_dir': PosixPath('zipformer/exp/v5'), 'bpe_model': 'data/lang_bpe_500/bpe.model', 'lang_dir': PosixPath('data/lang_bpe_500'), 'decoding_method': 'greedy_search', 'beam_size': 4, 'beam': 20.0, 'ngram_lm_scale': 0.01, 'max_contexts': 8, 'max_states': 64, 'context_size': 2, 'max_sym_per_frame': 1, 'num_paths': 200, 'nbest_scale': 0.5, 'num_encoder_layers': '2,2,3,4,3,2', 'downsampling_factor': '1,2,4,8,4,2', 'feedforward_dim': '512,768,1024,1536,1024,768', 'num_heads': '4,4,4,8,4,4', 'encoder_dim': '192,256,384,512,384,256', 'query_head_dim': '32', 'value_head_dim': '12', 'pos_head_dim': '4', 'pos_dim': 48, 'encoder_unmasked_dim': '192,192,256,256,256,192', 'cnn_module_kernel': '31,31,15,15,15,31', 'decoder_dim': 512, 'joiner_dim': 512, 'causal': False, 'chunk_size': '16,32,64,-1', 'left_context_frames': '64,128,256,-1', 'manifest_dir': PosixPath('data/manifests'), 'max_duration': 500, 'bucketing_sampler': True, 'num_buckets': 30, 'concatenate_cuts': False, 'duration_factor': 1.0, 'gap': 1.0, 'on_the_fly_feats': False, 'shuffle': True, 'return_cuts': True, 'num_workers': 2, 'enable_spec_aug': True, 'spec_aug_time_warp_factor': 80, 'enable_musan': True, 'res_dir': PosixPath('zipformer/exp/v5/greedy_search'), 'suffix': 'epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model', 'blank_id': 0, 'unk_id': 2, 'vocab_size': 500}
+2023-06-15 15:55:01,031 INFO [decode.py:693] About to create model
+2023-06-15 15:55:01,633 INFO [decode.py:760] Calculating the averaged model over epoch range from 28 (excluded) to 50
+2023-06-15 15:55:05,547 INFO [decode.py:794] Number of model parameters: 65549011
+2023-06-15 15:55:05,547 INFO [asr_datamodule.py:361] About to get dev cuts
+2023-06-15 15:55:05,550 INFO [asr_datamodule.py:366] About to get test cuts
+2023-06-15 15:55:08,961 INFO [decode.py:572] batch 0/?, cuts processed until now is 30
+2023-06-15 15:55:22,921 INFO [zipformer.py:1728] name=None, attn_weights_entropy = tensor([3.8087, 3.2133, 3.5770, 3.0500], device='cuda:0')
+2023-06-15 15:55:27,056 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/greedy_search/recogs-dev-greedy_search-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model.txt
+2023-06-15 15:55:27,088 INFO [utils.py:562] [dev-greedy_search] %WER 6.74% [1228 / 18226, 178 ins, 443 del, 607 sub ]
+2023-06-15 15:55:27,156 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/greedy_search/errs-dev-greedy_search-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model.txt
+2023-06-15 15:55:27,158 INFO [decode.py:617] 
+For dev, WER of different settings are:
+greedy_search	6.74	best for dev
+
+2023-06-15 15:55:28,580 INFO [decode.py:572] batch 0/?, cuts processed until now is 40
+2023-06-15 15:55:49,101 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/greedy_search/recogs-test-greedy_search-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model.txt
+2023-06-15 15:55:49,150 INFO [utils.py:562] [test-greedy_search] %WER 6.16% [1751 / 28430, 197 ins, 725 del, 829 sub ]
+2023-06-15 15:55:49,248 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/greedy_search/errs-test-greedy_search-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model.txt
+2023-06-15 15:55:49,290 INFO [decode.py:617] 
+For test, WER of different settings are:
+greedy_search	6.16	best for test
+
+2023-06-15 15:55:49,290 INFO [decode.py:825] Done!

decoding_results/regular_transducer/greedy_search/wer-summary-dev-greedy_search-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+greedy_search	6.74

decoding_results/regular_transducer/greedy_search/wer-summary-test-greedy_search-epoch-50-avg-22-context-2-max-sym-per-frame-1-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+greedy_search	6.16

decoding_results/regular_transducer/modified_beam_search/log-decode-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model-2023-06-15-15-56-46

@@ -0,0 +1,8 @@
+2023-06-15 15:56:46,258 INFO [decode.py:675] Decoding started
+2023-06-15 15:56:46,260 INFO [decode.py:681] Device: cuda:0
+2023-06-15 15:56:46,268 INFO [decode.py:691] {'best_train_loss': inf, 'best_valid_loss': inf, 'best_train_epoch': -1, 'best_valid_epoch': -1, 'batch_idx_train': 0, 'log_interval': 50, 'reset_interval': 200, 'valid_interval': 3000, 'feature_dim': 80, 'subsampling_factor': 4, 'warm_step': 2000, 'env_info': {'k2-version': '1.24.3', 'k2-build-type': 'Debug', 'k2-with-cuda': True, 'k2-git-sha1': '38211604d6a24b15f320578a1a38f6c12d7a711c', 'k2-git-date': 'Mon Jun 12 10:59:44 2023', 'lhotse-version': '1.15.0.dev+git.f1fd23d.clean', 'torch-version': '2.0.0+cu117', 'torch-cuda-available': True, 'torch-cuda-version': '11.7', 'python-version': '3.8', 'icefall-git-branch': 'ted/zipformer', 'icefall-git-sha1': '323a299-dirty', 'icefall-git-date': 'Tue Jun 13 04:47:15 2023', 'icefall-path': '/exp/draj/jsalt2023/icefall', 'k2-path': '/exp/draj/jsalt2023/k2/k2/python/k2/__init__.py', 'lhotse-path': '/exp/draj/jsalt2023/lhotse/lhotse/__init__.py', 'hostname': 'r2n04', 'IP address': '10.1.2.4'}, 'epoch': 50, 'iter': 0, 'avg': 22, 'use_averaged_model': True, 'exp_dir': PosixPath('zipformer/exp/v5'), 'bpe_model': 'data/lang_bpe_500/bpe.model', 'lang_dir': PosixPath('data/lang_bpe_500'), 'decoding_method': 'modified_beam_search', 'beam_size': 4, 'beam': 20.0, 'ngram_lm_scale': 0.01, 'max_contexts': 8, 'max_states': 64, 'context_size': 2, 'max_sym_per_frame': 1, 'num_paths': 200, 'nbest_scale': 0.5, 'num_encoder_layers': '2,2,3,4,3,2', 'downsampling_factor': '1,2,4,8,4,2', 'feedforward_dim': '512,768,1024,1536,1024,768', 'num_heads': '4,4,4,8,4,4', 'encoder_dim': '192,256,384,512,384,256', 'query_head_dim': '32', 'value_head_dim': '12', 'pos_head_dim': '4', 'pos_dim': 48, 'encoder_unmasked_dim': '192,192,256,256,256,192', 'cnn_module_kernel': '31,31,15,15,15,31', 'decoder_dim': 512, 'joiner_dim': 512, 'causal': False, 'chunk_size': '16,32,64,-1', 'left_context_frames': '64,128,256,-1', 'manifest_dir': PosixPath('data/manifests'), 'max_duration': 500, 'bucketing_sampler': True, 'num_buckets': 30, 'concatenate_cuts': False, 'duration_factor': 1.0, 'gap': 1.0, 'on_the_fly_feats': False, 'shuffle': True, 'return_cuts': True, 'num_workers': 2, 'enable_spec_aug': True, 'spec_aug_time_warp_factor': 80, 'enable_musan': True, 'res_dir': PosixPath('zipformer/exp/v5/modified_beam_search'), 'suffix': 'epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model', 'blank_id': 0, 'unk_id': 2, 'vocab_size': 500}
+2023-06-15 15:56:46,268 INFO [decode.py:693] About to create model
+2023-06-15 15:56:46,927 INFO [decode.py:760] Calculating the averaged model over epoch range from 28 (excluded) to 50
+2023-06-15 15:56:51,104 INFO [decode.py:794] Number of model parameters: 65549011
+2023-06-15 15:56:51,104 INFO [asr_datamodule.py:361] About to get dev cuts
+2023-06-15 15:56:51,106 INFO [asr_datamodule.py:366] About to get test cuts

decoding_results/regular_transducer/modified_beam_search/log-decode-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model-2023-06-15-15-58-01

@@ -0,0 +1,27 @@
+2023-06-15 15:58:01,483 INFO [decode.py:675] Decoding started
+2023-06-15 15:58:01,484 INFO [decode.py:681] Device: cuda:0
+2023-06-15 15:58:01,492 INFO [decode.py:691] {'best_train_loss': inf, 'best_valid_loss': inf, 'best_train_epoch': -1, 'best_valid_epoch': -1, 'batch_idx_train': 0, 'log_interval': 50, 'reset_interval': 200, 'valid_interval': 3000, 'feature_dim': 80, 'subsampling_factor': 4, 'warm_step': 2000, 'env_info': {'k2-version': '1.24.3', 'k2-build-type': 'Debug', 'k2-with-cuda': True, 'k2-git-sha1': '38211604d6a24b15f320578a1a38f6c12d7a711c', 'k2-git-date': 'Mon Jun 12 10:59:44 2023', 'lhotse-version': '1.15.0.dev+git.f1fd23d.clean', 'torch-version': '2.0.0+cu117', 'torch-cuda-available': True, 'torch-cuda-version': '11.7', 'python-version': '3.8', 'icefall-git-branch': 'ted/zipformer', 'icefall-git-sha1': '323a299-dirty', 'icefall-git-date': 'Tue Jun 13 04:47:15 2023', 'icefall-path': '/exp/draj/jsalt2023/icefall', 'k2-path': '/exp/draj/jsalt2023/k2/k2/python/k2/__init__.py', 'lhotse-path': '/exp/draj/jsalt2023/lhotse/lhotse/__init__.py', 'hostname': 'r2n04', 'IP address': '10.1.2.4'}, 'epoch': 50, 'iter': 0, 'avg': 22, 'use_averaged_model': True, 'exp_dir': PosixPath('zipformer/exp/v5'), 'bpe_model': 'data/lang_bpe_500/bpe.model', 'lang_dir': PosixPath('data/lang_bpe_500'), 'decoding_method': 'modified_beam_search', 'beam_size': 4, 'beam': 20.0, 'ngram_lm_scale': 0.01, 'max_contexts': 8, 'max_states': 64, 'context_size': 2, 'max_sym_per_frame': 1, 'num_paths': 200, 'nbest_scale': 0.5, 'num_encoder_layers': '2,2,3,4,3,2', 'downsampling_factor': '1,2,4,8,4,2', 'feedforward_dim': '512,768,1024,1536,1024,768', 'num_heads': '4,4,4,8,4,4', 'encoder_dim': '192,256,384,512,384,256', 'query_head_dim': '32', 'value_head_dim': '12', 'pos_head_dim': '4', 'pos_dim': 48, 'encoder_unmasked_dim': '192,192,256,256,256,192', 'cnn_module_kernel': '31,31,15,15,15,31', 'decoder_dim': 512, 'joiner_dim': 512, 'causal': False, 'chunk_size': '16,32,64,-1', 'left_context_frames': '64,128,256,-1', 'manifest_dir': PosixPath('data/manifests'), 'max_duration': 500, 'bucketing_sampler': True, 'num_buckets': 30, 'concatenate_cuts': False, 'duration_factor': 1.0, 'gap': 1.0, 'on_the_fly_feats': False, 'shuffle': True, 'return_cuts': True, 'num_workers': 2, 'enable_spec_aug': True, 'spec_aug_time_warp_factor': 80, 'enable_musan': True, 'res_dir': PosixPath('zipformer/exp/v5/modified_beam_search'), 'suffix': 'epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model', 'blank_id': 0, 'unk_id': 2, 'vocab_size': 500}
+2023-06-15 15:58:01,492 INFO [decode.py:693] About to create model
+2023-06-15 15:58:02,075 INFO [decode.py:760] Calculating the averaged model over epoch range from 28 (excluded) to 50
+2023-06-15 15:58:05,955 INFO [decode.py:794] Number of model parameters: 65549011
+2023-06-15 15:58:05,956 INFO [asr_datamodule.py:361] About to get dev cuts
+2023-06-15 15:58:05,958 INFO [asr_datamodule.py:366] About to get test cuts
+2023-06-15 15:58:15,345 INFO [decode.py:572] batch 0/?, cuts processed until now is 30
+2023-06-15 15:59:30,951 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/modified_beam_search/recogs-dev-beam_size_4-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 15:59:30,981 INFO [utils.py:562] [dev-beam_size_4] %WER 6.54% [1192 / 18226, 188 ins, 402 del, 602 sub ]
+2023-06-15 15:59:31,044 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/modified_beam_search/errs-dev-beam_size_4-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 15:59:31,045 INFO [decode.py:617] 
+For dev, WER of different settings are:
+beam_size_4	6.54	best for dev
+
+2023-06-15 15:59:36,799 INFO [decode.py:572] batch 0/?, cuts processed until now is 40
+2023-06-15 16:01:10,193 INFO [decode.py:572] batch 20/?, cuts processed until now is 1063
+2023-06-15 16:01:25,571 INFO [zipformer.py:1728] name=None, attn_weights_entropy = tensor([1.9471, 2.3907, 3.3421, 3.0589], device='cuda:0')
+2023-06-15 16:01:28,902 INFO [decode.py:588] The transcripts are stored in zipformer/exp/v5/modified_beam_search/recogs-test-beam_size_4-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 16:01:28,948 INFO [utils.py:562] [test-beam_size_4] %WER 6.00% [1707 / 28430, 204 ins, 699 del, 804 sub ]
+2023-06-15 16:01:29,043 INFO [decode.py:601] Wrote detailed error stats to zipformer/exp/v5/modified_beam_search/errs-test-beam_size_4-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model.txt
+2023-06-15 16:01:29,044 INFO [decode.py:617] 
+For test, WER of different settings are:
+beam_size_4	6.0	best for test
+
+2023-06-15 16:01:29,044 INFO [decode.py:825] Done!

decoding_results/regular_transducer/modified_beam_search/wer-summary-dev-beam_size_4-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+beam_size_4	6.54

decoding_results/regular_transducer/modified_beam_search/wer-summary-test-beam_size_4-epoch-50-avg-22-modified_beam_search-beam-size-4-use-averaged-model.txt

@@ -0,0 +1,2 @@
+settings	WER
+beam_size_4	6.0

exp/regular_transducer/decode.sh

@@ -0,0 +1,12 @@
+export CUDA_VISIBLE_DEVICES="0"
+for m in greedy_search beam_search fast_beam_search modified_beam_search; do
+  ./zipformer/decode.py \
+      --epoch 50 \
+      --avg 22 \
+      --use-averaged-model True \
+      --exp-dir zipformer/exp \
+      --max-duration 500 \
+      --causal 0 \
+      --decoding-method $m \
+      --beam-size 4
+done

exp/regular_transducer/export.sh

@@ -0,0 +1,6 @@
+./zipformer/export.py \
+  --exp-dir ./zipformer/exp \
+  --bpe-model data/lang_bpe_500/bpe.model \
+  --epoch 50 \
+  --avg 22 \
+  --jit 1

exp/regular_transducer/jit_script.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:254527e341deea8496867da1ea086efa2a33e2ec3c624bb5dc3727bb3f03e9c2
+size 264940478

exp/regular_transducer/pretrained.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f533966f4811c5c3d343b307dca956f9b9d63accbf7035b9bcfc4102c584d2e5
+size 262605742

exp/regular_transducer/tensorboard/events.out.tfevents.1686808592.r2n01.160747.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512da5ba3cbf0b75738c65c1afc259a9b13907f3e8242b704971dca0b9341bed
+size 272322

exp/regular_transducer/train.sh

@@ -0,0 +1,10 @@
+export CUDA_VISIBLE_DEVICES="0,1,2,3"
+./zipformer/train.py \
+  --world-size 4 \
+  --num-epochs 50 \
+  --start-epoch 1 \
+  --use-fp16 1 \
+  --exp-dir zipformer/exp \
+  --causal 0 \
+  --max-duration 1000 \
+  --master-port 12345