Probably no dropout applied to the encoder in the RNN encoder/decoder ASR
Hi,

When trying to apply dropout on the rnn encoder, I get this warning:
```
/lib/python3.6/site-packages/torch/nn/modules/rnn.py:50: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  "num_layers={}".format(dropout, num_layers))
```
Indeed, [after a quick check in the code](https://github.com/espnet/espnet/blob/master/espnet/nets/pytorch_backend/rnn/encoders.py#L35):
```python
rnn = torch.nn.LSTM(inputdim, cdim, dropout=dropout, num_layers=1, bidirectional=bidir,
                                batch_first=True) if "lstm" in typ \\
                else torch.nn.GRU(inputdim, cdim, dropout=dropout, num_layers=1, bidirectional=bidir, batch_first=True)
```

diff --git a/espnet/nets/pytorch_backend/rnn/encoders.py b/espnet/nets/pytorch_backend/rnn/encoders.py
index fe26275f8..77d5a6f0d 100644
--- a/espnet/nets/pytorch_backend/rnn/encoders.py
+++ b/espnet/nets/pytorch_backend/rnn/encoders.py
@@ -32,10 +32,12 @@ class RNNP(torch.nn.Module):
                 inputdim = idim
             else:
                 inputdim = hdim
-            rnn = torch.nn.LSTM(inputdim, cdim, dropout=dropout, num_layers=1, bidirectional=bidir,
-                                batch_first=True) if "lstm" in typ \\
-                else torch.nn.GRU(inputdim, cdim, dropout=dropout, num_layers=1, bidirectional=bidir, batch_first=True)
+
+            RNN = torch.nn.LSTM if "lstm" in typ else torch.nn.GRU
+            rnn = RNN(inputdim, cdim, num_layers=1, bidirectional=bidir, batch_first=True)
+
             setattr(self, "%s%d" % ("birnn" if bidir else "rnn", i), rnn)
+
             # bottleneck layer to merge
             if bidir:
                 setattr(self, "bt%d" % i, torch.nn.Linear(2 * cdim, hdim))
@@ -47,6 +49,7 @@ class RNNP(torch.nn.Module):
         self.subsample = subsample
         self.typ = typ
         self.bidir = bidir
+        self.dropout = dropout
 
     def forward(self, xs_pad, ilens, prev_state=None):
         """RNNP forward
@@ -74,12 +77,11 @@ class RNNP(torch.nn.Module):
                 ys_pad = ys_pad[:, ::sub]
                 ilens = [int(i + 1) // sub for i in ilens]
             # (sum _utt frame_utt) x dim
-            projected = getattr(self, 'bt' + str(layer)
-                                )(ys_pad.contiguous().view(-1, ys_pad.size(2)))
-            if layer == self.elayers - 1:
-                xs_pad = projected.view(ys_pad.size(0), ys_pad.size(1), -1)
-            else:
-                xs_pad = torch.tanh(projected.view(ys_pad.size(0), ys_pad.size(1), -1))
+            projection_layer = getattr(self, 'bt%d' % layer)
+            projected = projection_layer(ys_pad.contiguous().view(-1, ys_pad.size(2)))
+            xs_pad = projected.view(ys_pad.size(0), ys_pad.size(1), -1)
+            if layer < self.elayers - 1:
+                xs_pad = torch.tanh(F.dropout(xs_pad, p=self.dropout))
 
         return xs_pad, ilens, elayer_states  # x: utt list of frame x dim
 
['espnet/nets/pytorch_backend/rnn/encoders.py']