<commit_before>--[[ Convolutional Encoder similar to the one described in https://arxiv.org/abs/1611.02344.



--]]

local CNNEncoder, parent = torch.class('onmt.CNNEncoder', 'nn.Container')

local options = {
  {
    '-cnn_layers', 2,
    [[Number of convolutional layers in the encoder.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  },
  {
    '-cnn_kernel', 3,
    [[Kernel size for convolutions. Same in each layer.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  },
  {
    '-cnn_size', 500,
    [[Number of output units per convolutional layer. Same in each layer.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  },
  {
    '-use_pos_emb', true,
    [[Add positional embeddings to word embeddings.]],
    {
      structural = 0
    }
  },
  {
    '-max_pos', 50,
    [[Maximum value for positional indexes.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  }

  -- TODO : attention network
}


function CNNEncoder.declareOpts(cmd)
  cmd:setCmdLineOptions(options)
end


function CNNEncoder:__init(args, inputNetwork)

  parent.__init(self)

  self.inputNet = inputNetwork
  self.args = args

  self.args.numStates = 1

  local convInSize = inputNetwork.inputSize
  local convOutSize = args.cnn_size

  local input = nn.Identity()()

  -- Compute input network.
  local inLayer = self.inputNet(input)

  if self.args.use_pos_emb then
    local posEmb = onmt.PositionEmbedding(2, args.max_pos, convInSize)(input)
    inLayer = nn.CAddTable()({inLayer, posEmb})
  end

  local curLayer = inLayer

  for layer_idx=1,args.cnn_layers do

    if self.args.dropout_input and layer_idx == 1 then
      curLayer = nn.Dropout(self.args.dropout)(curLayer)
    end
<commit_msg>CNN encoder: apply dropout at each convolutional layer<commit_after>--[[ Convolutional Encoder similar to the one described in https://arxiv.org/abs/1611.02344.



--]]

local CNNEncoder, parent = torch.class('onmt.CNNEncoder', 'nn.Container')

local options = {
  {
    '-cnn_layers', 2,
    [[Number of convolutional layers in the encoder.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  },
  {
    '-cnn_kernel', 3,
    [[Kernel size for convolutions. Same in each layer.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  },
  {
    '-cnn_size', 500,
    [[Number of output units per convolutional layer. Same in each layer.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  },
  {
    '-use_pos_emb', true,
    [[Add positional embeddings to word embeddings.]],
    {
      structural = 0
    }
  },
  {
    '-max_pos', 50,
    [[Maximum value for positional indexes.]],
    {
      valid = onmt.utils.ExtendedCmdLine.isUInt(),
      structural = 0
    }
  }

  -- TODO : attention network
}


function CNNEncoder.declareOpts(cmd)
  cmd:setCmdLineOptions(options)
end


function CNNEncoder:__init(args, inputNetwork)

  parent.__init(self)

  self.inputNet = inputNetwork
  self.args = args

  self.args.numStates = 1

  local convInSize = inputNetwork.inputSize
  local convOutSize = args.cnn_size

  local input = nn.Identity()()

  -- Compute input network.
  local inLayer = self.inputNet(input)

  if self.args.use_pos_emb then
    local posEmb = onmt.PositionEmbedding(2, args.max_pos, convInSize)(input)
    inLayer = nn.CAddTable()({inLayer, posEmb})
  end

  local curLayer = inLayer

  for layer_idx=1,args.cnn_layers do

    if self.args.dropout_input or layer_idx > 1 then
      curLayer = nn.Dropout(self.args.dropout)(curLayer)
    end

{'fraction_non_alphanumeric': 0.12085244132721877, 'fraction_numerical': 0.013487995683841382, 'mean_word_length': 2.830578512396694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5600968', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1206, 'n_words': 289}