<commit_before>local ClassNLLCriterion, parent = torch.class('nn.ClassNLLCriterion', 'nn.Criterion')

function ClassNLLCriterion:__init()
   parent.__init(self)
end

function ClassNLLCriterion:forward(input, target)
   self.output = -input[target]
   return self.output
end

function ClassNLLCriterion:backward(input, target)
   self.gradInput:resizeAs(input)
   self.gradInput:zero()
   self.gradInput[target] = -1
   return self.gradInput
end<commit_msg>ClassNLLCriterion now handles batches<commit_after>local ClassNLLCriterion, parent = torch.class('nn.ClassNLLCriterion', 'nn.Criterion')

function ClassNLLCriterion:__init()
   parent.__init(self)
   self.sizeAverage = true
end

function ClassNLLCriterion:forward(input, target)
   if input:dim() == 1 then
      self.output = -input[target]
   else
      local output = 0
      for i=1,target:size(1) do
         output = output - input[i][target[i]]
      end
      if self.sizeAverage then
         output = output / target:size(1)
      end
      self.output = output
   end
   return self.output
end

function ClassNLLCriterion:backward(input, target)
   self.gradInput:resizeAs(input)
   self.gradInput:zero()

  if input:dim() == 1 then
      self.gradInput[target] = -1
   else
      local z = -1
      if self.sizeAverage then
         z = z / target:size(1)
      end
      local gradInput = self.gradInput
      for i=1,target:size(1) do
         gradInput[i][target[i]] = z
      end
   end

   return self.gradInput
end
{'fraction_non_alphanumeric': 0.10349462365591398, 'fraction_numerical': 0.008064516129032258, 'mean_word_length': 3.315942028985507, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7638934', 'n_tokens_mistral': 479, 'n_tokens_neox': 482, 'n_words': 108}