<commit_before>local THNN = require 'nn.THNN'
local SparseLinear, parent = torch.class('nn.SparseLinear', 'nn.Module')

local NO_LAST_INPUT = 0
local ONE_LAST_INPUT = 1
local ACC_MULTIPLE_TIMES = 2

function SparseLinear:__init(inputSize, outputSize, doGradInput)
   parent.__init(self)

   self.weightDecay = 0
   self.doGradInput = doGradInput or false
   self.weight = torch.Tensor(outputSize, inputSize):zero()
   self.bias = torch.Tensor(outputSize):zero()
   self.gradWeight = torch.Tensor(outputSize, inputSize):zero()
   self.gradBias = torch.Tensor(outputSize):zero()

   assert(type(self.doGradInput) == type(true))

   self.lastInput = nil
   self.sparseUpdate = NO_LAST_INPUT
   self.formatted_input = nil

   -- state
   self.gradInput:resize(inputSize)
   self.output:resize(outputSize)

   self:reset()
end

function SparseLinear:reset(stdv)
   if stdv then
      stdv = stdv * math.sqrt(3)
   else
      stdv = 1./math.sqrt(self.weight:size(2))
   end
   self.weight:uniform(-stdv, stdv)
   self.bias:uniform(-stdv, stdv):mul(0.000001)
end

function SparseLinear:reshapeInput(input)
   if type(input) == 'table' then
      return input, true, false
   else
      if input:dim() == 2 then
         return {input}, false, false
      else
         return input, true, true
      end
   end
end

function SparseLinear:updateOutput(input)<commit_msg>SparseLinear: gradInput should be a table by default<commit_after>local THNN = require 'nn.THNN'
local SparseLinear, parent = torch.class('nn.SparseLinear', 'nn.Module')

local NO_LAST_INPUT = 0
local ONE_LAST_INPUT = 1
local ACC_MULTIPLE_TIMES = 2

function SparseLinear:__init(inputSize, outputSize, doGradInput)
   parent.__init(self)

   self.weightDecay = 0
   self.doGradInput = doGradInput or false
   self.weight = torch.Tensor(outputSize, inputSize):zero()
   self.bias = torch.Tensor(outputSize):zero()
   self.gradWeight = torch.Tensor(outputSize, inputSize):zero()
   self.gradBias = torch.Tensor(outputSize):zero()

   assert(type(self.doGradInput) == type(true))

   self.lastInput = nil
   self.sparseUpdate = NO_LAST_INPUT
   self.formatted_input = nil

   -- state
   self.gradInput = {}
   self.output:resize(outputSize)

   self:reset()
end

function SparseLinear:reset(stdv)
   if stdv then
      stdv = stdv * math.sqrt(3)
   else
      stdv = 1./math.sqrt(self.weight:size(2))
   end
   self.weight:uniform(-stdv, stdv)
   self.bias:uniform(-stdv, stdv):mul(0.000001)
end

function SparseLinear:reshapeInput(input)
   if type(input) == 'table' then
      return input, true, false
   else
      if input:dim() == 2 then
         return {input}, false, false
      else
         return input, true, true
      end
   end
end

function SparseLinear:updateOutput(input)
{'fraction_non_alphanumeric': 0.11309090909090909, 'fraction_numerical': 0.01090909090909091, 'mean_word_length': 3.7595155709342563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4391809', 'n_tokens_mistral': 952, 'n_tokens_neox': 914, 'n_words': 202}