<commit_before>-- MSGD
-- Nesterov's momentum, see e.g. Sutskever et al., ICML 2013
-- Author: <NAME> (<EMAIL>)
require 'optim'

function optim.msgd(opfunc, w, config, state)   
   local config = config or {}   
   local state = state or config
   
   local lr = config.lr or 0
   local lrd = config.lrd or 0
   local lrp = config.lrp or 0
   local mmax = config.mommax or 1
   local mlrd = config.momdecay or 0
   local l2wd = config.l2wd or 0

   state.pversion = state.pversion or 0

   local mom = mmax
   if mom > 0 then
      if mlrd > 0 then
	 mom = math.min(mom, 1-0.5/(1+state.pversion/mlrd))
      end
      if not state.vt then
	 state.vt = w:clone():zero()
      end
      state.vt:mul(mom)
      w:add(state.vt)
   end   
   local fx,dfdx = opfunc(w)
   if l2wd ~= 0 then dfdx:add(l2wd,w) end
   local clr = lr
   if lrd > 0 and lrp > 0 then 
      clr = lr / math.pow(1+state.pversion*lrd,lrp)      
   end
   w:add(-clr,dfdx)<commit_msg>fix msgd mom variable init<commit_after>-- MSGD
-- Nesterov's momentum, see e.g. Sutskever et al., ICML 2013
-- Author: <NAME> (<EMAIL>)
require 'optim'

function optim.msgd(opfunc, w, config, state)   
   local config = config or {}   
   local state = state or config
   
   local lr = config.lr or 0
   local lrd = config.lrd or 0
   local lrp = config.lrp or 0
   local mom = config.mom or 0
   local mmax = config.mommax or 1
   local mlrd = config.momdecay or 0
   local l2wd = config.l2wd or 0

   state.pversion = state.pversion or 0

   if mom > 0 then
      if mlrd > 0 then
	 mom = math.min(mom, 1-0.5/(1+state.pversion/mlrd))
      end
      if not state.vt then
	 state.vt = w:clone():zero()
      end
      state.vt:mul(mom)
      w:add(state.vt)
   end   
   local fx,dfdx = opfunc(w)
   if l2wd ~= 0 then dfdx:add(l2wd,w) end
   local clr = lr
   if lrd > 0 and lrp > 0 then 
      clr = lr / math.pow(1+state.pversion*lrd,lrp)      
   end
   w:add(-clr,dfdx)
{'fraction_non_alphanumeric': 0.11358921161825726, 'fraction_numerical': 0.026452282157676348, 'mean_word_length': 2.269491525423729, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7212536', 'n_tokens_mistral': 803, 'n_tokens_neox': 745, 'n_words': 231}