<commit_before>hdf5_fns = ["data/train.hdf5", "data/test.hdf5"]
source_fns = ["data/train.txt", "data/test.txt"]
for i = 1:length(hdf5_fns)
  if !isfile(hdf5_fns[i])
    println("Data not found, use get-mnist.sh to generate HDF5 data")
    exit(1)
  else
    open(source_fns[i], "w") do s
      println(s, hdf5_fns[i])
    end
  end
end

#ENV["MOCHA_USE_NATIVE_EXT"] = "true"
#ENV["OMP_NUM_THREADS"] = 1
#blas_set_num_threads(1)
ENV["MOCHA_USE_CUDA"] = "true"

using Mocha

############################################################
# This is an example script for training a fully connected 
# network with dropout on mnist. 
#
# The network size is 784-1200-1200-10 with ReLU units 
# in the hidden layers and a softmax output layer.
# The parameters for training the network were chosen
# to reproduce the results from the original dropout paper: 
# http://arxiv.org/abs/1207.0580 
# and the corresponding newer JMLR paper: 
# http://jmlr.org/papers/volume15/srivastava14a/srivastava14a.pdf
#
# Our parameters slightly differ. This is mainly due to the
# fact that in the original dropout paper the weights are scaled
# by 0.5 after training whereas we scale them by 2 during training.
#
# The settings in this script should currently produce a model that 
# gets 100 errors (or 99 % accuracy) on the test set
# if you run it for the whole 2000 epochs (=600*2000 steps).
# This is slightly better than, but well within the error 
# bars of the JMLR paper. 
############################################################


# fix the random seed to make results reproducable
srand(12345678)

data_layer  = HDF5DataLayer(name="train-data", source=source_fns[1], batch_size=100)
fc1_layer   = InnerProductLayer(name="fc1", output_dim=1200, neuron=Neurons.ReLU(), weight_init = GaussianInitializer(std=0.01), bottoms=[:data], tops=[:fc1])
fc2_layer   = InnerProductLayer(name="fc2", output_dim=1200, neuron=Neurons.ReLU(), weight_init = GaussianInitializer(std=0.01), bottoms=[:fc1], tops=[:fc2])
fc3_layer   = InnerProductLayer(name="out", output_dim=10, bottoms=[:fc2], weight_init = ConstantInitializer(0), tops=[:out])
loss_layer  = SoftmaxLossLayer(name="loss", bottoms=[:out,:label])

# setup dropout for the different layers
drop_input  = DropoutLayer(name="drop_in", bottoms=[:data], ratio=0.2)
drop_fc1 = DropoutLayer(name="drop_fc1", bottoms=[:fc1], ratio=0.5)
drop_fc2  = DropoutLayer(name="drop_fc2", bottoms=[:fc2], ratio=0.5)

sys = System(CuDNNBackend())<commit_msg>make the dropout mnist example use constraints<commit_after>hdf5_fns = ["data/train.hdf5", "data/test.hdf5"]
source_fns = ["data/train.txt", "data/test.txt"]
for i = 1:length(hdf5_fns)
  if !isfile(hdf5_fns[i])
    println("Data not found, use get-mnist.sh to generate HDF5 data")
    exit(1)
  else
    open(source_fns[i], "w") do s
      println(s, hdf5_fns[i])
    end
  end
end

#ENV["MOCHA_USE_NATIVE_EXT"] = "true"
#ENV["OMP_NUM_THREADS"] = 1
#blas_set_num_threads(1)
ENV["MOCHA_USE_CUDA"] = "true"

using Mocha

############################################################
# This is an example script for training a fully connected 
# network with dropout on mnist. 
#
# The network size is 784-1200-1200-10 with ReLU units 
# in the hidden layers and a softmax output layer.
# The parameters for training the network were chosen
# to reproduce the results from the original dropout paper: 
# http://arxiv.org/abs/1207.0580 
# and the corresponding newer JMLR paper: 
# http://jmlr.org/papers/volume15/srivastava14a/srivastava14a.pdf
#
# Our parameters slightly differ. This is mainly due to the
# fact that in the original dropout paper the weights are scaled
# by 0.5 after training whereas we scale them by 2 during training.
#
# The settings in this script should currently produce a model that 
# gets 100 errors (or 99 % accuracy) on the test set
# if you run it for the whole 2000 epochs (=600*2000 steps).
# This is slightly better than, but well within the error 
# bars of the JMLR paper. 
############################################################


# fix the random seed to make results reproducable
srand(12345678)

data_layer  = HDF5DataLayer(name="train-data", source=source_fns[1], batch_size=100)
fc1_layer   = InnerProductLayer(name="fc1", output_dim=1200, neuron=Neurons.ReLU(),
                                weight_init = GaussianInitializer(std=0.01),
                                #weight_cons = L2Cons(4.5),
                                bottoms=[:data], tops=[:fc1])
fc2_layer   = InnerProductLayer(name="fc2", output_dim=1200, neuron=Neurons.ReLU(),
                                weight_init = GaussianInitializer(std=0.01),
                                weight_cons = L2Cons(4.5),
                                bottoms=[:fc1], tops=[:fc2])
fc3_layer   = InnerProductLayer(name="out", output_dim=10, bottoms=[:fc2],
                                weight_init = ConstantInitializer(0),
                                weight_cons = L2Cons(4.5),
                                tops=[:out])
loss_layer  = SoftmaxLossLayer(name="loss", bottoms=[:out,:label])

# setup dropout for the different layers
drop_input  = DropoutLayer(name="drop_in", bottoms=[:data], ratio=0.2)
drop_fc1 = DropoutLayer(name="drop_fc1", bottoms=[:fc1], ratio=0.5)
drop_fc2  = DropoutLayer(name="drop_fc2", bottoms=[:fc2], ratio=0.5)

sys = System(CuDNNBackend())
{'fraction_non_alphanumeric': 0.157433314680097, 'fraction_numerical': 0.041969781757134866, 'mean_word_length': 4.072847682119205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2957198', 'n_tokens_mistral': 1911, 'n_tokens_neox': 1752, 'n_words': 483}