<commit_before>package hex;

import hex.Layer.*;
import jsr166y.CountedCompleter;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.Job.ValidatedJob;
import water.api.DocGen;
import water.api.Progress2;
import water.api.Request;
import water.api.RequestServer;
import water.fvec.*;
import water.util.RString;
import water.util.Utils;

import java.util.Arrays;

/**
 * Neural network.
 *
 * @author cypof
 */
public class NeuralNet extends ValidatedJob {
  static final int API_WEAVER = 1;
  public static DocGen.FieldDoc[] DOC_FIELDS;
  public static final String DOC_GET = "Neural Network";

  public enum ExecutionMode {
    Serial, Threaded_Hogwild, MapReduce_Hogwild
  }

  @API(help = "Execution Mode", filter = Default.class)
  public ExecutionMode mode = ExecutionMode.Threaded_Hogwild;

  @API(help = "Activation function", filter = Default.class)
  public Activation activation = Activation.Tanh;

  @API(help = "Input layer dropout ratio", filter = Default.class, dmin = 0, dmax = 1)
  public float input_dropout_ratio = 0;

  @API(help = "Hidden layer sizes, e.g. 1000, 1000. Grid search: (100, 100), (200, 200)", filter = Default.class)
  public int[] hidden = new int[] { 500 };

  @API(help = "Initial Weight Distribution", filter = Default.class, dmin = 0)
  public InitialWeightDistribution initial_weight_distribution = InitialWeightDistribution.UniformAdaptive;

  @API(help = "Uniform: -value...value, Normal: stddev)", filter = Default.class, dmin = 0)
  public double initial_weight_scale = 0.01;

  @API(help = "Learning rate", filter = Default.class, dmin = 0)
  public double rate = .005;

  @API(help = "Learning rate annealing: rate / (1 + rate_annealing * samples)", filter = Default.class)
  public double rate_annealing = 1 / 1e6;

  @API(help = "Constraint for squared sum of incoming weights per unit", filter = Default.class)
  public float max_w2 = 15;

  @API(help = "Momentum at the beginning of training", filter = Default.class)
  public double momentum_start = .5;

  @API(help = "Number of samples for which momentum increases", filter = Default.class)
  public long momentum_ramp = 300 * 60000;

  @API(help = "Momentum once the initial increase is over", filter = Default.class, dmin = 0)
  public double momentum_stable = .99;

  //TODO: add a ramp down to 0 for l1 and l2

  @API(help = "L1 regularization", filter = Default.class, dmin = 0)
  public double l1;

  @API(help = "L2 regularization", filter = Default.class, dmin = 0)
  public double l2 = .001;

  @API(help = "Loss function", filter =Default.class)
  private Layer.Loss loss = Layer.Loss.CrossEntropy;

  @API(help = "Fast mode (minor approximation)", filter = Default.class)
  public boolean fast_mode = true;<commit_msg>Update default parameters for Mnist DredNet - likely just temporary.<commit_after>package hex;

import hex.Layer.*;
import jsr166y.CountedCompleter;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.Job.ValidatedJob;
import water.api.DocGen;
import water.api.Progress2;
import water.api.Request;
import water.api.RequestServer;
import water.fvec.*;
import water.util.RString;
import water.util.Utils;

import java.util.Arrays;

/**
 * Neural network.
 *
 * @author cypof
 */
public class NeuralNet extends ValidatedJob {
  static final int API_WEAVER = 1;
  public static DocGen.FieldDoc[] DOC_FIELDS;
  public static final String DOC_GET = "Neural Network";

  public enum ExecutionMode {
    Serial, Threaded_Hogwild, MapReduce_Hogwild
  }

  @API(help = "Execution Mode", filter = Default.class)
  public ExecutionMode mode = ExecutionMode.Threaded_Hogwild;

  @API(help = "Activation function", filter = Default.class)
  public Activation activation = Activation.Tanh;

  @API(help = "Input layer dropout ratio", filter = Default.class, dmin = 0, dmax = 1)
  public float input_dropout_ratio = 0;

  @API(help = "Hidden layer sizes, e.g. 1000, 1000. Grid search: (100, 100), (200, 200)", filter = Default.class)
  public int[] hidden = new int[] { 1024, 1024, 2048 };

  @API(help = "Initial Weight Distribution", filter = Default.class, dmin = 0)
  public InitialWeightDistribution initial_weight_distribution = InitialWeightDistribution.UniformAdaptive;

  @API(help = "Uniform: -value...value, Normal: stddev)", filter = Default.class, dmin = 0)
  public double initial_weight_scale = 0.01;

  @API(help = "Learning rate", filter = Default.class, dmin = 0)
  public double rate = .01;

  @API(help = "Learning rate annealing: rate / (1 + rate_annealing * samples)", filter = Default.class)
  public double rate_annealing = 1 / 1e6;

  @API(help = "Constraint for squared sum of incoming weights per unit", filter = Default.class)
  public float max_w2 = 15;

  @API(help = "Momentum at the beginning of training", filter = Default.class)
  public double momentum_start = .5;

  @API(help = "Number of samples for which momentum increases", filter = Default.class)
  public long momentum_ramp = 30 * 60000;

  @API(help = "Momentum once the initial increase is over", filter = Default.class, dmin = 0)
  public double momentum_stable = .99;

  //TODO: add a ramp down to 0 for l1 and l2

  @API(help = "L1 regularization", filter = Default.class, dmin = 0)
  public double l1 = 0.00001;

  @API(help = "L2 regularization", filter = Default.class, dmin = 0)
  public double l2 = 0.0;

  @API(help = "Loss function", filter =Default.class)
  private Layer.Loss loss = Layer.Loss.CrossEntropy;

  @API(help = "Fast mode (minor approximation)", filter = Default.class)
  public boolean fast_mode = true;
{'fraction_non_alphanumeric': 0.11276710360926558, 'fraction_numerical': 0.028371341353923506, 'mean_word_length': 4.299714557564225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4801908', 'n_tokens_mistral': 1922, 'n_tokens_neox': 1713, 'n_words': 562}