<commit_before>'use strict';

import linearAlgebra from 'linear-algebra';

import { randn, dropoutLayer } from '../lib';

const { Matrix } = linearAlgebra();

export default class FullyConnectedLayer {
  constructor(nIn, nOut, opts = {}) {
    this.pDropout = opts.pDropout || (opts.pDropout = 0);
    this.activationFn = opts.activationFn;
    this.w = randn(nOut, nIn).mulEach(1 / Math.sqrt(nIn));
    this.b = randn(nOut, 1);
  }

  setInput(input, inputDropout, miniBatchSize) {
    const axis = 0;
    const bMask = new Matrix(this.b.ravel().map(v => {
      let results = [];
      for (let i = 0; i < miniBatchSize; i++) {
        results.push(v);
      }
      return results;
    }));
    this.input = input;
    this.output = this.w.dot(input).mulEach(1 - this.pDropout).plus(bMask)[this.activationFn](axis);
    this.yOut = this.output.getArgMax();
    this.inputDropout = dropoutLayer(inputDropout, this.pDropout);
    this.outputDropout = this.w.dot(this.inputDropout).plus(bMask)[this.activationFn](axis);
  }

  accuracy(y) {
    return this.yOut === y;
  }

  update(delta) {
    this.nb = new Matrix(delta.getSum(1)).trans();
    this.nw = delta.dot(this.input.trans());
  }<commit_msg>fix a bug of dropout not working properly<commit_after>'use strict';

import linearAlgebra from 'linear-algebra';

import { randn, dropoutLayer } from '../lib';

const { Matrix } = linearAlgebra();

export default class FullyConnectedLayer {
  constructor(nIn, nOut, opts = {}) {
    this.pDropout = opts.pDropout || (opts.pDropout = 0);
    this.activationFn = opts.activationFn;
    this.w = randn(nOut, nIn).mulEach(1 / Math.sqrt(nIn));
    this.b = randn(nOut, 1);
  }

  setInput(input, inputDropout, miniBatchSize) {
    const axis = 0;
    const bMask = new Matrix(this.b.ravel().map(v => {
      let results = [];
      for (let i = 0; i < miniBatchSize; i++) {
        results.push(v);
      }
      return results;
    }));
    this.input = input;
    this.output = this.w.dot(input).mulEach(1 - this.pDropout).plus(bMask)[this.activationFn](axis);
    this.yOut = this.output.getArgMax();
    this.inputDropout = dropoutLayer(inputDropout, this.pDropout);
    this.outputDropout = this.w.dot(this.inputDropout).plus(bMask)[this.activationFn](axis);
  }

  accuracy(y) {
    return this.yOut === y;
  }

  update(delta) {
    this.nb = new Matrix(delta.getSum(1)).trans();
    this.nw = delta.dot(this.inputDropout.trans());
  }
{'fraction_non_alphanumeric': 0.1614092585006145, 'fraction_numerical': 0.005735354362965997, 'mean_word_length': 3.2842105263157895, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5546108', 'n_tokens_mistral': 879, 'n_tokens_neox': 855, 'n_words': 163}