<commit_before>#include "graph/node_initializers.h"
#include "3rd_party/svd/svd.h"
#include "layers/word2vec_reader.h"

#include <stdint.h>
#include <algorithm>
#include <iterator>
#include <random>

namespace marian {

namespace inits {

float xor128() {
  static uint64_t x = 123456789;
  static uint64_t y = 362436069;
  static uint64_t z = 521288629;
  static uint64_t w = 88675123;
  uint64_t t;

  t = (x ^ (x << 11)) % 1000;
  x = y;
  y = z;
  z = w;
  w = (w ^ (w >> 19) ^ t ^ (t >> 8)) % 1000;
  return 0.1 * ((w % 1000) / 1000.f) - 0.05;
}

void zeros(Tensor t) {
  t->set(0.f);
}

void ones(Tensor t) {
  t->set(1.0f);
}

NodeInitializer from_value(float v) {
  return [v](Tensor t) { t->set(v); };
}

NodeInitializer diag(float val) {
  return [val](Tensor t) {
    if(t->shape()[0] == t->shape()[1] && t->shape()[2] == 1
       && t->shape()[3] == 1) {
      std::vector<float> vec(t->size(), 0);
      for(int i = 0; i < t->shape()[0]; ++i)
        vec[i * t->shape()[1] + i] = val;
      t->set(vec);
    }
  };
}

NodeInitializer normal(float scale, bool ortho /*= true*/) {
  return [scale](Tensor t) {
    distribution<std::normal_distribution<float>>(t, 0, scale);
  };
}

NodeInitializer uniform(float scale) {
  return [scale](Tensor t) {
    distribution<std::uniform_real_distribution<float>>(t, -scale, scale);
  };
}

void glorot_uniform(Tensor t) {
  float scale = sqrtf(6.0f / (t->shape()[0] + t->shape()[1]));
  distribution<std::uniform_real_distribution<float>>(t, -scale, scale);
}

void xorshift(Tensor t) {
  std::vector<float> vals(t->size());
  for(auto&& v : vals)
    v = xor128();
  t->set(vals);
}

void glorot_normal(Tensor t) {
  float scale = sqrtf(2.0f / (t->shape()[0] + t->shape()[1]));
  distribution<std::normal_distribution<float>>(t, 0, scale);
}

void svd(std::vector<float>& vec, Shape shape) {
  int rows = shape[0] * shape[2] * shape[3];
  int cols = shape[1];

  int n = std::min(rows, cols);
  int m = std::max(rows, cols);

  ABORT_IF(m % n != 0,
           "Matrix dimensions must be equal or multiples of each other");

  for(int i = 0; i < shape.elements(); i += n * n) {
    std::vector<float> t1(n);<commit_msg>Calculate Glorot init scale from trailing dims<commit_after>#include "graph/node_initializers.h"
#include "3rd_party/svd/svd.h"
#include "layers/word2vec_reader.h"

#include <stdint.h>
#include <algorithm>
#include <iterator>
#include <random>

namespace marian {

namespace inits {

float xor128() {
  static uint64_t x = 123456789;
  static uint64_t y = 362436069;
  static uint64_t z = 521288629;
  static uint64_t w = 88675123;
  uint64_t t;

  t = (x ^ (x << 11)) % 1000;
  x = y;
  y = z;
  z = w;
  w = (w ^ (w >> 19) ^ t ^ (t >> 8)) % 1000;
  return 0.1 * ((w % 1000) / 1000.f) - 0.05;
}

void zeros(Tensor t) {
  t->set(0.f);
}

void ones(Tensor t) {
  t->set(1.0f);
}

NodeInitializer from_value(float v) {
  return [v](Tensor t) { t->set(v); };
}

NodeInitializer diag(float val) {
  return [val](Tensor t) {
    if(t->shape()[0] == t->shape()[1] && t->shape()[2] == 1
       && t->shape()[3] == 1) {
      std::vector<float> vec(t->size(), 0);
      for(int i = 0; i < t->shape()[0]; ++i)
        vec[i * t->shape()[1] + i] = val;
      t->set(vec);
    }
  };
}

NodeInitializer normal(float scale, bool ortho /*= true*/) {
  return [scale](Tensor t) {
    distribution<std::normal_distribution<float>>(t, 0, scale);
  };
}

NodeInitializer uniform(float scale) {
  return [scale](Tensor t) {
    distribution<std::uniform_real_distribution<float>>(t, -scale, scale);
  };
}

void glorot_uniform(Tensor t) {
  float scale = sqrtf(6.0f / (t->shape()[-2] + t->shape()[-1]));
  distribution<std::uniform_real_distribution<float>>(t, -scale, scale);
}

void xorshift(Tensor t) {
  std::vector<float> vals(t->size());
  for(auto&& v : vals)
    v = xor128();
  t->set(vals);
}

void glorot_normal(Tensor t) {
  float scale = sqrtf(2.0f / (t->shape()[-2] + t->shape()[-1]));
  distribution<std::normal_distribution<float>>(t, 0, scale);
}

void svd(std::vector<float>& vec, Shape shape) {
  int rows = shape[0] * shape[2] * shape[3];
  int cols = shape[1];

  int n = std::min(rows, cols);
  int m = std::max(rows, cols);

  ABORT_IF(m % n != 0,
           "Matrix dimensions must be equal or multiples of each other");

  for(int i = 0; i < shape.elements(); i += n * n) {
    std::vector<float> t1(n);
{'fraction_non_alphanumeric': 0.20077572438968744, 'fraction_numerical': 0.04973762263289984, 'mean_word_length': 3.5056526207605345, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6178463', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1740, 'n_words': 409}