<commit_before>/*
Copyright (c) by respective owners including Yahoo!, Microsoft, and
individual contributors. All rights reserved.    Released under a BSD (revised)
license as described in the file LICENSE.
*/
#include <string>
#include "gd.h"
#include "vw.h"
#include "rand48.h"
#include <math.h>

using namespace std;
using namespace LEARNER;

#define NORM2 (m+1)

struct update_data {
    struct OjaNewton *ON;
    float g;
    float sketch_cnt;
    float norm2_x;
    float *Zx;
    float *AZx;
    float *delta;
    float bdelta;
    float prediction;
};

struct OjaNewton {
    vw* all;
    int m;
    int epoch_size;
    float alpha;
    int cnt;
    int t;

    float *ev;
    float *b;
    float *D;
    float **A;
	float **K;

    float *zv;
    float *vv;
    float *tmp;

    example **buffer;
    float *weight_buffer;
    struct update_data data;

    float learning_rate_cnt;
    bool normalize;
    bool random_init;

	void initialize_Z() //TODO: use weight_paramters::set_default for initialization
	{  weight_parameters& weights = *(all->weights);
       uint32_t length = 1 << all->num_bits;
	   if (normalize) { // initialize normalization part
		 for (weight_parameters::iterator i = weights.begin(NORM2); i != weights.end(NORM2); ++i)
		   *i = 0.1f;
	    }
        if(!random_init) {
            // simple initialization
		  weight_parameters::iterator iter = weights.begin(0) + 1;
		  for (int i = 1; i <= m; ++i, ++iter)
		  { weight_parameters::iterator::w_iter j = iter.begin() + i;
			*j = 1.f;
		  }
        }
	    else {
            // more complicated initialization: orthgonal basis of a random matrix

	    const double PI2 = 2.f * 3.1415927f;
		for (weight_parameters::iterator i = weights.begin(0); i != weights.end(); ++i)
		  for (weight_parameters::iterator::w_iter j = i.begin() + 1; j != i.end(m + 1); ++j)
		  {  double r1 = frand48() + 1e-6;
		     double r2 = frand48();
			 *j = sqrt(-2.f * log(r1)) * cos(PI2 * r2);
		  }

            // Gram-Schmidt
          for (int j = 1; j <= m; j++) {
             for (int k = 1; k <= j - 1; k++) {
	            double tmp = 0;

				weight_parameters::iterator w_j = weights.begin(j);
				weight_parameters::iterator w_k = weights.begin(k);
		        for (; w_j != weights.end(j); ++w_j, ++w_k) 
		          tmp += (*w_j) * (*w_k);
				w_j = weights.begin(j);
				w_k = weights.begin(k);
				for (; w_j != weights.end(j); ++w_j, ++w_k)
		          *w_j -= (float)tmp * (*w_k);
	         }
	        double norm = 0;
			weight_parameters::iterator w_j = weights.begin(j);
			for (; w_j != weights.end(j); ++w_j)<commit_msg>reverting OjaNewton..adding 1e-6 to frand48() gives more errors<commit_after>/*
Copyright (c) by respective owners including Yahoo!, Microsoft, and
individual contributors. All rights reserved.    Released under a BSD (revised)
license as described in the file LICENSE.
*/
#include <string>
#include "gd.h"
#include "vw.h"
#include "rand48.h"
#include <math.h>

using namespace std;
using namespace LEARNER;

#define NORM2 (m+1)

struct update_data {
    struct OjaNewton *ON;
    float g;
    float sketch_cnt;
    float norm2_x;
    float *Zx;
    float *AZx;
    float *delta;
    float bdelta;
    float prediction;
};

struct OjaNewton {
    vw* all;
    int m;
    int epoch_size;
    float alpha;
    int cnt;
    int t;

    float *ev;
    float *b;
    float *D;
    float **A;
	float **K;

    float *zv;
    float *vv;
    float *tmp;

    example **buffer;
    float *weight_buffer;
    struct update_data data;

    float learning_rate_cnt;
    bool normalize;
    bool random_init;

	void initialize_Z() //TODO: use weight_paramters::set_default for initialization
	{  weight_parameters& weights = *(all->weights);
       uint32_t length = 1 << all->num_bits;
	   if (normalize) { // initialize normalization part
		 for (weight_parameters::iterator i = weights.begin(NORM2); i != weights.end(NORM2); ++i)
		   *i = 0.1f;
	    }
        if(!random_init) {
            // simple initialization
		  weight_parameters::iterator iter = weights.begin(0) + 1;
		  for (int i = 1; i <= m; ++i, ++iter)
		  { weight_parameters::iterator::w_iter j = iter.begin() + i;
			*j = 1.f;
		  }
        }
	    else {
            // more complicated initialization: orthgonal basis of a random matrix

	    const double PI2 = 2.f * 3.1415927f;
		for (weight_parameters::iterator i = weights.begin(0); i != weights.end(); ++i)
		  for (weight_parameters::iterator::w_iter j = i.begin() + 1; j != i.end(m + 1); ++j)
		  {  double r1 = frand48();
		     double r2 = frand48();
			 *j = sqrt(-2.f * log(r1)) * cos(PI2 * r2);
		  }

            // Gram-Schmidt
          for (int j = 1; j <= m; j++) {
             for (int k = 1; k <= j - 1; k++) {
	            double tmp = 0;

				weight_parameters::iterator w_j = weights.begin(j);
				weight_parameters::iterator w_k = weights.begin(k);
		        for (; w_j != weights.end(j); ++w_j, ++w_k) 
		          tmp += (*w_j) * (*w_k);
				w_j = weights.begin(j);
				w_k = weights.begin(k);
				for (; w_j != weights.end(j); ++w_j, ++w_k)
		          *w_j -= (float)tmp * (*w_k);
	         }
	        double norm = 0;
			weight_parameters::iterator w_j = weights.begin(j);
			for (; w_j != weights.end(j); ++w_j)
{'fraction_non_alphanumeric': 0.14394371553527285, 'fraction_numerical': 0.01787412055523864, 'mean_word_length': 2.4788359788359786, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4119638', 'n_tokens_mistral': 2022, 'n_tokens_neox': 1811, 'n_words': 519}