<commit_before>/*
Copyright (c) by respective owners including Yahoo!, Microsoft, and
individual contributors. All rights reserved.  Released under a BSD (revised)
license as described in the file LICENSE.
 */
#include <fstream>
#include <iostream>
using namespace std;

#ifndef _WIN32
#include <unistd.h>
#endif
#include <stdlib.h>
#include <stdint.h>
#include <math.h>
#include <algorithm>

#include "parse_regressor.h"
#include "loss_functions.h"
#include "io_buf.h"
#include "memory.h"
#include "rand48.h"
#include "global_data.h"

/* Define the last version where files are backward compatible. */
#define LAST_COMPATIBLE_VERSION "6.1.3"
#define VERSION_FILE_WITH_CUBIC "6.1.3"

void initialize_regressor(vw& all)
{
  // Regressor is already initialized.
  if (all.reg.weight_vector != NULL) {
    return;
  }

  size_t length = ((size_t)1) << all.num_bits;
  all.reg.weight_mask = (all.reg.stride * length) - 1;
  all.reg.weight_vector = (weight *)calloc_or_die(all.reg.stride*length, sizeof(weight));
  if (all.reg.weight_vector == NULL)
    {
      cerr << all.program_name << ": Failed to allocate weight array with " << all.num_bits << " bits: try decreasing -b <bits>" << endl;
      throw exception();
    }
  if (all.random_weights)
    {
      for (size_t j = 0; j < length; j++)
	all.reg.weight_vector[j*all.reg.stride] = (float)(frand48() - 0.5);
    }
  if (all.initial_weight != 0.)
    for (size_t j = 0; j < all.reg.stride*length; j+=all.reg.stride)
      all.reg.weight_vector[j] = all.initial_weight;
}

const size_t buf_size = 512;

void save_load_header(vw& all, io_buf& model_file, bool read, bool text)
{

  char buff[buf_size];
  char buff2[buf_size];
  uint32_t text_len;

  if (model_file.files.size() > 0)
    {
      uint32_t v_length = (uint32_t)version.to_string().length()+1;
      text_len = sprintf(buff, "Version %s\n", version.to_string().c_str());
      memcpy(buff2,version.to_string().c_str(),v_length);
      if (read)
	v_length = buf_size;
      bin_text_read_write(model_file, buff2, v_length, 
			  "", read, 
			  buff, text_len, text);
      version_struct v_tmp(buff2);
      if (v_tmp < LAST_COMPATIBLE_VERSION)
	{
	  cout << "Model has possibly incompatible version! " << v_tmp.to_string() << endl;
	  throw exception();
	}
      
      char model = 'm';
      bin_text_read_write_fixed(model_file,&model,1,
				"file is not a model file", read, 
				"", 0, text);
      
      text_len = sprintf(buff, "Min label:%f\n", all.sd->min_label);
      bin_text_read_write_fixed(model_file,(char*)&all.sd->min_label, sizeof(all.sd->min_label), 
				"", read, 
				buff, text_len, text);
      
      text_len = sprintf(buff, "Max label:%f\n", all.sd->max_label);
      bin_text_read_write_fixed(model_file,(char*)&all.sd->max_label, sizeof(all.sd->max_label), 
				"", read, 
				buff, text_len, text);
      
      text_len = sprintf(buff, "bits:%d\n", (int)all.num_bits);
      uint32_t local_num_bits = all.num_bits;
      bin_text_read_write_fixed(model_file,(char *)&local_num_bits, sizeof(local_num_bits), 
				"", read, 
				buff, text_len, text);
      if (all.default_bits != true && all.num_bits != local_num_bits)
	{
	  cout << "vw: -b bits mismatch: command-line " << all.num_bits << " != " << local_num_bits << " stored in model" << endl;
	  exit(1);
	}
      all.default_bits = false;
      all.num_bits = local_num_bits;
      
      uint32_t pair_len = (uint32_t)all.pairs.size();
      text_len = sprintf(buff, "%d pairs: ", (int)pair_len);
      bin_text_read_write_fixed(model_file,(char *)&pair_len, sizeof(pair_len), 
				"", read, 
				buff, text_len, text);
      for (size_t i = 0; i < pair_len; i++)
	{
	  char pair[2];
	  if (!read)
	    {
	      memcpy(pair,all.pairs[i].c_str(),2);
	      text_len = sprintf(buff, "%s ", all.pairs[i].c_str());
	    }
	  bin_text_read_write_fixed(model_file, pair,2, <commit_msg>parse_regressor.cc: revert exit(1) to throw exception()<commit_after>/*
Copyright (c) by respective owners including Yahoo!, Microsoft, and
individual contributors. All rights reserved.  Released under a BSD (revised)
license as described in the file LICENSE.
 */
#include <fstream>
#include <iostream>
using namespace std;

#ifndef _WIN32
#include <unistd.h>
#endif
#include <stdlib.h>
#include <stdint.h>
#include <math.h>
#include <algorithm>

#include "parse_regressor.h"
#include "loss_functions.h"
#include "io_buf.h"
#include "memory.h"
#include "rand48.h"
#include "global_data.h"

/* Define the last version where files are backward compatible. */
#define LAST_COMPATIBLE_VERSION "6.1.3"
#define VERSION_FILE_WITH_CUBIC "6.1.3"

void initialize_regressor(vw& all)
{
  // Regressor is already initialized.
  if (all.reg.weight_vector != NULL) {
    return;
  }

  size_t length = ((size_t)1) << all.num_bits;
  all.reg.weight_mask = (all.reg.stride * length) - 1;
  all.reg.weight_vector = (weight *)calloc_or_die(all.reg.stride*length, sizeof(weight));
  if (all.reg.weight_vector == NULL)
    {
      cerr << all.program_name << ": Failed to allocate weight array with " << all.num_bits << " bits: try decreasing -b <bits>" << endl;
      throw exception();
    }
  if (all.random_weights)
    {
      for (size_t j = 0; j < length; j++)
	all.reg.weight_vector[j*all.reg.stride] = (float)(frand48() - 0.5);
    }
  if (all.initial_weight != 0.)
    for (size_t j = 0; j < all.reg.stride*length; j+=all.reg.stride)
      all.reg.weight_vector[j] = all.initial_weight;
}

const size_t buf_size = 512;

void save_load_header(vw& all, io_buf& model_file, bool read, bool text)
{

  char buff[buf_size];
  char buff2[buf_size];
  uint32_t text_len;

  if (model_file.files.size() > 0)
    {
      uint32_t v_length = (uint32_t)version.to_string().length()+1;
      text_len = sprintf(buff, "Version %s\n", version.to_string().c_str());
      memcpy(buff2,version.to_string().c_str(),v_length);
      if (read)
	v_length = buf_size;
      bin_text_read_write(model_file, buff2, v_length, 
			  "", read, 
			  buff, text_len, text);
      version_struct v_tmp(buff2);
      if (v_tmp < LAST_COMPATIBLE_VERSION)
	{
	  cout << "Model has possibly incompatible version! " << v_tmp.to_string() << endl;
	  throw exception();
	}
      
      char model = 'm';
      bin_text_read_write_fixed(model_file,&model,1,
				"file is not a model file", read, 
				"", 0, text);
      
      text_len = sprintf(buff, "Min label:%f\n", all.sd->min_label);
      bin_text_read_write_fixed(model_file,(char*)&all.sd->min_label, sizeof(all.sd->min_label), 
				"", read, 
				buff, text_len, text);
      
      text_len = sprintf(buff, "Max label:%f\n", all.sd->max_label);
      bin_text_read_write_fixed(model_file,(char*)&all.sd->max_label, sizeof(all.sd->max_label), 
				"", read, 
				buff, text_len, text);
      
      text_len = sprintf(buff, "bits:%d\n", (int)all.num_bits);
      uint32_t local_num_bits = all.num_bits;
      bin_text_read_write_fixed(model_file,(char *)&local_num_bits, sizeof(local_num_bits), 
				"", read, 
				buff, text_len, text);
      if (all.default_bits != true && all.num_bits != local_num_bits)
	{
	  cout << "vw: -b bits mismatch: command-line " << all.num_bits << " != " << local_num_bits << " stored in model" << endl;
	  throw exception();
	}
      all.default_bits = false;
      all.num_bits = local_num_bits;
      
      uint32_t pair_len = (uint32_t)all.pairs.size();
      text_len = sprintf(buff, "%d pairs: ", (int)pair_len);
      bin_text_read_write_fixed(model_file,(char *)&pair_len, sizeof(pair_len), 
				"", read, 
				buff, text_len, text);
      for (size_t i = 0; i < pair_len; i++)
	{
	  char pair[2];
	  if (!read)
	    {
	      memcpy(pair,all.pairs[i].c_str(),2);
	      text_len = sprintf(buff, "%s ", all.pairs[i].c_str());
	    }
	  bin_text_read_write_fixed(model_file, pair,2, 
{'fraction_non_alphanumeric': 0.1514066496163683, 'fraction_numerical': 0.012020460358056265, 'mean_word_length': 3.505184331797235, 'pattern_counts': {'":': 2, '<': 87, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5502865', 'n_tokens_mistral': 3062, 'n_tokens_neox': 2911, 'n_words': 610}