<commit_before>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2009 <NAME>
 * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
 */

#include "features/ImplicitWeightedSpecFeatures.h"
#include "lib/io.h"

CImplicitWeightedSpecFeatures::CImplicitWeightedSpecFeatures(CStringFeatures<uint16_t>* str, bool normalize) : CDotFeatures()
{
	ASSERT(str);
	SG_REF(strings)
	strings=str;
	normalization_factors=NULL;
	spec_weights=NULL;
	num_strings = str->get_num_vectors();
	alphabet_size = str->get_original_num_symbols();
	degree=str->get_order();
	set_wd_weights();

	SG_DEBUG("WEIGHTED SPEC alphasz=%d, size=%d, num_str=%d\n", alphabet_size,
			spec_size, num_strings);

	if (normalize)
		compute_normalization_const();
}

void CImplicitWeightedSpecFeatures::compute_normalization_const()
{
	float64_t* factors=new float64_t[num_strings];

	for (int32_t i=0; i<num_strings; i++)
		factors[i]=1.0/CMath::sqrt(dot(i,i));

	normalization_factors=factors;
	//CMath::display_vector(normalization_factors, num_strings, "n");
}<commit_msg>fix same bug also in implicit wd spec features<commit_after>/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2009 <NAME>
 * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
 */

#include "features/ImplicitWeightedSpecFeatures.h"
#include "lib/io.h"

CImplicitWeightedSpecFeatures::CImplicitWeightedSpecFeatures(CStringFeatures<uint16_t>* str, bool normalize) : CDotFeatures()
{
	ASSERT(str);
	strings=str;
	SG_REF(strings)
	normalization_factors=NULL;
	spec_weights=NULL;
	num_strings = str->get_num_vectors();
	alphabet_size = str->get_original_num_symbols();
	degree=str->get_order();
	set_wd_weights();

	SG_DEBUG("WEIGHTED SPEC alphasz=%d, size=%d, num_str=%d\n", alphabet_size,
			spec_size, num_strings);

	if (normalize)
		compute_normalization_const();
}

void CImplicitWeightedSpecFeatures::compute_normalization_const()
{
	float64_t* factors=new float64_t[num_strings];

	for (int32_t i=0; i<num_strings; i++)
		factors[i]=1.0/CMath::sqrt(dot(i,i));

	normalization_factors=factors;
	//CMath::display_vector(normalization_factors, num_strings, "n");
}
{'fraction_non_alphanumeric': 0.11719053162592162, 'fraction_numerical': 0.015521924718665115, 'mean_word_length': 6.4293948126801155, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '125078', 'n_tokens_mistral': 932, 'n_tokens_neox': 866, 'n_words': 208}