<commit_before>/*
 * This software is distributed under BSD 3-clause license (see LICENSE file).
 *
 * Authors: <NAME>, <NAME>, <NAME>, <NAME>,
 *          <NAME>, <NAME>, <NAME>, <NAME>,
 *          <NAME>, <NAME>, <NAME>, <NAME>,
 *          <NAME>, <NAME>, <NAME>,
 *          <NAME>
 */

#include <shogun/base/some.h>
#include <shogun/features/DenseFeatures.h>
#include <shogun/preprocessor/DensePreprocessor.h>
#include <shogun/io/SGIO.h>
#include <shogun/base/Parameter.h>
#include <shogun/mathematics/Math.h>
#include <shogun/mathematics/eigen3.h>
#include <shogun/mathematics/linalg/LinalgNamespace.h>
#include <algorithm>
#include <string.h>

namespace shogun {

template<class ST> CDenseFeatures<ST>::CDenseFeatures(int32_t size) : CDotFeatures(size)
{
	init();
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(const CDenseFeatures & orig) :
		CDotFeatures(orig)
{
	init();
	set_feature_matrix(orig.feature_matrix);
	initialize_cache();

	if (orig.m_subset_stack != NULL)
	{
		SG_UNREF(m_subset_stack);
		m_subset_stack=new CSubsetStack(*orig.m_subset_stack);
		SG_REF(m_subset_stack);
	}
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(SGMatrix<ST> matrix) :
		CDotFeatures()
{
	init();
	set_feature_matrix(matrix);
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(ST* src, int32_t num_feat, int32_t num_vec) :
		CDotFeatures()
{
	init();
	set_feature_matrix(SGMatrix<ST>(src, num_feat, num_vec));
}
template<class ST> CDenseFeatures<ST>::CDenseFeatures(CFile* loader) :
		CDotFeatures()
{
	init();
	load(loader);
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(CDotFeatures* features) :
		CDotFeatures()
{
	init();

	auto num_feat = features->get_dim_feature_space();
	auto num_vec = features->get_num_vectors();

	ASSERT(num_feat>0 && num_vec>0)
	feature_matrix = SGMatrix<ST>(num_feat, num_vec);
	for (auto i = 0; i < num_vec; i++)
	{
		SGVector<float64_t> v = features->get_computed_dot_feature_vector(i);
		ASSERT(num_feat==v.vlen)

		for (auto j = 0; j < num_feat; j++)
			feature_matrix.matrix[i * int64_t(num_feat) + j] = (ST) v.vector[j];
	}
	num_features = num_feat;
	num_vectors = num_vec;
}

template<class ST> CFeatures* CDenseFeatures<ST>::duplicate() const
{
	return new CDenseFeatures<ST>(*this);
}

template<class ST> CDenseFeatures<ST>::~CDenseFeatures()
{
	free_features();
}

template<class ST> void CDenseFeatures<ST>::free_features()
{
	free_feature_matrix();
	SG_UNREF(feature_cache);
}

template<class ST> void CDenseFeatures<ST>::free_feature_matrix()
{
	m_subset_stack->remove_all_subsets();
	feature_matrix=SGMatrix<ST>();
	num_vectors = 0;
	num_features = 0;
}

template<class ST> ST* CDenseFeatures<ST>::get_feature_vector(int32_t num, int32_t& len, bool& dofree)
{
	/* index conversion for subset, only for array access */
	int32_t real_num=m_subset_stack->subset_idx_conversion(num);

	len = num_features;

	ST* feat = NULL;
	dofree = false;

	if (feature_matrix.matrix)
	{
		feat = &feature_matrix.matrix[real_num * int64_t(num_features)];
	}
	else
	{
		if (feature_cache)
			feat = feature_cache->lock_entry(real_num);

		if (!feat)
			feat = feature_cache->set_entry(real_num);
		if (!feat)
		{
			dofree = true;
			feat = compute_feature_vector(num, len, feat);
		}
	}

	if (get_num_preprocessors())
	{
		SGVector<ST> feat_vec(feat, len, false);

		for (auto i = 0; i < get_num_preprocessors(); i++)
		{
			auto preprocessor =
				get_preprocessor(i)->as<CDensePreprocessor<ST>>();
			// temporary hack
			SGVector<ST> applied =
				preprocessor->apply_to_feature_vector(feat_vec);

			if (i == 0)
				free_feature_vector(feat_vec.vector, num, dofree);
			feat_vec = applied;
		}

		feat = SG_MALLOC(ST, feat_vec.vlen);
		sg_memcpy(feat, feat_vec.vector, feat_vec.vlen * sizeof(ST));
		dofree = true;
		len = feat_vec.vlen;
	}
	return feat;
}

template<class ST> SGVector<ST> CDenseFeatures<ST>::get_feature_vector(int32_t num)
{
	/* index conversion for subset, only for array access */
	int32_t real_num=m_subset_stack->subset_idx_conversion(num);

	if (num >= get_num_vectors())
	{
		SG_ERROR("Index out of bounds (number of vectors %d, you "
		"requested %d)\n", get_num_vectors(), real_num);<commit_msg>Add missing template keyword<commit_after>/*
 * This software is distributed under BSD 3-clause license (see LICENSE file).
 *
 * Authors: <NAME>, <NAME>, <NAME>, <NAME>,
 *          <NAME>, <NAME>, <NAME>, <NAME>,
 *          <NAME>, <NAME>, <NAME>, <NAME>,
 *          <NAME>, <NAME>, <NAME>,
 *          <NAME>
 */

#include <shogun/base/some.h>
#include <shogun/features/DenseFeatures.h>
#include <shogun/preprocessor/DensePreprocessor.h>
#include <shogun/io/SGIO.h>
#include <shogun/base/Parameter.h>
#include <shogun/mathematics/Math.h>
#include <shogun/mathematics/eigen3.h>
#include <shogun/mathematics/linalg/LinalgNamespace.h>
#include <algorithm>
#include <string.h>

namespace shogun {

template<class ST> CDenseFeatures<ST>::CDenseFeatures(int32_t size) : CDotFeatures(size)
{
	init();
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(const CDenseFeatures & orig) :
		CDotFeatures(orig)
{
	init();
	set_feature_matrix(orig.feature_matrix);
	initialize_cache();

	if (orig.m_subset_stack != NULL)
	{
		SG_UNREF(m_subset_stack);
		m_subset_stack=new CSubsetStack(*orig.m_subset_stack);
		SG_REF(m_subset_stack);
	}
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(SGMatrix<ST> matrix) :
		CDotFeatures()
{
	init();
	set_feature_matrix(matrix);
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(ST* src, int32_t num_feat, int32_t num_vec) :
		CDotFeatures()
{
	init();
	set_feature_matrix(SGMatrix<ST>(src, num_feat, num_vec));
}
template<class ST> CDenseFeatures<ST>::CDenseFeatures(CFile* loader) :
		CDotFeatures()
{
	init();
	load(loader);
}

template<class ST> CDenseFeatures<ST>::CDenseFeatures(CDotFeatures* features) :
		CDotFeatures()
{
	init();

	auto num_feat = features->get_dim_feature_space();
	auto num_vec = features->get_num_vectors();

	ASSERT(num_feat>0 && num_vec>0)
	feature_matrix = SGMatrix<ST>(num_feat, num_vec);
	for (auto i = 0; i < num_vec; i++)
	{
		SGVector<float64_t> v = features->get_computed_dot_feature_vector(i);
		ASSERT(num_feat==v.vlen)

		for (auto j = 0; j < num_feat; j++)
			feature_matrix.matrix[i * int64_t(num_feat) + j] = (ST) v.vector[j];
	}
	num_features = num_feat;
	num_vectors = num_vec;
}

template<class ST> CFeatures* CDenseFeatures<ST>::duplicate() const
{
	return new CDenseFeatures<ST>(*this);
}

template<class ST> CDenseFeatures<ST>::~CDenseFeatures()
{
	free_features();
}

template<class ST> void CDenseFeatures<ST>::free_features()
{
	free_feature_matrix();
	SG_UNREF(feature_cache);
}

template<class ST> void CDenseFeatures<ST>::free_feature_matrix()
{
	m_subset_stack->remove_all_subsets();
	feature_matrix=SGMatrix<ST>();
	num_vectors = 0;
	num_features = 0;
}

template<class ST> ST* CDenseFeatures<ST>::get_feature_vector(int32_t num, int32_t& len, bool& dofree)
{
	/* index conversion for subset, only for array access */
	int32_t real_num=m_subset_stack->subset_idx_conversion(num);

	len = num_features;

	ST* feat = NULL;
	dofree = false;

	if (feature_matrix.matrix)
	{
		feat = &feature_matrix.matrix[real_num * int64_t(num_features)];
	}
	else
	{
		if (feature_cache)
			feat = feature_cache->lock_entry(real_num);

		if (!feat)
			feat = feature_cache->set_entry(real_num);
		if (!feat)
		{
			dofree = true;
			feat = compute_feature_vector(num, len, feat);
		}
	}

	if (get_num_preprocessors())
	{
		SGVector<ST> feat_vec(feat, len, false);

		for (auto i = 0; i < get_num_preprocessors(); i++)
		{
			auto preprocessor =
				get_preprocessor(i)->template as<CDensePreprocessor<ST>>();
			// temporary hack
			SGVector<ST> applied =
				preprocessor->apply_to_feature_vector(feat_vec);

			if (i == 0)
				free_feature_vector(feat_vec.vector, num, dofree);
			feat_vec = applied;
		}

		feat = SG_MALLOC(ST, feat_vec.vlen);
		sg_memcpy(feat, feat_vec.vector, feat_vec.vlen * sizeof(ST));
		dofree = true;
		len = feat_vec.vlen;
	}
	return feat;
}

template<class ST> SGVector<ST> CDenseFeatures<ST>::get_feature_vector(int32_t num)
{
	/* index conversion for subset, only for array access */
	int32_t real_num=m_subset_stack->subset_idx_conversion(num);

	if (num >= get_num_vectors())
	{
		SG_ERROR("Index out of bounds (number of vectors %d, you "
		"requested %d)\n", get_num_vectors(), real_num);
{'fraction_non_alphanumeric': 0.14761904761904762, 'fraction_numerical': 0.007619047619047619, 'mean_word_length': 5.292883895131086, 'pattern_counts': {'":': 0, '<': 131, '<?xml version=': 0, '>': 151, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5201027', 'n_tokens_mistral': 3341, 'n_tokens_neox': 3144, 'n_words': 589}