<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <assert.h>
#include <yepCore.h>
#include <yepMath.h>
#include <yepLibrary.h>

/* The naive implementation of entropy computation using log function for LibM */
double compute_entropy_naive(const double *pPointer, size_t pLength) {
	double entropy = 0.0;
	for (; pLength != 0; pLength--) {
		const double p = *pPointer++;
		entropy -= p * log(p);
	}
	return entropy;
}

/* The implementation of entropy computation using vector log and dot-product functions from Yeppp! library */
/* To avoid allocating a large array for logarithms (and also to benefit from cache locality) the logarithms are computed on small blocks of the input array */
/* The size of the block used to compute the logarithm */
#define BLOCK_SIZE 1024
double compute_entropy_yeppp(const double *p, size_t length) {
	enum YepStatus status;
	Yep64f entropy = 0.0;
	/* The small array for computed logarithms of the part of the input array */
	YEP_ALIGN(8) Yep64f logP[BLOCK_SIZE];
	YepSize index, blockLength;<commit_msg>Minor update to C Entropy example<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <assert.h>
#include <yepCore.h>
#include <yepMath.h>
#include <yepLibrary.h>

/* The naive implementation of entropy computation using log function for LibM */
double compute_entropy_naive(const double *pPointer, size_t pLength) {
	double entropy = 0.0;
	for (; pLength != 0; pLength--) {
		const double p = *pPointer++;
		entropy -= p * log(p);
	}
	return entropy;
}

/* The implementation of entropy computation using vector log and dot-product functions from Yeppp! library */
/* To avoid allocating a large array for logarithms (and also to benefit from cache locality) the logarithms are computed on small blocks of the input array */
/* The size of the block used to compute the logarithm */
#define BLOCK_SIZE 1024
double compute_entropy_yeppp(const double *p, size_t length) {
	enum YepStatus status;
	Yep64f entropy = 0.0;
	/* The small array for computed logarithms of the part of the input array */
	YEP_ALIGN(64) Yep64f logP[BLOCK_SIZE];
	YepSize index, blockLength;
{'fraction_non_alphanumeric': 0.0953262378528459, 'fraction_numerical': 0.013419713095788986, 'mean_word_length': 4.601036269430052, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4123660', 'n_tokens_mistral': 709, 'n_tokens_neox': 668, 'n_words': 266}