<commit_before>#include "ccv.h"
#include <omp.h>

static inline int __ccv_run_sgf_feature(ccv_sgf_feature_t* feature, int* step, int** i32c8)
{
#define pf_at(i) (*(i32c8[feature->pz[i]] + feature->px[i] + feature->py[i] * step[feature->pz[i]]))
#define nf_at(i) (*(i32c8[feature->nz[i]] + feature->nx[i] + feature->ny[i] * step[feature->nz[i]]))
	int pmin = pf_at(0), nmax = nf_at(0);
	/* check if every point in P > every point in N, and take a shortcut */
	if ( pmin <= nmax )
		return 0;
	int i;
	for ( i = 1; i < feature->size; ++i )
	{
		if ( feature->pz[i] >= 0 )
		{
			int p = pf_at(i);
			if ( p < pmin )
			{
				if ( p <= nmax )
					return 0;
				pmin = p;
			}
		}
		if ( feature->nz[i] >= 0 )
		{
			int n = nf_at(i);
			if ( n > nmax )
			{
				if ( pmin <= n )
					return 0;
				nmax = n;
			}
		}
	}
#undef pf_at
#undef nf_at
	return 1;
}

static int __ccv_prepare_background_data(ccv_sgf_classifier_cascade_t* cascade, char** bgfiles, int bgnum, int** negdata, int negnum)
{
	int t, i, j, k, x, y;
	int negperbg = negnum / bgnum + 1;
	int negtotal = 0;
	int isizs0 = cascade->size.width * cascade->size.height * 8;
	int isizs1 = ((cascade->size.width >> 1) - HOG_BORDER_SIZE) * ((cascade->size.height >> 1) - HOG_BORDER_SIZE) * 8;
	int steps[] = { cascade->size.width * 8, ((cascade->size.width >> 1) - HOG_BORDER_SIZE) * 8 };
	printf("Preparing negative data ...  0%%");
	CvMemStorage* parent = cvCreateMemStorage(NULL);
	int* idcheck = (int*)cvAlloc( negnum * sizeof(idcheck[0]) );
	CvRNG rng = cvRNG( (int64)idcheck );
	CvMat* imgs0 = cvCreateMat( cascade->size.height + HOG_BORDER_SIZE * 2, cascade->size.width + HOG_BORDER_SIZE * 2, CV_8UC1 );
	CvMat* imgs1 = cvCreateMat( imgs0->rows >> 1, imgs0->cols >> 1, CV_8UC1 );
	int rneg = negtotal;
	for ( t = 0; negtotal < negnum; ++t )
	{
		for ( i = 0; i < bgnum; ++i )
		{
			negperbg = ( t < 2 ) ? (negnum - negtotal) / ( bgnum - i) + 1 : negnum - negtotal;
			IplImage* image = cvLoadImage( bgfiles[i], CV_LOAD_IMAGE_GRAYSCALE );
			if ( image == NULL )
			{
				printf("\n%s file corrupted\n", bgfiles[i]);
				continue;
			}
			if ( t % 2 != 0 )
				cvFlip( image, NULL, 1 );
			CvMemStorage* storage = cvCreateChildMemStorage( parent );
			CvSeq* detected = cvSGFDetectObjects( image, &cascade, 1, storage, 0, 0, cascade->size );<commit_msg>modified:   ccv_sgf.c<commit_after>#include "ccv.h"
#include <omp.h>

static inline int __ccv_run_sgf_feature(ccv_sgf_feature_t* feature, int* step, int** i32c8)
{
#define pf_at(i) (*(i32c8[feature->pz[i]] + feature->px[i] + feature->py[i] * step[feature->pz[i]]))
#define nf_at(i) (*(i32c8[feature->nz[i]] + feature->nx[i] + feature->ny[i] * step[feature->nz[i]]))
	int pmin = pf_at(0), nmax = nf_at(0);
	/* check if every point in P > every point in N, and take a shortcut */
	if (pmin <= nmax)
		return 0;
	int i;
	for (i = 1; i < feature->size; i++)
	{
		if (feature->pz[i] >= 0)
		{
			int p = pf_at(i);
			if (p < pmin)
			{
				if (p <= nmax)
					return 0;
				pmin = p;
			}
		}
		if (feature->nz[i] >= 0)
		{
			int n = nf_at(i);
			if (n > nmax)
			{
				if (pmin <= n)
					return 0;
				nmax = n;
			}
		}
	}
#undef pf_at
#undef nf_at
	return 1;
}

#define HOG_BORDER_SIZE (2)

static int __ccv_prepare_background_data(ccv_sgf_classifier_cascade_t* cascade, char** bgfiles, int bgnum, int** negdata, int negnum)
{
	int t, i, j, k, x, y;
	int negperbg = negnum / bgnum + 1;
	int negtotal = 0;
	int isizs0 = cascade->size.width * cascade->size.height * 8;
	int isizs1 = ((cascade->size.width >> 1) - HOG_BORDER_SIZE) * ((cascade->size.height >> 1) - HOG_BORDER_SIZE) * 8;
	int steps[] = { cascade->size.width * 8, ((cascade->size.width >> 1) - HOG_BORDER_SIZE) * 8 };
	printf("preparing negative data ...  0%%");
	int* idcheck = (int*)malloc(negnum * sizeof(int));
	CvRNG rng = cvRNG((int64)idcheck);
	CvMat* imgs0 = cvCreateMat( cascade->size.height + HOG_BORDER_SIZE * 2, cascade->size.width + HOG_BORDER_SIZE * 2, CV_8UC1 );
	CvMat* imgs1 = cvCreateMat( imgs0->rows >> 1, imgs0->cols >> 1, CV_8UC1 );
	int rneg = negtotal;
	for (t = 0; negtotal < negnum; ++t)
	{
		for (i = 0; i < bgnum; ++i)
		{
			negperbg = ( t < 2 ) ? (negnum - negtotal) / ( bgnum - i) + 1 : negnum - negtotal;
			IplImage* image = cvLoadImage( bgfiles[i], CV_LOAD_IMAGE_GRAYSCALE );
			if ( image == NULL )
			{
				printf("\n%s file corrupted\n", bgfiles[i]);
				continue;
			}
			if ( t % 2 != 0 )
				cvFlip( image, NULL, 1 );
			CvMemStorage* storage = cvCreateChildMemStorage( parent );
			CvSeq* detected = cvSGFDetectObjects( image, &cascade, 1, storage, 0, 0, cascade->size );
{'fraction_non_alphanumeric': 0.17694486816299848, 'fraction_numerical': 0.02397036391370669, 'mean_word_length': 3.297752808988764, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7168867', 'n_tokens_mistral': 2207, 'n_tokens_neox': 1967, 'n_words': 475}