<commit_before>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* gc.c - Handles recycling data hash memory                               *
*                                                                         *
* Updates:                                                                *
**************************************************************************/

#include "ministry.h"


// does not have it's own config - owned by mem.c


__attribute__((hot)) int gc_hash_list( DHASH **list, DHASH **flist, PRED **plist, unsigned int idx, int thresh )
{
	int lock = 0, freed = 0;
	DHASH *h, *prev, *next;
	PTLIST *pts;

	for( prev = NULL, h = *list; h; h = next )
	{
		next = h->next;

		if( h->valid == 0 )
		{
			if( !lock )
			{
				lock_table( idx );
				lock = 1;
			}

			// remove s
			if( prev )
				prev->next = next;
			else
				*list = next;

			// update the free list
			h->next = *flist;
			*flist  = h;

			// clear any waiting data points
			if( h->type == DATA_TYPE_STATS )
			{
				lock_stats( h );

				pts = h->in.points;
				h->in.points = NULL;

				unlock_stats( h );

				if( pts )
					mem_free_point_list( pts );
			}

			// clear any predictor block
			if( h->predict )
			{
				h->predict->next = *plist;
				*plist = h->predict;
				h->predict = NULL;
			}
		}
		else if( h->empty > thresh )
		{
			// unset the valid flag in the first pass<commit_msg>fixes gc behaviour for histo<commit_after>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* gc.c - Handles recycling data hash memory                               *
*                                                                         *
* Updates:                                                                *
**************************************************************************/

#include "ministry.h"


// does not have it's own config - owned by mem.c


__attribute__((hot)) int gc_hash_list( DHASH **list, DHASH **flist, PRED **plist, unsigned int idx, int thresh )
{
	int lock = 0, freed = 0;
	DHASH *h, *prev, *next;
	PTLIST *pts;

	for( prev = NULL, h = *list; h; h = next )
	{
		next = h->next;

		if( h->valid == 0 )
		{
			if( !lock )
			{
				lock_table( idx );
				lock = 1;
			}

			// remove s
			if( prev )
				prev->next = next;
			else
				*list = next;

			// update the free list
			h->next = *flist;
			*flist  = h;

			// clear any waiting data points
			if( h->type == DATA_TYPE_STATS )
			{
				lock_stats( h );

				pts = h->in.points;
				h->in.points = NULL;

				unlock_stats( h );

				if( pts )
					mem_free_point_list( pts );
			}
			else if( h->type == DATA_TYPE_HISTO )
			{
				lock_histo( h );

				if( h->in.hist.counts )
				{
					free( h->in.hist.counts );
					free( h->proc.hist.counts );

					h->in.hist.counts   = NULL;
					h->proc.hist.counts = NULL;
				}

				unlock_histo( h );
			}

			// clear any predictor block
			if( h->predict )
			{
				h->predict->next = *plist;
				*plist = h->predict;
				h->predict = NULL;
			}

			// go no further - we need to set
			// prev
			continue;
		}

		prev = h;

		if( h->empty > thresh )
		{
			// unset the valid flag in the first pass
{'fraction_non_alphanumeric': 0.1968026460859978, 'fraction_numerical': 0.005512679162072767, 'mean_word_length': 1.4703880190605854, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1501010', 'n_tokens_mistral': 1347, 'n_tokens_neox': 1060, 'n_words': 307}