<commit_before>/*-------------------------------------------------------------------------
 *
 * ginbulk.c
 *    routines for fast build of inverted index 
 *
 *
 * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 * IDENTIFICATION
 *          $PostgreSQL: pgsql/src/backend/access/gin/ginbulk.c,v 1.3 2006/07/14 14:52:16 momjian Exp $
 *-------------------------------------------------------------------------
 */

#include "postgres.h"
#include "access/gin.h"

#define DEF_NENTRY	2048
#define DEF_NPTR	4

void
ginInitBA(BuildAccumulator *accum) {
	accum->maxdepth = 1;
	accum->stackpos = 0;
	accum->entries = NULL;
	accum->stack = NULL;
	accum->allocatedMemory = 0;
	accum->entryallocator = NULL;
}

static EntryAccumulator*
EAAllocate( BuildAccumulator *accum ) {
	if ( accum->entryallocator == NULL || accum->length>=DEF_NENTRY ) {
		accum->entryallocator = palloc(sizeof(EntryAccumulator)*DEF_NENTRY);
		accum->allocatedMemory += sizeof(EntryAccumulator)*DEF_NENTRY;
		accum->length = 0;
	}

	accum->length++;
	return accum->entryallocator + accum->length - 1;
}

/*
 * Stores heap item pointer. For robust, it checks that
 * item pointer are ordered
 */
static void
ginInsertData(BuildAccumulator *accum, EntryAccumulator *entry, ItemPointer heapptr) {
	if ( entry->number >= entry->length ) {
		accum->allocatedMemory += sizeof(ItemPointerData) * entry->length;
		entry->length *= 2;
		entry->list    = (ItemPointerData*)repalloc(entry->list,
											sizeof(ItemPointerData)*entry->length);
	}

	if ( entry->shouldSort==FALSE ) {  
		int res = compareItemPointers( entry->list + entry->number - 1, heapptr );

		Assert( res != 0 );

		if ( res > 0 )
			entry->shouldSort=TRUE;
	}

	entry->list[ entry->number ] = *heapptr;
	entry->number++;
}

static Datum
getDatumCopy(BuildAccumulator *accum, Datum value) {
	Form_pg_attribute *att = accum->ginstate->tupdesc->attrs;
	Datum 	newvalue;
	int data_length = 0;
	void *ptr;

	if ( att[0]->attbyval ) {
		store_att_byval(&newvalue, value, att[0]->attlen);
	} else {
		/* pass-by-reference */
		if (att[0]->attlen == -1) {
			/* varlena */
			data_length = VARATT_SIZE(DatumGetPointer(value));
		} else if (att[0]->attlen == -2) {
			/* c-string */
			data_length = strlen(DatumGetCString(value)) + 1;
		} else {
			/* fixed-length pass-by-reference */
			Assert(att[0]->attlen > 0);
			data_length = att[0]->attlen;
		}

		ptr = palloc( data_length );
		memcpy(ptr, DatumGetPointer(value), data_length);
		newvalue = PointerGetDatum(ptr);
	}

	accum->allocatedMemory+=data_length;

	return newvalue;
}

/*
 * Find/store one entry from indexed value.
 */
static void
ginInsertEntry(BuildAccumulator *accum, ItemPointer heapptr,  Datum entry) {
	EntryAccumulator	*ea = accum->entries, *pea = NULL;
	int res = 0;
	uint32	depth = 1;

	while( ea ) {
		res = compareEntries(accum->ginstate, entry, ea->value);
		if ( res == 0 ) 
			break;  /* found */
		else {
			pea = ea;
			if ( res < 0 ) 
				ea = ea->left;
			else
				ea = ea->right;
		}
		depth++;
	}<commit_msg>Fix getDatumCopy(): don't use store_att_byval to copy into a Datum variable (this accounts for regression failures on PPC64, and in fact won't work on any big-endian machine).  Get rid of hardwired knowledge about datum size rules; make it look just like datumCopy().<commit_after>/*-------------------------------------------------------------------------
 *
 * ginbulk.c
 *    routines for fast build of inverted index 
 *
 *
 * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 * IDENTIFICATION
 *          $PostgreSQL: pgsql/src/backend/access/gin/ginbulk.c,v 1.4 2006/07/16 00:54:22 tgl Exp $
 *-------------------------------------------------------------------------
 */

#include "postgres.h"

#include "access/gin.h"
#include "utils/datum.h"


#define DEF_NENTRY	2048
#define DEF_NPTR	4

void
ginInitBA(BuildAccumulator *accum) {
	accum->maxdepth = 1;
	accum->stackpos = 0;
	accum->entries = NULL;
	accum->stack = NULL;
	accum->allocatedMemory = 0;
	accum->entryallocator = NULL;
}

static EntryAccumulator*
EAAllocate( BuildAccumulator *accum ) {
	if ( accum->entryallocator == NULL || accum->length>=DEF_NENTRY ) {
		accum->entryallocator = palloc(sizeof(EntryAccumulator)*DEF_NENTRY);
		accum->allocatedMemory += sizeof(EntryAccumulator)*DEF_NENTRY;
		accum->length = 0;
	}

	accum->length++;
	return accum->entryallocator + accum->length - 1;
}

/*
 * Stores heap item pointer. For robust, it checks that
 * item pointer are ordered
 */
static void
ginInsertData(BuildAccumulator *accum, EntryAccumulator *entry, ItemPointer heapptr) {
	if ( entry->number >= entry->length ) {
		accum->allocatedMemory += sizeof(ItemPointerData) * entry->length;
		entry->length *= 2;
		entry->list    = (ItemPointerData*)repalloc(entry->list,
											sizeof(ItemPointerData)*entry->length);
	}

	if ( entry->shouldSort==FALSE ) {  
		int res = compareItemPointers( entry->list + entry->number - 1, heapptr );

		Assert( res != 0 );

		if ( res > 0 )
			entry->shouldSort=TRUE;
	}

	entry->list[ entry->number ] = *heapptr;
	entry->number++;
}

/*
 * This is basically the same as datumCopy(), but we duplicate some code
 * to avoid computing the datum size twice.
 */
static Datum
getDatumCopy(BuildAccumulator *accum, Datum value) {
	Form_pg_attribute *att = accum->ginstate->tupdesc->attrs;
	Datum		res;

	if (att[0]->attbyval)
		res = value;
	else
	{
		Size		realSize;
		char	   *s;

		realSize = datumGetSize(value, false, att[0]->attlen);

		s = (char *) palloc(realSize);
		memcpy(s, DatumGetPointer(value), realSize);
		res = PointerGetDatum(s);

		accum->allocatedMemory += realSize;
	}
	return res;
}

/*
 * Find/store one entry from indexed value.
 */
static void
ginInsertEntry(BuildAccumulator *accum, ItemPointer heapptr,  Datum entry) {
	EntryAccumulator	*ea = accum->entries, *pea = NULL;
	int res = 0;
	uint32	depth = 1;

	while( ea ) {
		res = compareEntries(accum->ginstate, entry, ea->value);
		if ( res == 0 ) 
			break;  /* found */
		else {
			pea = ea;
			if ( res < 0 ) 
				ea = ea->left;
			else
				ea = ea->right;
		}
		depth++;
	}
{'fraction_non_alphanumeric': 0.17923776112262796, 'fraction_numerical': 0.017700526231860948, 'mean_word_length': 3.030848329048843, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4647420', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2097, 'n_words': 513}