<commit_before>/*****************************************************************
 **
 ** strings.c
 **
 **
 **
 ***************
 ** September **
 *****************************************************************/

// ===============================================================
//  Includes
// ===============================================================

#include <stdlib.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>

#include "../common/debugging.h"
#include "mem.h"
#include "gc.h"
#include "types.h"
#include "objects.h"
#include "strings.h"
#include "runtime.h"
#include "vm.h"

// ===============================================================
//  Hashing implementation
// ===============================================================

// The current string hash implementation, Bernstein's DJB2 hash.
uint32_t cstring_hash(const char *c_string) {
	uint32_t hash = 5381;
	uint8_t *str = (uint8_t*)c_string;
	uint8_t c;
	while ((c = *str++))
		hash = ((hash << 5) + hash) ^ c;

	return hash;
}

// ===============================================================
//  Strings
// ===============================================================

SepString *sepstr_for(const char *c_string) {
	// look in the cache and returned interned string if something's there
	uint32_t hash = cstring_hash(c_string);
	if (rt.string_cache) {
		PropertyEntry *entry = props_find_entry_raw(rt.string_cache, c_string, hash);
		if (entry) {
			log("strcache", "Returning cached string: '%s'", c_string);
			return entry->name;
		}
	}

	// create and intern a new string
	SepString *string = mem_allocate(sepstr_allocation_size(c_string));
	sepstr_init(string, c_string);

	// we know the hash already
	string->hash = hash;

	// strings might be used before the string_cache is initialized
	// but if it is, intern the string
	if (rt.string_cache) {
		props_add_prop(rt.string_cache, string, &st_field, SEPV_NOTHING);
	} else {
		// if its not, prevent it from being GC'd
		gc_register(str_to_sepv(string));
	}

	log("strcache", "Returning new string: '%s'", c_string);
	return string;
}

SepString *sepstr_new(const char *c_string) {
	// create new string, without interning
	SepString *string = mem_allocate(sepstr_allocation_size(c_string));
	sepstr_init(string, c_string);

	// register it to avoid accidentally GC'ing it away<commit_msg>String cache now uses the string itself as the value of the property.<commit_after>/*****************************************************************
 **
 ** strings.c
 **
 **
 **
 ***************
 ** September **
 *****************************************************************/

// ===============================================================
//  Includes
// ===============================================================

#include <stdlib.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>

#include "../common/debugging.h"
#include "mem.h"
#include "gc.h"
#include "types.h"
#include "objects.h"
#include "strings.h"
#include "runtime.h"
#include "vm.h"

// ===============================================================
//  Hashing implementation
// ===============================================================

// The current string hash implementation, Bernstein's DJB2 hash.
uint32_t cstring_hash(const char *c_string) {
	uint32_t hash = 5381;
	uint8_t *str = (uint8_t*)c_string;
	uint8_t c;
	while ((c = *str++))
		hash = ((hash << 5) + hash) ^ c;

	return hash;
}

// ===============================================================
//  Strings
// ===============================================================

SepString *sepstr_for(const char *c_string) {
	// look in the cache and returned interned string if something's there
	uint32_t hash = cstring_hash(c_string);
	if (rt.string_cache) {
		PropertyEntry *entry = props_find_entry_raw(rt.string_cache, c_string, hash);
		if (entry) {
			log("strcache", "Returning cached string: '%s'", c_string);
			return entry->name;
		}
	}

	// create and intern a new string
	SepString *string = mem_allocate(sepstr_allocation_size(c_string));
	sepstr_init(string, c_string);

	// we know the hash already
	string->hash = hash;

	// strings might be used before the string_cache is initialized
	// but if it is, intern the string
	if (rt.string_cache) {
		// the name and the value of the property in the string cache are the same
		props_add_prop(rt.string_cache, string, &st_field, str_to_sepv(string));
	} else {
		// if its not, prevent it from being GC'd
		gc_register(str_to_sepv(string));
	}

	log("strcache", "Returning new string: '%s'", c_string);
	return string;
}

SepString *sepstr_new(const char *c_string) {
	// create new string, without interning
	SepString *string = mem_allocate(sepstr_allocation_size(c_string));
	sepstr_init(string, c_string);

	// register it to avoid accidentally GC'ing it away
{'fraction_non_alphanumeric': 0.32335947172926127, 'fraction_numerical': 0.006190672719768881, 'mean_word_length': 5.3442408376963355, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3071509', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1416, 'n_words': 378}