<commit_before>/*
 * tré – Copyright (c) 2005–2008,2010,2012–2013 <NAME> <<EMAIL>>
 */

#include <string.h>
#include <strings.h>
#include <stdlib.h>
#include <stdio.h>

#include "config.h"
#include "atom.h"
#include "list.h"
#include "error.h"
#include "util.h"
#include "symbol.h"
#include "alloc.h"

size_t num_symbols;

struct tresymbol_page {
	struct tresymbol_page * entries[256];
	size_t atom;
	size_t num_entries;
	char * name;
};

struct tresymbol_root {
	treptr package;
	struct tresymbol_page * root;
};

struct tresymbol_root tresymbol_roots[MAX_PACKAGES];

struct tresymbol_page *
tresymbolpage_alloc ()
{
	void * r = malloc (sizeof (struct tresymbol_page));
	if (! r) {
		fprintf (stderr, "tresymbolpage: out of memory");
		exit (-1);
	}
	bzero (r, sizeof (struct tresymbol_page));

	return (struct tresymbol_page *) r;
}

struct tresymbol_page *
tresymbolpage_find_root (treptr package)
{
	long i;

	DOTIMES(i, MAX_PACKAGES)
		if (tresymbol_roots[i].package == package)
			return tresymbol_roots[i].root;

	DOTIMES(i, MAX_PACKAGES)
		if (tresymbol_roots[i].package == 0) {
			tresymbol_roots[i].package = package;
			return tresymbol_roots[i].root;
		}

	return NULL;
}

void
tresymbolpage_set_package (size_t i, treptr package)
{
	tresymbol_roots[i].package = package;
}

struct tresymbol_page *
tresymbolpage_add_rec (struct tresymbol_page * p, char * name, treptr atom, char * np)
{
	size_t x = (size_t) (unsigned char) *np;

	p->num_entries++;

	if (* np) {
		if (p->entries[x] == NULL)
			p->entries[x] = tresymbolpage_alloc ();

		return tresymbolpage_add_rec (p->entries[x], name, atom, ++np);
	}

	if (p->name) {
		printf ("tresymbol_page: '%s' already set", name);
		exit (-1);
	}

	p->name = name;
	p->atom = atom;

    return p;
}

struct tresymbol_page *
tresymbolpage_add (treptr atom)
{
	char * name = TRESYMBOL_NAME(atom);
    return tresymbolpage_add_rec (tresymbolpage_find_root (TRESYMBOL_PACKAGE(atom)), name, atom, name);
}

treptr tresymbolpage_find_rec (struct tresymbol_page * p, char * np);

treptr
tresymbolpage_find_rec (struct tresymbol_page * p, char * np)
{
	size_t x = (size_t) (unsigned char) *np;

	if (x && p->entries[x] == NULL)
		return treptr_invalid;

	if (x == 0)
		return p->name ? p->atom : treptr_invalid;

	return tresymbolpage_find_rec (p->entries[x], ++np);
}

treptr
tresymbolpage_find (char * name, treptr package)
{
	return tresymbolpage_find_rec (tresymbolpage_find_root (package), name);
}

size_t
tresymbolpage_remove_rec (struct tresymbol_page * p, char * np)
{
	size_t x = (size_t) (unsigned char) *np;

	if (x) {
		if (tresymbolpage_remove_rec (p->entries[x], ++np) == 0) {
			free (p->entries[x]);
			p->entries[x] = NULL;
		}
	} else {
		p->name = NULL;
		p->atom = 0;
	}

	return --p->num_entries;
}

void
tresymbolpage_remove (treptr atom)
{
	(void) tresymbolpage_remove_rec (tresymbolpage_find_root (TRESYMBOL_PACKAGE(atom)), TRESYMBOL_NAME(atom));
}

void
tresymbolpage_init ()
{
	long i;

	bzero (&tresymbol_roots, sizeof (tresymbol_roots));

	for (i = 0; i < MAX_PACKAGES; i++)
		tresymbol_roots[i].root = tresymbolpage_alloc ();
}

char *
tresymbol_add (char * symbol)<commit_msg>Copy symbol string just to be safe.<commit_after>/*
 * tré – Copyright (c) 2005–2008,2010,2012–2013 <NAME> <<EMAIL>>
 */

#include <string.h>
#include <strings.h>
#include <stdlib.h>
#include <stdio.h>

#include "config.h"
#include "atom.h"
#include "list.h"
#include "error.h"
#include "util.h"
#include "symbol.h"
#include "alloc.h"

size_t num_symbols;

struct tresymbol_page {
	struct tresymbol_page * entries[256];
	size_t atom;
	size_t num_entries;
	char * name;
};

struct tresymbol_root {
	treptr package;
	struct tresymbol_page * root;
};

struct tresymbol_root tresymbol_roots[MAX_PACKAGES];

struct tresymbol_page *
tresymbolpage_alloc ()
{
	void * r = malloc (sizeof (struct tresymbol_page));
	if (! r) {
		fprintf (stderr, "tresymbolpage: out of memory");
		exit (-1);
	}
	bzero (r, sizeof (struct tresymbol_page));

	return (struct tresymbol_page *) r;
}

struct tresymbol_page *
tresymbolpage_find_root (treptr package)
{
	long i;

	DOTIMES(i, MAX_PACKAGES)
		if (tresymbol_roots[i].package == package)
			return tresymbol_roots[i].root;

	DOTIMES(i, MAX_PACKAGES)
		if (tresymbol_roots[i].package == 0) {
			tresymbol_roots[i].package = package;
			return tresymbol_roots[i].root;
		}

	return NULL;
}

void
tresymbolpage_set_package (size_t i, treptr package)
{
	tresymbol_roots[i].package = package;
}

struct tresymbol_page *
tresymbolpage_add_rec (struct tresymbol_page * p, char * name, treptr atom, char * np)
{
	size_t x = (size_t) (unsigned char) *np;

	p->num_entries++;

	if (* np) {
		if (p->entries[x] == NULL)
			p->entries[x] = tresymbolpage_alloc ();

		return tresymbolpage_add_rec (p->entries[x], name, atom, ++np);
	}

	if (p->name) {
		printf ("tresymbol_page: '%s' already set", name);
		exit (-1);
	}

	p->name = tresymbol_add (name);
	p->atom = atom;

    return p;
}

struct tresymbol_page *
tresymbolpage_add (treptr atom)
{
	char * name = TRESYMBOL_NAME(atom);
    return tresymbolpage_add_rec (tresymbolpage_find_root (TRESYMBOL_PACKAGE(atom)), name, atom, name);
}

treptr tresymbolpage_find_rec (struct tresymbol_page * p, char * np);

treptr
tresymbolpage_find_rec (struct tresymbol_page * p, char * np)
{
	size_t x = (size_t) (unsigned char) *np;

	if (x && p->entries[x] == NULL)
		return treptr_invalid;

	if (x == 0)
		return p->name ? p->atom : treptr_invalid;

	return tresymbolpage_find_rec (p->entries[x], ++np);
}

treptr
tresymbolpage_find (char * name, treptr package)
{
	return tresymbolpage_find_rec (tresymbolpage_find_root (package), name);
}

size_t
tresymbolpage_remove_rec (struct tresymbol_page * p, char * np)
{
	size_t x = (size_t) (unsigned char) *np;

	if (x) {
		if (tresymbolpage_remove_rec (p->entries[x], ++np) == 0) {
			free (p->entries[x]);
			p->entries[x] = NULL;
		}
	} else {
        tresymbol_free (p->name);
		p->name = NULL;
		p->atom = 0;
	}

	return --p->num_entries;
}

void
tresymbolpage_remove (treptr atom)
{
	(void) tresymbolpage_remove_rec (tresymbolpage_find_root (TRESYMBOL_PACKAGE(atom)), TRESYMBOL_NAME(atom));
}

void
tresymbolpage_init ()
{
	long i;

	bzero (&tresymbol_roots, sizeof (tresymbol_roots));

	for (i = 0; i < MAX_PACKAGES; i++)
		tresymbol_roots[i].root = tresymbolpage_alloc ();
}

char *
tresymbol_add (char * symbol)
{'fraction_non_alphanumeric': 0.14317042606516292, 'fraction_numerical': 0.009398496240601503, 'mean_word_length': 4.680604982206406, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '877991', 'n_tokens_mistral': 2607, 'n_tokens_neox': 2498, 'n_words': 578}