<commit_before>#include <YCommon/Headers/stdincludes.h>
#include "StringTable.h"

#include <YCommon/YContainers/AtomicHashTable.h>

namespace YEngine { namespace YCore {

namespace {
  YCommon::YContainers::AtomicHashTable gHashTable;
  size_t max_string_size = 0;
}

size_t GetAllocationSize(size_t max_string_size, size_t table_size) {
  return YCommon::YContainers::AtomicHashTable::GetAllocationSize(
      table_size, max_string_size);
}

void StringTable::Initialize(size_t max_string_size, size_t table_size,
                             void* buffer, size_t buffer_size) {
  gHashTable.Init(buffer, buffer_size, table_size, max_string_size);
  max_string_size = max_string_size;
}

void StringTable::Terminate() {
  gHashTable.Reset();
  max_string_size = 0;
}

uint64_t StringTable::AddString(const char* string, size_t string_size) {
  YDEBUG_CHECK(string_size <= max_string_size,
               "Cannot exceed maximum string size: %u > %u",
               static_cast<uint32_t>(string_size),
               static_cast<uint32_t>(max_string_size));
  uint64_t hash_key = 0;
  gHashTable.Insert(string, string_size, string, string_size, &hash_key);
  return hash_key;
}

const char* StringTable::StringLookup(uint64_t string_hash) {
  const void* value = gHashTable.GetValue(string_hash);
  YDEBUG_CHECK(value != NULL, "Cannot lookup string which does not exist!");
  return static_cast<const char*>(value);
}

}} // namespace YEngine { namespace YCore {
<commit_msg>Fixed string table initialization.<commit_after>#include <YCommon/Headers/stdincludes.h>
#include "StringTable.h"

#include <YCommon/YContainers/AtomicHashTable.h>

namespace YEngine { namespace YCore {

namespace {
  YCommon::YContainers::AtomicHashTable gHashTable;
  size_t gMaxStringSize = 0;
}

size_t GetAllocationSize(size_t max_string_size, size_t table_size) {
  return YCommon::YContainers::AtomicHashTable::GetAllocationSize(
      table_size, max_string_size);
}

void StringTable::Initialize(size_t max_string_size, size_t table_size,
                             void* buffer, size_t buffer_size) {
  YDEBUG_CHECK(max_string_size > 0 && table_size > 0,
               "Invalid StringTable initialization values.");
  gHashTable.Init(buffer, buffer_size, table_size, max_string_size);
  gMaxStringSize = max_string_size;
}

void StringTable::Terminate() {
  gHashTable.Reset();
  gMaxStringSize = 0;
}

uint64_t StringTable::AddString(const char* string, size_t string_size) {
  YDEBUG_CHECK(string_size <= gMaxStringSize,
               "Cannot exceed maximum string size: %u > %u",
               static_cast<uint32_t>(string_size),
               static_cast<uint32_t>(gMaxStringSize));
  uint64_t hash_key = 0;
  gHashTable.Insert(string, string_size, string, string_size, &hash_key);
  return hash_key;
}

const char* StringTable::StringLookup(uint64_t string_hash) {
  const void* value = gHashTable.GetValue(string_hash);
  YDEBUG_CHECK(value != NULL, "Cannot lookup string which does not exist!");
  return static_cast<const char*>(value);
}

}} // namespace YEngine { namespace YCore {

{'fraction_non_alphanumeric': 0.10246433203631647, 'fraction_numerical': 0.009079118028534372, 'mean_word_length': 4.734200743494424, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1756215', 'n_tokens_mistral': 1017, 'n_tokens_neox': 1000, 'n_words': 207}