<commit_before>#include "StatCache.hpp"
#include "MemAllocHeap.hpp"
#include "MemAllocLinear.hpp"
#include "Stats.hpp"

#include <algorithm>

namespace t2
{

void StatCacheInit(StatCache* self, MemAllocLinear* allocator, MemAllocHeap* heap)
{
  self->m_Allocator      = allocator;
  self->m_Heap           = heap;
  HashTableInit(&self->m_Files, heap);
  ReadWriteLockInit(&self->m_HashLock);
}

void StatCacheDestroy(StatCache* self)
{
  HashTableDestroy(&self->m_Files);
  ReadWriteLockDestroy(&self->m_HashLock);
}

static void StatCacheInsert(StatCache* self, uint32_t hash, const char* path, const FileInfo& info)
{
  ReadWriteLockWrite(&self->m_HashLock);

  HashTableInsert(&self->m_Files, hash, path, info);

  ReadWriteUnlockWrite(&self->m_HashLock);
}

void StatCacheMarkDirty(StatCache* self, const char* path, uint32_t hash)
{
  ReadWriteLockWrite(&self->m_HashLock);

  if (FileInfo* fi = HashTableLookup(&self->m_Files, hash, path))
  {
    fi->m_Flags = FileInfo::kFlagDirty;
  }

  ReadWriteUnlockWrite(&self->m_HashLock);
}

FileInfo StatCacheStat(StatCache* self, const char* path, uint32_t hash)
{
  ReadWriteLockRead(&self->m_HashLock);

  const FileInfo* fi = HashTableLookup(&self->m_Files, hash, path);

  if (fi != nullptr && 0 == (fi->m_Flags & FileInfo::kFlagDirty))
  {
    FileInfo result = *fi;
    ReadWriteUnlockRead(&self->m_HashLock);
    return result;
  }

  ReadWriteUnlockRead(&self->m_HashLock);
<commit_msg>Don't add pointers with uncertain lifetimes into long lived stat cache hash table<commit_after>#include "StatCache.hpp"
#include "MemAllocHeap.hpp"
#include "MemAllocLinear.hpp"
#include "Stats.hpp"

#include <algorithm>

namespace t2
{

void StatCacheInit(StatCache* self, MemAllocLinear* allocator, MemAllocHeap* heap)
{
  self->m_Allocator      = allocator;
  self->m_Heap           = heap;
  HashTableInit(&self->m_Files, heap);
  ReadWriteLockInit(&self->m_HashLock);
}

void StatCacheDestroy(StatCache* self)
{
  HashTableDestroy(&self->m_Files);
  ReadWriteLockDestroy(&self->m_HashLock);
}

static void StatCacheInsert(StatCache* self, uint32_t hash, const char* path, const FileInfo& info)
{
  ReadWriteLockWrite(&self->m_HashLock);
  HashTableInsert(&self->m_Files, hash, StrDup(self->m_Allocator, path), info);
  ReadWriteUnlockWrite(&self->m_HashLock);
}

void StatCacheMarkDirty(StatCache* self, const char* path, uint32_t hash)
{
  ReadWriteLockWrite(&self->m_HashLock);

  if (FileInfo* fi = HashTableLookup(&self->m_Files, hash, path))
  {
    fi->m_Flags = FileInfo::kFlagDirty;
  }

  ReadWriteUnlockWrite(&self->m_HashLock);
}

FileInfo StatCacheStat(StatCache* self, const char* path, uint32_t hash)
{
  ReadWriteLockRead(&self->m_HashLock);

  const FileInfo* fi = HashTableLookup(&self->m_Files, hash, path);

  if (fi != nullptr && 0 == (fi->m_Flags & FileInfo::kFlagDirty))
  {
    FileInfo result = *fi;
    ReadWriteUnlockRead(&self->m_HashLock);
    return result;
  }

  ReadWriteUnlockRead(&self->m_HashLock);

{'fraction_non_alphanumeric': 0.13333333333333333, 'fraction_numerical': 0.005360134003350084, 'mean_word_length': 5.421505376344086, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1953665', 'n_tokens_mistral': 1047, 'n_tokens_neox': 1029, 'n_words': 203}