<commit_before>// strtable.h            see license.txt for copyright and terms of use
// implements a collection of immutable strings with unique representatives

#ifndef STRTABLE_H
#define STRTABLE_H

#include "strhash.h"     // StringHash
#include "str.h"         // rostring

// fwd
class Flatten;

// global string table for use during flattening/unflattening;
// it's up to clients to manage this ptr, as this module doesn't
// do anything besides define it into existence
// (this isn't the ideal solution..)
extern class StringTable *flattenStrTable;


// the type of references to strings in a string table; the pointer
// can be used directly in equality comparisons, because several calls
// to 'add' return the same pointer; and it points to the represented
// string (null-terminated), so it can be printed directly, etc.
typedef char const *StringRef;


class StringTable {
  // types
public: // must be public for MSVC++
  // constants
  enum {
    rackSize = 16000,      // size of one rack
    longThreshold = 1000,  // minimum length of a "long" string
  };

private:
  // some of the strings stored in the table
  struct Rack {
    Rack *next;            // (owner) next rack, if any; for deallocation
    int usedBytes;         // # of bytes of 'data' that are used
    char data[rackSize];   // data where strings are stored

  public:
    Rack(Rack *n) : next(n), usedBytes(0) {}
    int availBytes() const { return rackSize - usedBytes; }
    char *nextByte() { return data + usedBytes; }
  };

  // stores long strings
  struct LongString {<commit_msg>style adjustment<commit_after>// strtable.h            see license.txt for copyright and terms of use
// implements a collection of immutable strings with unique representatives

#ifndef STRTABLE_H
#define STRTABLE_H

#include "strhash.h"     // StringHash
#include "str.h"         // rostring

// fwd
class Flatten;

// global string table for use during flattening/unflattening;
// it's up to clients to manage this ptr, as this module doesn't
// do anything besides define it into existence
// (this isn't the ideal solution..)
extern class StringTable *flattenStrTable;


// the type of references to strings in a string table; the pointer
// can be used directly in equality comparisons, because several calls
// to 'add' return the same pointer; and it points to the represented
// string (null-terminated), so it can be printed directly, etc.
typedef char const *StringRef;


class StringTable {
public:      // constants
  enum {
    rackSize = 16000,      // size of one rack
    longThreshold = 1000,  // minimum length of a "long" string
  };

private:     // types
  // some of the strings stored in the table
  struct Rack {
    Rack *next;            // (owner) next rack, if any; for deallocation
    int usedBytes;         // # of bytes of 'data' that are used
    char data[rackSize];   // data where strings are stored

  public:
    Rack(Rack *n) : next(n), usedBytes(0) {}
    int availBytes() const { return rackSize - usedBytes; }
    char *nextByte() { return data + usedBytes; }
  };

  // stores long strings
  struct LongString {
{'fraction_non_alphanumeric': 0.09006707122325136, 'fraction_numerical': 0.0063877355477483235, 'mean_word_length': 2.8906832298136647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4842191', 'n_tokens_mistral': 905, 'n_tokens_neox': 859, 'n_words': 388}