<commit_before>#include "strm.h"
#include "khash.h"
#include <pthread.h>

#if defined(NO_READONLY_DATA_CHECK) || defined(_WIN32) || defined(__CYGWIN__)

static inline int
readonly_data_p(const char* s)
{
  return 0;
}

#elif defined(__APPLE__) && defined(__MACH__)

#include <mach-o/getsect.h>

static inline int
readonly_data_p(const char* p)
{
  return (void*)get_etext() < (void*)p && (void*)p < (void*)get_edata();
}

#else

extern char _etext[];
extern char __init_array_start[];

static inline int
readonly_data_p(const char* p)
{
  return _etext < p && p < (char*)&__init_array_start;
}
#endif

struct sym_key {
  const char *ptr;
  strm_int len;
};

static khint_t
sym_hash(struct sym_key key)
{
  const char *s = key.ptr;
  khint_t h;
  strm_int len = key.len;

  h = *s++;
  while (len--) {
    h = (h << 5) - h + (khint_t)*s++;
  }
  return h;
}

static khint_t
sym_eq(struct sym_key a, struct sym_key b)
{
  if (a.len != b.len) return FALSE;
  if (memcmp(a.ptr, b.ptr, a.len) == 0) return TRUE;
  return FALSE;
}

KHASH_INIT(sym, struct sym_key, strm_string, 1, sym_hash, sym_eq);

static pthread_mutex_t sym_mutex = PTHREAD_MUTEX_INITIALIZER;
static khash_t(sym) *sym_table;


#if __BYTE_ORDER == __LITTLE_ENDIAN
# define VALP_PTR(p) ((char*)p)
#else
/* big endian */
# define VALP_PTR(p) (((char*)p)+4)
#endif
#define VAL_PTR(v) VALP_PTR(&v)

static strm_string
str_new(const char* p, strm_int len, int foreign)
{
  strm_value tag;
  strm_value val;
  char* s;

  if (!p) goto mkbuf;
  if (len < 6) {
    tag = STRM_TAG_STRING_I;
    val = 0;
    s = VAL_PTR(val)+1;
    memcpy(s, p, len);
    s[-1] = len;
  }
  else if (len == 6) {
    tag = STRM_TAG_STRING_6;
    val = 0;
    s = VAL_PTR(val);
    memcpy(s, p, len);
  }
  else {
    struct strm_string* str;

    if (p && (foreign || readonly_data_p(p))) {
      tag = STRM_TAG_STRING_F;
      str = malloc(sizeof(struct strm_string));
      str->ptr = p;
    }
    else {
      char *buf;

    mkbuf:
      tag = STRM_TAG_STRING_O;
      str = malloc(sizeof(struct strm_string)+len+1);
      buf = (char*)&str[1];
      if (p) {
        memcpy(buf, p, len);
      }
      else {
        memset(buf, 0, len);
      }
      buf[len] = '\0';
      str->ptr = buf;
    }
    str->len = len;
    val = (strm_value)str;
  }
  return tag | (val & STRM_VAL_MASK);
}

static strm_string
str_intern(const char *p, strm_int len)
{
  khiter_t k;
  struct sym_key key;
  int ret;
  strm_string str;

  if (len <= 6) {
    return str_new(p, len, 0);
  }
  if (!sym_table) {
    sym_table = kh_init(sym);
  }
  key.ptr = p;
  key.len = len;
  k = kh_put(sym, sym_table, key, &ret);

  if (ret == 0) {               /* found */
    return kh_value(sym_table, k);
  }
  str = str_new(p, len, 1);
  kh_key(sym_table, k).ptr = strm_str_ptr(str);
  kh_value(sym_table, k) = str;

  return str;
}

#ifndef STRM_STR_INTERN_LIMIT
#define STRM_STR_INTERN_LIMIT 64
#endif

strm_string
strm_str_new(const char* p, strm_int len)
{
  if (!strm_event_loop_started) {
    /* single thread mode */
    if (p && (len < STRM_STR_INTERN_LIMIT || readonly_data_p(p))) {
      return str_intern(p, len);
    }
  }
  return str_new(p, len, 0);
}

strm_string
strm_str_intern(const char* p, strm_int len)
{
  strm_string str;

  assert(p!=NULL);
  if (!strm_event_loop_started) {<commit_msg>intern strings may not be foreign<commit_after>#include "strm.h"
#include "khash.h"
#include <pthread.h>

#if defined(NO_READONLY_DATA_CHECK) || defined(_WIN32) || defined(__CYGWIN__)

static inline int
readonly_data_p(const char* s)
{
  return 0;
}

#elif defined(__APPLE__) && defined(__MACH__)

#include <mach-o/getsect.h>

static inline int
readonly_data_p(const char* p)
{
  return (void*)get_etext() < (void*)p && (void*)p < (void*)get_edata();
}

#else

extern char _etext[];
extern char __init_array_start[];

static inline int
readonly_data_p(const char* p)
{
  return _etext < p && p < (char*)&__init_array_start;
}
#endif

struct sym_key {
  const char *ptr;
  strm_int len;
};

static khint_t
sym_hash(struct sym_key key)
{
  const char *s = key.ptr;
  khint_t h;
  strm_int len = key.len;

  h = *s++;
  while (len--) {
    h = (h << 5) - h + (khint_t)*s++;
  }
  return h;
}

static khint_t
sym_eq(struct sym_key a, struct sym_key b)
{
  if (a.len != b.len) return FALSE;
  if (memcmp(a.ptr, b.ptr, a.len) == 0) return TRUE;
  return FALSE;
}

KHASH_INIT(sym, struct sym_key, strm_string, 1, sym_hash, sym_eq);

static pthread_mutex_t sym_mutex = PTHREAD_MUTEX_INITIALIZER;
static khash_t(sym) *sym_table;


#if __BYTE_ORDER == __LITTLE_ENDIAN
# define VALP_PTR(p) ((char*)p)
#else
/* big endian */
# define VALP_PTR(p) (((char*)p)+4)
#endif
#define VAL_PTR(v) VALP_PTR(&v)

static strm_string
str_new(const char* p, strm_int len, int foreign)
{
  strm_value tag;
  strm_value val;
  char* s;

  if (!p) goto mkbuf;
  if (len < 6) {
    tag = STRM_TAG_STRING_I;
    val = 0;
    s = VAL_PTR(val)+1;
    memcpy(s, p, len);
    s[-1] = len;
  }
  else if (len == 6) {
    tag = STRM_TAG_STRING_6;
    val = 0;
    s = VAL_PTR(val);
    memcpy(s, p, len);
  }
  else {
    struct strm_string* str;

    if (p && (foreign || readonly_data_p(p))) {
      tag = STRM_TAG_STRING_F;
      str = malloc(sizeof(struct strm_string));
      str->ptr = p;
    }
    else {
      char *buf;

    mkbuf:
      tag = STRM_TAG_STRING_O;
      str = malloc(sizeof(struct strm_string)+len+1);
      buf = (char*)&str[1];
      if (p) {
        memcpy(buf, p, len);
      }
      else {
        memset(buf, 0, len);
      }
      buf[len] = '\0';
      str->ptr = buf;
    }
    str->len = len;
    val = (strm_value)str;
  }
  return tag | (val & STRM_VAL_MASK);
}

static strm_string
str_intern(const char *p, strm_int len)
{
  khiter_t k;
  struct sym_key key;
  int ret;
  strm_string str;

  if (len <= 6) {
    return str_new(p, len, 0);
  }
  if (!sym_table) {
    sym_table = kh_init(sym);
  }
  key.ptr = p;
  key.len = len;
  k = kh_put(sym, sym_table, key, &ret);

  if (ret == 0) {               /* found */
    return kh_value(sym_table, k);
  }
  str = str_new(p, len, 0);
  kh_key(sym_table, k).ptr = strm_str_ptr(str);
  kh_value(sym_table, k) = str;

  return str;
}

#ifndef STRM_STR_INTERN_LIMIT
#define STRM_STR_INTERN_LIMIT 64
#endif

strm_string
strm_str_new(const char* p, strm_int len)
{
  if (!strm_event_loop_started) {
    /* single thread mode */
    if (p && (len < STRM_STR_INTERN_LIMIT || readonly_data_p(p))) {
      return str_intern(p, len);
    }
  }
  return str_new(p, len, 0);
}

strm_string
strm_str_intern(const char* p, strm_int len)
{
  strm_string str;

  assert(p!=NULL);
  if (!strm_event_loop_started) {
{'fraction_non_alphanumeric': 0.1473147314731473, 'fraction_numerical': 0.007500750075007501, 'mean_word_length': 3.1564837905236907, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6559024', 'n_tokens_mistral': 2884, 'n_tokens_neox': 2805, 'n_words': 655}