<commit_before>#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <stdlib.h>
#include <stdbool.h>
#include <stdio.h>

#if defined(TREE_SITTER_TEST)

void *ts_record_malloc(size_t);
void *ts_record_calloc(size_t, size_t);
void *ts_record_realloc(void *, size_t);
void ts_record_free(void *);
bool ts_toggle_allocation_recording(bool);

static inline void *ts_malloc(size_t size) {
  return ts_record_malloc(size);
}

static inline void *ts_calloc(size_t count, size_t size) {
  return ts_record_calloc(count, size);
}

static inline void *ts_realloc(void *buffer, size_t size) {
  return ts_record_realloc(buffer, size);
}

static inline void ts_free(void *buffer) {
  ts_record_free(buffer);
}

#else

#include <stdlib.h>

static inline bool ts_toggle_allocation_recording(bool value) {
  (void)value;
  return false;
}

static inline void *ts_malloc(size_t size) {
  void *result = malloc(size);
  if (size > 0 && !result) {
    fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size);
    exit(1);
  }
  return result;
}

static inline void *ts_calloc(size_t count, size_t size) {
  void *result = calloc(count, size);
  if (count > 0 && !result) {
    fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size);
    exit(1);
  }
  return result;
}

static inline void *ts_realloc(void *buffer, size_t size) {
  void *result = realloc(buffer, size);
  if (size > 0 && !result) {
    fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size);
    exit(1);
  }
  return result;
}

static inline void ts_free(void *buffer) {
  free(buffer);
}

#endif

#ifdef __cplusplus
}
#endif

#endif  // TREE_SITTER_ALLOC_H_
<commit_msg>add custom allocation override<commit_after>#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_

#ifdef __cplusplus
extern "C" {
#endif

#include <stdlib.h>
#include <stdbool.h>
#include <stdio.h>

#if defined(TREE_SITTER_TEST)

void *ts_record_malloc(size_t);
void *ts_record_calloc(size_t, size_t);
void *ts_record_realloc(void *, size_t);
void ts_record_free(void *);
bool ts_toggle_allocation_recording(bool);

static inline void *ts_malloc(size_t size) {
  return ts_record_malloc(size);
}

static inline void *ts_calloc(size_t count, size_t size) {
  return ts_record_calloc(count, size);
}

static inline void *ts_realloc(void *buffer, size_t size) {
  return ts_record_realloc(buffer, size);
}

static inline void ts_free(void *buffer) {
  ts_record_free(buffer);
}

#else

#include <stdlib.h>

static inline bool ts_toggle_allocation_recording(bool value) {
  (void)value;
  return false;
}

#ifndef ts_malloc
#define ts_malloc(_sz)       ts_malloc_dflt(_sz)
#endif
#ifndef ts_calloc
#define ts_calloc(_cnt,_sz)  ts_calloc_dflt(_cnt,_sz)
#endif
#ifndef ts_realloc
#define ts_realloc(_ptr,_sz) ts_realloc_dflt(_ptr,_sz)
#endif
#ifndef ts_free
#define ts_free(_ptr)        ts_free_dflt(_ptr)
#endif

static inline void *ts_malloc_dflt(size_t size) {
  void *result = malloc(size);
  if (size > 0 && !result) {
    fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size);
    exit(1);
  }
  return result;
}

static inline void *ts_calloc_dflt(size_t count, size_t size) {
  void *result = calloc(count, size);
  if (count > 0 && !result) {
    fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size);
    exit(1);
  }
  return result;
}

static inline void *ts_realloc_dflt(void *buffer, size_t size) {
  void *result = realloc(buffer, size);
  if (size > 0 && !result) {
    fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size);
    exit(1);
  }
  return result;
}

static inline void ts_free_dflt(void *buffer) {
  free(buffer);
}

#endif

#ifdef __cplusplus
}
#endif

#endif  // TREE_SITTER_ALLOC_H_

{'fraction_non_alphanumeric': 0.12026490066225165, 'fraction_numerical': 0.0031788079470198675, 'mean_word_length': 5.06099518459069, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2830095', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1437, 'n_words': 349}