<commit_before>#include <assert.h>
#include <ctype.h>
#include <limits.h>
#include <stdbool.h>
#include <string.h>
#include <stdio.h>
#include "runtime/alloc.h"
#include "runtime/atomic.h"
#include "runtime/subtree.h"
#include "runtime/length.h"
#include "runtime/language.h"
#include "runtime/error_costs.h"

typedef struct {
  Length start;
  Length old_end;
  Length new_end;
} Edit;

TSStateId TS_TREE_STATE_NONE = USHRT_MAX;

static const uint32_t MAX_TREE_POOL_SIZE = 0;

static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}};

// ExternalScannerState

void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) {
  self->length = length;
  if (length > sizeof(self->short_data)) {
    self->long_data = ts_malloc(length);
    memcpy(self->long_data, data, length);
  } else {
    memcpy(self->short_data, data, length);
  }
}

void ts_external_scanner_state_delete(ExternalScannerState *self) {
  if (self->length > sizeof(self->short_data)) {
    ts_free(self->long_data);
  }
}

const char *ts_external_scanner_state_data(const ExternalScannerState *self) {
  if (self->length > sizeof(self->short_data)) {
    return self->long_data;
  } else {
    return self->short_data;
  }
}

bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalScannerState *b) {
  return a == b || (
    a->length == b->length &&
    !memcmp(ts_external_scanner_state_data(a), ts_external_scanner_state_data(b), a->length)
  );
}

// SubtreeArray

bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) {
  Subtree **contents = NULL;
  if (self.capacity > 0) {
    contents = ts_calloc(self.capacity, sizeof(Subtree *));
    memcpy(contents, self.contents, self.size * sizeof(Subtree *));
    for (uint32_t i = 0; i < self.size; i++) {
      ts_subtree_retain(contents[i]);
    }
  }

  dest->size = self.size;
  dest->capacity = self.capacity;
  dest->contents = contents;
  return true;<commit_msg>Revert "Fix another const mismatch warning"<commit_after>#include <assert.h>
#include <ctype.h>
#include <limits.h>
#include <stdbool.h>
#include <string.h>
#include <stdio.h>
#include "runtime/alloc.h"
#include "runtime/atomic.h"
#include "runtime/subtree.h"
#include "runtime/length.h"
#include "runtime/language.h"
#include "runtime/error_costs.h"

typedef struct {
  Length start;
  Length old_end;
  Length new_end;
} Edit;

TSStateId TS_TREE_STATE_NONE = USHRT_MAX;

static const uint32_t MAX_TREE_POOL_SIZE = 0;

static const ExternalScannerState empty_state = {.length = 0, .short_data = {0}};

// ExternalScannerState

void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length) {
  self->length = length;
  if (length > sizeof(self->short_data)) {
    self->long_data = ts_malloc(length);
    memcpy(self->long_data, data, length);
  } else {
    memcpy(self->short_data, data, length);
  }
}

void ts_external_scanner_state_delete(ExternalScannerState *self) {
  if (self->length > sizeof(self->short_data)) {
    ts_free(self->long_data);
  }
}

const char *ts_external_scanner_state_data(const ExternalScannerState *self) {
  if (self->length > sizeof(self->short_data)) {
    return self->long_data;
  } else {
    return self->short_data;
  }
}

bool ts_external_scanner_state_eq(const ExternalScannerState *a, const ExternalScannerState *b) {
  return a == b || (
    a->length == b->length &&
    !memcmp(ts_external_scanner_state_data(a), ts_external_scanner_state_data(b), a->length)
  );
}

// SubtreeArray

bool ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest) {
  const Subtree **contents = NULL;
  if (self.capacity > 0) {
    contents = ts_calloc(self.capacity, sizeof(Subtree *));
    memcpy(contents, self.contents, self.size * sizeof(Subtree *));
    for (uint32_t i = 0; i < self.size; i++) {
      ts_subtree_retain(contents[i]);
    }
  }

  dest->size = self.size;
  dest->capacity = self.capacity;
  dest->contents = contents;
  return true;
{'fraction_non_alphanumeric': 0.13509471585244268, 'fraction_numerical': 0.00448654037886341, 'mean_word_length': 4.644163150492265, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3586741', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1423, 'n_words': 289}