<commit_before>#ifndef SIMDJSON_PARSEDJSON_H
#define SIMDJSON_PARSEDJSON_H

#include <math.h>
#include <inttypes.h>
#include <string.h>
#include <iomanip>
#include <iostream>

#include "simdjson/portability.h"
#include "simdjson/jsonformatutils.h"

#define JSONVALUEMASK 0xFFFFFFFFFFFFFF

#define DEFAULTMAXDEPTH 1024// a JSON document with a depth exceeding 1024 is probably de facto invalid


/************
 * The JSON is parsed to a tape, see the accompanying tape.md file
 * for documentation.
 ***********/
struct ParsedJson {
public:

  // create a ParsedJson container with zero capacity, call allocateCapacity to
  // allocate memory
  ParsedJson()
      : bytecapacity(0), depthcapacity(0), tapecapacity(0), stringcapacity(0),
        current_loc(0), n_structural_indexes(0),
        structural_indexes(NULL), tape(NULL), containing_scope_offset(NULL),
        ret_address(NULL), string_buf(NULL), current_string_buf_loc(NULL), isvalid(false) {}

  // if needed, allocate memory so that the object is able to process JSON
  // documents having up to len butes and maxdepth "depth"
  WARN_UNUSED
  inline bool allocateCapacity(size_t len, size_t maxdepth = DEFAULTMAXDEPTH) {
    if ((maxdepth == 0) || (len == 0)) {
      std::cerr << "capacities must be non-zero " << std::endl;
      return false;
    }
    if (len > 0) {
      if ((len <= bytecapacity) && (depthcapacity < maxdepth))
        return true;
      deallocate();
    }
    isvalid = false;
    bytecapacity = 0; // will only set it to len after allocations are a success
    n_structural_indexes = 0;
    uint32_t max_structures = ROUNDUP_N(len, 64) + 2 + 7;
    structural_indexes = new uint32_t[max_structures];
    size_t localtapecapacity = ROUNDUP_N(len, 64);
    size_t localstringcapacity = ROUNDUP_N(len, 64);
    string_buf = new uint8_t[localstringcapacity];
    tape = new uint64_t[localtapecapacity];
    containing_scope_offset = new uint32_t[maxdepth];
#ifdef SIMDJSON_USE_COMPUTED_GOTO
    ret_address = new void *[maxdepth];
#else
    ret_address = new char[maxdepth];
#endif
    if ((string_buf == NULL) || (tape == NULL) ||
        (containing_scope_offset == NULL) || (ret_address == NULL) || (structural_indexes == NULL)) {
      std::cerr << "Could not allocate memory" << std::endl;
      if(ret_address != NULL) delete[] ret_address;
      if(containing_scope_offset != NULL) delete[] containing_scope_offset;
      if(tape != NULL) delete[] tape;
      if(string_buf != NULL) delete[] string_buf;
      if(structural_indexes != NULL) delete[] structural_indexes;
      return false;
    }

    bytecapacity = len;
    depthcapacity = maxdepth;
    tapecapacity = localtapecapacity;
    stringcapacity = localstringcapacity;
    return true;
  }
<commit_msg>Tape capacity increase by 32 bytes to allow for expected overflow.<commit_after>#ifndef SIMDJSON_PARSEDJSON_H
#define SIMDJSON_PARSEDJSON_H

#include <math.h>
#include <inttypes.h>
#include <string.h>
#include <iomanip>
#include <iostream>

#include "simdjson/portability.h"
#include "simdjson/jsonformatutils.h"

#define JSONVALUEMASK 0xFFFFFFFFFFFFFF

#define DEFAULTMAXDEPTH 1024// a JSON document with a depth exceeding 1024 is probably de facto invalid


/************
 * The JSON is parsed to a tape, see the accompanying tape.md file
 * for documentation.
 ***********/
struct ParsedJson {
public:

  // create a ParsedJson container with zero capacity, call allocateCapacity to
  // allocate memory
  ParsedJson()
      : bytecapacity(0), depthcapacity(0), tapecapacity(0), stringcapacity(0),
        current_loc(0), n_structural_indexes(0),
        structural_indexes(NULL), tape(NULL), containing_scope_offset(NULL),
        ret_address(NULL), string_buf(NULL), current_string_buf_loc(NULL), isvalid(false) {}

  // if needed, allocate memory so that the object is able to process JSON
  // documents having up to len butes and maxdepth "depth"
  WARN_UNUSED
  inline bool allocateCapacity(size_t len, size_t maxdepth = DEFAULTMAXDEPTH) {
    if ((maxdepth == 0) || (len == 0)) {
      std::cerr << "capacities must be non-zero " << std::endl;
      return false;
    }
    if (len > 0) {
      if ((len <= bytecapacity) && (depthcapacity < maxdepth))
        return true;
      deallocate();
    }
    isvalid = false;
    bytecapacity = 0; // will only set it to len after allocations are a success
    n_structural_indexes = 0;
    uint32_t max_structures = ROUNDUP_N(len, 64) + 2 + 7;
    structural_indexes = new uint32_t[max_structures];
    size_t localtapecapacity = ROUNDUP_N(len, 64);
    size_t localstringcapacity = ROUNDUP_N(len + 32, 64);
    string_buf = new uint8_t[localstringcapacity];
    tape = new uint64_t[localtapecapacity];
    containing_scope_offset = new uint32_t[maxdepth];
#ifdef SIMDJSON_USE_COMPUTED_GOTO
    ret_address = new void *[maxdepth];
#else
    ret_address = new char[maxdepth];
#endif
    if ((string_buf == NULL) || (tape == NULL) ||
        (containing_scope_offset == NULL) || (ret_address == NULL) || (structural_indexes == NULL)) {
      std::cerr << "Could not allocate memory" << std::endl;
      if(ret_address != NULL) delete[] ret_address;
      if(containing_scope_offset != NULL) delete[] containing_scope_offset;
      if(tape != NULL) delete[] tape;
      if(string_buf != NULL) delete[] string_buf;
      if(structural_indexes != NULL) delete[] structural_indexes;
      return false;
    }

    bytecapacity = len;
    depthcapacity = maxdepth;
    tapecapacity = localtapecapacity;
    stringcapacity = localstringcapacity;
    return true;
  }

{'fraction_non_alphanumeric': 0.11454219030520646, 'fraction_numerical': 0.01400359066427289, 'mean_word_length': 3.9300884955752213, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3803396', 'n_tokens_mistral': 1840, 'n_tokens_neox': 1745, 'n_words': 485}