<commit_before>// TODO Remove this -- deprecated API and files

#ifndef SIMDJSON_JSONSTREAM_H
#define SIMDJSON_JSONSTREAM_H

#include "simdjson/document_stream.h"
#include "simdjson/padded_string.h"

namespace simdjson {

/**
 * @deprecated use document::stream instead.
 *
 * The main motivation for this piece of software is to achieve maximum speed and offer
 * good quality of life while parsing files containing multiple JSON documents.
 *
 * Since we want to offer flexibility and not restrict ourselves to a specific file
 * format, we support any file that contains any valid JSON documents separated by one
 * or more character that is considered a whitespace by the JSON spec.
 * Namely: space, nothing, linefeed, carriage return, horizontal tab.
 * Anything that is not whitespace will be parsed as a JSON document and could lead
 * to failure.
 *
 * To offer maximum parsing speed, our implementation processes the data inside the
 * buffer by batches and their size is defined by the parameter "batch_size".
 * By loading data in batches, we can optimize the time spent allocating data in the
 * parser and can also open the possibility of multi-threading.
 * The batch_size must be at least as large as the biggest document in the file, but
 * not too large in order to submerge the chached memory.  We found that 1MB is
 * somewhat a sweet spot for now.  Eventually, this batch_size could be fully
 * automated and be optimal at all times.
 *
 *  The template parameter (string_container) must
 * support the data() and size() methods, returning a pointer
 * to a char* and to the number of bytes respectively.
 * The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end
 * of the string, so if you do not use a padded_string container,
 * you have the responsability to overallocated. If you fail to
 * do so, your software may crash if you cross a page boundary,
 * and you should expect memory checkers to object.
 * Most users should use a simdjson::padded_string.
 */
template <class string_container> class JsonStream {
public:
  /* Create a JsonStream object that can be used to parse sequentially the valid
   * JSON documents found in the buffer "buf".
   *
   * The batch_size must be at least as large as the biggest document in the
   * file, but
   * not too large to submerge the cached memory.  We found that 1MB is
   * somewhat a sweet spot for now.
   *
   * The user is expected to call the following json_parse method to parse the
   * next<commit_msg>Minor typo.<commit_after>// TODO Remove this -- deprecated API and files

#ifndef SIMDJSON_JSONSTREAM_H
#define SIMDJSON_JSONSTREAM_H

#include "simdjson/document_stream.h"
#include "simdjson/padded_string.h"

namespace simdjson {

/**
 * @deprecated use document::stream instead.
 *
 * The main motivation for this piece of software is to achieve maximum speed and offer
 * good quality of life while parsing files containing multiple JSON documents.
 *
 * Since we want to offer flexibility and not restrict ourselves to a specific file
 * format, we support any file that contains any valid JSON documents separated by one
 * or more character that is considered a whitespace by the JSON spec.
 * Namely: space, nothing, linefeed, carriage return, horizontal tab.
 * Anything that is not whitespace will be parsed as a JSON document and could lead
 * to failure.
 *
 * To offer maximum parsing speed, our implementation processes the data inside the
 * buffer by batches and their size is defined by the parameter "batch_size".
 * By loading data in batches, we can optimize the time spent allocating data in the
 * parser and can also open the possibility of multi-threading.
 * The batch_size must be at least as large as the biggest document in the file, but
 * not too large in order to submerge the chached memory.  We found that 1MB is
 * somewhat a sweet spot for now.  Eventually, this batch_size could be fully
 * automated and be optimal at all times.
 *
 *  The template parameter (string_container) must
 * support the data() and size() methods, returning a pointer
 * to a char* and to the number of bytes respectively.
 * The simdjson parser may read up to SIMDJSON_PADDING bytes beyond the end
 * of the string, so if you do not use a padded_string container,
 * you have the responsability to overallocate. If you fail to
 * do so, your software may crash if you cross a page boundary,
 * and you should expect memory checkers to object.
 * Most users should use a simdjson::padded_string.
 */
template <class string_container> class JsonStream {
public:
  /* Create a JsonStream object that can be used to parse sequentially the valid
   * JSON documents found in the buffer "buf".
   *
   * The batch_size must be at least as large as the biggest document in the
   * file, but
   * not too large to submerge the cached memory.  We found that 1MB is
   * somewhat a sweet spot for now.
   *
   * The user is expected to call the following json_parse method to parse the
   * next
{'fraction_non_alphanumeric': 0.04817109734159504, 'fraction_numerical': 0.0007995202878273036, 'mean_word_length': 3.881951219512195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '899909', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1257, 'n_words': 740}