<commit_before><?php
require_once 'ParsingError.php';
require_once 'Listener.php';

class JsonStreamingParser_Parser {
  private $_state;
  const STATE_START_DOCUMENT     = 0;
  const STATE_DONE               = -1;
  const STATE_IN_ARRAY           = 1;
  const STATE_IN_OBJECT          = 2;
  const STATE_END_KEY            = 3;
  const STATE_AFTER_KEY          = 4;  
  const STATE_IN_STRING          = 5;
  const STATE_START_ESCAPE       = 6;
  const STATE_UNICODE            = 7;
  const STATE_IN_NUMBER          = 8;
  const STATE_IN_TRUE            = 9;
  const STATE_IN_FALSE           = 10;
  const STATE_IN_NULL            = 11;
  const STATE_AFTER_VALUE        = 12;

  const STACK_OBJECT             = 0;
  const STACK_ARRAY              = 1;
  const STACK_KEY                = 2;
  const STACK_STRING             = 3;
  private $_stack;

  private $_stream;
  private $_listener;
  private $_buffer;
  private $_buffer_size;
  private $_unicode_buffer;
  private $_unicode_high_codepoint;


  public function __construct($stream, $listener) {
    if (!is_resource($stream) || get_resource_type($stream) != 'stream') {
      throw new InvalidArgumentException("Argument is not a stream");
    }
    if (!in_array("JsonStreamingParser_Listener", class_implements(get_class($listener)))) {
      throw new InvalidArgumentException("Listener must implement JsonStreamingParser_Listener");
    }

    $this->_stream = $stream;
    $this->_listener = $listener;

    $this->_state = self::STATE_START_DOCUMENT;
    $this->_stack = array();

    $this->_buffer = '';
    $this->_buffer_size = 8192;
    $this->_unicode_buffer = array();
    $this->_unicode_high_codepoint = -1;
  }


  public function parse() {
    while (!feof($this->_stream)) {
      if(function_exists('stream_get_line')) {
        $line = $this->_unicode_str_split(stream_get_line($this->_stream, $this->_buffer_size));
      }
      else {
        $line = $this->_unicode_str_split(fgets($this->_stream, $this->_buffer_size));
      }
      foreach ($line as $c) {
        $this->_consume_char($c);
      }
    }
  }

  private function _unicode_str_split($bytes) {
    // FIXME we are still getting the error of basically reading across a
    //       unicode character boundary with this.
    return preg_split('//u', $bytes, -1, PREG_SPLIT_NO_EMPTY);
  }

  private function _consume_char($c) {
    // preg_match check skips whitespace
    if (preg_match('/\s/u', $c) &&
        !($this->_state === self::STATE_IN_STRING ||
          $this->_state === self::STATE_UNICODE ||
          $this->_state === self::STATE_START_ESCAPE ||
          $this->_state === self::STATE_IN_NUMBER ||
          $this->_state === self::STATE_START_DOCUMENT)) {
      return;
    }

    switch ($this->_state) {

      case self::STATE_START_DOCUMENT:
        $this->_listener->start_document();
        if ($c === '[') {
          $this->_start_array();
        } elseif ($c === '{') {<commit_msg>Fix #8<commit_after><?php
require_once 'ParsingError.php';
require_once 'Listener.php';

class JsonStreamingParser_Parser {
  private $_state;
  const STATE_START_DOCUMENT     = 0;
  const STATE_DONE               = -1;
  const STATE_IN_ARRAY           = 1;
  const STATE_IN_OBJECT          = 2;
  const STATE_END_KEY            = 3;
  const STATE_AFTER_KEY          = 4;  
  const STATE_IN_STRING          = 5;
  const STATE_START_ESCAPE       = 6;
  const STATE_UNICODE            = 7;
  const STATE_IN_NUMBER          = 8;
  const STATE_IN_TRUE            = 9;
  const STATE_IN_FALSE           = 10;
  const STATE_IN_NULL            = 11;
  const STATE_AFTER_VALUE        = 12;

  const STACK_OBJECT             = 0;
  const STACK_ARRAY              = 1;
  const STACK_KEY                = 2;
  const STACK_STRING             = 3;
  private $_stack;

  private $_stream;
  private $_listener;
  private $_buffer;
  private $_buffer_size;
  private $_unicode_buffer;
  private $_unicode_high_codepoint;


  public function __construct($stream, $listener) {
    if (!is_resource($stream) || get_resource_type($stream) != 'stream') {
      throw new InvalidArgumentException("Argument is not a stream");
    }
    if (!in_array("JsonStreamingParser_Listener", class_implements(get_class($listener)))) {
      throw new InvalidArgumentException("Listener must implement JsonStreamingParser_Listener");
    }

    $this->_stream = $stream;
    $this->_listener = $listener;

    $this->_state = self::STATE_START_DOCUMENT;
    $this->_stack = array();

    $this->_buffer = '';
    $this->_buffer_size = 8192;
    $this->_unicode_buffer = array();
    $this->_unicode_high_codepoint = -1;
  }


  public function parse() {
    while (!feof($this->_stream)) {
      if(function_exists('stream_get_line')) {
        $line = $this->_unicode_str_split(stream_get_line($this->_stream, $this->_buffer_size));
      }
      else {
        $line = $this->_unicode_str_split(fgets($this->_stream, $this->_buffer_size));
      }
      foreach ($line as $c) {
        $this->_consume_char($c);
      }
    }
  }

  private function _unicode_str_split($bytes) {
    // FIXME we are still getting the error of basically reading across a
    //       unicode character boundary with this.
    return preg_split('//', $bytes, -1, PREG_SPLIT_NO_EMPTY);
  }

  private function _consume_char($c) {
    // preg_match check skips whitespace
    if (preg_match('/\s/u', $c) &&
        !($this->_state === self::STATE_IN_STRING ||
          $this->_state === self::STATE_UNICODE ||
          $this->_state === self::STATE_START_ESCAPE ||
          $this->_state === self::STATE_IN_NUMBER ||
          $this->_state === self::STATE_START_DOCUMENT)) {
      return;
    }

    switch ($this->_state) {

      case self::STATE_START_DOCUMENT:
        $this->_listener->start_document();
        if ($c === '[') {
          $this->_start_array();
        } elseif ($c === '{') {
{'fraction_non_alphanumeric': 0.12774957698815567, 'fraction_numerical': 0.009306260575296108, 'mean_word_length': 2.5737605804111245, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5982378', 'n_tokens_mistral': 1938, 'n_tokens_neox': 1813, 'n_words': 381}