<commit_before>//#![deny(missing_docs)]
//#![cfg_attr(all(test, feature = "nightly"), feature(test))]
//#![cfg_attr(all(feature = "nightly"), feature(io))]

//! jed creates Json iterators over instances of io.Read

extern crate rustc_serialize;

use std::io::{ /*Chars,*/ Read };
use std::iter::Iterator;
use rustc_serialize::json::{ Json, Builder };

// workaround imports for std::io::Read::chars()
use std::error::Error;
use std::{ fmt, io, result, str };

/// An iterator over the Json elements of an io::Read stream
pub struct Iter<R> {
  inner: R
}

impl<R: Read> Iter<R> {
  /// Create a new Iter instance
  pub fn new(inner: R) -> Iter<R> {
    Iter { inner: inner }
  }
}

impl<R: Read> Iterator for Iter<R> {
  type Item = Json;

  fn next(&mut self) -> Option<Json> {

    fn chomp<R: Read>(mut chars: Chars<R>, buf: &mut String) -> Option<Json> { 
      match chars.next() {
        Some(Ok(c)) => {
          buf.push(c);
          match c {
            '}' | ']' => {
              let cbuf = buf.clone();
              match Builder::new(cbuf.chars()).build() {
                Ok(j) => Some(j),
                    _ => chomp(chars, buf)
              }
            }, _ =>
              chomp(chars, buf)
          }
        }, _ => None
      }
    }
    let ref mut inner = self.inner;
    chomp(Chars { inner: inner }, &mut String::new())
  }
}

/// work arounds until read::chars() stablizes

static UTF8_CHAR_WIDTH: [u8; 256] = [
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF
0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF
4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF
];

#[inline]
fn utf8_char_width(b: u8) -> usize {
    return UTF8_CHAR_WIDTH[b as usize] as usize;
}

struct Chars<R> {<commit_msg>sadly, the recusive impl of next() was liable to run into stack overflow issues. the current impl is somewhat more imperative but actually less code and uses more rust idioms<commit_after>//#![deny(missing_docs)]
//#![cfg_attr(all(test, feature = "nightly"), feature(test))]
//#![cfg_attr(all(feature = "nightly"), feature(io))]

//! jed creates Json iterators over instances of io.Read

extern crate rustc_serialize;

use std::io::{ /*Chars,*/ Read };
use std::iter::Iterator;
use rustc_serialize::json::{ Json, Builder };

// workaround imports for std::io::Read::chars()
use std::error::Error;
use std::{ fmt, io, result, str };

/// An iterator over the Json elements of an io::Read stream
pub struct Iter<R> {
  inner: R
}

impl<R: Read> Iter<R> {
  /// Create a new Iter instance
  pub fn new(inner: R) -> Iter<R> {
    Iter { inner: inner }
  }
}

impl<R: Read> Iterator for Iter<R> {
  type Item = Json;

  fn next(&mut self) -> Option<Json> {
    let ref mut inner = self.inner;
    let mut chars = Chars { inner: inner };
    let mut buf = String::new();
    while let Some(Ok(c)) = chars.next() {
      buf.push(c);
      match c {
        '}' | ']' =>
          match Builder::new(buf.chars()).build() {
            Ok(j) => return Some(j),
            _ => ()
          },
        _ => ()
      }
    }
    None
  }
}

/// work arounds until read::chars() stablizes

static UTF8_CHAR_WIDTH: [u8; 256] = [
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x1F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x3F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x5F
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // 0x7F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0x9F
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 0xBF
0,0,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // 0xDF
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, // 0xEF
4,4,4,4,4,0,0,0,0,0,0,0,0,0,0,0, // 0xFF
];

#[inline]
fn utf8_char_width(b: u8) -> usize {
    return UTF8_CHAR_WIDTH[b as usize] as usize;
}

struct Chars<R> {
{'fraction_non_alphanumeric': 0.24849606663581675, 'fraction_numerical': 0.12864414622859788, 'mean_word_length': 3.5505263157894738, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1812222', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2200, 'n_words': 356}