<commit_before>extern crate bytes;
#[macro_use]
extern crate combine;

extern crate futures;
extern crate partial_io;
extern crate tokio_io;

use std::str;
use std::io::Cursor;

use bytes::BytesMut;

use tokio_io::codec::Decoder;

use combine::range::{range, take};
use combine::{skip_many, Parser, many1};
use combine::easy::{self, Error as CombineError, Errors};
use combine::byte::digit;

#[derive(Debug)]
enum State {
    Nothing,
    ContentLength(usize),
}

#[derive(Debug)]
pub struct LanguageServerDecoder {
    state: State,
}

impl LanguageServerDecoder {
    pub fn new() -> LanguageServerDecoder {
        LanguageServerDecoder {
            state: State::Nothing,
        }
    }
}

fn combine_decode<'a, P, R>(
    mut parser: P,
    src: &'a [u8],
) -> Result<Option<(R, usize)>, Errors<usize, u8, String>>
where
    P: Parser<Input = easy::Stream<&'a [u8]>, Output = R>,
{
    match parser.parse(easy::Stream(&src[..])) {
        Ok((message, rest)) => Ok(Some((message, src.len() - rest.0.len()))),
        Err(err) => {
            return if err.errors
                .iter()
                .any(|err| *err == CombineError::end_of_input())
            {
                Ok(None)
            } else {
                Err(err.map_range(|r| {
                    str::from_utf8(r)
                        .ok()
                        .map_or_else(|| format!("{:?}", r), |s| s.to_string())
                }).map_position(|p| p.translate_position(&src[..])))
            }
        }
    }
}

macro_rules! decode {
    ($parser: expr, $src: expr) => {
        {
            let (output, removed_len) = {
                match combine_decode($parser, &$src[..])? {
                    None => return Ok(None),
                    Some(x) => x,
                }
            };
            $src.split_to(removed_len);
            output
        }
    };
}

impl Decoder for LanguageServerDecoder {
    type Item = String;
    type Error = Box<::std::error::Error + Send + Sync>;

    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
        loop {
            match self.state {
                State::Nothing => {
                    let value = decode!(
                        (
                            skip_many(range(&b"\r\n"[..])),
                            range(&b"Content-Length: "[..]),
                            many1(digit()),
                            range(&b"\r\n\r\n"[..]),
                        ).map(|t| t.2)
                            .and_then(|digits: Vec<u8>| unsafe {
                                String::from_utf8_unchecked(digits).parse::<usize>()
                            }),
                        src
                    );
<commit_msg>Avoid allocating a string in the tokio example<commit_after>extern crate bytes;
#[macro_use]
extern crate combine;

extern crate futures;
extern crate partial_io;
extern crate tokio_io;

use std::str;
use std::io::Cursor;

use bytes::BytesMut;

use tokio_io::codec::Decoder;

use combine::range::{range, recognize, take};
use combine::{skip_many, Parser};
use combine::easy::{self, Error as CombineError, Errors};
use combine::byte::digit;

#[derive(Debug)]
enum State {
    Nothing,
    ContentLength(usize),
}

#[derive(Debug)]
pub struct LanguageServerDecoder {
    state: State,
}

impl LanguageServerDecoder {
    pub fn new() -> LanguageServerDecoder {
        LanguageServerDecoder {
            state: State::Nothing,
        }
    }
}

fn combine_decode<'a, P, R>(
    mut parser: P,
    src: &'a [u8],
) -> Result<Option<(R, usize)>, Errors<usize, u8, String>>
where
    P: Parser<Input = easy::Stream<&'a [u8]>, Output = R>,
{
    match parser.parse(easy::Stream(&src[..])) {
        Ok((message, rest)) => Ok(Some((message, src.len() - rest.0.len()))),
        Err(err) => {
            return if err.errors
                .iter()
                .any(|err| *err == CombineError::end_of_input())
            {
                Ok(None)
            } else {
                Err(err.map_range(|r| {
                    str::from_utf8(r)
                        .ok()
                        .map_or_else(|| format!("{:?}", r), |s| s.to_string())
                }).map_position(|p| p.translate_position(&src[..])))
            }
        }
    }
}

macro_rules! decode {
    ($parser: expr, $src: expr) => {
        {
            let (output, removed_len) = {
                match combine_decode($parser, &$src[..])? {
                    None => return Ok(None),
                    Some(x) => x,
                }
            };
            $src.split_to(removed_len);
            output
        }
    };
}

impl Decoder for LanguageServerDecoder {
    type Item = String;
    type Error = Box<::std::error::Error + Send + Sync>;

    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
        loop {
            match self.state {
                State::Nothing => {
                    let value = decode!(
                        (
                            skip_many(range(&b"\r\n"[..])),
                            range(&b"Content-Length: "[..]),
                            recognize(digit()),
                            range(&b"\r\n\r\n"[..]),
                        ).map(|t| t.2)
                            .and_then(|digits: &[u8]| unsafe {
                                str::from_utf8_unchecked(digits).parse::<usize>()
                            }),
                        src
                    );

{'fraction_non_alphanumeric': 0.1541954231747185, 'fraction_numerical': 0.0032691609153650564, 'mean_word_length': 1.5590148698884758, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4373895', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1627, 'n_words': 337}