<commit_before>use token::Token;

use regex::Regex;
use context::ParseContext;

pub trait ESCharClasses {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
}

impl ESCharClasses for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }
}

pub struct Lexer<'a, I, C> {
    chars: I,
    cx: &'a C
}

impl<'a, I, C> Lexer<'a, I, C> where I: Iterator<Item=char>, C: ParseContext {
    pub fn new(chars: I, cx: &'a C) -> Lexer<'a, I, C> {
        Lexer { chars: chars, cx: cx }
    }
}

impl<'a, I, C> Iterator for Lexer<'a, I, C> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        let char = self.chars.by_ref().find(|c| !c.is_es_whitespace() );
<commit_msg>src/parser.rs:13:1: 15:2 error: the parameter type `C` may not live long enough [E0309]<commit_after>use token::Token;

use regex::Regex;
use context::ParseContext;

pub trait ESCharClasses {
    fn is_es_newline(self) -> bool;
    fn is_es_whitespace(self) -> bool;
}

impl ESCharClasses for char {
    fn is_es_newline(self) -> bool {
        match self {
            '\u{000a}' | '\u{000d}' | '\u{2028}' | '\u{2029}' => true,
            _ => false
        }
    }

    fn is_es_whitespace(self) -> bool {
        match self {
              '\u{0009}' | '\u{000b}' | '\u{000c}' | '\u{0020}' | '\u{00a0}'
            | '\u{1680}' | '\u{2000}' | '\u{2001}' | '\u{2002}' | '\u{2003}' | '\u{2004}'
            | '\u{2005}' | '\u{2006}' | '\u{2009}' | '\u{200a}' | '\u{202f}' | '\u{205f}'
            | '\u{3000}' | '\u{feff}' => true,
            _ => false
        }
    }
}

pub struct Lexer<'a, I, C: 'a> {
    chars: I,
    cx: &'a C
}

impl<'a, I, C> Lexer<'a, I, C> where I: Iterator<Item=char>, C: ParseContext {
    pub fn new(chars: I, cx: &'a C) -> Lexer<'a, I, C> {
        Lexer { chars: chars, cx: cx }
    }
}

impl<'a, I, C> Iterator for Lexer<'a, I, C> where I: Iterator<Item=char> {
    type Item = Token;

    fn next(&mut self) -> Option<Token> {
        let char = self.chars.by_ref().find(|c| !c.is_es_whitespace() );

{'fraction_non_alphanumeric': 0.21978444957659737, 'fraction_numerical': 0.06543494996150885, 'mean_word_length': 2.2245657568238215, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3568831', 'n_tokens_mistral': 1158, 'n_tokens_neox': 1046, 'n_words': 241}