<commit_before>//! Builds a token iterator for twee input.
//!
//! This is lexical analyser using [rustlex](#about-rustlex) to analyse twee input 
//! and create a lazy evaluating iterator over lexical tokens.
//!
//! # About Rustlex
//!
//! From rustlex' [readme](https://github.com/LeoTestard/rustlex/blob/master/README.md):
//!
//! > RustLex is a lexical analysers generator, i.e. a program that generate [lexical
//! analysers](http://en.wikipedia.org/wiki/Lexical_analysis) for use in compiler from a
//! description of the language using regular expressions.
//!
//! The Zwreec development team forked rustlex to enable support for some advanced 
//! features like preserving accurate line and column numbers for tokens and fix 
//! some limitations of the original project. You can find the fork on Github at 
//! [Farthen/rustlex](https://github.com/Farthen/rustlex).
//!
//! For more information about Rustlex, visit the original repository by
//! [LeoTestard](https://github.com/LeoTestard/rustlex).
//!
//! # The `TweeLexer` Struct
//!
//! The `TweeLexer` Struct and its accompaning implementations are autogenerated by 
//! rustlex' compiler plugin and cannot be documented using rustdoc. To get a better
//! impression of how it is supposed to work, take a look at the uncompiled [source
//! code](/src/zwreec/frontend/lexer.rs.html#308-819).
use std::io::{BufReader, Read};
use utils::extensions::{Peeking, PeekingExt, FilteringScan, FilteringScanExt};
use config::Config;

use self::Token::*;

pub struct LexerError;

/// Stores the state for the custom iterator `scan_filter()`
///
/// The `zwreec::utils::extensions` module defines a new iterator `FilteringScan`.
/// This struct stores the state for this iterator.
pub struct ScanState<'a> {
    cfg: &'a Config,
    current_text: String,
    current_text_location: (u64, u64),
    skip_next: bool,
}

/// Builds a Token iterator for twee input.
///
/// This function takes a twee input and uses the `TweeLexer` generated by rustlex 
/// to create a token iterator. The tokens are then cleaned using the `Peeking` 
/// iterator to provide lookahead and the `FilteringScan` iterator to merge adjacent
/// text tokens and combine variable assignment.
///
/// # Example
///
/// ```
/// # extern crate zwreec;
/// use std::io::Cursor;
///
/// let cfg = zwreec::config::Config::default_config();
/// let mut input = Cursor::new("::Start\nHello World".to_string().into_bytes());
///
/// let tokens = zwreec::frontend::lexer::lex(&cfg, &mut input);
/// ```
#[allow(unused_variables)]
pub fn lex<'a, R: Read>(cfg: &'a Config, input: &'a mut R) -> FilteringScan<Peeking<TweeLexer<BufReader<&'a mut R>>, Token>, ScanState<'a>, fn(&mut ScanState, (Token, Option<Token>)) -> Option<Token>>  {

    TweeLexer::new(BufReader::new(input)).peeking().scan_filter(
        ScanState {
            cfg: cfg,
            current_text: String::new(),
            current_text_location: (0, 0),
            skip_next: false,
        },
        {
            fn scan_fn(state: &mut ScanState, elem: (Token, Option<Token>)) -> Option<Token> {
                if state.skip_next {
                    state.skip_next = false;
                    return None;
                }

                match elem {
                    (TokError {location, message}, _) => {
                        if state.cfg.force {
                            error!("{}", message);
                        } else {
                            panic!("{}", message);
                        }
                        None
                    }
                    (TokText {location, text}, Some(TokText{ .. })) => {
                        if state.current_text.len() == 0 {
                            state.current_text_location = location;
                        }

                        state.current_text.push_str(&text);
                        None
                    }
                    (TokText {location, text}, _) => {
                        if state.current_text.len() == 0 {
                            state.current_text_location = location;
                        }

                        state.current_text.push_str(&text);
                        let val = TokText {location: state.current_text_location, text: state.current_text.clone()};
                        state.current_text.clear();
                        Some(val)
                    },
                    (TokVariable {location, name: var}, Some(TokAssign {op_name: op, ..} )) => {
                        state.skip_next = true;
                        Some(TokAssign {location: location, var_name: var, op_name: op} )
                    },
                    (x, _) => Some(x),<commit_msg>Clean up code<commit_after>//! Builds a token iterator for twee input.
//!
//! This is lexical analyser using [rustlex](#about-rustlex) to analyse twee input 
//! and create a lazy evaluating iterator over lexical tokens.
//!
//! # About Rustlex
//!
//! From rustlex' [readme](https://github.com/LeoTestard/rustlex/blob/master/README.md):
//!
//! > RustLex is a lexical analysers generator, i.e. a program that generate [lexical
//! analysers](http://en.wikipedia.org/wiki/Lexical_analysis) for use in compiler from a
//! description of the language using regular expressions.
//!
//! The Zwreec development team forked rustlex to enable support for some advanced 
//! features like preserving accurate line and column numbers for tokens and fix 
//! some limitations of the original project. You can find the fork on Github at 
//! [Farthen/rustlex](https://github.com/Farthen/rustlex).
//!
//! For more information about Rustlex, visit the original repository by
//! [LeoTestard](https://github.com/LeoTestard/rustlex).
//!
//! # The `TweeLexer` Struct
//!
//! The `TweeLexer` Struct and its accompaning implementations are autogenerated by 
//! rustlex' compiler plugin and cannot be documented using rustdoc. To get a better
//! impression of how it is supposed to work, take a look at the uncompiled [source
//! code](/src/zwreec/frontend/lexer.rs.html#308-819).
use std::io::{BufReader, Read};
use utils::extensions::{Peeking, PeekingExt, FilteringScan, FilteringScanExt};
use config::Config;

use self::Token::*;

pub struct LexerError;

/// Stores the state for the custom iterator `scan_filter()`
///
/// The `zwreec::utils::extensions` module defines a new iterator `FilteringScan`.
/// This struct stores the state for this iterator.
pub struct ScanState<'a> {
    cfg: &'a Config,
    current_text: String,
    current_text_location: (u64, u64),
    skip_next: bool,
}

/// Builds a Token iterator for twee input.
///
/// This function takes a twee input and uses the `TweeLexer` generated by rustlex 
/// to create a token iterator. The tokens are then cleaned using the `Peeking` 
/// iterator to provide lookahead and the `FilteringScan` iterator to merge adjacent
/// text tokens and combine variable assignment.
///
/// # Example
///
/// ```
/// # extern crate zwreec;
/// use std::io::Cursor;
///
/// let cfg = zwreec::config::Config::default_config();
/// let mut input = Cursor::new("::Start\nHello World".to_string().into_bytes());
///
/// let tokens = zwreec::frontend::lexer::lex(&cfg, &mut input);
/// ```
#[allow(unused_variables)]
pub fn lex<'a, R: Read>(cfg: &'a Config, input: &'a mut R) -> FilteringScan<Peeking<TweeLexer<BufReader<&'a mut R>>, Token>, ScanState<'a>, fn(&mut ScanState, (Token, Option<Token>)) -> Option<Token>>  {

    TweeLexer::new(BufReader::new(input)).peeking().scan_filter(
        ScanState {
            cfg: cfg,
            current_text: String::new(),
            current_text_location: (0, 0),
            skip_next: false,
        },
        {
            fn scan_fn(state: &mut ScanState, elem: (Token, Option<Token>)) -> Option<Token> {
                if state.skip_next {
                    state.skip_next = false;
                    return None;
                }

                match elem {
                    (TokError {location, message}, _) => {
                        error!("{}", message);
                        if !state.cfg.force {
                            panic!();
                        }
                        None
                    }
                    (TokText {location, text}, Some(TokText{ .. })) => {
                        if state.current_text.len() == 0 {
                            state.current_text_location = location;
                        }

                        state.current_text.push_str(&text);
                        None
                    }
                    (TokText {location, text}, _) => {
                        if state.current_text.len() == 0 {
                            state.current_text_location = location;
                        }

                        state.current_text.push_str(&text);
                        let val = TokText {location: state.current_text_location, text: state.current_text.clone()};
                        state.current_text.clear();
                        Some(val)
                    },
                    (TokVariable {location, name: var}, Some(TokAssign {op_name: op, ..} )) => {
                        state.skip_next = true;
                        Some(TokAssign {location: location, var_name: var, op_name: op} )
                    },
                    (x, _) => Some(x),
{'fraction_non_alphanumeric': 0.13198077949813133, 'fraction_numerical': 0.002989855846235985, 'mean_word_length': 2.0125442264393696, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 43, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1755767', 'n_tokens_mistral': 2691, 'n_tokens_neox': 2529, 'n_words': 757}