<commit_before>// The tokenizer ("reader", in Scheme parlance) for Unseemly.

extern crate regex;

use name::*;

custom_derive! {
    #[derive(Debug,PartialEq,Eq,Clone,Copy,Reifiable)]
    pub enum DelimChar { Paren, SquareBracket, CurlyBracket }
}

impl DelimChar {
    pub fn open(self) -> char {
        match self {
            Paren => '(',
            SquareBracket => '[',
            CurlyBracket => '{',
        }
    }
    pub fn close(self) -> char {
        match self {
            Paren => ')',
            SquareBracket => ']',
            CurlyBracket => '}',
        }
    }
}

use self::DelimChar::*;

#[derive(Debug, PartialEq, Eq)]
pub struct TokenTree {
    pub t: Vec<Token>,
}

pub type Token = Name;

impl Token {
    pub fn is_just(&self, s: &str) -> bool { self.is(s) }
}

impl TokenTree {
    pub fn to_string(&self) -> String {
        let mut result = "".to_owned();
        for t in &self.t {
            result += " ";
            result += &t.orig_sp();
        }
        result
    }
}

// A token may start with an open delimiter, or end with a close delmiter,
// but otherwise may not contain delimiters

const nondelim: &str = r"[^\[\]\(\)\{\}\s]";
const open: &str = r"[\[\(\{]";
const close: &str = r"[\]\)\}]";

pub fn delim(s: &str) -> DelimChar {
    match s {
        "(" | ")" => Paren,
        "[" | "]" => SquareBracket,
        "{" | "}" => CurlyBracket,
        _ => panic!("not a delimiter!"),
    }
}

pub fn read_tokens(s: &str) -> Result<TokenTree, String> {
    lazy_static! {
        static ref token : regex::Regex =
            regex::Regex::new(format!(r"(?P<open_all>(?P<main_o>{nd}*)(?P<open>{o}))|((?P<close>{c})(?P<main_c>{nd}*))|(?P<normal>{nd}+)",
                                      o = open, c = close, nd = nondelim)
                                      .as_str()).unwrap();
    }
    let mut flat_tokens = token.captures_iter(s);

    fn read_token_tree<'a>(
        flat_tokens: &mut regex::CaptureMatches<'a, 'a>,
    ) -> Result<(TokenTree, Option<(DelimChar, &'a str)>), String> {
        let mut this_level: Vec<Token> = vec![];
        loop {
            match flat_tokens.next() {
                None => return Ok((TokenTree { t: this_level }, None)),
                Some(c) => {
                    if let Some(normal) = c.name("normal") {
                        this_level.push(n(normal.as_str()));
                    } else if let (Some(_main), Some(o_del), Some(all)) =
                        (c.name("main_o"), c.name("open"), c.name("open_all"))
                    {
                        let (mut inside, last) = read_token_tree(flat_tokens)?;

                        if let Some(last) = last {
                            if format!("{}{}", last.1, o_del.as_str()) == all.as_str() {
                                this_level.push(n(all.as_str()));
                                this_level.append(&mut inside.t);
                                this_level.push(n(&format!(
                                    "{}{}",
                                    delim(o_del.as_str()).close(),
                                    last.1
                                )));
                            } else {
                                return Err(format!(
                                    "Unmatched delimiter names: \"{}\" is closed by \"{}\". \
                                     Remember(this tokenizer is weird)Remember",
                                    all.as_str(),
                                    last.1
                                ));
                            }
                        } else {
                            return Err(format!(
                                "Unclosed delimiter at EOF: \"{}\"",
                                o_del.as_str()
                            ));
                        }
                    } else if let (Some(main), Some(c_del)) = (c.name("main_c"), c.name("close")) {
                        return Ok((
                            TokenTree { t: this_level },
                            Some((delim(c_del.as_str()), main.as_str())),
                        ));
                    } else {
                        icp!()
                    }
                }
            }
        }
    }

    let (tt, leftover) = read_token_tree(&mut flat_tokens)?;

    match leftover {
        None => Ok(tt),
        Some(l) => Err(format!("Read error: leftover {:#?}", l)),
    }
}

#[test]
fn simple_reading() {
    assert_eq!(read_tokens(""), Ok(tokens!()));
    assert_eq!(read_tokens("asdf"), Ok(tokens!("asdf")));
    assert_eq!(
        read_tokens("a s d-f d - f && a\na    8888"),
        Ok(tokens!("a" "s" "d-f" "d" "-" "f" "&&" "a" "a" "8888"))
    );
}
#[test]
fn nested_reading() {
    assert_eq!(read_tokens("()"), Ok(tokens!(("";))));
    assert_eq!(read_tokens("a ()"), Ok(tokens!("a" ("";))));
    assert_eq!(read_tokens("(b)"), Ok(tokens!(("";"b"))));
    assert_eq!(read_tokens("f x(c)x"), Ok(tokens!("f" ("x";"c"))));
    assert_eq!(read_tokens("f x[d]x"), Ok(tokens!("f" ["x";"d"])));
    assert_eq!(read_tokens("$-[()]$- x"), Ok(tokens!(["$-";("";)] "x")));
    assert_eq!(
        read_tokens("(#(5 6)# -[yy foo()foo aa]-)"),
        Ok(tokens!((""; ("#"; "5" "6") ["-"; "yy" ("foo";) "aa"])))
    )
}<commit_msg>Remove the old reader.<commit_after>use name::*;

custom_derive! {
    #[derive(Debug,PartialEq,Eq,Clone,Copy,Reifiable)]
    pub enum DelimChar { Paren, SquareBracket, CurlyBracket }
}

impl DelimChar {
    pub fn open(self) -> char {
        match self {
            Paren => '(',
            SquareBracket => '[',
            CurlyBracket => '{',
        }
    }
    pub fn close(self) -> char {
        match self {
            Paren => ')',
            SquareBracket => ']',
            CurlyBracket => '}',
        }
    }
}

use self::DelimChar::*;

#[derive(Debug, PartialEq, Eq)]
pub struct TokenTree {
    pub t: Vec<Token>,
}

pub type Token = Name;

impl Token {
    pub fn is_just(&self, s: &str) -> bool { self.is(s) }
}

impl TokenTree {
    pub fn to_string(&self) -> String {
        let mut result = "".to_owned();
        for t in &self.t {
            result += " ";
            result += &t.orig_sp();
        }
        result
    }
}

pub fn delim(s: &str) -> DelimChar {
    match s {
        "(" | ")" => Paren,
        "[" | "]" => SquareBracket,
        "{" | "}" => CurlyBracket,
        _ => panic!("not a delimiter!"),
    }
}
{'fraction_non_alphanumeric': 0.18239578490624517, 'fraction_numerical': 0.0023245002324500234, 'mean_word_length': 1.4861325115562403, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1617038', 'n_tokens_mistral': 2065, 'n_tokens_neox': 2031, 'n_words': 409}