<commit_before>use ast;
use std::{io, mem};
use std::borrow::Borrow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::File;
use std::rc::Rc;

////////////////////////////////////////////////////////////////////////////////
// Interned strings
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Name(u32);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct RcString(Rc<String>);

impl From<String> for RcString {
    fn from(s: String) -> Self {
        RcString(Rc::new(s))
    }
}

impl<'a> From<&'a str> for RcString {
    fn from(s: &str) -> Self {
        RcString::from(String::from(s))
    }
}

impl Borrow<str> for RcString {
    fn borrow(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
struct StrInterner {
    map: RefCell<HashMap<RcString, Name>>,
    vec: RefCell<Vec<RcString>>,
}

impl StrInterner {
    fn new() -> Self {
        StrInterner {
            map: RefCell::new(HashMap::new()),
            vec: RefCell::new(Vec::new()),
        }
    }

    fn intern(&self, string: &str) -> Name {
        let mut map = self.map.borrow_mut();
        let mut vec = self.vec.borrow_mut();

        if let Some(&idx) = map.get(string) {
            return idx;
        }

        let new_idx = Name(vec.len() as u32);
        let rc_str = RcString::from(string);
        map.insert(rc_str.clone(), new_idx);
        vec.push(rc_str);
        new_idx
    }
}

fn interner() -> Rc<StrInterner> {
    thread_local!(static KEY: Rc<StrInterner> = Rc::new(StrInterner::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Lexer
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn new(start: usize, end: usize, kind: TokenKind) -> Self {
        Token { kind: kind, span: Span::new(start, end) }
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Symbol(Name),
    StrLit(String),
    ParenL,
    ParenR,
    BraceL,
    BraceR,
    BracketL,
    BracketR,
    Semicolon,
    Eof,
    Invalid,
}

/// A reference to a contiguous region of a source file.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Span {
    pub start: usize,
    pub end: usize,
}

impl Span {
    pub fn new(start: usize, end: usize) -> Self {
        Span { start: start, end: end }
    }

    pub fn point(pos: usize) -> Self {
        Span { start: pos, end: pos }
    }
}

#[derive(Clone, Debug)]
pub struct Lexer<'src> {
    source: &'src str,
    position: usize,
    token_start: usize,
    current: Option<char>,
}

impl<'src> Lexer<'src> {
    pub fn new(source: &'src str) -> Self {
        let mut lexer = Lexer { source: source, position: 0, token_start: 0, current: None };
        lexer.current = lexer.char_at(0);
        lexer
    }

    pub fn next(&mut self) -> Token {
        use self::TokenKind::*;

        self.token_start = self.position;

        let c = match self.current {
            Some(c) => c,
            None => return Token::new(self.position, self.position, TokenKind::Eof),
        };

        self.advance();

        let kind = match c {
            '"' => StrLit(self.lex_string()),
            '(' => ParenL,
            ')' => ParenR,
            '{' => BraceL,
            '}' => BraceR,
            '[' => BracketL,
            ']' => BracketR,
            ';' => Semicolon,
            _ => Invalid,
        };

        Token::new(self.token_start, self.position, kind)
    }

    /// Lex a string, assuming the initial " has already been consumed.
    fn lex_string(&mut self) -> String {
        let mut string = String::new();

        loop {
            let c = match self.current {
                Some(c) => c,
                None => {
                    errors().report(Span::point(self.position),
                                    ErrorLevel::Error,
                                    "unterminated string literal");
                    errors().report(Span::point(self.token_start),
                                    ErrorLevel::Note,
                                    "string literal began here");
                    break;
                }
            };

            let escape_start = self.position;
            self.advance();

            match c {
                '\\' => {
                    let escaped = match self.current { Some(c) => c, None => break };
                    self.advance();

                    string.push(match escaped {
                        '\\' | '"' => escaped,
                        'n' => '\n',
                        'r' => '\r',
                        't' => '\t',
                        c => {
                            errors().report(Span::new(escape_start, self.position),<commit_msg>Fix missing errors for character escape before end of file.<commit_after>use ast;
use std::{io, mem};
use std::borrow::Borrow;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fs::File;
use std::rc::Rc;

////////////////////////////////////////////////////////////////////////////////
// Interned strings
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub struct Name(u32);

#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
struct RcString(Rc<String>);

impl From<String> for RcString {
    fn from(s: String) -> Self {
        RcString(Rc::new(s))
    }
}

impl<'a> From<&'a str> for RcString {
    fn from(s: &str) -> Self {
        RcString::from(String::from(s))
    }
}

impl Borrow<str> for RcString {
    fn borrow(&self) -> &str {
        &self.0
    }
}

#[derive(Debug)]
struct StrInterner {
    map: RefCell<HashMap<RcString, Name>>,
    vec: RefCell<Vec<RcString>>,
}

impl StrInterner {
    fn new() -> Self {
        StrInterner {
            map: RefCell::new(HashMap::new()),
            vec: RefCell::new(Vec::new()),
        }
    }

    fn intern(&self, string: &str) -> Name {
        let mut map = self.map.borrow_mut();
        let mut vec = self.vec.borrow_mut();

        if let Some(&idx) = map.get(string) {
            return idx;
        }

        let new_idx = Name(vec.len() as u32);
        let rc_str = RcString::from(string);
        map.insert(rc_str.clone(), new_idx);
        vec.push(rc_str);
        new_idx
    }
}

fn interner() -> Rc<StrInterner> {
    thread_local!(static KEY: Rc<StrInterner> = Rc::new(StrInterner::new()));
    KEY.with(|k| k.clone())
}

////////////////////////////////////////////////////////////////////////////////
// Lexer
////////////////////////////////////////////////////////////////////////////////

#[derive(Clone, Debug)]
pub struct Token {
    pub kind: TokenKind,
    pub span: Span,
}

impl Token {
    pub fn new(start: usize, end: usize, kind: TokenKind) -> Self {
        Token { kind: kind, span: Span::new(start, end) }
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub enum TokenKind {
    Symbol(Name),
    StrLit(String),
    ParenL,
    ParenR,
    BraceL,
    BraceR,
    BracketL,
    BracketR,
    Semicolon,
    Eof,
    Invalid,
}

/// A reference to a contiguous region of a source file.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Span {
    pub start: usize,
    pub end: usize,
}

impl Span {
    pub fn new(start: usize, end: usize) -> Self {
        Span { start: start, end: end }
    }

    pub fn point(pos: usize) -> Self {
        Span { start: pos, end: pos }
    }
}

#[derive(Clone, Debug)]
pub struct Lexer<'src> {
    source: &'src str,
    position: usize,
    token_start: usize,
    current: Option<char>,
}

impl<'src> Lexer<'src> {
    pub fn new(source: &'src str) -> Self {
        let mut lexer = Lexer { source: source, position: 0, token_start: 0, current: None };
        lexer.current = lexer.char_at(0);
        lexer
    }

    pub fn next(&mut self) -> Token {
        use self::TokenKind::*;

        self.token_start = self.position;

        let c = match self.current {
            Some(c) => c,
            None => return Token::new(self.position, self.position, TokenKind::Eof),
        };

        self.advance();

        let kind = match c {
            '"' => StrLit(self.lex_string()),
            '(' => ParenL,
            ')' => ParenR,
            '{' => BraceL,
            '}' => BraceR,
            '[' => BracketL,
            ']' => BracketR,
            ';' => Semicolon,
            _ => Invalid,
        };

        Token::new(self.token_start, self.position, kind)
    }

    /// Lex a string, assuming the initial " has already been consumed.
    fn lex_string(&mut self) -> String {
        let mut string = String::new();

        loop {
            let c = match self.current {
                Some(c) => c,
                None => {
                    errors().report(Span::point(self.position),
                                    ErrorLevel::Error,
                                    "unterminated string literal");
                    errors().report(Span::point(self.token_start),
                                    ErrorLevel::Note,
                                    "string literal began here");
                    break;
                }
            };

            let escape_start = self.position;
            self.advance();

            match c {
                '\\' => {
                    let escaped = match self.current { Some(c) => c, None => continue };
                    self.advance();

                    string.push(match escaped {
                        '\\' | '"' => escaped,
                        'n' => '\n',
                        'r' => '\r',
                        't' => '\t',
                        c => {
                            errors().report(Span::new(escape_start, self.position),
{'fraction_non_alphanumeric': 0.20224381448462386, 'fraction_numerical': 0.001602724631874186, 'mean_word_length': 1.4428676290677758, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2702651', 'n_tokens_mistral': 3002, 'n_tokens_neox': 2987, 'n_words': 716}