<commit_before>/*
 * MRustC - Rust Compiler
 * - By <NAME> (Mutabah/thePowersGang)
 *
 * parse/lex.cpp
 * - Lexer (converts input file to token stream)
 */
#include "lex.hpp"
#include "tokentree.hpp"
#include "parseerror.hpp"
#include "../common.hpp"
#include <cassert>
#include <iostream>
#include <cstdlib>  // strtol
#include <typeinfo>
#include <algorithm>    // std::count
#include <cctype>
//#define TRACE_CHARS
//#define TRACE_RAW_TOKENS

Lexer::Lexer(const ::std::string& filename, AST::Edition edition, ParseState ps):
    TokenStream(ps),
    m_path(filename.c_str()),
    m_line(1),
    m_line_ofs(0),
    m_istream_fp(filename != "-" ? new std::ifstream(filename.c_str()) : nullptr),
    m_istream(filename != "-" ? *m_istream_fp : std::cin),
    m_last_char_valid(false),
    m_edition(edition),
    m_hygiene( Ident::Hygiene::new_scope() )
{
    if( m_istream_fp )
    {
        if( !m_istream_fp->is_open() )
        {
            throw ::std::runtime_error("Unable to open file '" + filename + "'");
        }
        // Consume the BOM
        if( this->getc_byte() == '\xef' )
        {
            if( this->getc_byte() != '\xbb' ) {
                throw ::std::runtime_error("Incomplete BOM - missing \\xBB in second position");<commit_msg>Lex - include missing <limits> header<commit_after>/*
 * MRustC - Rust Compiler
 * - By <NAME> (Mutabah/thePowersGang)
 *
 * parse/lex.cpp
 * - Lexer (converts input file to token stream)
 */
#include "lex.hpp"
#include "tokentree.hpp"
#include "parseerror.hpp"
#include "../common.hpp"
#include <cassert>
#include <iostream>
#include <cstdlib>  // strtol
#include <typeinfo>
#include <algorithm>    // std::count
#include <limits>       // std::numeric_limits
#include <cctype>
//#define TRACE_CHARS
//#define TRACE_RAW_TOKENS

Lexer::Lexer(const ::std::string& filename, AST::Edition edition, ParseState ps):
    TokenStream(ps),
    m_path(filename.c_str()),
    m_line(1),
    m_line_ofs(0),
    m_istream_fp(filename != "-" ? new std::ifstream(filename.c_str()) : nullptr),
    m_istream(filename != "-" ? *m_istream_fp : std::cin),
    m_last_char_valid(false),
    m_edition(edition),
    m_hygiene( Ident::Hygiene::new_scope() )
{
    if( m_istream_fp )
    {
        if( !m_istream_fp->is_open() )
        {
            throw ::std::runtime_error("Unable to open file '" + filename + "'");
        }
        // Consume the BOM
        if( this->getc_byte() == '\xef' )
        {
            if( this->getc_byte() != '\xbb' ) {
                throw ::std::runtime_error("Incomplete BOM - missing \\xBB in second position");
{'fraction_non_alphanumeric': 0.1673736984188199, 'fraction_numerical': 0.0015426147319706903, 'mean_word_length': 3.0979462875197474, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761373', 'n_tokens_mistral': 944, 'n_tokens_neox': 920, 'n_words': 186}