<commit_before>use ast::{TIdent, TNumber, TString, TSemicolon, TComment, TColon, TDot, TEqual, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TBinOp, TAssignOp, TUnaryOp, TArrow};
use ast::{OpAdd, OpSub, OpMul, OpDiv, OpMod};
use ast::{BitAnd, BitOr, BitXor, BitShl, BitShr};
use ast::{CompEqual, CompStrictEqual, CompNotEqual, CompStrictNotEqual, CompLessThan, CompGreaterThan, CompLessThanOrEqual, CompGreaterThanOrEqual};
use ast::{LogAnd, LogOr};
use ast::{UnaryNot};
use ast::{BinNum, BinBit, BinComp, BinLog};
use ast::{Token, TokenData};
use std::io::{BufReader, BufferedReader, Buffer, IoResult, EndOfFile};
use std::strbuf::StrBuf;
use std::char::{from_u32, is_whitespace};
use std::num::from_str_radix;
#[deriving(Clone)]
#[deriving(Eq)]
#[deriving(Show)]
/// The type of string used
pub enum StringType {
	/// `"` - Double-quoted
	DoubleQuote,
	/// `'` - Single-quoted
	SingleQuote
}
#[deriving(Clone)]
#[deriving(Eq)]
#[deriving(Show)]
/// The type of comment used
pub enum CommentType {
	/// `/*...*/` - A multi-line comment
	MultiLineComment,
	/// `//...` - A single-line comment
	SingleLineComment
}
#[deriving(Clone, Eq, Show)]
/// The type of number used
pub enum NumberType {
	/// A decimal number, such as `3.1415`
	DecimalNumber,
	/// A hexadecimal number, such as `0xFF00FF`
	HexadecimalNumber,
	/// An octal number, such as `011`
	OctalNumber
}
/// A Javascript lexer
pub struct Lexer<B> {
	/// The list of tokens generated so far
	pub tokens : Vec<Token>,
	/// The string buffer for identities
	ident_buffer : StrBuf,
	/// The string buffer for strings<commit_msg>Remove outdated deriving traits and shorting the deriving metadata<commit_after>use ast::{TIdent, TNumber, TString, TSemicolon, TComment, TColon, TDot, TEqual, TOpenParen, TCloseParen, TComma, TOpenBlock, TCloseBlock, TOpenArray, TCloseArray, TQuestion, TBinOp, TAssignOp, TUnaryOp, TArrow};
use ast::{OpAdd, OpSub, OpMul, OpDiv, OpMod};
use ast::{BitAnd, BitOr, BitXor, BitShl, BitShr};
use ast::{CompEqual, CompStrictEqual, CompNotEqual, CompStrictNotEqual, CompLessThan, CompGreaterThan, CompLessThanOrEqual, CompGreaterThanOrEqual};
use ast::{LogAnd, LogOr};
use ast::{UnaryNot};
use ast::{BinNum, BinBit, BinComp, BinLog};
use ast::{Token, TokenData};
use std::io::{BufReader, BufferedReader, Buffer, IoResult, EndOfFile};
use std::strbuf::StrBuf;
use std::char::{from_u32, is_whitespace};
use std::num::from_str_radix;
#[deriving(Clone, Eq)]
/// The type of string used
pub enum StringType {
	/// `"` - Double-quoted
	DoubleQuote,
	/// `'` - Single-quoted
	SingleQuote
}
#[deriving(Clone, Eq)]
/// The type of comment used
pub enum CommentType {
	/// `/*...*/` - A multi-line comment
	MultiLineComment,
	/// `//...` - A single-line comment
	SingleLineComment
}
#[deriving(Clone, Eq)]
/// The type of number used
pub enum NumberType {
	/// A decimal number, such as `3.1415`
	DecimalNumber,
	/// A hexadecimal number, such as `0xFF00FF`
	HexadecimalNumber,
	/// An octal number, such as `011`
	OctalNumber
}
/// A Javascript lexer
pub struct Lexer<B> {
	/// The list of tokens generated so far
	pub tokens : Vec<Token>,
	/// The string buffer for identities
	ident_buffer : StrBuf,
	/// The string buffer for strings
{'fraction_non_alphanumeric': 0.14715821812596006, 'fraction_numerical': 0.007987711213517665, 'mean_word_length': 4.92, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4757163', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1155, 'n_words': 328}