<commit_before>use std::mem;

use num::bigint;
use num::bigint::ToBigInt;
use num::traits::{Zero, One};

use encoding::{Encoding, DecoderTrap};
use encoding::all::ISO_8859_1;

use bertterm::BertTerm;
use error::{Result, BertError};

static BERT_MAGIC_NUMBER: u8 = 131;
static SMALL_INTEGER_EXT: u8 = 97;
static INTEGER_EXT: u8 = 98;
static FLOAT_EXT: u8 = 99;
static ATOM_EXT: u8 = 100;
static SMALL_ATOM_EXT: u8 = 115;
static SMALL_TUPLE_EXT: u8 = 104;
static LARGE_TUPLE_EXT: u8 = 105;
static NIL_EXT: u8 = 106;
static STRING_EXT: u8 = 107;
static LIST_EXT: u8 = 108;
static BINARY_EXT: u8 = 109;
static SMALL_BIG_EXT: u8 = 110;
static LARGE_BIG_EXT: u8 = 111;
static ATOM_UTF8_EXT: u8 = 118;
static SMALL_ATOM_UTF8_EXT: u8 = 119;
static NEW_FLOAT_EXT: u8 = 70;

#[derive(Debug)]
pub struct Parser {
    contents: Vec<u8>,
    pos: usize,
}

impl Parser {
    pub fn new(contents: Vec<u8>) -> Parser {
        Parser { contents: contents, pos: 0 }
    }

    pub fn parse(&mut self) -> Result<BertTerm> {
        let () = self.magic_number()?;
        let term = self.bert_term()?;
        if self.pos == self.contents.len() {
            return Ok(term);
        } else {
            return Err(BertError::ExtraData(self.pos));
        }
    }

    // Parsers
    fn magic_number(&mut self) -> Result<()> {
        let offset = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(offset));
        } else {
            return Ok(());
        }
    }

    fn bert_term(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        let tag = self.eat_u8()?;
        if tag == SMALL_INTEGER_EXT {
            self.small_integer()
        } else if tag == INTEGER_EXT {
            self.integer()
        } else if tag == FLOAT_EXT {
            self.old_float()
        } else if tag == NEW_FLOAT_EXT {<commit_msg>Replace position comparison with eof() call<commit_after>use std::mem;

use num::bigint;
use num::bigint::ToBigInt;
use num::traits::{Zero, One};

use encoding::{Encoding, DecoderTrap};
use encoding::all::ISO_8859_1;

use bertterm::BertTerm;
use error::{Result, BertError};

static BERT_MAGIC_NUMBER: u8 = 131;
static SMALL_INTEGER_EXT: u8 = 97;
static INTEGER_EXT: u8 = 98;
static FLOAT_EXT: u8 = 99;
static ATOM_EXT: u8 = 100;
static SMALL_ATOM_EXT: u8 = 115;
static SMALL_TUPLE_EXT: u8 = 104;
static LARGE_TUPLE_EXT: u8 = 105;
static NIL_EXT: u8 = 106;
static STRING_EXT: u8 = 107;
static LIST_EXT: u8 = 108;
static BINARY_EXT: u8 = 109;
static SMALL_BIG_EXT: u8 = 110;
static LARGE_BIG_EXT: u8 = 111;
static ATOM_UTF8_EXT: u8 = 118;
static SMALL_ATOM_UTF8_EXT: u8 = 119;
static NEW_FLOAT_EXT: u8 = 70;

#[derive(Debug)]
pub struct Parser {
    contents: Vec<u8>,
    pos: usize,
}

impl Parser {
    pub fn new(contents: Vec<u8>) -> Parser {
        Parser { contents: contents, pos: 0 }
    }

    pub fn parse(&mut self) -> Result<BertTerm> {
        let () = self.magic_number()?;
        let term = self.bert_term()?;
        if self.eof() {
            return Ok(term);
        } else {
            return Err(BertError::ExtraData(self.pos));
        }
    }

    // Parsers
    fn magic_number(&mut self) -> Result<()> {
        let offset = self.pos;
        let magic = self.eat_u8()?;
        if magic != BERT_MAGIC_NUMBER {
            return Err(BertError::InvalidMagicNumber(offset));
        } else {
            return Ok(());
        }
    }

    fn bert_term(&mut self) -> Result<BertTerm> {
        let offset = self.pos;
        let tag = self.eat_u8()?;
        if tag == SMALL_INTEGER_EXT {
            self.small_integer()
        } else if tag == INTEGER_EXT {
            self.integer()
        } else if tag == FLOAT_EXT {
            self.old_float()
        } else if tag == NEW_FLOAT_EXT {
{'fraction_non_alphanumeric': 0.1284259984338293, 'fraction_numerical': 0.03967632471939441, 'mean_word_length': 2.6704980842911876, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5344311', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1370, 'n_words': 348}