<commit_before>use std::fmt;


///Struct which containing the current position
#[derive(Clone, Copy, Show, PartialEq)]
pub struct SourcePosition {
    ///Current line of the input
    pub line: i32,
    ///Current column of the input
    pub column: i32
}
impl SourcePosition {
    fn start() -> SourcePosition {
        SourcePosition { line: 1, column: 1 }
    }

    fn update(&mut self, c: &char) {
        self.column += 1;
        if *c == '\n' {
            self.column = 1;
            self.line += 1;
        }
    }
}

///Enum used to store information about an error that has occured
#[derive(Clone, PartialEq, Show)]
pub enum Error {
    ///Error indicating an unexpected token has been encountered in the stream
    Unexpected(char),
    ///Error indicating that the parser expected something else
    Expected(String),
    ///Generic message
    Message(String)
}

///Enum used to indicate if a stream has had any elements consumed
#[derive(Clone, PartialEq, Show, Copy)]
pub enum Consumed {
    ///Flag indicating that the parser has consumed elements
    Consumed,
    ///Flag indicating that the parser did not consume any elements
    Empty
}

impl Consumed {
    pub fn combine(&mut self, other: Consumed) {
        if other == Consumed::Consumed {
            *self = Consumed::Consumed;
        }
    }
}
///Struct which hold information about an error that occured at a specific position.
///Can hold multiple instances of `Error` if more that one error occured at the position.
#[derive(Clone, Show, PartialEq)]
pub struct ParseError {
    ///The position where the error occured
    pub position: SourcePosition,
    ///Flag indicating wether the parser had consumed any elements from the stream before the error
    ///occured
    pub consumed: Consumed,
    ///A vector containing specific information on what errors occured at `position`
    pub errors: Vec<Error>
}

impl ParseError {
    pub fn new(position: SourcePosition, consumed: Consumed, error: Error) -> ParseError {
        ParseError { position: position, consumed: consumed, errors: vec![error] }
    }
    pub fn add_message(&mut self, message: String) {
        self.add_error(Error::Message(message));
    }
    pub fn add_error(&mut self, message: Error) {
        //Don't add duplicate errors
        if self.errors.iter().find(|msg| **msg == message).is_none() {
            self.errors.push(message);
        }
    }
    pub fn merge(mut self, other: ParseError) -> ParseError {
        for message in other.errors.into_iter() {
            self.add_error(message);
        }
        self
    }
}

impl fmt::String for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(writeln!(f, "Parse error at {}", self.position));
        for error in self.errors.iter() {
            try!(writeln!(f, "{}", error));
        }
        Ok(())
    }
}<commit_msg>Fixed errors being reported at the wrong places<commit_after>use std::fmt;


///Struct which containing the current position
#[derive(Clone, Copy, Show, Eq, PartialEq, Ord, PartialOrd)]
pub struct SourcePosition {
    ///Current line of the input
    pub line: i32,
    ///Current column of the input
    pub column: i32
}
impl SourcePosition {
    fn start() -> SourcePosition {
        SourcePosition { line: 1, column: 1 }
    }

    fn update(&mut self, c: &char) {
        self.column += 1;
        if *c == '\n' {
            self.column = 1;
            self.line += 1;
        }
    }
}

///Enum used to store information about an error that has occured
#[derive(Clone, PartialEq, Show)]
pub enum Error {
    ///Error indicating an unexpected token has been encountered in the stream
    Unexpected(char),
    ///Error indicating that the parser expected something else
    Expected(String),
    ///Generic message
    Message(String)
}

///Enum used to indicate if a stream has had any elements consumed
#[derive(Clone, PartialEq, Show, Copy)]
pub enum Consumed {
    ///Flag indicating that the parser has consumed elements
    Consumed,
    ///Flag indicating that the parser did not consume any elements
    Empty
}

impl Consumed {
    pub fn combine(&mut self, other: Consumed) {
        if other == Consumed::Consumed {
            *self = Consumed::Consumed;
        }
    }
}
///Struct which hold information about an error that occured at a specific position.
///Can hold multiple instances of `Error` if more that one error occured at the position.
#[derive(Clone, Show, PartialEq)]
pub struct ParseError {
    ///The position where the error occured
    pub position: SourcePosition,
    ///Flag indicating wether the parser had consumed any elements from the stream before the error
    ///occured
    pub consumed: Consumed,
    ///A vector containing specific information on what errors occured at `position`
    pub errors: Vec<Error>
}

impl ParseError {
    pub fn new(position: SourcePosition, consumed: Consumed, error: Error) -> ParseError {
        ParseError { position: position, consumed: consumed, errors: vec![error] }
    }
    pub fn add_message(&mut self, message: String) {
        self.add_error(Error::Message(message));
    }
    pub fn add_error(&mut self, message: Error) {
        //Don't add duplicate errors
        if self.errors.iter().find(|msg| **msg == message).is_none() {
            self.errors.push(message);
        }
    }
    pub fn merge(mut self, other: ParseError) -> ParseError {
        use std::cmp::Ordering;
        //Only keep the errors which occured after consuming the most amount of data
        match self.position.cmp(&other.position) {
            Ordering::Less => other,
            Ordering::Greater => self,
            Ordering::Equal => {
                for message in other.errors.into_iter() {
                    self.add_error(message);
                }
                self
            }
        }
    }
}

impl fmt::String for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        try!(writeln!(f, "Parse error at {}", self.position));
        for error in self.errors.iter() {
            try!(writeln!(f, "{}", error));
        }
        Ok(())
    }
}
{'fraction_non_alphanumeric': 0.10991869918699188, 'fraction_numerical': 0.002926829268292683, 'mean_word_length': 2.49290176036343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2710851', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1677, 'n_words': 630}