<commit_before>use read_token;
use range::Range;
use std::rc::Rc;

use {
    MetaData,
    MetaReader,
    ParseError,
};

/// Stores information about text.
pub struct Text {
    /// Whether to allow empty string.
    pub allow_empty: bool,
    /// Which property to set if text is read.
    pub property: Option<Rc<String>>,
}

impl Text {
    /// Parses text.
    pub fn parse<M>(
        &self,
        meta_reader: &mut M,
        state: &M::State,
        chars: &[char],
        offset: usize
    ) -> Result<(Range, M::State), (Range, ParseError)>
        where M: MetaReader
    {
        if let Some(range) = read_token::string(chars, offset) {
            if !self.allow_empty && range.length == 2 {
                Err((range, ParseError::EmptyTextNotAllowed))
            } else {
                match read_token::parse_string(
                    chars, offset, range.next_offset()) {
                    // Focus range to invalid string format.
                    Err(err) => Err((err.range(),
                        ParseError::ParseStringError(err))),
                    Ok(text) => {
                        if let Some(ref property) = self.property {
                            match meta_reader.data(
                                MetaData::String(property.clone(), text),
                                state,
                                range
                            ) {
                                Err(err) => Err((range, err)),
                                Ok(state) => Ok((range, state)),
                            }
                        } else {
                            Ok((range, state.clone()))
                        }
                    }
                }
            }
        } else {
            Err((Range::new(offset, 0), ParseError::ExpectedText))
        }
    }
}<commit_msg>Added unit tests for `Text`<commit_after>use read_token;
use range::Range;
use std::rc::Rc;

use {
    MetaData,
    MetaReader,
    ParseError,
};

/// Stores information about text.
pub struct Text {
    /// Whether to allow empty string.
    pub allow_empty: bool,
    /// Which property to set if text is read.
    pub property: Option<Rc<String>>,
}

impl Text {
    /// Parses text.
    pub fn parse<M>(
        &self,
        meta_reader: &mut M,
        state: &M::State,
        chars: &[char],
        offset: usize
    ) -> Result<(Range, M::State), (Range, ParseError)>
        where M: MetaReader
    {
        if let Some(range) = read_token::string(chars, offset) {
            if !self.allow_empty && range.length == 2 {
                Err((range, ParseError::EmptyTextNotAllowed))
            } else {
                match read_token::parse_string(
                    chars, offset, range.next_offset()) {
                    // Focus range to invalid string format.
                    Err(err) => Err((err.range(),
                        ParseError::ParseStringError(err))),
                    Ok(text) => {
                        if let Some(ref property) = self.property {
                            match meta_reader.data(
                                MetaData::String(property.clone(), text),
                                state,
                                range
                            ) {
                                Err(err) => Err((range, err)),
                                Ok(state) => Ok((range, state)),
                            }
                        } else {
                            Ok((range, state.clone()))
                        }
                    }
                }
            }
        } else {
            Err((Range::new(offset, 0), ParseError::ExpectedText))
        }
    }
}

#[cfg(test)]
mod tests {
    use super::super::*;
    use range::Range;
    use std::rc::Rc;

    #[test]
    fn expected_text() {
        let text = "23";
        let chars: Vec<char> = text.chars().collect();
        let mut tokenizer = Tokenizer::new();
        let s = TokenizerState::new();
        let text = Text {
            allow_empty: true,
            property: None
        };
        let res = text.parse(&mut tokenizer, &s, &chars, 0);
        assert_eq!(res, Err((Range::new(0, 0), ParseError::ExpectedText)));
    }

    #[test]
    fn empty_string() {
        let text = "\"\"";
        let chars: Vec<char> = text.chars().collect();
        let mut tokenizer = Tokenizer::new();
        let s = TokenizerState::new();
        let text = Text {
            allow_empty: false,
            property: None
        };
        let res = text.parse(&mut tokenizer, &s, &chars, 0);
        assert_eq!(res, Err((Range::new(0, 2), ParseError::EmptyTextNotAllowed)));
    }

    #[test]
    fn successful() {
        let text = "foo \"hello\"";
        let chars: Vec<char> = text.chars().collect();
        let mut tokenizer = Tokenizer::new();
        let s = TokenizerState::new();
        let foo: Rc<String> = Rc::new("foo".into());
        let text = Text {
            allow_empty: true,
            property: Some(foo.clone())
        };
        let res = text.parse(&mut tokenizer, &s, &chars[4..], 4);
        assert_eq!(res, Ok((Range::new(4, 7), TokenizerState(1))));
        assert_eq!(tokenizer.tokens.len(), 1);
        assert_eq!(&tokenizer.tokens[0].0,
            &MetaData::String(foo.clone(), "hello".into()));
    }
}
{'fraction_non_alphanumeric': 0.13395522388059702, 'fraction_numerical': 0.0037313432835820895, 'mean_word_length': 1.3606340819022458, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5468315', 'n_tokens_mistral': 1492, 'n_tokens_neox': 1442, 'n_words': 349}