<commit_before>use std::str::Chars;

pub enum Protocol {
    Version(uint),
    Binary(Vec<u8>),
    Text(~str)
}

pub enum TextProtocol {
    Key(~str),
    Value(Vec<u8>)
}

// The TextProtocol's parser. All we need to spit out is a key->value combination.
// Because we allow streaming within the protocol, we'll only return each
// key->value pair once we have fully parsed both.
//
// There could be an option to receive the raw data as we get it for the value,
// but not the key.
//
// ```rust
// Parser::new()
// ```
pub struct Parser<'a> {
    input: &'a str,
    iter: Chars<'a>
}

impl<'a> Parser<'a> {
    pub fn new(input: &'a str) -> Parser<'a> {
        Parser {
            input: input,
            iter: input.chars()
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn new_parser() {
        let mut parser = Parser::new("hello world");
        assert_eq!(parser.iter.next().unwrap(), 'h');
    }
}<commit_msg>Got the text parser working<commit_after>use std::str::{Chars, from_char};

pub enum Protocol {
    Version(uint),
    Binary(Vec<u8>),
    Text(~str)
}

pub enum TextProtocol {
    Key(~str),
    Value(Vec<u8>)
}

#[deriving(Eq,Show)]
pub enum TextAst {
    Empty,
    KeyVal(~str, ~str)
}

// The TextProtocol's parser. All we need to spit out is a key->value combination.
// Because we allow streaming within the protocol, we'll only return each
// key->value pair once we have fully parsed both.
//
// There could be an option to receive the raw data as we get it for the value,
// but not the key.
//
// ```rust
// Parser::new("hello world")
// ```
pub struct Parser<'a> {
    input: &'a str,
    iter: Chars<'a>
}

impl<'a> Parser<'a> {
    pub fn new(input: &'a str) -> Parser<'a> {
        Parser {
            input: input,
            iter: input.chars()
        }
    }

    pub fn parse(&mut self) -> TextAst {

        if self.input.len() == 0 {
            return Empty;
        }

        let mut key = "".to_owned();
        let mut value = "".to_owned();
        let mut c = self.iter.next().unwrap();

        if c.is_alphanumeric() {
            key = key.append(from_char(c));
            while c != '[' {
                c = self.iter.next().unwrap();

                if c != '[' {
                    key = key.append(from_char(c));
                }
            }

            // Start the value
            if c == '[' {
                while c != ']' {
                    c = self.iter.next().unwrap();

                    if c != ']' {
                        value = value.append(from_char(c));
                    }
                }
            }
        }

        KeyVal(key, value)
    }

}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn new_parser() {
        let mut parser = Parser::new("hello world");
        assert_eq!(parser.iter.next().unwrap(), 'h');
    }

    #[test]
    fn parser_parse_empty_string() {
        let mut parser = Parser::new("");
        assert_eq!(parser.parse(), Empty);
    }

    #[test]
    fn parse_simple_kv() {
        let mut parser = Parser::new("Version[1]");
        assert_eq!(parser.parse(), KeyVal("Version".to_owned(), "1".to_owned()));
    }
}
{'fraction_non_alphanumeric': 0.1598373475132937, 'fraction_numerical': 0.002189552705661558, 'mean_word_length': 1.8528099910793934, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6502540', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1039, 'n_words': 289}