<commit_before>use std::str;
use std::str::FromStr;
use nom::*;
use nom::IResult::*;
use nock::Noun;

// XXX: Is there a nicer way to ignore the results from eg. newline than
// the superfluous chain!?
named!(gap<()>, alt!(
        chain!(newline, || { () })
      | chain!(tag!(" ") ~ tag!(" ") ~ multispace, || { () })
    )
);

// TODO: Handle separator dots
named!(atom<Noun>,
  map_res!(
    map_res!(
      digit,
      str::from_utf8
    ),
    FromStr::from_str
  )
);

named!(wtcl<Noun>,
   chain!(
       tag!("?:") ~
       x: alt!(
           chain!(
               gap ~
               p: atom ~
               gap ~
               q: atom ~
               gap ~
               r: atom,
               || { n![6, p, q, r] })
        | delimited!(
            tag!("("),
            chain!(
               p: atom ~
               space ~
               q: atom ~
               space ~
               r: atom,
               || { n![6, p, q, r] }),
            tag!(")"))
        ),
    || { x }
    )
);

named!(hoon<Noun>,
    alt!(
        atom
      | wtcl
    )
);


#[cfg(test)]
mod test {
    use nom::IResult;
    use nock::Noun;
    use super::{gap, atom, wtcl};

    #[test]
    fn test_parse_hoon() {
        assert_eq!(atom(&b"1234"[..]), IResult::Done(&b""[..], Noun::Atom(1234)));

        assert_eq!(gap(&b"  "[..]), IResult::Done(&b""[..], ()));
        assert_eq!(gap(&b"    "[..]), IResult::Done(&b""[..], ()));
        assert_eq!(gap(&b"\n"[..]), IResult::Done(&b""[..], ()));
        assert!(gap(&b" "[..]) != IResult::Done(&b""[..], ()));

        assert_eq!(wtcl(&b"?:(1 2 3)"[..]), IResult::Done(&b""[..], n![6, 1, 2, 3]));
        assert_eq!(wtcl(&b"?:  1\n  2\n3"[..]), IResult::Done(&b""[..], n![6, 1, 2, 3]));
    }
}<commit_msg>Hoon parser does gaps<commit_after>use std::str;
use std::str::FromStr;
use nom::*;
use nom::IResult::*;
use nom::Err::*;
use nock::Noun;

/// Match at least two spaces or one newline.
pub fn gap(input: &[u8]) -> IResult<&[u8], &[u8]> {
    let mut spaces = 0;
    let mut idx = 0;
    for item in input.iter() {
        if *item == '\t' as u8 {
            // Physical tabs are banned.
            // XXX: Should have own ErrorKind tag.
            return Error(Position(ErrorKind::MultiSpace, input));
        }

        if *item == '\n' as u8 {
            spaces += 2;
        } else if is_space(*item) {
            spaces += 1;
        }

        if !is_space(*item) && *item != '\r' as u8 && *item != '\n' as u8 {
            break;
        }

        idx += 1;
    }

    if spaces < 2 {
        return Error(Position(ErrorKind::MultiSpace, input));
    } else {
        return Done(&input[idx..], &input[0..idx]);
    }
}

// TODO: Handle separator dots
named!(atom<Noun>,
  map_res!(
    map_res!(
      digit,
      str::from_utf8
    ),
    FromStr::from_str
  )
);

named!(pqr<(Noun, Noun, Noun)>,
    alt!(
        chain!(
            p: preceded!(gap, atom) ~
            q: preceded!(gap, atom) ~
            r: preceded!(gap, atom),
            || { (p, q, r) })
        | delimited!(
            tag!("("),
            chain!(
                p: atom ~
                multispace ~
                q: atom ~
                multispace ~
                r: atom,
                || { (p, q, r) }),
            tag!(")"))
    )
);

named!(wtcl<Noun>,
   chain!(
       tag!("?:")
     ~ a: pqr
     , || { let (p, q, r) = a.clone(); n![6, p, q, r] }
    )
);

named!(hoon<Noun>,
    alt!(
        atom
      | wtcl
    )
);


#[cfg(test)]
mod test {
    use nom::IResult;
    use nock::Noun;
    use super::{gap, atom, pqr, wtcl};

    #[test]
    fn test_parse_hoon() {
        assert_eq!(atom(&b"1234"[..]),
                   IResult::Done(&b""[..], Noun::Atom(1234)));

        assert!(gap(&b"  "[..]).is_done());
        assert!(gap(&b"    "[..]).is_done());
        assert!(gap(&b"\n"[..]).is_done());
        assert!(gap(&b"\n  "[..]).is_done());
        assert!(gap(&b"  \n  "[..]).is_done());
        assert!(!gap(&b" "[..]).is_done());

        assert_eq!(pqr(&b"(1 2 3)"[..]),
                   IResult::Done(&b""[..], (Noun::Atom(1), Noun::Atom(2), Noun::Atom(3))));
        assert_eq!(pqr(&b"\n1\n2\n3"[..]),
                   IResult::Done(&b""[..], (Noun::Atom(1), Noun::Atom(2), Noun::Atom(3))));
        assert_eq!(pqr(&b"  1  2  3"[..]),
                   IResult::Done(&b""[..], (Noun::Atom(1), Noun::Atom(2), Noun::Atom(3))));

        assert_eq!(wtcl(&b"?:(1 2 3)"[..]),
                   IResult::Done(&b""[..], n![6, 1, 2, 3]));
        assert_eq!(wtcl(&b"?:(1   2\n3)"[..]),
                   IResult::Done(&b""[..], n![6, 1, 2, 3]));
        assert_eq!(wtcl(&b"?:  1\n  2\n3"[..]),
                   IResult::Done(&b""[..], n![6, 1, 2, 3]));
    }
}
{'fraction_non_alphanumeric': 0.2409233997901364, 'fraction_numerical': 0.018467995802728225, 'mean_word_length': 1.6700280112044819, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7336611', 'n_tokens_mistral': 1842, 'n_tokens_neox': 1740, 'n_words': 351}