<commit_before>//! Parse hoon source into AST twig

use std::str;
use std::str::FromStr;
use nom::*;
use nom::IResult::*;
use nom::Err::*;

use twig::{Twig, Rune, Odor};

#[inline]
pub fn is_lowercase(chr: u8) -> bool {
    chr >= 'a' as u8 && chr <= 'z' as u8
}

/// Match at least two spaces or one newline.
pub fn long_space(input: &[u8]) -> IResult<&[u8], &[u8]> {
    let mut spaces = 0;
    let mut idx = 0;
    for item in input.iter() {
        if *item == '\t' as u8 {
            // Physical tabs are banned.
            // XXX: Should have own ErrorKind tag.
            return Error(Position(ErrorKind::MultiSpace, input));
        }

        if *item == '\n' as u8 {
            spaces += 2;
        } else if *item == ' ' as u8 {
            spaces += 1;
        }

        if *item != ' ' as u8 && *item != '\r' as u8 && *item != '\n' as u8 {
            break;
        }

        idx += 1;
    }

    if spaces < 2 {
        Error(Position(ErrorKind::MultiSpace, input))
    } else {
        Done(&input[idx..], &input[0..idx])
    }
}

pub fn ident(input:&[u8]) -> IResult<&[u8], &[u8]> {
    for (idx, item) in input.iter().enumerate() {
        if idx == 0 {
            // TODO: Should we only accept lowercase chars?
            if !is_alphabetic(*item) {
                return Error(Position(ErrorKind::Alpha, input))
            }
        } else {
            if !is_alphabetic(*item) && !is_digit(*item) && *item != '-' as u8 {
                return Done(&input[idx..], &input[0..idx])
            }
        }
    }
    Error(Position(ErrorKind::Alpha, input))
}

named!(comment<&[u8]>,
    chain!(
        tag!("::") ~
        x: take_until_and_consume!("\n"),
        || { x }
    )
);

// A valid gap is any sequence of long spaces and comments.
named!(gap< Vec<&[u8]> >,
    many1!(
        alt!(
            long_space
          | comment
        )
    )
);

// TODO: Handle separator dots
// TODO: Handle other odors than ud.
named!(ud<Twig>,
  map_res!(
    map_res!(
      map_res!(
        digit,
        str::from_utf8
      ),
      FromStr::from_str
    ),
    |x| Ok::<Twig, ()>(Twig::Cell(
            box Twig::Rune(Rune::dtzy),
            box Twig::Atom(Odor::ud, x)))
  )
);

// TODO: Don't have two different-named identifier parsers that differ just on
// having the from_utf8...
/// Parse an identifier name.
named!(id<&str>,
    map_res!(
        ident,
        str::from_utf8
    )
);

/// Terminator for an arbitrary-length tall rune.
named!(tall_terminator<()>,
    chain!(
        gap ~
        tag!("=="),
      || ()
    )
);


macro_rules! tall_rune_args {
    ($first: tt) => {
        chain!(
            x: $first ~
            gap,
         || { x }
         )
    };
    ($first: tt, $($rest: tt),+) => {
        chain!(
            x: $first ~
            gap ~
            xs: tall_rune_args!($($rest),*),
         || { Twig::Cell(box x, box xs) }
         )
    };
}

macro_rules! wide_rune_args {
    ($first: tt) => {
        chain!(
            x: $first ~
            tag!(")"),
         || { x }
         )
    };
    ($first: tt, $($rest: tt),+) => {
        chain!(
            x: $first ~
            tag!(" ") ~
            xs: wide_rune_args!($($rest),*),
         || { Twig::Cell(box x, box xs) }
         )
    };
}

/// A standard rune that may have either a wide or a tall form.
macro_rules! rune {
    ($name:ident, $($parser: tt),+) => {
        chain!(
            tag!(Rune::$name.glyph()) ~
            args: alt!(
                chain!(
                    tag!("(") ~
                    args: wide_rune_args($($parser),*),
                    || { args }
                ) |
                chain!(
                    gap ~
                    args: tall_rune_args($($parser),*),
                    || { args }
                )
            ),
            || { Twig::Cell(box Twig::Rune(Rune::$name), box args) }
        )
    }
}

/// Parse a Hoon expression into an AST.
named!(pub ream<Twig>,
    alt!(
        rune!(brhp, ream) |
        ud
        /*
      | Brhp
      | Dtls
      | Dtts
      | Ktts
      | Tsgr
      | Tsls
      | Wtcl
      */<commit_msg>Apparently you need an extra magic parameter for parser macros<commit_after>//! Parse hoon source into AST twig

use std::str;
use std::str::FromStr;
use nom::*;
use nom::IResult::*;
use nom::Err::*;

use twig::{Twig, Rune, Odor};

#[inline]
pub fn is_lowercase(chr: u8) -> bool {
    chr >= 'a' as u8 && chr <= 'z' as u8
}

/// Match at least two spaces or one newline.
pub fn long_space(input: &[u8]) -> IResult<&[u8], &[u8]> {
    let mut spaces = 0;
    let mut idx = 0;
    for item in input.iter() {
        if *item == '\t' as u8 {
            // Physical tabs are banned.
            // XXX: Should have own ErrorKind tag.
            return Error(Position(ErrorKind::MultiSpace, input));
        }

        if *item == '\n' as u8 {
            spaces += 2;
        } else if *item == ' ' as u8 {
            spaces += 1;
        }

        if *item != ' ' as u8 && *item != '\r' as u8 && *item != '\n' as u8 {
            break;
        }

        idx += 1;
    }

    if spaces < 2 {
        Error(Position(ErrorKind::MultiSpace, input))
    } else {
        Done(&input[idx..], &input[0..idx])
    }
}

pub fn ident(input:&[u8]) -> IResult<&[u8], &[u8]> {
    for (idx, item) in input.iter().enumerate() {
        if idx == 0 {
            // TODO: Should we only accept lowercase chars?
            if !is_alphabetic(*item) {
                return Error(Position(ErrorKind::Alpha, input))
            }
        } else {
            if !is_alphabetic(*item) && !is_digit(*item) && *item != '-' as u8 {
                return Done(&input[idx..], &input[0..idx])
            }
        }
    }
    Error(Position(ErrorKind::Alpha, input))
}

named!(comment<&[u8]>,
    chain!(
        tag!("::") ~
        x: take_until_and_consume!("\n"),
        || { x }
    )
);

// A valid gap is any sequence of long spaces and comments.
named!(gap< Vec<&[u8]> >,
    many1!(
        alt!(
            long_space
          | comment
        )
    )
);

// TODO: Handle separator dots
// TODO: Handle other odors than ud.
named!(ud<Twig>,
  map_res!(
    map_res!(
      map_res!(
        digit,
        str::from_utf8
      ),
      FromStr::from_str
    ),
    |x| Ok::<Twig, ()>(Twig::Cell(
            box Twig::Rune(Rune::dtzy),
            box Twig::Atom(Odor::ud, x)))
  )
);

// TODO: Don't have two different-named identifier parsers that differ just on
// having the from_utf8...
/// Parse an identifier name.
named!(id<&str>,
    map_res!(
        ident,
        str::from_utf8
    )
);

/// Terminator for an arbitrary-length tall rune.
named!(tall_terminator<()>,
    chain!(
        gap ~
        tag!("=="),
      || ()
    )
);


macro_rules! tall_rune_args {
    ($i: expr, $first: tt) => {
        chain!($i,
            x: $first ~
            gap,
         || { x }
         )
    };
    ($i: expr, $first: tt, $($rest: tt),+) => {
        chain!($i,
            x: $first ~
            gap ~
            xs: tall_rune_args!($($rest),*),
         || { Twig::Cell(box x, box xs) }
         )
    };
}

macro_rules! wide_rune_args {
    ($i: expr, $first: tt) => {
        chain!($i,
            x: $first ~
            tag!(")"),
         || { x }
         )
    };
    ($i: expr, $first: tt, $($rest: tt),+) => {
        chain!($i,
            x: $first ~
            tag!(" ") ~
            xs: wide_rune_args!($($rest),*),
         || { Twig::Cell(box x, box xs) }
         )
    };
}

/// A standard rune that may have either a wide or a tall form.
macro_rules! rune {
    ($i: expr, $name:ident, $($parser: tt),+) => {
        chain!($i,
            tag!(Rune::$name.glyph()) ~
            args: alt!(
                chain!(
                    tag!("(") ~
                    args: wide_rune_args!($($parser),*),
                    || { args }
                ) |
                chain!(
                    gap ~
                    args: tall_rune_args!($($parser),*),
                    || { args }
                )
            ),
            || { Twig::Cell(box Twig::Rune(Rune::$name), box args) }
        )
    }
}

/// Parse a Hoon expression into an AST.
named!(pub ream<Twig>,
    alt!(
        rune!(brhp, ream) |
        ud
        /*
      | Brhp
      | Dtls
      | Dtts
      | Ktts
      | Tsgr
      | Tsls
      | Wtcl
      */
{'fraction_non_alphanumeric': 0.17457868502254925, 'fraction_numerical': 0.007358177070970804, 'mean_word_length': 1.4727112676056338, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '286141', 'n_tokens_mistral': 2912, 'n_tokens_neox': 2823, 'n_words': 689}