<commit_before>use nom::{ErrorKind, IResult, Needed, alphanumeric};

use ssl::syntax;

/// Parse a module separator and a module name.
named!(module_sep_n_name,
  do_parse!(
    char!('.') >>
    name: alphanumeric >>
    (name)
  )
);

/// Parse a module path.
///
/// foo
/// foo.bar
/// foo.bar.zoo
named!(module_path<&[u8], Vec<&[u8]>>,
  do_parse!(
    // recognize at least one module name
    base: alphanumeric >>
    // recognize the rest of the path, if any
    rest: many0!(module_sep_n_name) >>

    ({
      let mut rest = rest.clone(); // FIXME: meh?
      rest.insert(0, base);
      rest
    })
  )
);

/// Parse a module list.
///
///     ( item0, item1, item2, …)
named!(module_list<&[u8], Vec<Vec<&[u8]>>>,
  ws!(
    delimited!(char!('('),
               separated_list!(char!(','), module_path),
               char!(')')
    )
  )
);

// /// Parse an export list.
// named!(export_list<&[u8], ExportList>,
//   ws!(do_parse!(
//     tag!("export") >>
//     modules: module_list >>
//     (ExportList {
//       exported_list: modules.into_iter().collect()
//     })
//   ))
// );

#[test]
fn test_module_sep_n_name() {
  assert_eq!(module_sep_n_name(&b".foo"[..]), IResult::Done(&b""[..], &b"foo"[..]));
  assert_eq!(module_sep_n_name(&b".foo.bar"[..]), IResult::Done(&b".bar"[..], &b"foo"[..]));
  assert_eq!(module_sep_n_name(&b"foo"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(module_sep_n_name(&b" .foo"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(module_sep_n_name(&b"."[..]), IResult::Incomplete(Needed::Unknown));
}

#[test]
fn test_module_path_simple() {
  assert_eq!(module_path(&b"foo"[..]), IResult::Done(&b""[..], vec![&b"foo"[..]]));
  assert_eq!(module_path(&b"foo "[..]), IResult::Done(&b" "[..], vec![&b"foo"[..]]));
  assert_eq!(module_path(&b"foo."[..]), IResult::Incomplete(Needed::Unknown));
  assert_eq!(module_path(&b" foo"[..]), IResult::Error(ErrorKind::AlphaNumeric));
}

#[test]
fn test_module_path_several() {
  assert_eq!(module_path(&b"foo.bar.zoo"[..]), IResult::Done(&b""[..], vec![&b"foo"[..], &b"bar"[..], &b"zoo"[..]]));
}

#[test]
fn test_module_list() {
  assert_eq!(module_list(&b"(foo,bar,zoo.woo)"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(module_list(&b"( foo,bar,zoo.woo  )"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(module_list(&b"( foo, bar ,   zoo.woo  )"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(module_list(&b"(,bar,zoo.woo)"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(module_list(&b"("[..]), IResult::Incomplete(Needed::Unknown));
  assert_eq!(module_list(&b"  ("[..]), IResult::Incomplete(Needed::Unknown));
}<commit_msg>export_list parser. #24<commit_after>use nom::{ErrorKind, IResult, Needed, alphanumeric};

use ssl::syntax;

/// Parse a module separator and a module name.
named!(module_sep_n_name,
  do_parse!(
    char!('.') >>
    name: alphanumeric >>
    (name)
  )
);

/// Parse a module path.
///
/// foo
/// foo.bar
/// foo.bar.zoo
named!(module_path<&[u8], Vec<&[u8]>>,
  do_parse!(
    // recognize at least one module name
    base: alphanumeric >>
    // recognize the rest of the path, if any
    rest: many0!(module_sep_n_name) >>

    ({
      let mut rest = rest.clone(); // FIXME: meh?
      rest.insert(0, base);
      rest
    })
  )
);

/// Parse a module list.
///
///     ( item0, item1, item2, …)
named!(module_list<&[u8], Vec<Vec<&[u8]>>>,
  ws!(
    delimited!(char!('('),
               separated_list!(char!(','), module_path),
               char!(')')
    )
  )
);

/// Parse an export list.
named!(export_list<&[u8], Vec<Vec<&[u8]>>>,
  ws!(do_parse!(
    tag!("export") >>
    modules: module_list >>
    (modules)
  ))
);

#[test]
fn test_module_sep_n_name() {
  assert_eq!(module_sep_n_name(&b".foo"[..]), IResult::Done(&b""[..], &b"foo"[..]));
  assert_eq!(module_sep_n_name(&b".foo.bar"[..]), IResult::Done(&b".bar"[..], &b"foo"[..]));
  assert_eq!(module_sep_n_name(&b"foo"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(module_sep_n_name(&b" .foo"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(module_sep_n_name(&b"."[..]), IResult::Incomplete(Needed::Unknown));
}

#[test]
fn test_module_path_simple() {
  assert_eq!(module_path(&b"foo"[..]), IResult::Done(&b""[..], vec![&b"foo"[..]]));
  assert_eq!(module_path(&b"foo "[..]), IResult::Done(&b" "[..], vec![&b"foo"[..]]));
  assert_eq!(module_path(&b"foo."[..]), IResult::Incomplete(Needed::Unknown));
  assert_eq!(module_path(&b" foo"[..]), IResult::Error(ErrorKind::AlphaNumeric));
}

#[test]
fn test_module_path_several() {
  assert_eq!(module_path(&b"foo.bar.zoo"[..]), IResult::Done(&b""[..], vec![&b"foo"[..], &b"bar"[..], &b"zoo"[..]]));
}

#[test]
fn test_module_list() {
  assert_eq!(module_list(&b"(foo,bar,zoo.woo)"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(module_list(&b" ( foo,bar,zoo.woo  ) "[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(module_list(&b"( foo, bar ,   zoo.woo  )"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(module_list(&b"(,bar,zoo.woo)"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(module_list(&b"("[..]), IResult::Incomplete(Needed::Unknown));
  assert_eq!(module_list(&b"  ("[..]), IResult::Incomplete(Needed::Unknown));
}

#[test]
fn test_export_list() {
  assert_eq!(export_list(&b"export (foo,bar,zoo.woo)"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(export_list(&b"   export ( foo,bar,zoo.woo  )  "[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(export_list(&b"export ( foo, bar ,   zoo.woo  )"[..]), IResult::Done(&b""[..], vec![vec![&b"foo"[..]], vec![&b"bar"[..]], vec![&b"zoo"[..], &b"woo"[..]]]));
  assert_eq!(export_list(&b"export (,bar,zoo.woo)"[..]), IResult::Error(ErrorKind::Char));
  assert_eq!(export_list(&b"export ("[..]), IResult::Incomplete(Needed::Unknown));
  assert_eq!(export_list(&b"export   ("[..]), IResult::Incomplete(Needed::Unknown));
}
{'fraction_non_alphanumeric': 0.30728600279633367, 'fraction_numerical': 0.003573093055771322, 'mean_word_length': 4.401006711409396, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7274703', 'n_tokens_mistral': 2700, 'n_tokens_neox': 2622, 'n_words': 344}