<commit_before>#[macro_use] extern crate scan_rules;
#[macro_use] mod util;

use scan_rules::ScanError as SE;
use scan_rules::ScanErrorKind as SEK;

#[test]
fn test_repeating() {
    assert_match!(
        scan!("[]"; ("[", [ let ns: i32 ]?, "]") => ns),
        Ok(ref ns) if *ns == vec![]
    );

    assert_match!(
        scan!("[]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![]
    );

    assert_match!(
        scan!("[]"; ("[", [ let ns: i32 ]+, "]") => ns),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("[0]"; ("[", [ let ns: i32 ]?, "]") => ns),
        Ok(ref ns) if *ns == vec![0]
    );

    assert_match!(
        scan!("[0]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![0]
    );

    assert_match!(
        scan!("[0]"; ("[", [ let ns: i32 ]+, "]") => ns),
        Ok(ref ns) if *ns == vec![0]
    );

    assert_match!(
        scan!("[0 1]"; ("[", [ let ns: i32 ]?, "]") => ns),
        Err(SE { ref at, kind: SEK::LiteralMismatch }) if at.offset() == 3
    );

    assert_match!(
        scan!("[0 1]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("[0 1]"; ("[", [ let ns: i32 ]+, "]") => ns),
        Ok(ref ns) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("[0 1 2 3]"; ("[", [ let _: i32 ]*, "]") => ()),
        Ok(())
    );

    assert_match!(
        scan!("[0 1 2 3]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![0, 1, 2, 3]
    );

    assert_match!(
        scan!("[0 1 2 3]"; ("[", [ let xs: i32, let ys: i32 ]*, "]") => (xs, ys)),
        Ok((ref xs, ref ys)) if &**xs == [0, 2] && &**ys == [1, 3]
    );

    assert_match!(
        scan!("[[0 1] [2 3]]"; ("[", ["[", [let nss: i32]*, "]"]*, "]") => nss),
        Ok(ref ns) if *ns == vec![vec![0, 1], vec![2, 3]]
    );

    assert_match!(
        scan!("[0 [1 2] 3 [4 5]]"; ("[", [let xs: i32, "[", [let yss: i32]*, "]"]*, "]") => (xs, yss)),
        Ok((ref xs, ref yss)) if *xs == vec![0, 3] && *yss == vec![vec![1, 2], vec![4, 5]]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{2}, ..tail) => (ns, tail)),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{2}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0 1 2"; ([ let ns: i32 ]{2}, ..tail) => (ns, tail)),
        Ok((ref ns, " 2")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0 1 2"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1, 2]
    );

    assert_match!(
        scan!("0 1 2 3"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Ok((ref ns, " 3")) if *ns == vec![0, 1, 2]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{,3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0]
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{,3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]<commit_msg>Added missing tests for repeating patterns.<commit_after>#[macro_use] extern crate scan_rules;
#[macro_use] mod util;

use scan_rules::ScanError as SE;
use scan_rules::ScanErrorKind as SEK;

#[test]
fn test_repeating() {
    assert_match!(
        scan!("[]"; ("[", [ let ns: i32 ]?, "]") => ns),
        Ok(ref ns) if *ns == vec![]
    );

    assert_match!(
        scan!("[]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![]
    );

    assert_match!(
        scan!("[]"; ("[", [ let ns: i32 ]+, "]") => ns),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("[0]"; ("[", [ let ns: i32 ]?, "]") => ns),
        Ok(ref ns) if *ns == vec![0]
    );

    assert_match!(
        scan!("[0]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![0]
    );

    assert_match!(
        scan!("[0]"; ("[", [ let ns: i32 ]+, "]") => ns),
        Ok(ref ns) if *ns == vec![0]
    );

    assert_match!(
        scan!("[0 1]"; ("[", [ let ns: i32 ]?, "]") => ns),
        Err(SE { ref at, kind: SEK::LiteralMismatch }) if at.offset() == 3
    );

    assert_match!(
        scan!("[0 1]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("[0 1]"; ("[", [ let ns: i32 ]+, "]") => ns),
        Ok(ref ns) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("[0 1 2 3]"; ("[", [ let _: i32 ]*, "]") => ()),
        Ok(())
    );

    assert_match!(
        scan!("[0 1 2 3]"; ("[", [ let ns: i32 ]*, "]") => ns),
        Ok(ref ns) if *ns == vec![0, 1, 2, 3]
    );

    assert_match!(
        scan!("[0 1 2 3]"; ("[", [ let xs: i32, let ys: i32 ]*, "]") => (xs, ys)),
        Ok((ref xs, ref ys)) if &**xs == [0, 2] && &**ys == [1, 3]
    );

    assert_match!(
        scan!("[[0 1] [2 3]]"; ("[", ["[", [let nss: i32]*, "]"]*, "]") => nss),
        Ok(ref ns) if *ns == vec![vec![0, 1], vec![2, 3]]
    );

    assert_match!(
        scan!("[0 [1 2] 3 [4 5]]"; ("[", [let xs: i32, "[", [let yss: i32]*, "]"]*, "]") => (xs, yss)),
        Ok((ref xs, ref yss)) if *xs == vec![0, 3] && *yss == vec![vec![1, 2], vec![4, 5]]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{2}, ..tail) => (ns, tail)),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{2}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0 1 2"; ([ let ns: i32 ]{2}, ..tail) => (ns, tail)),
        Ok((ref ns, " 2")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{2,}, ..tail) => (ns, tail)),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{2,}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0 1 2"; ([ let ns: i32 ]{2,}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1, 2]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Err(SE { ref at, kind: SEK::Missing }) if at.offset() == 1
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]
    );

    assert_match!(
        scan!("0 1 2"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1, 2]
    );

    assert_match!(
        scan!("0 1 2 3"; ([ let ns: i32 ]{2, 3}, ..tail) => (ns, tail)),
        Ok((ref ns, " 3")) if *ns == vec![0, 1, 2]
    );

    assert_match!(
        scan!("0"; ([ let ns: i32 ]{,3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0]
    );

    assert_match!(
        scan!("0 1"; ([ let ns: i32 ]{,3}, ..tail) => (ns, tail)),
        Ok((ref ns, "")) if *ns == vec![0, 1]
{'fraction_non_alphanumeric': 0.2864064602960969, 'fraction_numerical': 0.04562584118438762, 'mean_word_length': 2.0885286783042396, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4396505', 'n_tokens_mistral': 3205, 'n_tokens_neox': 2968, 'n_words': 811}