<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import pytest  # noqa
from parglare import Parser, GLRParser, Grammar

parsers = pytest.mark.parametrize("parser_class", [Parser, GLRParser])


@parsers
def test_layout_whitespaces(parser_class):
    grammar = r"""
    S: K EOF;
    K: A | B;
    A: 'a' A | 'a';
    B: 'b' B | 'b';
    LAYOUT: WS | EMPTY;
    WS: /\s+/;
    """
    g = Grammar.from_string(grammar)

    in_str = """aaa a    aaaa
    aa    aa a aaa

    aaa
    """

    parser = parser_class(g, debug=True)
    result = parser.parse(in_str)
    assert result


@parsers
def test_layout_simple_comments(parser_class):
    grammar = r"""
    S: K EOF;
    K: A | B;
    A: 'a' A | 'a';
    B: 'b' B | 'b';
    LAYOUT: LayoutItem | LAYOUT LayoutItem;
    LayoutItem: WS | Comment | EMPTY;
    WS: /\s+/;
    Comment: /\/\/.*/;
    """
    g = Grammar.from_string(grammar)

    in_str = """aaa a    aaaa
    aa    aa a aaa // This is a comment

    aaa
    """

    parser = parser_class(g)
    parser.parse(in_str)


@parsers
def test_layout_nested_comments(parser_class):
    grammar = """
    S: K EOF;
    K: 'a' B | 'a' C;
    B: 'b' | B 'b';
    C: 'c' | C 'c';

    LAYOUT: LayoutItem | LAYOUT LayoutItem;
    LayoutItem: WS | Comment | EMPTY;
    WS: /\s+/;
    Comment: '/*' CorNCs '*/' | /\/\/.*/;
    CorNCs: CorNC | CorNCs CorNC | EMPTY;
    CorNC: Comment | NotComment | WS;
    NotComment: /((\*[^\/])|[^\s*\/]|\/[^\*])+/;
    """
    g = Grammar.from_string(grammar)

    in_str = """<commit_msg>Extended layout test with comment starting at the beginning of the input.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import pytest  # noqa
from parglare import Parser, GLRParser, Grammar

parsers = pytest.mark.parametrize("parser_class", [Parser, GLRParser])


@parsers
def test_layout_whitespaces(parser_class):
    grammar = r"""
    S: K EOF;
    K: A | B;
    A: 'a' A | 'a';
    B: 'b' B | 'b';
    LAYOUT: WS | EMPTY;
    WS: /\s+/;
    """
    g = Grammar.from_string(grammar)

    in_str = """aaa a    aaaa
    aa    aa a aaa

    aaa
    """

    parser = parser_class(g, debug=True)
    result = parser.parse(in_str)
    assert result


@parsers
def test_layout_simple_comments(parser_class):
    grammar = r"""
    S: K EOF;
    K: A | B;
    A: 'a' A | 'a';
    B: 'b' B | 'b';
    LAYOUT: LayoutItem | LAYOUT LayoutItem;
    LayoutItem: WS | Comment | EMPTY;
    WS: /\s+/;
    Comment: /\/\/.*/;
    """
    g = Grammar.from_string(grammar)

    in_str = """aaa a    aaaa
    aa    aa a aaa // This is a comment

    aaa
    """

    parser = parser_class(g)
    parser.parse(in_str)


@parsers
def test_layout_nested_comments(parser_class):
    grammar = """
    S: K EOF;
    K: 'a' B | 'a' C;
    B: 'b' | B 'b';
    C: 'c' | C 'c';

    LAYOUT: LayoutItem | LAYOUT LayoutItem;
    LayoutItem: WS | Comment | EMPTY;
    WS: /\s+/;
    Comment: '/*' CorNCs '*/' | /\/\/.*/;
    CorNCs: CorNC | CorNCs CorNC | EMPTY;
    CorNC: Comment | NotComment | WS;
    NotComment: /((\*[^\/])|[^\s*\/]|\/[^\*])+/;
    """
    g = Grammar.from_string(grammar)

    in_str = """//Line comment at beginning
{'fraction_non_alphanumeric': 0.17047231779793556, 'fraction_numerical': 0.0006255864873318737, 'mean_word_length': 2.31055900621118, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '485311', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1256, 'n_words': 327}