<commit_before># -*- coding: utf-8 -*-

"""
IDL Ref:
    https://google.github.io/flatbuffers/flatbuffers_grammar.html
"""



import collections
import operator
import os
import sys
import types
from ply import lex, yacc
from .lexer import *  # noqa
from .exc import FbsParserError, FbsGrammerError
from thriftpy._compat import urlopen, urlparse
from fbs.fbs import FBSType, FBSPayload
from thriftpy.thrift import gen_init

def p_error(p):
    if p is None:
        raise FbsGrammerError('Grammer error at EOF')
    raise FbsGrammerError('Grammer error %r at line %d' %
                             (p.value, p.lineno))


def p_start(p):
    '''start : include_seq body_seq'''


def p_include_seq(p):
    '''include_seq : include_seq include_one
                    | include_one
                    |'''

def p_body_seq(p):
    '''body_seq : body_seq body
                    | body
                    |'''

def p_include_one(p):
    '''include_one : INCLUDE LITERAL ';' '''
    fbs = fbs_stack[-1]
    if fbs.__fbs_file__ is None:
        raise FbsParserError('Unexcepted include statement while loading'
                                'from file like object.')
    replace_include_dirs = [os.path.dirname(fbs.__fbs_file__)] \
        + include_dirs_
    for include_dir in replace_include_dirs:
        path = os.path.join(include_dir, p[2])
        if os.path.exists(path):
            child = parse(path)
            setattr(fbs, child.__name__, child)
            _add_fbs_meta('includes', child)
            return
    raise FbsParserError(('Couldn\'t include fbs %s in any '
                             'directories provided') % p[2])

def p_body(p):
    '''body : namespace
            | typedef
            | enum
            | root
            | file_extension
            | file_identifier
            | attribute
            | object
            |'''

def p_namespace(p):
    '''namespace : NAMESPACE IDENTIFIER ';' '''
    setattr(fbs_stack[-1], 'namespace', p[2])

def p_root(p):
    '''root : ROOT_TYPE IDENTIFIER ';' '''
    setattr(fbs_stack[-1], 'root', p[2])

def p_file_extension(p):
    '''file_extension : FILE_EXTENSION LITERAL ';' '''
    setattr(fbs_stack[-1], 'file_extension', p[2])

def p_file_identifier(p):
    '''file_identifier : FILE_IDENTIFIER LITERAL ';' '''
    setattr(fbs_stack[-1], 'file_identifier', p[2])

def p_metadata(p):
    '''metadata : '(' metadata_seq ')'
                |'''
    if len(p) > 2:
        p[0] = p[2]
    else:
        p[0] = []

def p_metadata_seq(p):
    '''metadata_seq : metadata_item ',' metadata_seq
                    | metadata_item
                    |'''
    _parse_seq(p)

def p_metadata_item(p):
    '''metadata_item : IDENTIFIER
                     | IDENTIFIER ':' scalar'''
    if len(p) == 4:
        p[0] = [p[1], p[3]]
    else:
        p[0] = [p[1]]

def p_attribute(p):
    '''attribute : ATTRIBUTE LITERAL ';' '''

def p_object(p):
    '''object : '{' field_seq '}' '''

def p_typedef(p):
    '''typedef : table
       typedef : struct '''
    if (p[1]):
        setattr(fbs_stack[-1], p[1][0], p[1][1])

def p_enum(p):  # noqa
    '''enum : ENUM IDENTIFIER metadata '{' enum_seq '}'
       enum : ENUM IDENTIFIER ':' simple_base_type metadata '{' enum_seq '}'
       enum : union'''
    if len(p) == 9:
        val = _make_enum(p[2], p[7])
    elif len(p) == 7:
        val = _make_enum(p[2], p[5])
    else:
        val = p[1]
    if (len(p) > 2):
        setattr(fbs_stack[-1], p[2], val)
        _add_fbs_meta('enums', val)


def p_enum_seq(p):
    '''enum_seq : enum_item ',' enum_seq
                | enum_item enum_seq
                |'''
    _parse_seq(p)


def p_enum_item(p):
    '''enum_item : IDENTIFIER '=' INTCONSTANT
                 | IDENTIFIER
                 |'''
    if len(p) == 4:
        p[0] = [p[1], p[3]]
    elif len(p) == 2:
        p[0] = [p[1], None]


def p_struct(p):
    '''struct : STRUCT IDENTIFIER metadata '{' field_seq '}' '''
    val = _make_empty_struct(p[2])
    setattr(fbs_stack[-1], p[1], val)
    val = _fill_in_struct(val, p[5])
    _add_fbs_meta('structs', val)

def p_table(p):
    '''table : TABLE IDENTIFIER metadata '{' field_seq '}' '''
    val = _make_empty_struct(p[2])
    setattr(fbs_stack[-1], p[1], val)
    val = _fill_in_struct(val, p[5])
    _add_fbs_meta('tables', val)

def p_union(p):
    '''union : UNION IDENTIFIER metadata '{' enum_seq '}' '''
    val = _make_empty_struct(p[2])
    setattr(fbs_stack[-1], p[1], val)
    # Needs new code.
    #val = _fill_in_struct(val, p[5])<commit_msg>Treat unions same way as enums<commit_after># -*- coding: utf-8 -*-

"""
IDL Ref:
    https://google.github.io/flatbuffers/flatbuffers_grammar.html
"""



import collections
import operator
import os
import sys
import types
from ply import lex, yacc
from .lexer import *  # noqa
from .exc import FbsParserError, FbsGrammerError
from thriftpy._compat import urlopen, urlparse
from fbs.fbs import FBSType, FBSPayload
from thriftpy.thrift import gen_init

def p_error(p):
    if p is None:
        raise FbsGrammerError('Grammer error at EOF')
    raise FbsGrammerError('Grammer error %r at line %d' %
                             (p.value, p.lineno))


def p_start(p):
    '''start : include_seq body_seq'''


def p_include_seq(p):
    '''include_seq : include_seq include_one
                    | include_one
                    |'''

def p_body_seq(p):
    '''body_seq : body_seq body
                    | body
                    |'''

def p_include_one(p):
    '''include_one : INCLUDE LITERAL ';' '''
    fbs = fbs_stack[-1]
    if fbs.__fbs_file__ is None:
        raise FbsParserError('Unexcepted include statement while loading'
                                'from file like object.')
    replace_include_dirs = [os.path.dirname(fbs.__fbs_file__)] \
        + include_dirs_
    for include_dir in replace_include_dirs:
        path = os.path.join(include_dir, p[2])
        if os.path.exists(path):
            child = parse(path)
            setattr(fbs, child.__name__, child)
            _add_fbs_meta('includes', child)
            return
    raise FbsParserError(('Couldn\'t include fbs %s in any '
                             'directories provided') % p[2])

def p_body(p):
    '''body : namespace
            | typedef
            | enum
            | root
            | file_extension
            | file_identifier
            | attribute
            | object
            |'''

def p_namespace(p):
    '''namespace : NAMESPACE IDENTIFIER ';' '''
    setattr(fbs_stack[-1], 'namespace', p[2])

def p_root(p):
    '''root : ROOT_TYPE IDENTIFIER ';' '''
    setattr(fbs_stack[-1], 'root', p[2])

def p_file_extension(p):
    '''file_extension : FILE_EXTENSION LITERAL ';' '''
    setattr(fbs_stack[-1], 'file_extension', p[2])

def p_file_identifier(p):
    '''file_identifier : FILE_IDENTIFIER LITERAL ';' '''
    setattr(fbs_stack[-1], 'file_identifier', p[2])

def p_metadata(p):
    '''metadata : '(' metadata_seq ')'
                |'''
    if len(p) > 2:
        p[0] = p[2]
    else:
        p[0] = []

def p_metadata_seq(p):
    '''metadata_seq : metadata_item ',' metadata_seq
                    | metadata_item
                    |'''
    _parse_seq(p)

def p_metadata_item(p):
    '''metadata_item : IDENTIFIER
                     | IDENTIFIER ':' scalar'''
    if len(p) == 4:
        p[0] = [p[1], p[3]]
    else:
        p[0] = [p[1]]

def p_attribute(p):
    '''attribute : ATTRIBUTE LITERAL ';' '''

def p_object(p):
    '''object : '{' field_seq '}' '''

def p_typedef(p):
    '''typedef : table
       typedef : struct '''
    if (p[1]):
        setattr(fbs_stack[-1], p[1][0], p[1][1])

def p_enum(p):  # noqa
    '''enum : ENUM IDENTIFIER metadata '{' enum_seq '}'
       enum : ENUM IDENTIFIER ':' simple_base_type metadata '{' enum_seq '}'
       enum : union'''
    if len(p) == 9:
        val = _make_enum(p[2], p[7])
    elif len(p) == 7:
        val = _make_enum(p[2], p[5])
    else:
        val = p[1]
    if (len(p) > 2):
        setattr(fbs_stack[-1], p[2], val)
        _add_fbs_meta('enums', val)


def p_enum_seq(p):
    '''enum_seq : enum_item ',' enum_seq
                | enum_item enum_seq
                |'''
    _parse_seq(p)


def p_enum_item(p):
    '''enum_item : IDENTIFIER '=' INTCONSTANT
                 | IDENTIFIER
                 |'''
    if len(p) == 4:
        p[0] = [p[1], p[3]]
    elif len(p) == 2:
        p[0] = [p[1], None]


def p_struct(p):
    '''struct : STRUCT IDENTIFIER metadata '{' field_seq '}' '''
    val = _make_empty_struct(p[2])
    setattr(fbs_stack[-1], p[1], val)
    val = _fill_in_struct(val, p[5])
    _add_fbs_meta('structs', val)

def p_table(p):
    '''table : TABLE IDENTIFIER metadata '{' field_seq '}' '''
    val = _make_empty_struct(p[2])
    setattr(fbs_stack[-1], p[1], val)
    val = _fill_in_struct(val, p[5])
    _add_fbs_meta('tables', val)

def p_union(p):
    '''union : UNION IDENTIFIER metadata '{' enum_seq '}' '''
    val = _make_enum(p[2], p[5])
    setattr(fbs_stack[-1], p[1], val)
{'fraction_non_alphanumeric': 0.15129762562120375, 'fraction_numerical': 0.01258972943125345, 'mean_word_length': 2.281159420289855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3472711', 'n_tokens_mistral': 3309, 'n_tokens_neox': 3240, 'n_words': 689}