<commit_before>from __future__ import print_function, unicode_literals
import sys
import os
from collections import OrderedDict
from itertools import chain
from parglare.grammar import ProductionRHS, AUGSYMBOL, \
    ASSOC_LEFT, ASSOC_RIGHT, STOP, StringRecognizer, RegExRecognizer, \
    Grammar, EMPTY, NonTerminal
from parglare.exceptions import GrammarError, SRConflict, RRConflict
from parglare.closure import closure, LR_1
from parglare.termui import prints, s_header, h_print, a_print, s_emph
from parglare.tables.persist import load_table, save_table
if sys.version < '3':
    text = unicode  # NOQA
else:
    text = str


SHIFT = 0
REDUCE = 1
ACCEPT = 2

# Tables construction algorithms
SLR = 0
LALR = 1


def create_load_table(grammar, itemset_type=LR_1, start_production=1,
                      prefer_shifts=False, prefer_shifts_over_empty=True,
                      force_create=False):
    """
    Construct table by loading from file if present and newer than the grammar.
    If table file is older than the grammar or non-existent calculate the table
    and save to file.

    Arguments:
    see create_table

    force_create(bool): If set to True table will be created even if table file
        exists.

    """

    table_file_name = None
    if grammar.file_path:
        file_basename, _ = os.path.splitext(grammar.file_path)
        table_file_name = "{}.pgt".format(file_basename)

    create_table_file = True

    if not force_create:
        if grammar.file_path:
            file_basename, _ = os.path.splitext(grammar.file_path)
            table_file_name = "{}.pgt".format(file_basename)

            if os.path.exists(table_file_name):
                create_table_file = False
                table_mtime = os.path.getmtime(table_file_name)
                # Check if older than any of the grammar files
                for g_file_name in grammar.imported_files.keys():
                    if os.path.getmtime(g_file_name) > table_mtime:
                        create_table_file = True
                        break

    if create_table_file or force_create:
        table = create_table(grammar, itemset_type, start_production,
                             prefer_shifts, prefer_shifts_over_empty)
        if table_file_name:
            save_table(table_file_name, table)
    else:
        table = load_table(table_file_name, grammar)

    return table


def create_table(grammar, itemset_type=LR_1, start_production=1,
                 prefer_shifts=False, prefer_shifts_over_empty=True):
    """
    Arguments:
    grammar (Grammar):
    itemset_type(int) - SRL=0 LR_1=1. By default LR_1.
    start_production(int) - The production which defines start state.
        By default 1 - first production from the grammar.
    prefer_shifts(bool) - Conflict resolution strategy which favours SHIFT over
        REDUCE (gready). By default False.
    prefer_shifts_over_empty(bool) - Conflict resolution strategy which favours
        SHIFT over REDUCE of EMPTY. By default False. If prefer_shifts is
        `True` this param is ignored.
    """

    first_sets = first(grammar)

    # Check for states with GOTO links but without SHIFT links.
    # This is invalid as the GOTO link will never be traversed.<commit_msg>Do not fail during table saving in case of IO errors.<commit_after>from __future__ import print_function, unicode_literals
import sys
import os
from collections import OrderedDict
from itertools import chain
from parglare.grammar import ProductionRHS, AUGSYMBOL, \
    ASSOC_LEFT, ASSOC_RIGHT, STOP, StringRecognizer, RegExRecognizer, \
    Grammar, EMPTY, NonTerminal
from parglare.exceptions import GrammarError, SRConflict, RRConflict
from parglare.closure import closure, LR_1
from parglare.termui import prints, s_header, h_print, a_print, s_emph
from parglare.tables.persist import load_table, save_table
if sys.version < '3':
    text = unicode  # NOQA
else:
    text = str


SHIFT = 0
REDUCE = 1
ACCEPT = 2

# Tables construction algorithms
SLR = 0
LALR = 1


def create_load_table(grammar, itemset_type=LR_1, start_production=1,
                      prefer_shifts=False, prefer_shifts_over_empty=True,
                      force_create=False):
    """
    Construct table by loading from file if present and newer than the grammar.
    If table file is older than the grammar or non-existent calculate the table
    and save to file.

    Arguments:
    see create_table

    force_create(bool): If set to True table will be created even if table file
        exists.

    """

    table_file_name = None
    if grammar.file_path:
        file_basename, _ = os.path.splitext(grammar.file_path)
        table_file_name = "{}.pgt".format(file_basename)

    create_table_file = True

    if not force_create:
        if grammar.file_path:
            file_basename, _ = os.path.splitext(grammar.file_path)
            table_file_name = "{}.pgt".format(file_basename)

            if os.path.exists(table_file_name):
                create_table_file = False
                table_mtime = os.path.getmtime(table_file_name)
                # Check if older than any of the grammar files
                for g_file_name in grammar.imported_files.keys():
                    if os.path.getmtime(g_file_name) > table_mtime:
                        create_table_file = True
                        break

    if create_table_file or force_create:
        table = create_table(grammar, itemset_type, start_production,
                             prefer_shifts, prefer_shifts_over_empty)
        if table_file_name:
            try:
                save_table(table_file_name, table)
            except (IOError, PermissionError):
                pass
    else:
        table = load_table(table_file_name, grammar)

    return table


def create_table(grammar, itemset_type=LR_1, start_production=1,
                 prefer_shifts=False, prefer_shifts_over_empty=True):
    """
    Arguments:
    grammar (Grammar):
    itemset_type(int) - SRL=0 LR_1=1. By default LR_1.
    start_production(int) - The production which defines start state.
        By default 1 - first production from the grammar.
    prefer_shifts(bool) - Conflict resolution strategy which favours SHIFT over
        REDUCE (gready). By default False.
    prefer_shifts_over_empty(bool) - Conflict resolution strategy which favours
        SHIFT over REDUCE of EMPTY. By default False. If prefer_shifts is
        `True` this param is ignored.
    """

    first_sets = first(grammar)

    # Check for states with GOTO links but without SHIFT links.
    # This is invalid as the GOTO link will never be traversed.
{'fraction_non_alphanumeric': 0.06324348742659239, 'fraction_numerical': 0.004818551422978467, 'mean_word_length': 2.7044060234244283, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822753', 'n_tokens_mistral': 2060, 'n_tokens_neox': 1924, 'n_words': 578}