<commit_before>import os, logging
from io import open
from plyplus import Grammar, ParseError

class Parser(object):

    def __init__(self, try_ply=True, cwd=None, expand_includes=True):
        self.try_ply = try_ply
        self.cwd = cwd
        self.expand_includes = True

        gf = os.path.join(os.path.dirname(__file__), "mapfile.g")
        grammar_text = open(gf).read()

        self.g = Grammar(grammar_text, engine='pearley')
        if self.try_ply:
            self.ply_g = Grammar(grammar_text, engine='ply')

    def strip_quotes(self, s):
        return s.strip("'").strip('"')

    def load_includes(self, text):

        lines = text.split('\n')
        includes = {}

        for idx, l in enumerate(lines):
            if l.strip().lower().startswith('include'):
                if '#' in l:
                    l = l[:l.index('#')]

                parts = [p for p in l.split()]

                assert (len(parts) == 2)
                assert (parts[0].lower() == 'include')
                fn = os.path.join(self.cwd, self.strip_quotes(parts[1]))
                include_text = self.open_file(fn)
                # recursively load any further includes
                includes[idx] = self.load_includes(include_text)
    
        for idx, txt in includes.items():
            rm = lines.pop(idx) # remove the original include
            lines.insert(idx, txt)

        return '\n'.join(lines)

    def open_file(self, fn):
        return open(fn, "r", encoding="utf-8").read() # specify unicode for Python 2.7

    def parse_file(self, fn):

        if not self.cwd:
            self.cwd = os.path.dirname(fn)

        text = self.open_file(fn)
        return self.parse(text)

    def parse(self, text):

        if self.expand_includes:
            text = self.load_includes(text)

        text += '\n'

        if self.try_ply:
            try:
                return self.ply_g.parse(text)
            except ParseError as ex:
                logging.exception(ex)
                pass

        return self.g.parse(text)<commit_msg>Remove logging statement<commit_after>import os, logging
from io import open
from plyplus import Grammar, ParseError

class Parser(object):

    def __init__(self, try_ply=True, cwd=None, expand_includes=True):
        self.try_ply = try_ply
        self.cwd = cwd
        self.expand_includes = True

        gf = os.path.join(os.path.dirname(__file__), "mapfile.g")
        grammar_text = open(gf).read()

        self.g = Grammar(grammar_text, engine='pearley')
        if self.try_ply:
            self.ply_g = Grammar(grammar_text, engine='ply')

    def strip_quotes(self, s):
        return s.strip("'").strip('"')

    def load_includes(self, text):

        lines = text.split('\n')
        includes = {}

        for idx, l in enumerate(lines):
            if l.strip().lower().startswith('include'):
                if '#' in l:
                    l = l[:l.index('#')]

                parts = [p for p in l.split()]

                assert (len(parts) == 2)
                assert (parts[0].lower() == 'include')
                fn = os.path.join(self.cwd, self.strip_quotes(parts[1]))
                include_text = self.open_file(fn)
                # recursively load any further includes
                includes[idx] = self.load_includes(include_text)
    
        for idx, txt in includes.items():
            rm = lines.pop(idx) # remove the original include
            lines.insert(idx, txt)

        return '\n'.join(lines)

    def open_file(self, fn):
        return open(fn, "r", encoding="utf-8").read() # specify unicode for Python 2.7

    def parse_file(self, fn):

        self.cwd = os.path.dirname(fn)

        text = self.open_file(fn)
        return self.parse(text)

    def parse(self, text):

        if self.expand_includes:
            text = self.load_includes(text)

        text += '\n'

        if self.try_ply:
            try:
                return self.ply_g.parse(text)
            except ParseError as ex:
                #logging.exception(ex)
                pass

        return self.g.parse(text)
{'fraction_non_alphanumeric': 0.12314792324508136, 'fraction_numerical': 0.002914743745445713, 'mean_word_length': 1.841959972394755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1958019', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1219, 'n_words': 290}