<commit_before># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt

"""Parser.py: a main for invoking code in coverage/parser.py"""

from __future__ import division

import collections
import glob
import optparse
import os
import re
import sys
import textwrap

import disgen

from coverage.parser import PythonParser
from coverage.python import get_python_source

opcode_counts = collections.Counter()


class ParserMain(object):
    """A main for code parsing experiments."""

    def main(self, args):
        """A main function for trying the code from the command line."""

        parser = optparse.OptionParser()
        parser.add_option(
            "-d", action="store_true", dest="dis",
            help="Disassemble"
            )
        parser.add_option(
            "-H", action="store_true", dest="histogram",
            help="Count occurrences of opcodes"
            )
        parser.add_option(
            "-R", action="store_true", dest="recursive",
            help="Recurse to find source files"
            )
        parser.add_option(
            "-s", action="store_true", dest="source",
            help="Show analyzed source"
            )
        parser.add_option(
            "-t", action="store_true", dest="tokens",
            help="Show tokens"
            )

        options, args = parser.parse_args()
        if options.recursive:
            if args:
                root = args[0]
            else:
                root = "."
            for root, _, _ in os.walk(root):
                for f in glob.glob(root + "/*.py"):
                    self.one_file(options, f)
        elif not args:
            parser.print_help()
        else:
            self.one_file(options, args[0])

        if options.histogram:
            total = sum(opcode_counts.values())
            print("{0} total opcodes".format(total))
            for opcode, number in opcode_counts.most_common():
                print("{0:20s} {1:6d}  {2:.1%}".format(opcode, number, number/total))

    def one_file(self, options, filename):
        """Process just one file."""
        # `filename` can have a line number suffix. In that case, extract those
        # lines, dedent them, and use that.  This is for trying test cases
        # embedded in the test files.
        match = re.search(r"^(.*):(\d+)-(\d+)$", filename)
        if match:
            filename, start, end = match.groups()
            start, end = int(start), int(end)
        else:
            start = end = None

        try:
            text = get_python_source(filename)
            if start is not None:
                lines = text.splitlines(True)
                text = textwrap.dedent("".join(lines[start-1:end]).replace("\\\\", "\\"))
            pyparser = PythonParser(text, filename=filename, exclude=r"no\s*cover")
        except Exception as err:
            print("%s" % (err,))
            return

        if options.dis:
            print("Main code:")
            self.disassemble(pyparser.byte_parser, histogram=options.histogram)

        arcs = pyparser.arcs()

        if options.source or options.tokens:
            pyparser.show_tokens = options.tokens
            pyparser.parse_source()

            if options.source:
                arc_chars = self.arc_ascii_art(arcs)
                if arc_chars:
                    arc_width = max(len(a) for a in arc_chars.values())

                exit_counts = pyparser.exit_counts()

                for lineno, ltext in enumerate(pyparser.lines, start=1):<commit_msg>Fix lab/parser.py, which wasn't showing enough arcs<commit_after># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt

"""Parser.py: a main for invoking code in coverage/parser.py"""

from __future__ import division

import collections
import glob
import optparse
import os
import re
import sys
import textwrap

import disgen

from coverage.parser import PythonParser
from coverage.python import get_python_source

opcode_counts = collections.Counter()


class ParserMain(object):
    """A main for code parsing experiments."""

    def main(self, args):
        """A main function for trying the code from the command line."""

        parser = optparse.OptionParser()
        parser.add_option(
            "-d", action="store_true", dest="dis",
            help="Disassemble"
            )
        parser.add_option(
            "-H", action="store_true", dest="histogram",
            help="Count occurrences of opcodes"
            )
        parser.add_option(
            "-R", action="store_true", dest="recursive",
            help="Recurse to find source files"
            )
        parser.add_option(
            "-s", action="store_true", dest="source",
            help="Show analyzed source"
            )
        parser.add_option(
            "-t", action="store_true", dest="tokens",
            help="Show tokens"
            )

        options, args = parser.parse_args()
        if options.recursive:
            if args:
                root = args[0]
            else:
                root = "."
            for root, _, _ in os.walk(root):
                for f in glob.glob(root + "/*.py"):
                    self.one_file(options, f)
        elif not args:
            parser.print_help()
        else:
            self.one_file(options, args[0])

        if options.histogram:
            total = sum(opcode_counts.values())
            print("{0} total opcodes".format(total))
            for opcode, number in opcode_counts.most_common():
                print("{0:20s} {1:6d}  {2:.1%}".format(opcode, number, number/total))

    def one_file(self, options, filename):
        """Process just one file."""
        # `filename` can have a line number suffix. In that case, extract those
        # lines, dedent them, and use that.  This is for trying test cases
        # embedded in the test files.
        match = re.search(r"^(.*):(\d+)-(\d+)$", filename)
        if match:
            filename, start, end = match.groups()
            start, end = int(start), int(end)
        else:
            start = end = None

        try:
            text = get_python_source(filename)
            if start is not None:
                lines = text.splitlines(True)
                text = textwrap.dedent("".join(lines[start-1:end]).replace("\\\\", "\\"))
            pyparser = PythonParser(text, filename=filename, exclude=r"no\s*cover")
            pyparser.parse_source()
        except Exception as err:
            print("%s" % (err,))
            return

        if options.dis:
            print("Main code:")
            self.disassemble(pyparser.byte_parser, histogram=options.histogram)

        arcs = pyparser.arcs()

        if options.source or options.tokens:
            pyparser.show_tokens = options.tokens
            pyparser.parse_source()

            if options.source:
                arc_chars = self.arc_ascii_art(arcs)
                if arc_chars:
                    arc_width = max(len(a) for a in arc_chars.values())

                exit_counts = pyparser.exit_counts()

                for lineno, ltext in enumerate(pyparser.lines, start=1):
{'fraction_non_alphanumeric': 0.11561561561561562, 'fraction_numerical': 0.003822003822003822, 'mean_word_length': 1.911005164878824, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4460060', 'n_tokens_mistral': 2078, 'n_tokens_neox': 2012, 'n_words': 536}