<commit_before>#!/usr/bin/env python2.5
# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

"""Refactoring framework.

Used as a main program, this can refactor any number of files and/or
recursively descend down directories.  Imported as a module, this
provides infrastructure to write your own refactoring tool.
"""

__author__ = "<NAME> <<EMAIL>>"


# Python imports
import os
import sys
import difflib
import optparse
import logging
from collections import defaultdict
from itertools import chain

# Local imports
from .pgen2 import driver
from .pgen2 import tokenize

from . import pytree
from . import patcomp
from . import fixes
from . import pygram

def main(fixer_dir, args=None):
    """Main program.

    Args:
        fixer_dir: directory where fixer modules are located.
        args: optional; a list of command line arguments. If omitted,
              sys.argv[1:] is used.

    Returns a suggested exit status (0, 1, 2).
    """
    # Set up option parser
    parser = optparse.OptionParser(usage="refactor.py [options] file|dir ...")
    parser.add_option("-d", "--doctests_only", action="store_true",
                      help="Fix up doctests only")
    parser.add_option("-f", "--fix", action="append", default=[],
                      help="Each FIX specifies a transformation; default all")
    parser.add_option("-l", "--list-fixes", action="store_true",
                      help="List available transformations (fixes/fix_*.py)")
    parser.add_option("-p", "--print-function", action="store_true",
                      help="Modify the grammar so that print() is a function")
    parser.add_option("-v", "--verbose", action="store_true",
                      help="More verbose logging")
    parser.add_option("-w", "--write", action="store_true",
                      help="Write back modified files")

    # Parse command line arguments
    options, args = parser.parse_args(args)
    if options.list_fixes:
        print "Available transformations for the -f/--fix option:"
        for fixname in get_all_fix_names(fixer_dir):
            print fixname
        if not args:
            return 0
    if not args:
        print >>sys.stderr, "At least one file or directory argument required."
        print >>sys.stderr, "Use --help to show usage."
        return 2

    # Set up logging handler
    if sys.version_info < (2, 4):
        hdlr = logging.StreamHandler()
        fmt = logging.Formatter('%(name)s: %(message)s')
        hdlr.setFormatter(fmt)
        logging.root.addHandler(hdlr)
    else:
        logging.basicConfig(format='%(name)s: %(message)s', level=logging.INFO)

    # Initialize the refactoring tool
    rt = RefactoringTool(fixer_dir, options)

    # Refactor all files and directories passed as arguments
    if not rt.errors:
        rt.refactor_args(args)
        rt.summarize()

    # Return error status (0 if rt.errors is zero)
    return int(bool(rt.errors))


def get_all_fix_names(fixer_dir):
    """Return a sorted list of all available fix names."""
    fix_names = []
    names = os.listdir(fixer_dir)
    names.sort()
    for name in names:
        if name.startswith("fix_") and name.endswith(".py"):
            fix_names.append(name[4:-3])
    fix_names.sort()
    return fix_names

def get_head_types(pat):
    """ Accepts a pytree Pattern Node and returns a set
        of the pattern types which will match first. """

    if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)):
        # NodePatters must either have no type and no content
        #   or a type and content -- so they don't get any farther
        # Always return leafs
        return set([pat.type])

    if isinstance(pat, pytree.NegatedPattern):
        if pat.content:
            return get_head_types(pat.content)
        return set([None]) # Negated Patterns don't have a type

    if isinstance(pat, pytree.WildcardPattern):
        # Recurse on each node in content
        r = set()
        for p in pat.content:
            for x in p:
                r.update(get_head_types(x))
        return r

    raise Exception("Oh no! I don't understand pattern %s" %(pat))

def get_headnode_dict(fixer_list):
    """ Accepts a list of fixers and returns a dictionary
        of head node type --> fixer list.  """
    head_nodes = defaultdict(list)
    for fixer in fixer_list:
        if not fixer.pattern:
            head_nodes[None].append(fixer)
            continue
        for t in get_head_types(fixer.pattern):
            head_nodes[t].append(fixer)
    return head_nodes


class RefactoringTool(object):

    def __init__(self, fixer_dir, options):
        """Initializer.

        Args:
            fixer_dir: directory in which to find fixer modules.
            options: an optparse.Values instance.
        """
        self.fixer_dir = fixer_dir
        self.options = options
        self.errors = []
        self.logger = logging.getLogger("RefactoringTool")
        self.fixer_log = []
        if self.options.print_function:
            del pygram.python_grammar.keywords["print"]
        self.driver = driver.Driver(pygram.python_grammar,
                                    convert=pytree.convert,
                                    logger=self.logger)
        self.pre_order, self.post_order = self.get_fixers()

        self.pre_order = get_headnode_dict(self.pre_order)
        self.post_order = get_headnode_dict(self.post_order)

        self.files = []  # List of files that were or should be modified

    def get_fixers(self):
        """Inspects the options to load the requested patterns and handlers.

        Returns:
          (pre_order, post_order), where pre_order is the list of fixers that
          want a pre-order AST traversal, and post_order is the list that want
          post-order traversal.
        """
        fixer_pkg = ".".join(self.fixer_dir.split(os.path.sep))
        pre_order_fixers = []
        post_order_fixers = []<commit_msg>On windows, lib2to3 should accept / and \ in the fixer_dir argument Needed to let the tests pass<commit_after>#!/usr/bin/env python2.5
# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

"""Refactoring framework.

Used as a main program, this can refactor any number of files and/or
recursively descend down directories.  Imported as a module, this
provides infrastructure to write your own refactoring tool.
"""

__author__ = "<NAME> <<EMAIL>>"


# Python imports
import os
import sys
import difflib
import optparse
import logging
from collections import defaultdict
from itertools import chain

# Local imports
from .pgen2 import driver
from .pgen2 import tokenize

from . import pytree
from . import patcomp
from . import fixes
from . import pygram

def main(fixer_dir, args=None):
    """Main program.

    Args:
        fixer_dir: directory where fixer modules are located.
        args: optional; a list of command line arguments. If omitted,
              sys.argv[1:] is used.

    Returns a suggested exit status (0, 1, 2).
    """
    # Set up option parser
    parser = optparse.OptionParser(usage="refactor.py [options] file|dir ...")
    parser.add_option("-d", "--doctests_only", action="store_true",
                      help="Fix up doctests only")
    parser.add_option("-f", "--fix", action="append", default=[],
                      help="Each FIX specifies a transformation; default all")
    parser.add_option("-l", "--list-fixes", action="store_true",
                      help="List available transformations (fixes/fix_*.py)")
    parser.add_option("-p", "--print-function", action="store_true",
                      help="Modify the grammar so that print() is a function")
    parser.add_option("-v", "--verbose", action="store_true",
                      help="More verbose logging")
    parser.add_option("-w", "--write", action="store_true",
                      help="Write back modified files")

    # Parse command line arguments
    options, args = parser.parse_args(args)
    if options.list_fixes:
        print "Available transformations for the -f/--fix option:"
        for fixname in get_all_fix_names(fixer_dir):
            print fixname
        if not args:
            return 0
    if not args:
        print >>sys.stderr, "At least one file or directory argument required."
        print >>sys.stderr, "Use --help to show usage."
        return 2

    # Set up logging handler
    if sys.version_info < (2, 4):
        hdlr = logging.StreamHandler()
        fmt = logging.Formatter('%(name)s: %(message)s')
        hdlr.setFormatter(fmt)
        logging.root.addHandler(hdlr)
    else:
        logging.basicConfig(format='%(name)s: %(message)s', level=logging.INFO)

    # Initialize the refactoring tool
    rt = RefactoringTool(fixer_dir, options)

    # Refactor all files and directories passed as arguments
    if not rt.errors:
        rt.refactor_args(args)
        rt.summarize()

    # Return error status (0 if rt.errors is zero)
    return int(bool(rt.errors))


def get_all_fix_names(fixer_dir):
    """Return a sorted list of all available fix names."""
    fix_names = []
    names = os.listdir(fixer_dir)
    names.sort()
    for name in names:
        if name.startswith("fix_") and name.endswith(".py"):
            fix_names.append(name[4:-3])
    fix_names.sort()
    return fix_names

def get_head_types(pat):
    """ Accepts a pytree Pattern Node and returns a set
        of the pattern types which will match first. """

    if isinstance(pat, (pytree.NodePattern, pytree.LeafPattern)):
        # NodePatters must either have no type and no content
        #   or a type and content -- so they don't get any farther
        # Always return leafs
        return set([pat.type])

    if isinstance(pat, pytree.NegatedPattern):
        if pat.content:
            return get_head_types(pat.content)
        return set([None]) # Negated Patterns don't have a type

    if isinstance(pat, pytree.WildcardPattern):
        # Recurse on each node in content
        r = set()
        for p in pat.content:
            for x in p:
                r.update(get_head_types(x))
        return r

    raise Exception("Oh no! I don't understand pattern %s" %(pat))

def get_headnode_dict(fixer_list):
    """ Accepts a list of fixers and returns a dictionary
        of head node type --> fixer list.  """
    head_nodes = defaultdict(list)
    for fixer in fixer_list:
        if not fixer.pattern:
            head_nodes[None].append(fixer)
            continue
        for t in get_head_types(fixer.pattern):
            head_nodes[t].append(fixer)
    return head_nodes


class RefactoringTool(object):

    def __init__(self, fixer_dir, options):
        """Initializer.

        Args:
            fixer_dir: directory in which to find fixer modules.
            options: an optparse.Values instance.
        """
        self.fixer_dir = fixer_dir
        self.options = options
        self.errors = []
        self.logger = logging.getLogger("RefactoringTool")
        self.fixer_log = []
        if self.options.print_function:
            del pygram.python_grammar.keywords["print"]
        self.driver = driver.Driver(pygram.python_grammar,
                                    convert=pytree.convert,
                                    logger=self.logger)
        self.pre_order, self.post_order = self.get_fixers()

        self.pre_order = get_headnode_dict(self.pre_order)
        self.post_order = get_headnode_dict(self.post_order)

        self.files = []  # List of files that were or should be modified

    def get_fixers(self):
        """Inspects the options to load the requested patterns and handlers.

        Returns:
          (pre_order, post_order), where pre_order is the list of fixers that
          want a pre-order AST traversal, and post_order is the list that want
          post-order traversal.
        """
        fixer_pkg = self.fixer_dir.replace(os.path.sep, ".")
        if os.path.altsep:
            fixer_pkg = self.fixer_dir.replace(os.path.altsep, ".")
        pre_order_fixers = []
        post_order_fixers = []
{'fraction_non_alphanumeric': 0.09874208665625257, 'fraction_numerical': 0.0032886623365945902, 'mean_word_length': 2.4371291325233115, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '885023', 'n_tokens_mistral': 3576, 'n_tokens_neox': 3466, 'n_words': 1085}