<commit_before>#!/usr/bin/env python3
import ast
import argparse
import sys
import os
import re


IGNORE_DIRECTORIES = ["__pycache__", ".git"]
SEARCH_DEFAULT, SEARCH_LINENO, SEARCH_REGEX = range(3)


def echo(*args):
    """A poor man's print"""
    output = " ".join([str(i) for i in args]) + "\n"
    sys.stdout.write(output)


class SourceCode(object):
    def __init__(self, filename, offset=1):
        """
        offset is the difference between the number of the first line of the file and 0.  It's probably 1.
        """
        self.offset = offset
        self.filename = filename
        self.lines = []
        with open(filename, "r") as f:
            self.lines = f.readlines()

    def __repr__(self):
        return "SourceCode('{}', {})".format(self.filename, self.offset)

    def line(self, lineno):
        return self.lines[lineno - self.offset]

    @property
    def numlines(self):
        return len(self.lines)

    def format_line(self, lineno):
        return "{}:  {}".format(str(lineno).rjust(len(str(self.numlines))), self.line(lineno))


def make_matcher(search_type, look_for):
    def _regex(node):
        try: lineno = node.lineno
        except: return None
        for attr in node._fields:
            try:
                if look_for.match(getattr(node, attr)):
                    return lineno
            except:
                pass

    def _default(node):
        try: lineno = node.lineno
        except: return None
        for attr in node._fields:
            if type(look_for) is str and getattr(node, attr) == look_for:
                return lineno       

    def _lineno(node):
        try: lineno = node.lineno
        except: return None
        return lineno if lineno == look_for else None

    # Convert look_for to the type that the matcher functions need.
    # Doing it here rather than inside the matcher function because it only needs to happen once
    look_for = {SEARCH_DEFAULT: lambda x: x,
                SEARCH_LINENO: int,
                SEARCH_REGEX: re.compile}[search_type](look_for)
    return {SEARCH_DEFAULT: _default,
            SEARCH_LINENO: _lineno,
            SEARCH_REGEX: _regex}[search_type]


def walk(node, matcher, history=None):
    history = [] if history is None else history[:]
    matches = []

    def append_to_history(item):
        try: history.append(item.lineno)
        except AttributeError: pass

    def pop_from_history():
        try: history.pop()
        except IndexError: pass

    children = list(ast.iter_child_nodes(node))
    match = matcher(node)

    if match:
        matches.append(match)
        matches = list(set(matches + history))

    if len(children) == 0:
        return set(matches)

    append_to_history(node)
    for i in children:
        append_to_history(i)
        matches.extend(walk(i, matcher, history))
        pop_from_history()
    pop_from_history()

    return set(matches)


def find_top_level(tree, depth=1):
    matches = [i.lineno for i in ast.iter_child_nodes(tree) if isinstance(i, (ast.FunctionDef, ast.ClassDef))]
    return matches


def find_context(tree, look_for, matcher):
    """
    Return a list of lines indicating branches that lead to the object we're looking for
    """
    # TODO: Add something to parse look_for, for example FooClass.bar should only match bar
    #       function calls that are attributes of a FooClass instance
    matches = walk(tree, matcher)
    return sorted(list(matches))


def parse_source(filename):
    with open(filename, "r") as f:
        tree = ast.parse(f.read(), filename=filename)
    return tree


def main(look_for, files, search_type=SEARCH_DEFAULT, recursive=False,
         ignore=IGNORE_DIRECTORIES, verbose=False, definitions=False):
    """
    search_type can be default|lineno|regex
    """
    if recursive:
        files = os.walk(files)
    else:
        files = [('', [], [files])]  # match format generated by os.walk

    skipped_files = {}

    for (directory, _, filenames) in files:
        abspath = os.path.abspath(directory)
        ignore = [os.path.abspath(i) for i in ignore]
        if abspath in ignore:
            continue
        for fn in filenames:
            if os.path.abspath(fn) in ignore:
                continue
            source_file = os.path.join(directory, fn)
            try:
                source = SourceCode(source_file)
                tree = parse_source(source_file)
                if definitions:
                    context = find_top_level(tree)
                else:
                    context = find_context(tree, look_for, make_matcher(search_type, look_for))
            except KeyboardInterrupt as e:
                sys.exit(1)
            except Exception as e:
                skipped_files[source_file] = e
                continue
            if len(context) == 0:
                continue
            if recursive:
                echo("\n{}".format(source_file))
            echo("\n" + "".join([source.format_line(i) for i in context]))

        if verbose and skipped_files:
            echo("Skipped these files due to errors:\n{}".format(
                "\n".join(["{}: {}".format(key, skipped_files[key]) for key in skipped_files])))


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("look_for", help="object to look for in the file")
    parser.add_argument("path", help="file (or directory if -r option) to look in")
    parser.add_argument("-r", "--recursive", action="store_true", help="recursively search directory")
    parser.add_argument("-n", "--search-line", dest="search_type", action="store_const", const=SEARCH_LINENO,
                        help="search by line number")
    parser.add_argument("-e", "--search-regex", dest="search_type", action="store_const", const=SEARCH_REGEX,
                        help="search by regexp")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="display information about errors and skipped files")
    parser.add_argument("-i", "--ignore",
                        help=("comma-separated list of files and directories "
                              "to ignore, default is {}".format(IGNORE_DIRECTORIES)))
    parser.add_argument("-d", "--definitions", action="store_true",
                        help=("just look for class and function defintions.  The look_for argument "
                              "should be an integer indicating the maxmimum depth of search"))
    args = parser.parse_args()

    if args.ignore:
        ignore = args.ignore.split(",") + IGNORE_DIRECTORIES
    else:
        ignore = IGNORE_DIRECTORIES

    main(args.look_for,
         args.path,
         search_type=args.search_type or SEARCH_DEFAULT,
         recursive=args.recursive,
         ignore=ignore,
         verbose=args.verbose,
         definitions=args.definitions)<commit_msg>changed command line argument to accept multiple files to search, moved verbose message out of for loop to after for loop finishes<commit_after>#!/usr/bin/env python3
import ast
import argparse
import sys
import os
import re


IGNORE_DIRECTORIES = ["__pycache__", ".git"]
SEARCH_DEFAULT, SEARCH_LINENO, SEARCH_REGEX = range(3)


def echo(*args):
    """A poor man's print"""
    output = " ".join([str(i) for i in args]) + "\n"
    sys.stdout.write(output)


class SourceCode(object):
    def __init__(self, filename, offset=1):
        """
        offset is the difference between the number of the first line of the file and 0.  It's probably 1.
        """
        self.offset = offset
        self.filename = filename
        self.lines = []
        with open(filename, "r") as f:
            self.lines = f.readlines()

    def __repr__(self):
        return "SourceCode('{}', {})".format(self.filename, self.offset)

    def line(self, lineno):
        return self.lines[lineno - self.offset]

    @property
    def numlines(self):
        return len(self.lines)

    def format_line(self, lineno):
        return "{}:  {}".format(str(lineno).rjust(len(str(self.numlines))), self.line(lineno))


def make_matcher(search_type, look_for):
    def _regex(node):
        try: lineno = node.lineno
        except: return None
        for attr in node._fields:
            try:
                if look_for.match(getattr(node, attr)):
                    return lineno
            except:
                pass

    def _default(node):
        try: lineno = node.lineno
        except: return None
        for attr in node._fields:
            if type(look_for) is str and getattr(node, attr) == look_for:
                return lineno       

    def _lineno(node):
        try: lineno = node.lineno
        except: return None
        return lineno if lineno == look_for else None

    # Convert look_for to the type that the matcher functions need.
    # Doing it here rather than inside the matcher function because it only needs to happen once
    look_for = {SEARCH_DEFAULT: lambda x: x,
                SEARCH_LINENO: int,
                SEARCH_REGEX: re.compile}[search_type](look_for)
    return {SEARCH_DEFAULT: _default,
            SEARCH_LINENO: _lineno,
            SEARCH_REGEX: _regex}[search_type]


def walk(node, matcher, history=None):
    history = [] if history is None else history[:]
    matches = []

    def append_to_history(item):
        try: history.append(item.lineno)
        except AttributeError: pass

    def pop_from_history():
        try: history.pop()
        except IndexError: pass

    children = list(ast.iter_child_nodes(node))
    match = matcher(node)

    if match:
        matches.append(match)
        matches = list(set(matches + history))

    if len(children) == 0:
        return set(matches)

    append_to_history(node)
    for i in children:
        append_to_history(i)
        matches.extend(walk(i, matcher, history))
        pop_from_history()
    pop_from_history()

    return set(matches)


def find_top_level(tree, depth=1):
    matches = [i.lineno for i in ast.iter_child_nodes(tree) if isinstance(i, (ast.FunctionDef, ast.ClassDef))]
    return matches


def find_context(tree, look_for, matcher):
    """
    Return a list of lines indicating branches that lead to the object we're looking for
    """
    # TODO: Add something to parse look_for, for example FooClass.bar should only match bar
    #       function calls that are attributes of a FooClass instance
    matches = walk(tree, matcher)
    return sorted(list(matches))


def parse_source(filename):
    with open(filename, "r") as f:
        tree = ast.parse(f.read(), filename=filename)
    return tree


def main(look_for, files, search_type=SEARCH_DEFAULT, recursive=False,
         ignore=IGNORE_DIRECTORIES, verbose=False, definitions=False):

    files = os.walk(files) if recursive else [(os.curdir, [], [f]) for f in files]  # format returned by os.walk
    skipped_files = {}

    for (directory, _, filenames) in files:
        abspath = os.path.abspath(directory)
        ignore = [os.path.abspath(i) for i in ignore]
        if abspath in ignore:
            continue
        for fn in filenames:
            if os.path.abspath(fn) in ignore:
                continue
            source_file = os.path.join(directory, fn)
            try:
                source = SourceCode(source_file)
                tree = parse_source(source_file)
                if definitions:
                    context = find_top_level(tree)
                else:
                    context = find_context(tree, look_for, make_matcher(search_type, look_for))
            except KeyboardInterrupt as e:
                sys.exit(1)
            except Exception as e:
                skipped_files[source_file] = e
                continue
            if len(context) == 0:
                continue
            if recursive or len(files) > 1:
                echo("\n{}".format(source_file))
            echo("\n" + "".join([source.format_line(i) for i in context]))

    if verbose and skipped_files:
        echo("Skipped these files due to errors:\n{}".format(
            "\n".join(["{}: {}".format(key, skipped_files[key]) for key in skipped_files])))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Find lines in a Python source file and the context they're in."
    )
    parser.add_argument("look_for", help="object to look for in the file")
    parser.add_argument("paths", help="files (or directories if -r option) to look in", nargs="+")
    parser.add_argument("-r", "--recursive", action="store_true", help="recursively search directory")
    parser.add_argument("-n", "--search-line", dest="search_type", action="store_const", const=SEARCH_LINENO,
                        help="search by line number")
    parser.add_argument("-e", "--search-regex", dest="search_type", action="store_const", const=SEARCH_REGEX,
                        help="search by regexp")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="display information about errors and skipped files")
    parser.add_argument("-i", "--ignore",
                        help=("comma-separated list of files and directories "
                              "to ignore, default is {}".format(IGNORE_DIRECTORIES)))
    parser.add_argument("-d", "--definitions", action="store_true",
                        help=("just look for class and function defintions.  The look_for argument "
                              "should be an integer indicating the maxmimum depth of search"))
    args = parser.parse_args()

    if args.ignore:
        ignore = args.ignore.split(",") + IGNORE_DIRECTORIES
    else:
        ignore = IGNORE_DIRECTORIES

    main(args.look_for,
         args.paths,
         search_type=args.search_type or SEARCH_DEFAULT,
         recursive=args.recursive,
         ignore=ignore,
         verbose=args.verbose,
         definitions=args.definitions)
{'fraction_non_alphanumeric': 0.10638908829863604, 'fraction_numerical': 0.0013639626704953338, 'mean_word_length': 2.3383656841600766, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7072789', 'n_tokens_mistral': 4117, 'n_tokens_neox': 3946, 'n_words': 1125}