<commit_before>import ast, os
import yaml


__all__ = ["api_crawler", "differ"]


def differ(old_version, new_version):
    with open(old_version, "r") as f:
        old = f.read()
    with open(new_version, "r") as f:
        new = f.read()
    old_version = yaml.load(old)
    new_version = yaml.load(new)

    # Crawl using this
    files = set(old_version) & set(new_version)
    file_difference = set(old_version) - set(new_version)

    # Classes
    old_classes = {x: list(old_version[x]["classes"].keys()) for x in files}
    new_classes = {x: list(new_version[x]["classes"].keys()) for x in files}
    classes = {}
    class_difference = {}
    pruned_difference = {}
    for x in files:
        classes[x] = set(old_classes[x]) & set(new_classes[x])
        class_difference[x] = set(old_classes[x]) - set(new_classes[x])
    for x in class_difference.keys():
        if class_difference[x]:
            pruned_difference[x] = class_difference[x]

    return {"files": file_difference, "classes": pruned_difference}


class APICrawler(object):
    exclude = ("tests", "static", "sampledata", "mplexplorer")

    def __init__(self, directory):
        self.directory = directory

    def is_public(self, name):
        if name.startswith("_", 0, 1) and name != "__init__":
            return False
        else:
            return True

    def is_function(self, ast_node):
        if isinstance(ast_node, ast.FunctionDef) and ast_node.col_offset == 0:
            return True
        else:
            return False

    def is_class(self, ast_node):
        if isinstance(ast_node, ast.ClassDef):
            return True
        else:
            return False

    def get_classes(self, source):
        parsed = ast.parse(source)
        classes = [node for node in ast.walk(parsed) if self.is_class(node) and self.is_public(node.name)]
        class_defs = {}
        for x in classes:
            class_defs[x.name] = {}
            methods = []
            for y in x.body:
                if type(y) == ast.FunctionDef and self.is_public(y.name):
                    methods.append(y.name)
            class_defs[x.name]["methods"] = methods
        return class_defs

    def get_functions(self, source):
        parsed = ast.parse(source)
        functions = [node for node in ast.walk(parsed) if self.is_function(node) and self.is_public(node.name)]
        functions = [x.name for x in functions]
        return functions

    def get_filenames(self, directory):
        files = []
        tree = os.walk(directory, topdown=True, followlinks=False)
        for dirpath, dirnames, filenames in tree:
            for folder in self.exclude:
                if folder in dirpath:
                    break
            else:
                for name in filenames:
                    if name.endswith(".py") and not name.startswith("_"):
                        files.append(dirpath + "/" + name)
        return files

    def get_files_dict(self, filenames):
        files_dict = {}
        for x in filenames:
            with open(x, "r") as f:
                source = f.read()
                files_dict[x] = {"classes": {}, "functions": []}
                files_dict[x]["classes"] = self.get_classes(source)
                files_dict[x]["functions"] = self.get_functions(source)
        return files_dict

    def dump_yaml(self, output_file):
        files = self.get_filenames(self.directory)
        files_dict = self.get_files_dict(files)
        with open(output_file, "w") as stream:
            yaml.dump(files_dict, stream, default_flow_style=False)


api_crawler = APICrawler
<commit_msg>Combine diffs into ine dict.<commit_after>import ast, os
import yaml


__all__ = ["api_crawler", "differ"]


def differ(old_version, new_version):
    with open(old_version, "r") as f:
        old = f.read()
    with open(new_version, "r") as f:
        new = f.read()
    old_version = yaml.load(old)
    new_version = yaml.load(new)
    union = []
    diff = []

    files_union = set(old_version) & set(new_version)
    files_diff = set(old_version) - set(new_version)

    union += list(files_union)
    diff += list(files_diff)

    classes_old = [{x: {"classes": list(old_version[x]["classes"].keys())}} for x in files_union]
    classes_new = [{x: {"classes": list(new_version[x]["classes"].keys())}} for x in files_union]
    for x, y in zip(classes_old, classes_new):
        assert(x.keys() == y.keys())
        old_value = list(x.values())[0]["classes"]
        new_value = list(y.values())[0]["classes"]
        x[list(x.keys())[0]] = list(set(old_value) & set(new_value))
        y[list(y.keys())[0]] = list(set(old_value) - set(new_value))
    classes_union = [x for x in classes_old if list(x.values())[0]]
    classes_diff = [x for x in classes_new if list(x.values())[0]]
    union += classes_union
    diff += classes_diff
    return diff


class APICrawler(object):
    exclude = ("tests", "static", "sampledata", "mplexplorer")

    def __init__(self, directory):
        self.directory = directory

    def is_public(self, name):
        if name.startswith("_", 0, 1) and name != "__init__":
            return False
        else:
            return True

    def is_function(self, ast_node):
        if isinstance(ast_node, ast.FunctionDef) and ast_node.col_offset == 0:
            return True
        else:
            return False

    def is_class(self, ast_node):
        if isinstance(ast_node, ast.ClassDef):
            return True
        else:
            return False

    def get_classes(self, source):
        parsed = ast.parse(source)
        classes = [node for node in ast.walk(parsed) if self.is_class(node) and self.is_public(node.name)]
        class_defs = {}
        for x in classes:
            class_defs[x.name] = {}
            methods = []
            for y in x.body:
                if type(y) == ast.FunctionDef and self.is_public(y.name):
                    methods.append(y.name)
            class_defs[x.name]["methods"] = methods
        return class_defs

    def get_functions(self, source):
        parsed = ast.parse(source)
        functions = [node for node in ast.walk(parsed) if self.is_function(node) and self.is_public(node.name)]
        functions = [x.name for x in functions]
        return functions

    def get_filenames(self, directory):
        files = []
        tree = os.walk(directory, topdown=True, followlinks=False)
        for dirpath, dirnames, filenames in tree:
            for folder in self.exclude:
                if folder in dirpath:
                    break
            else:
                for name in filenames:
                    if name.endswith(".py") and not name.startswith("_"):
                        files.append(dirpath + "/" + name)
        return files

    def get_files_dict(self, filenames):
        files_dict = {}
        for x in filenames:
            with open(x, "r") as f:
                source = f.read()
                files_dict[x] = {"classes": {}, "functions": []}
                files_dict[x]["classes"] = self.get_classes(source)
                files_dict[x]["functions"] = self.get_functions(source)
        return files_dict

    def dump_yaml(self, output_file):
        files = self.get_filenames(self.directory)
        files_dict = self.get_files_dict(files)
        with open(output_file, "w") as stream:
            yaml.dump(files_dict, stream, default_flow_style=False)


api_crawler = APICrawler

{'fraction_non_alphanumeric': 0.11972114224426866, 'fraction_numerical': 0.0016087947446038343, 'mean_word_length': 2.2590650939274792, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7286471', 'n_tokens_mistral': 2364, 'n_tokens_neox': 2287, 'n_words': 558}