<commit_before>from pygments import lexers, token, util
import networkx
import os
import subprocess


def run(project_id, repo_path, cursor, **options):
    query = 'SELECT * FROM projects WHERE id = ' + str(project_id)
    cursor.execute(query)

    record = cursor.fetchone()
    language = record[5]

    ctags_process = subprocess.Popen(
        ['ctags', '-Rx', "--languages={0}".format(language), repo_path],
        stdout=subprocess.PIPE
    )

    ctags_output = ctags_process.stdout.readlines()
    file_names = get_language_files(ctags_output)

    graph = networkx.Graph()
    lexer = lexers.get_lexer_by_name(language)
    build_graph(file_names, graph, lexer)

    result = networkx.is_connected(graph)
    return result, int(result)


def build_graph(file_names, graph, lexer):
    """
    for each file in the set of files
        create a node and add it to the graph
        open the file
        read the contents into memory
        get a list of tokens from the lexer
        for each token in the resulting tokens
            check if the token is defining a symbol
            if true, add the symbol to the file node

    for each file in the set of files
        open the file
        read the contents into memory
        get a list of token from the lexer
        for each token in the resulting tokens
            check if the token is using a symbol
            if true:
                search the graph for the node that has the symbol definition
                create a relationship from the current file to the node with
                the symbol definition
    for each node in the graph
        check if there exists a relationship with any other node in the graph

    if there is a node without any relationships, fail the architecture test
    """
    for file_name in file_names:
        node = Node(file_name)
        graph.add_node(node)
        try:
            with open(file_name, 'r', encoding='utf-8') as file:
                contents = file.read()

            tokens = lexer.get_tokens(contents)
            for item in tokens:
                token_type = item[0]
                if token_type in [token.Name.Function, token.Name.Class]:
                    node.add_symbol(item[1], token_type)
            if 'DEBUG' in os.environ:
                print(node)
        except FileNotFoundError as e:
            continue
        except UnicodeDecodeError:
            continue

    for origin_node in graph.nodes():
        try:
            with open(origin_node.path, 'r', encoding='utf-8') as file:
                contents = file.read()

            tokens = lexer.get_tokens(contents)
            for item in tokens:
                token_type = item[0]
                if token_type not in [token.Name.Function, token.Name.Class]:
                    for node in graph.nodes_iter():
                        if node.has_symbol(item[1]):
                            graph.add_edge(origin_node, node)
        except FileNotFoundError as e:
            print("Not found")
        except UnicodeDecodeError:
            continue


def find_node_by_name(graph, name):
    for node, data in graph.nodes_iter(data=True):
        if data['name'] is name:
            return node
    return None


def get_language_files(ctags_output):
    result = set()

    for line in ctags_output:
        fields = line.decode('utf-8').split()
        result.add(fields[3])

    return result


class Node():
    def __init__(self, path):
        self.path = path
        self.symbols = set()

    def add_symbol(self, symbol, symbol_type=token.Generic):
        self.symbols.add((symbol_type, symbol))

    def has_symbol(self, item):
        for symbol in self.symbols:
            if item == symbol[1]:
                return True
        return False

    def __hash__(self):
        return hash(self.path)

    def __eq__(self, other):
        return self.path == other.path

    def __str__(self):
        result = '\r'
        for symbol in self.symbols:
            symbol_str = '{0}: {1}\n'.format(symbol[0], symbol[1])
            result += symbol_str

        return "{0}\n{1}\n{2}".format(
            self.path,
            '=' * len(self.path),
            result
        )

if __name__ == '__main__':
    print('Attribute plugins are not meant to be executed directly.')<commit_msg>Calculate connectedness of architecture graph<commit_after>from pygments import lexers, token, util
import networkx
import os
import subprocess


def run(project_id, repo_path, cursor, **options):
    query = 'SELECT * FROM projects WHERE id = ' + str(project_id)
    cursor.execute(query)

    record = cursor.fetchone()
    language = record[5]

    ctags_process = subprocess.Popen(
        ['ctags', '-Rx', "--languages={0}".format(language), repo_path],
        stdout=subprocess.PIPE
    )

    ctags_output = ctags_process.stdout.readlines()
    file_names = get_language_files(ctags_output)

    graph = networkx.Graph()
    lexer = lexers.get_lexer_by_name(language)
    build_graph(file_names, graph, lexer)

    result = get_connectedness(graph)
    return result > options.get('connectedness', 0.75), result


def build_graph(file_names, graph, lexer):
    """
    for each file in the set of files
        create a node and add it to the graph
        open the file
        read the contents into memory
        get a list of tokens from the lexer
        for each token in the resulting tokens
            check if the token is defining a symbol
            if true, add the symbol to the file node

    for each file in the set of files
        open the file
        read the contents into memory
        get a list of token from the lexer
        for each token in the resulting tokens
            check if the token is using a symbol
            if true:
                search the graph for the node that has the symbol definition
                create a relationship from the current file to the node with
                the symbol definition
    """
    for file_name in file_names:
        node = Node(file_name)
        graph.add_node(node)
        try:
            with open(file_name, 'r', encoding='utf-8') as file:
                contents = file.read()

            tokens = lexer.get_tokens(contents)
            for item in tokens:
                token_type = item[0]
                if token_type in [token.Name.Function, token.Name.Class]:
                    node.add_symbol(item[1], token_type)
            if 'DEBUG' in os.environ:
                print(node)
        except FileNotFoundError as e:
            continue
        except UnicodeDecodeError:
            continue

    for origin_node in graph.nodes():
        try:
            with open(origin_node.path, 'r', encoding='utf-8') as file:
                contents = file.read()

            tokens = lexer.get_tokens(contents)
            for item in tokens:
                token_type = item[0]
                if token_type not in [token.Name.Function, token.Name.Class]:
                    for node in graph.nodes_iter():
                        if node.has_symbol(item[1]):
                            graph.add_edge(origin_node, node)
        except FileNotFoundError as e:
            print("Not found: " + origin_node.path)
        except UnicodeDecodeError:
            continue


def get_connectedness(graph):
    node_degrees = graph.degree()
    zero_degrees = len(
        [degree for key, degree in node_degrees.items() if degree is 0]
    )

    return 1 - (zero_degrees / len(node_degrees))


def find_node_by_name(graph, name):
    for node, data in graph.nodes_iter(data=True):
        if data['name'] is name:
            return node
    return None


def get_language_files(ctags_output):
    # TODO: Fix issue where ctags output has unstable fields
    result = set()

    for line in ctags_output:
        fields = line.decode('utf-8').split()
        result.add(fields[3])

    return result


class Node():
    def __init__(self, path):
        self.path = path
        self.symbols = set()

    def add_symbol(self, symbol, symbol_type=token.Generic):
        self.symbols.add((symbol_type, symbol))

    def has_symbol(self, item):
        for symbol in self.symbols:
            if item == symbol[1]:
                return True
        return False

    def __hash__(self):
        return hash(self.path)

    def __eq__(self, other):
        return self.path == other.path

    def __str__(self):
        result = '\r'
        for symbol in self.symbols:
            symbol_str = '{0}: {1}\n'.format(symbol[0], symbol[1])
            result += symbol_str

        return "{0}\n{1}\n{2}".format(
            self.path,
            '=' * len(self.path),
            result
        )

if __name__ == '__main__':
    import json
    import mysql.connector
    import sys

    os.environ['DEBUG'] = '1'

    with open('../../config.json', 'r') as file:
        config = json.load(file)

    mysql_config = config['options']['datasource']

    connection = mysql.connector.connect(**mysql_config)
    connection.connect()

    cursor = connection.cursor()
    result = run(sys.argv[1], sys.argv[2], cursor, threshold=0.75)
    cursor.close()

    connection.close()

    print(result)
{'fraction_non_alphanumeric': 0.08845820701602425, 'fraction_numerical': 0.005088783022953659, 'mean_word_length': 2.0235679214402618, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2112101', 'n_tokens_mistral': 2639, 'n_tokens_neox': 2521, 'n_words': 819}