<commit_before>__author__ = 'kevin'

import sys
from subprocess import check_output
import os


class Call():

    input_functions = ['canonicalize_file_name', 'catgets', 'confstr', 'ctermid', 'ctermid', 'cuserid', 'dgettext',
                       'dngettext', 'fgetc', 'fgetc_unlocked', 'fgets', 'fgets_unlocked', 'fpathconf', 'fread',
                       'fread_unlocked', 'fscanf', 'getc', 'getchar', 'getchar_unlocked', 'getc_unlocked',
                       'get_current_dir_name', 'getcwd', 'getdelim', '__getdelim', 'getdelim', 'getdents', 'getenv',
                       'gethostbyaddr', 'gethostbyname', 'gethostbyname2', 'gethostent', 'gethostid', 'getline',
                       'getline', 'getlogin', 'getlogin_r', 'getmsg', 'getopt', '_getopt_internal', 'getopt_long',
                       'getopt_long_only', 'getpass', 'getpmsg', 'gets', 'gettext', 'getw', 'getwd', 'ngettext',
                       'pathconf', 'pread', 'pread64', 'ptsname', 'ptsname_r', 'read', 'readdir', 'readlink', 'readv',
                       'realpath', 'recv', 'recv_from', 'recvmesg', 'scanf', '__secure_getenv', 'signal', 'sysconf',
                       'ttyname', 'ttyname_r', 'vfscanf', 'vscanf']

    output_functions = ['dprintf', 'fprintf', 'fputc', 'fputchar_unlocked', 'fputc_unlocked', 'fputs', 'fputs_unlocked',
                        'fwrite', 'fwrite_unlocked', 'perror', 'printf', 'psignal', 'putc', 'putchar', 'putc_unlocked',
                        'putenv', 'putmsg', 'putpmsg', 'puts', 'putw', 'pwrite', 'pwrite64', 'send', 'sendmsg',
                        'sendto', 'setenv', 'sethostid', 'setlogin', 'ungetc', 'vdprintf', 'vfprintf', 'vsyslog',
                        'write', 'writev']

    def __init__(self, clfow_line):
        split_line = clfow_line.split(Cflow.indent)

        self.function_info = split_line[-1]
        self.level = len(split_line) - 1

    def __str__(self):
        return "Level: " + str(self.level) + " - " + self.function_info

    def is_input_function(self):
        is_input = self._leaf_belongs_to(Call.input_functions)
        return is_input

    def is_output_function(self):
        is_output = self._leaf_belongs_to(Call.output_functions)
        return is_output

    def _leaf_belongs_to(self, function_set):
        belongs = False

        if self.is_leaf():
            belongs = self.leaf_function_name in function_set

        return belongs

    def is_leaf(self):
        """Checks if the function call represented by this object is a leaf in the call graph generated by cflow.
        Being a leaf means that the function is not defined in the files given to cflow as arguments to analyze.
        """
        return self.function_info.endswith("()")

    @property
    def leaf_function_name(self):
        return self.function_info[:-2]


class Cflow():
    """Wraps around some of cflow's command line interface"""

    indent = "    "

    _file_patterns = ["*.c", "*.h"]
    _cflow_exec = "cflow"
    
    def __init__(self, source_dir):
        self.source_dir = source_dir

    def _get_cflow_args(self):
        dirs_found = []

        for path, dirs, files in os.walk(os.path.abspath(self.source_dir)):
            for dir in dirs:
                for pattern in Cflow._file_patterns:
                    dirs_found.append(os.path.join(path, dir, pattern))

        return dirs_found

    def _build_cflow_command(self):
        cflow_args = self._get_cflow_args()
        command = "cd {0}; ".format(self.source_dir) + Cflow._cflow_exec + ' ' + ' '.join(cflow_args) + ";"

        return command

    def get_text_call_graph(self):
        cmd_cflow = self._build_cflow_command()
        cflow_output = check_output(cmd_cflow, shell=True).decode("utf-8")

        return cflow_output


class Stack():

    _collection = list()

    def __init__(self):
        pass

    def push(self, data):
        self._collection.append(data)

    def pop(self):
        return self._collection.pop()

    @property
    def top(self):
        return self._collection[-1]


def main(args):
    source_dir = args[1]

    cflow = Cflow(source_dir)
    text_call_graph = cflow.get_text_call_graph()

    main_call = Call(text_call_graph.splitlines()[0])

    parent = Stack()
    parent.push(main_call)
    previous = main_call

    entry_points = list()
    exit_points = list()

    for i, line in enumerate(text_call_graph.splitlines()):
        if i != 0:
            current = Call(line)

            if current.level > previous.level:
                parent.push(previous)
            elif current.level < previous.level:
                for i in range(current.level - previous.level):
                    parent.pop()

            if current.is_input_function():
                entry_points.append(parent.top)

            if current.is_output_function():
                exit_points.append(parent.top)

            previous = current

    print("entry points: " + str(len(entry_points)))
    print("exit points: " + str(len(exit_points)))

if __name__ == '__main__':
    main(sys.argv)<commit_msg>io_points.py no longer calls cflow but uses its output as a file given as an argument.<commit_after>__author__ = 'kevin'

import sys
from subprocess import check_output
import os


class Call():

    input_functions = ['canonicalize_file_name', 'catgets', 'confstr', 'ctermid', 'ctermid', 'cuserid', 'dgettext',
                       'dngettext', 'fgetc', 'fgetc_unlocked', 'fgets', 'fgets_unlocked', 'fpathconf', 'fread',
                       'fread_unlocked', 'fscanf', 'getc', 'getchar', 'getchar_unlocked', 'getc_unlocked',
                       'get_current_dir_name', 'getcwd', 'getdelim', '__getdelim', 'getdelim', 'getdents', 'getenv',
                       'gethostbyaddr', 'gethostbyname', 'gethostbyname2', 'gethostent', 'gethostid', 'getline',
                       'getline', 'getlogin', 'getlogin_r', 'getmsg', 'getopt', '_getopt_internal', 'getopt_long',
                       'getopt_long_only', 'getpass', 'getpmsg', 'gets', 'gettext', 'getw', 'getwd', 'ngettext',
                       'pathconf', 'pread', 'pread64', 'ptsname', 'ptsname_r', 'read', 'readdir', 'readlink', 'readv',
                       'realpath', 'recv', 'recv_from', 'recvmesg', 'scanf', '__secure_getenv', 'signal', 'sysconf',
                       'ttyname', 'ttyname_r', 'vfscanf', 'vscanf']

    output_functions = ['dprintf', 'fprintf', 'fputc', 'fputchar_unlocked', 'fputc_unlocked', 'fputs', 'fputs_unlocked',
                        'fwrite', 'fwrite_unlocked', 'perror', 'printf', 'psignal', 'putc', 'putchar', 'putc_unlocked',
                        'putenv', 'putmsg', 'putpmsg', 'puts', 'putw', 'pwrite', 'pwrite64', 'send', 'sendmsg',
                        'sendto', 'setenv', 'sethostid', 'setlogin', 'ungetc', 'vdprintf', 'vfprintf', 'vsyslog',
                        'write', 'writev']

    def __init__(self, clfow_line):
        split_line = clfow_line.split(Cflow.indent)

        self.function_info = split_line[-1]
        self.level = len(split_line) - 1

    def __str__(self):
        return "Level: " + str(self.level) + " - " + self.function_info

    def is_input_function(self):
        is_input = self._leaf_belongs_to(Call.input_functions)
        return is_input

    def is_output_function(self):
        is_output = self._leaf_belongs_to(Call.output_functions)
        return is_output

    def _leaf_belongs_to(self, function_set):
        belongs = False

        if self.is_leaf():
            belongs = self.leaf_function_name in function_set

        return belongs

    def is_leaf(self):
        """Checks if the function call represented by this object is a leaf in the call graph generated by cflow.
        Being a leaf means that the function is not defined in the files given to cflow as arguments to analyze.
        """
        return self.function_info.endswith("()")

    @property
    def leaf_function_name(self):
        return self.function_info[:-2]


class Cflow():
    """Wraps around some of cflow's command line interface"""

    indent = "    "

    _file_patterns = ["*.c", "*.h"]
    _cflow_exec = "cflow"
    
    def __init__(self, source_dir):
        self.source_dir = source_dir

    def _get_cflow_args(self):
        dirs_found = []

        for path, dirs, files in os.walk(os.path.abspath(self.source_dir)):
            for dir in dirs:
                for pattern in Cflow._file_patterns:
                    dirs_found.append(os.path.join(path, dir, pattern))

        return dirs_found

    def _build_cflow_command(self):
        cflow_args = self._get_cflow_args()
        command = "cd {0}; ".format(self.source_dir) + Cflow._cflow_exec + ' ' + ' '.join(cflow_args) + ";"

        return command

    def get_text_call_graph(self):
        cmd_cflow = self._build_cflow_command()
        cflow_output = check_output(cmd_cflow, shell=True).decode("utf-8")

        return cflow_output


class Stack():

    _collection = list()

    def __init__(self):
        pass

    def push(self, data):
        self._collection.append(data)

    def pop(self):
        return self._collection.pop()

    @property
    def top(self):
        return self._collection[-1]


def main(args):
    call_graph_file = args[1]

    entry_points = list()
    exit_points = list()

    with open(call_graph_file) as call_graph_file:
        for i, line in enumerate(call_graph_file):
            if i == 0:
                main_call = Call(line)

                parent = Stack()
                parent.push(main_call)
                previous = main_call
            else:
                current = Call(line)

                if current.level > previous.level:
                    parent.push(previous)
                elif current.level < previous.level:
                    for i in range(current.level - previous.level):
                        parent.pop()

                if current.is_input_function():
                    entry_points.append(parent.top)

                if current.is_output_function():
                    exit_points.append(parent.top)

                previous = current

    print("entry points: " + str(len(entry_points)))
    print("exit points: " + str(len(exit_points)))

if __name__ == '__main__':
    main(sys.argv)
{'fraction_non_alphanumeric': 0.12916707162989602, 'fraction_numerical': 0.002624161726115269, 'mean_word_length': 2.4231536926147705, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2913718', 'n_tokens_mistral': 3315, 'n_tokens_neox': 3187, 'n_words': 715}