<commit_before>import os
import re
import time

from twisted.internet import defer, reactor
from twisted.trial.runner import filenameToModule
from twisted.python import usage, reflect

from ooni import geoip
from ooni.tasks import Measurement
from ooni.utils import log, checkForRoot
from ooni import otime
from ooni.settings import config

from ooni import errors as e

from inspect import getmembers
from StringIO import StringIO

class NoTestCasesFound(Exception):
    pass

def get_test_methods(item, method_prefix="test_"):
    """
    Look for test_ methods in subclasses of NetTestCase
    """
    test_cases = []
    try:
        assert issubclass(item, NetTestCase)
        methods = reflect.prefixedMethodNames(item, method_prefix)
        test_methods = []
        for method in methods:
            test_methods.append(method_prefix + method)
        if test_methods:
            test_cases.append((item, test_methods))
    except (TypeError, AssertionError):
        pass
    return test_cases

def loadNetTestString(net_test_string):
    """
    Load NetTest from a string.
    WARNING input to this function *MUST* be sanitized and *NEVER* be
    untrusted.
    Failure to do so will result in code exec.

    net_test_string:

        a string that contains the net test to be run.
    """
    net_test_file_object = StringIO(net_test_string)

    ns = {}
    test_cases = []
    exec net_test_file_object.read() in ns
    for item in ns.itervalues():
        test_cases.extend(get_test_methods(item))

    if not test_cases:
        raise NoTestCasesFound

    return test_cases

def loadNetTestFile(net_test_file):
    """
    Load NetTest from a file.
    """
    test_cases = []
    module = filenameToModule(net_test_file)
    for __, item in getmembers(module):
        test_cases.extend(get_test_methods(item))

    if not test_cases:
        raise NoTestCasesFound

    return test_cases

def getTestClassFromFile(net_test_file):
    """
    Will return the first class that is an instance of NetTestCase.

    XXX this means that if inside of a test there are more than 1 test case
        then we will only run the first one.
    """
    module = filenameToModule(net_test_file)
    for __, item in getmembers(module):
        try:
            assert issubclass(item, NetTestCase)
            return item
        except (TypeError, AssertionError):
            pass

def getOption(opt_parameter, required_options, type='text'):
    """
    Arguments:
        usage_options: a list as should be the optParameters of an UsageOptions class.

        required_options: a list containing the strings of the options that are
            required.

        type: a string containing the type of the option.

    Returns:
        a dict containing
            {
                'description': the description of the option,
                'default': the default value of the option,
                'required': True|False if the option is required or not,
                'type': the type of the option ('text' or 'file')
            }
    """
    option_name, _, default, description = opt_parameter
    if option_name in required_options:
        required = True
    else:
        required = False

    return {'description': description,
        'value': default, 'required': required,
        'type': type
    }

def getArguments(test_class):
    arguments = {}
    if test_class.inputFile:
        option_name = test_class.inputFile[0]
        arguments[option_name] = getOption(test_class.inputFile,
                test_class.requiredOptions, type='file')
    try:
        list(test_class.usageOptions.optParameters)
    except AttributeError:
        return arguments

    for opt_parameter in test_class.usageOptions.optParameters:
        option_name = opt_parameter[0]
        opt_type="text"
        if opt_parameter[3].lower().startswith("file"):
            opt_type="file"
        arguments[option_name] = getOption(opt_parameter,
                test_class.requiredOptions, type=opt_type)

    return arguments

def test_class_name_to_name(test_class_name):
    return test_class_name.lower().replace(' ','_')

def getNetTestInformation(net_test_file):
    """
    Returns a dict containing:

    {
        'id': the test filename excluding the .py extension,
        'name': the full name of the test,
        'description': the description of the test,
        'version': version number of this test,
        'arguments': a dict containing as keys the supported arguments and as
            values the argument description.
    }
    """
    test_class = getTestClassFromFile(net_test_file)

    test_id = test_class_name_to_name(test_class.name)
    information = {'id': test_id,
        'name': test_class.name,
        'description': test_class.description,
        'version': test_class.version,
        'arguments': getArguments(test_class),
        'path': net_test_file
    }
    return information

class NetTestLoader(object):
    method_prefix = 'test'

    def __init__(self, options, test_file=None, test_string=None):
        self.options = options
        test_cases = None

        if test_file:
            test_cases = loadNetTestFile(test_file)
        elif test_string:
            test_cases = loadNetTestString(test_string)

        if test_cases:
            self.setupTestCases(test_cases)

    @property
    def testDetails(self):
        from ooni import __version__ as software_version

        client_geodata = {}
        if config.probe_ip.address and (config.privacy.includeip or \
                config.privacy.includeasn or \
                config.privacy.includecountry or \
                config.privacy.includecity):
            log.msg("We will include some geo data in the report")
            try:
                client_geodata = geoip.IPToLocation(config.probe_ip.address)
            except e.GeoIPDataFilesNotFound:
                log.err("Unable to find the geoip data files")
                client_geodata = {'city': None, 'countrycode': None, 'asn': None}

        if config.privacy.includeip:
            client_geodata['ip'] = config.probe_ip.address
        else:
            client_geodata['ip'] = "127.0.0.1"

        # Here we unset all the client geodata if the option to not include then
        # has been specified
        if client_geodata and not config.privacy.includeasn:
            client_geodata['asn'] = 'AS0'
        elif 'asn' in client_geodata:
            # XXX this regexp should probably go inside of geodata
            client_geodata['asn'] = client_geodata['asn']
            log.msg("Your AS number is: %s" % client_geodata['asn'])
        else:
            client_geodata['asn'] = None

        if (client_geodata and not config.privacy.includecity) \
                or ('city' not in client_geodata):
            client_geodata['city'] = None

        if (client_geodata and not config.privacy.includecountry) \
                or ('countrycode' not in client_geodata):
            client_geodata['countrycode'] = None

        test_details = {'start_time': time.time(),
            'probe_asn': client_geodata['asn'],
            'probe_cc': client_geodata['countrycode'],
            'probe_ip': client_geodata['ip'],
            'test_name': self.testName,
            'test_version': self.testVersion,
            'software_name': 'ooniprobe',
            'software_version': software_version,
            'options': self.options
        }
        return test_details


    def _parseNetTestOptions(self, klass):
        """
        Helper method to assemble the options into a single UsageOptions object
        """
        usage_options = klass.usageOptions

        if not hasattr(usage_options, 'optParameters'):
            usage_options.optParameters = []
        else:
            for parameter in usage_options.optParameters:
                if len(parameter) == 5:
                    parameter.pop()

        if klass.inputFile:
            usage_options.optParameters.append(klass.inputFile)

        if klass.baseParameters:
            for parameter in klass.baseParameters:
                usage_options.optParameters.append(parameter)

        if klass.baseFlags:
            if not hasattr(usage_options, 'optFlags'):
                usage_options.optFlags = []
            for flag in klass.baseFlags:
                usage_options.optFlags.append(flag)

        return usage_options

    @property
    def usageOptions(self):
        usage_options = None
        for test_class, test_method in self.testCases:
            if not usage_options:
                usage_options = self._parseNetTestOptions(test_class)
            else:
                assert usage_options == test_class.usageOptions
        return usage_options

    def loadNetTestString(self, net_test_string):
        """
        Load NetTest from a string.
        WARNING input to this function *MUST* be sanitized and *NEVER* take
        untrusted input.
        Failure to do so will result in code exec.

        net_test_string:

            a string that contains the net test to be run.
        """
        net_test_file_object = StringIO(net_test_string)

        ns = {}
        test_cases = []
        exec net_test_file_object.read() in ns
        for item in ns.itervalues():
            test_cases.extend(self._get_test_methods(item))

        if not test_cases:
            raise NoTestCasesFound

        self.setupTestCases(test_cases)

    def loadNetTestFile(self, net_test_file):
        """
        Load NetTest from a file.
        """
        test_cases = []
        module = filenameToModule(net_test_file)
        for __, item in getmembers(module):
            test_cases.extend(self._get_test_methods(item))

        if not test_cases:
            raise NoTestCasesFound

        self.setupTestCases(test_cases)

    def setupTestCases(self, test_cases):
        """
        Creates all the necessary test_cases (a list of tuples containing the
        NetTestCase (test_class, test_method))

        example:
            [(test_classA, test_method1),
            (test_classA, test_method2),
            (test_classA, test_method3),
            (test_classA, test_method4),
            (test_classA, test_method5),

            (test_classB, test_method1),
            (test_classB, test_method2)]

        Note: the inputs must be valid for test_classA and test_classB.

        net_test_file:
            is either a file path or a file like object that will be used to
            generate the test_cases.
        """
        test_class, _ = test_cases[0]
        self.testVersion = test_class.version
        self.testName = test_class_name_to_name(test_class.name)
        self.testCases = test_cases

    def checkOptions(self):
        """
        Call processTest and processOptions methods of each NetTestCase
        """
        test_classes = set([])
        for test_class, test_method in self.testCases:
            test_classes.add(test_class)

        for klass in test_classes:
            options = self.usageOptions()
            options.parseOptions(self.options)

            if options:
                klass.localOptions = options

            test_instance = klass()
            if test_instance.requiresRoot:
                checkForRoot()
            test_instance._checkRequiredOptions()
            test_instance._checkValidOptions()

            inputs = test_instance.getInputProcessor()
            if not inputs:
                inputs = [None]
            klass.inputs = inputs

    def _get_test_methods(self, item):
        """
        Look for test_ methods in subclasses of NetTestCase
        """
        test_cases = []
        try:
            assert issubclass(item, NetTestCase)
            methods = reflect.prefixedMethodNames(item, self.method_prefix)
            test_methods = []
            for method in methods:
                test_methods.append(self.method_prefix + method)
            if test_methods:
                test_cases.append((item, test_methods))
        except (TypeError, AssertionError):
            pass
        return test_cases

class NetTestState(object):
    def __init__(self, allTasksDone):
        """
        This keeps track of the state of a running NetTests case.

        Args:
            allTasksDone is a deferred that will get fired once all the NetTest
            cases have reached a final done state.
        """
        self.doneTasks = 0
        self.tasks = 0

        self.completedScheduling = False
        self.allTasksDone = allTasksDone

    def taskCreated(self):
        self.tasks += 1

    def checkAllTasksDone(self):
        log.debug("Checking all tasks for completion %s == %s" %
                  (self.doneTasks, self.tasks))
        if self.completedScheduling and \
                self.doneTasks == self.tasks:
            self.allTasksDone.callback(self.doneTasks)

    def taskDone(self):<commit_msg>Write the number of currently running tasks to a text file<commit_after>import os
import re
import time

from twisted.internet import defer, reactor
from twisted.trial.runner import filenameToModule
from twisted.python import usage, reflect

from ooni import geoip
from ooni.tasks import Measurement
from ooni.utils import log, checkForRoot
from ooni import otime
from ooni.settings import config

from ooni import errors as e

from inspect import getmembers
from StringIO import StringIO

class NoTestCasesFound(Exception):
    pass

def get_test_methods(item, method_prefix="test_"):
    """
    Look for test_ methods in subclasses of NetTestCase
    """
    test_cases = []
    try:
        assert issubclass(item, NetTestCase)
        methods = reflect.prefixedMethodNames(item, method_prefix)
        test_methods = []
        for method in methods:
            test_methods.append(method_prefix + method)
        if test_methods:
            test_cases.append((item, test_methods))
    except (TypeError, AssertionError):
        pass
    return test_cases

def loadNetTestString(net_test_string):
    """
    Load NetTest from a string.
    WARNING input to this function *MUST* be sanitized and *NEVER* be
    untrusted.
    Failure to do so will result in code exec.

    net_test_string:

        a string that contains the net test to be run.
    """
    net_test_file_object = StringIO(net_test_string)

    ns = {}
    test_cases = []
    exec net_test_file_object.read() in ns
    for item in ns.itervalues():
        test_cases.extend(get_test_methods(item))

    if not test_cases:
        raise NoTestCasesFound

    return test_cases

def loadNetTestFile(net_test_file):
    """
    Load NetTest from a file.
    """
    test_cases = []
    module = filenameToModule(net_test_file)
    for __, item in getmembers(module):
        test_cases.extend(get_test_methods(item))

    if not test_cases:
        raise NoTestCasesFound

    return test_cases

def getTestClassFromFile(net_test_file):
    """
    Will return the first class that is an instance of NetTestCase.

    XXX this means that if inside of a test there are more than 1 test case
        then we will only run the first one.
    """
    module = filenameToModule(net_test_file)
    for __, item in getmembers(module):
        try:
            assert issubclass(item, NetTestCase)
            return item
        except (TypeError, AssertionError):
            pass

def getOption(opt_parameter, required_options, type='text'):
    """
    Arguments:
        usage_options: a list as should be the optParameters of an UsageOptions class.

        required_options: a list containing the strings of the options that are
            required.

        type: a string containing the type of the option.

    Returns:
        a dict containing
            {
                'description': the description of the option,
                'default': the default value of the option,
                'required': True|False if the option is required or not,
                'type': the type of the option ('text' or 'file')
            }
    """
    option_name, _, default, description = opt_parameter
    if option_name in required_options:
        required = True
    else:
        required = False

    return {'description': description,
        'value': default, 'required': required,
        'type': type
    }

def getArguments(test_class):
    arguments = {}
    if test_class.inputFile:
        option_name = test_class.inputFile[0]
        arguments[option_name] = getOption(test_class.inputFile,
                test_class.requiredOptions, type='file')
    try:
        list(test_class.usageOptions.optParameters)
    except AttributeError:
        return arguments

    for opt_parameter in test_class.usageOptions.optParameters:
        option_name = opt_parameter[0]
        opt_type="text"
        if opt_parameter[3].lower().startswith("file"):
            opt_type="file"
        arguments[option_name] = getOption(opt_parameter,
                test_class.requiredOptions, type=opt_type)

    return arguments

def test_class_name_to_name(test_class_name):
    return test_class_name.lower().replace(' ','_')

def getNetTestInformation(net_test_file):
    """
    Returns a dict containing:

    {
        'id': the test filename excluding the .py extension,
        'name': the full name of the test,
        'description': the description of the test,
        'version': version number of this test,
        'arguments': a dict containing as keys the supported arguments and as
            values the argument description.
    }
    """
    test_class = getTestClassFromFile(net_test_file)

    test_id = test_class_name_to_name(test_class.name)
    information = {'id': test_id,
        'name': test_class.name,
        'description': test_class.description,
        'version': test_class.version,
        'arguments': getArguments(test_class),
        'path': net_test_file
    }
    return information

class NetTestLoader(object):
    method_prefix = 'test'

    def __init__(self, options, test_file=None, test_string=None):
        self.options = options
        test_cases = None

        if test_file:
            test_cases = loadNetTestFile(test_file)
        elif test_string:
            test_cases = loadNetTestString(test_string)

        if test_cases:
            self.setupTestCases(test_cases)

    @property
    def testDetails(self):
        from ooni import __version__ as software_version

        client_geodata = {}
        if config.probe_ip.address and (config.privacy.includeip or \
                config.privacy.includeasn or \
                config.privacy.includecountry or \
                config.privacy.includecity):
            log.msg("We will include some geo data in the report")
            try:
                client_geodata = geoip.IPToLocation(config.probe_ip.address)
            except e.GeoIPDataFilesNotFound:
                log.err("Unable to find the geoip data files")
                client_geodata = {'city': None, 'countrycode': None, 'asn': None}

        if config.privacy.includeip:
            client_geodata['ip'] = config.probe_ip.address
        else:
            client_geodata['ip'] = "127.0.0.1"

        # Here we unset all the client geodata if the option to not include then
        # has been specified
        if client_geodata and not config.privacy.includeasn:
            client_geodata['asn'] = 'AS0'
        elif 'asn' in client_geodata:
            # XXX this regexp should probably go inside of geodata
            client_geodata['asn'] = client_geodata['asn']
            log.msg("Your AS number is: %s" % client_geodata['asn'])
        else:
            client_geodata['asn'] = None

        if (client_geodata and not config.privacy.includecity) \
                or ('city' not in client_geodata):
            client_geodata['city'] = None

        if (client_geodata and not config.privacy.includecountry) \
                or ('countrycode' not in client_geodata):
            client_geodata['countrycode'] = None

        test_details = {'start_time': time.time(),
            'probe_asn': client_geodata['asn'],
            'probe_cc': client_geodata['countrycode'],
            'probe_ip': client_geodata['ip'],
            'test_name': self.testName,
            'test_version': self.testVersion,
            'software_name': 'ooniprobe',
            'software_version': software_version,
            'options': self.options
        }
        return test_details


    def _parseNetTestOptions(self, klass):
        """
        Helper method to assemble the options into a single UsageOptions object
        """
        usage_options = klass.usageOptions

        if not hasattr(usage_options, 'optParameters'):
            usage_options.optParameters = []
        else:
            for parameter in usage_options.optParameters:
                if len(parameter) == 5:
                    parameter.pop()

        if klass.inputFile:
            usage_options.optParameters.append(klass.inputFile)

        if klass.baseParameters:
            for parameter in klass.baseParameters:
                usage_options.optParameters.append(parameter)

        if klass.baseFlags:
            if not hasattr(usage_options, 'optFlags'):
                usage_options.optFlags = []
            for flag in klass.baseFlags:
                usage_options.optFlags.append(flag)

        return usage_options

    @property
    def usageOptions(self):
        usage_options = None
        for test_class, test_method in self.testCases:
            if not usage_options:
                usage_options = self._parseNetTestOptions(test_class)
            else:
                assert usage_options == test_class.usageOptions
        return usage_options

    def loadNetTestString(self, net_test_string):
        """
        Load NetTest from a string.
        WARNING input to this function *MUST* be sanitized and *NEVER* take
        untrusted input.
        Failure to do so will result in code exec.

        net_test_string:

            a string that contains the net test to be run.
        """
        net_test_file_object = StringIO(net_test_string)

        ns = {}
        test_cases = []
        exec net_test_file_object.read() in ns
        for item in ns.itervalues():
            test_cases.extend(self._get_test_methods(item))

        if not test_cases:
            raise NoTestCasesFound

        self.setupTestCases(test_cases)

    def loadNetTestFile(self, net_test_file):
        """
        Load NetTest from a file.
        """
        test_cases = []
        module = filenameToModule(net_test_file)
        for __, item in getmembers(module):
            test_cases.extend(self._get_test_methods(item))

        if not test_cases:
            raise NoTestCasesFound

        self.setupTestCases(test_cases)

    def setupTestCases(self, test_cases):
        """
        Creates all the necessary test_cases (a list of tuples containing the
        NetTestCase (test_class, test_method))

        example:
            [(test_classA, test_method1),
            (test_classA, test_method2),
            (test_classA, test_method3),
            (test_classA, test_method4),
            (test_classA, test_method5),

            (test_classB, test_method1),
            (test_classB, test_method2)]

        Note: the inputs must be valid for test_classA and test_classB.

        net_test_file:
            is either a file path or a file like object that will be used to
            generate the test_cases.
        """
        test_class, _ = test_cases[0]
        self.testVersion = test_class.version
        self.testName = test_class_name_to_name(test_class.name)
        self.testCases = test_cases

    def checkOptions(self):
        """
        Call processTest and processOptions methods of each NetTestCase
        """
        test_classes = set([])
        for test_class, test_method in self.testCases:
            test_classes.add(test_class)

        for klass in test_classes:
            options = self.usageOptions()
            options.parseOptions(self.options)

            if options:
                klass.localOptions = options

            test_instance = klass()
            if test_instance.requiresRoot:
                checkForRoot()
            test_instance._checkRequiredOptions()
            test_instance._checkValidOptions()

            inputs = test_instance.getInputProcessor()
            if not inputs:
                inputs = [None]
            klass.inputs = inputs

    def _get_test_methods(self, item):
        """
        Look for test_ methods in subclasses of NetTestCase
        """
        test_cases = []
        try:
            assert issubclass(item, NetTestCase)
            methods = reflect.prefixedMethodNames(item, self.method_prefix)
            test_methods = []
            for method in methods:
                test_methods.append(self.method_prefix + method)
            if test_methods:
                test_cases.append((item, test_methods))
        except (TypeError, AssertionError):
            pass
        return test_cases

class NetTestState(object):
    def __init__(self, allTasksDone):
        """
        This keeps track of the state of a running NetTests case.

        Args:
            allTasksDone is a deferred that will get fired once all the NetTest
            cases have reached a final done state.
        """
        self.doneTasks = 0
        self.tasks = 0

        self.completedScheduling = False
        self.allTasksDone = allTasksDone

    def taskCreated(self):
        self.tasks += 1

    def checkAllTasksDone(self):
        log.debug("Checking all tasks for completion %s == %s" %
                  (self.doneTasks, self.tasks))
        with open('task_count_log.txt', 'a+') as f:
            diff = int(self.tasks) - int(self.doneTasks)
            f.write("%s, %s, %s, %s\n" % (otime.timestamp(), self.doneTasks, self.tasks, diff))

        if self.completedScheduling and \
                self.doneTasks == self.tasks:
            self.allTasksDone.callback(self.doneTasks)

    def taskDone(self):
{'fraction_non_alphanumeric': 0.07734383965114945, 'fraction_numerical': 0.0017595532264851011, 'mean_word_length': 2.203921568627451, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3255039', 'n_tokens_mistral': 7295, 'n_tokens_neox': 7111, 'n_words': 1971}