<commit_before>"""
Portal selenium tests base module.

"""


from functools import wraps
import traceback
from datetime import datetime
import logging
import time
import os
import json
import pprint
from collections import namedtuple
import re

from selenium import webdriver
import pytest


# The selenium logger is just too noisy, it shows the whole terribly huge
# base64 encoded json message, which contains the whole visible map,
# and it has over 10000 lines on my console, so it will be hidden for now.
# The huge info is shown as logging.DEBUG message.
logging.getLogger('selenium.webdriver.remote').setLevel(logging.ERROR)

# configuration data instance, keep here to have it valid
# during the entire test suite run, it's initialized just
# once per entire run
CONFIG = {}

# Class used for runing information from the acme.getCurrentPose() function
Pose = namedtuple("pose", ['alt', 'lon', 'lat'])


def make_screenshot(browser, fname, index):
    ss_dir = CONFIG["screenshots_dir"]

    if not os.path.exists(ss_dir):
        os.makedirs(ss_dir)

    fname = "{}{}{}_{}_{}".format(ss_dir,
                                  os.path.sep,
                                  datetime.now().isoformat(),
                                  fname,
                                  index)
    #print "Making screenshot: " + fname
    browser.save_screenshot('{}.png'.format(fname))
    return fname


def screenshot_on_error(test):
    """
    Decorator for test functions to make screenshots on error.

    The wrapper runs the test. When it fails, then it makes
    a screenshot in the CONFIG["screenshots_dir"] directory.

    """
    @wraps(test)
    def wrapper(*args, **kwargs):
        try:
            test(*args, **kwargs)
        except:
            test_obj = args[0]
            test_name = test_obj.current_method
            fname = make_screenshot(test_obj.browser, test_name, 0)
            with open("{}.log".format(fname), "w") as flog:
                flog.write(traceback.format_exc())
            raise
    return wrapper


def load_configuration():
    """
    Read the test suite JSON configuration from a file.
    The file path is read from a env variable.
    Set the global CONFIG object.

    """
    global CONFIG
    portal_config_env = "PORTAL_TESTS_CONFIG"
    try:
        config_file = os.environ[portal_config_env]
    except KeyError:
        m = "Can't load tests config file, set '%s' accordingly." % portal_config_env
        pytest.exit(m)

    print ("Loading configuration from "
           "env PORTAL_TESTS_CONFIG: '%s' ..." % config_file)
    f = open(config_file, 'r')
    CONFIG = json.load(f)
    # for jenkins, the following variables are missing from the configuration
    # file, so figure them out and set into CONFIG accordingly
    config_vars = dict(extensions_dir="extensions", screenshots_dir="tests_results")
    for var_key, var_value in config_vars.items():
        if not CONFIG.get(var_key, None):
            # this will fail if not set up properly
            workspace = os.environ["WORKSPACE"]
            CONFIG[var_key] = os.path.join(workspace, var_value)

    print "CONFIG: test suite configuration:"
    pprint.pprint(CONFIG)
    return CONFIG


def set_env_variables():
    """
    Sets env variables according to values from the config file.

    """
    global CONFIG
    if "env_vars" in CONFIG:
        print "Setting env. variables ..."
        for var in CONFIG["env_vars"]:
            name = var["name"]
            old = os.environ.get(name, None)
            if var["extend_current"] and old:
                os.environ[name] = old + ":" + var["value"]
            else:
                os.environ[name] = var["value"]
            print ("Set env '%s' old value: '%s' current value: '%s'" %
                  (name, old, os.environ[name]))


def prepare_environment():
    """
    Load the test suite configuration and prepare
    the environment.
    Run executables before the test suite requires.

    """
    global CONFIG
    if CONFIG:
        print "\nCONFIG is initialized, not loading anything ..."
    else:
        print "\nCONFIG is NOT initialized, running preparation ..."
        load_configuration()
        set_env_variables()
        for command in CONFIG["executables"]:
            print "Current working directory: '%s'" % os.getcwd()
            print "Running '%s' ..." % command
            r = os.system(command)
            print "exit status: %s" % r


class TestBase(object):
    """
    Base unit test class, with all the utilities for starting Chrome.

    Test suite configuration is loaded once per test suite from here
    with this class initialization.

    """

    @classmethod
    def setup_class(cls):
        """
        Setup any state specific to the execution of the given module.

        """
        prepare_environment()

    @classmethod
    def get_config(cls):
        """
        Return the configuration object.
        If client module does just import, it will import the symbol
        before it is initialized, hence empty.

        Returns:
            configuration object

        """
        global CONFIG
        return CONFIG

    @classmethod
    def _get_extensions_options(cls, config_chrome_section):
        """
        Returns ChromeOptions object with extensions paths.

        The Chrome browser started by Selenium doesn't have any plugins loaded.
        We need plugins for kiosk and display, so we need to provide the list
        in the ChromeOptions object.

        The path can be relative, however it needs to load packed extensions.

        Args:
            config_chrome_section: corresponding chrome section from the
                configuration file describing the chrome configuration
                we want to load

        Returns:
            ChromeOptions object with extensions declarations

        """
        op = webdriver.ChromeOptions()
        # if the path doesn't exist, let it fail, being clever here
        # could lead to some unexpected surprises of picking up undesired browser
        op.binary_location = config_chrome_section["binary_path"]

        for chrome_argument in config_chrome_section["arguments"]:
            op.add_argument(chrome_argument)

        for ext_name in config_chrome_section["extensions"]:
            ext_dir = CONFIG["extensions_dir"]
            # be less verbose now
            print "Loading extension {} from {}".format(ext_name, ext_dir)
            op.add_extension('{}/{}.crx'.format(ext_dir, ext_name))
        return op

    @classmethod
    def run_browser(cls, config_chrome_section):
        """
        Runs browser with proper driver path and extensions.

        Args:
            config_chrome_section: corresponding chrome section from the
                configuration file describing the chrome configuration
                we want to load

        Returns:
            selenium browser driver handler

        """
        #print "Starting browser for configuration (from \"chromes\" section):"
        #pprint.pprint(config_chrome_section)
        capabilities = webdriver.DesiredCapabilities.CHROME.copy()
        options = cls._get_extensions_options(config_chrome_section)
        # remote or local chrome
        if config_chrome_section["remote"]:
            # TODO
            # UNTESTED
            # e.g. 'http://localhost:4444/wd/hub'
            uri = config_chrome_section["uri"]
            browser = webdriver.chrome.webdriver.RemoteWebDriver(command_executor=uri,
                                                                 desired_capabilities=capabilities)
            print("Remote webdriver connecting to %s") % uri
        else:
            driver = config_chrome_section["chrome_driver"]["path"]
            # Set environment variable for Chrome.
            # Chrome driver needs to have an environment variable set,
            # this must be set to the path to the webdriver file.
            os.environ["webdriver.chrome.driver"] = driver
            browser = webdriver.Chrome(executable_path=driver,
                                       chrome_options=options,
                                       desired_capabilities=capabilities)
            print("Remote webdriver connecting to %s") % uri
        return browser

    def setup_method(self, method):
        """
        Base method called before every test case method is called.

        """
        self.config = self.get_config()
        self.current_method = method.__name__
        print "current test: %s" % self.current_method
<commit_msg>Added remote capabilities<commit_after>"""
Portal selenium tests base module.

"""


from functools import wraps
import traceback
from datetime import datetime
import logging
import time
import os
import json
import pprint
from collections import namedtuple
import re

from selenium import webdriver
import pytest


# The selenium logger is just too noisy, it shows the whole terribly huge
# base64 encoded json message, which contains the whole visible map,
# and it has over 10000 lines on my console, so it will be hidden for now.
# The huge info is shown as logging.DEBUG message.
logging.getLogger('selenium.webdriver.remote').setLevel(logging.ERROR)

# configuration data instance, keep here to have it valid
# during the entire test suite run, it's initialized just
# once per entire run
CONFIG = {}

# Class used for runing information from the acme.getCurrentPose() function
Pose = namedtuple("pose", ['alt', 'lon', 'lat'])


def make_screenshot(browser, fname, index):
    ss_dir = CONFIG["screenshots_dir"]

    if not os.path.exists(ss_dir):
        os.makedirs(ss_dir)

    fname = "{}{}{}_{}_{}".format(ss_dir,
                                  os.path.sep,
                                  datetime.now().isoformat(),
                                  fname,
                                  index)
    #print "Making screenshot: " + fname
    browser.save_screenshot('{}.png'.format(fname))
    return fname


def screenshot_on_error(test):
    """
    Decorator for test functions to make screenshots on error.

    The wrapper runs the test. When it fails, then it makes
    a screenshot in the CONFIG["screenshots_dir"] directory.

    """
    @wraps(test)
    def wrapper(*args, **kwargs):
        try:
            test(*args, **kwargs)
        except:
            test_obj = args[0]
            test_name = test_obj.current_method
            fname = make_screenshot(test_obj.browser, test_name, 0)
            with open("{}.log".format(fname), "w") as flog:
                flog.write(traceback.format_exc())
            raise
    return wrapper


def load_configuration():
    """
    Read the test suite JSON configuration from a file.
    The file path is read from a env variable.
    Set the global CONFIG object.

    """
    global CONFIG
    portal_config_env = "PORTAL_TESTS_CONFIG"
    try:
        config_file = os.environ[portal_config_env]
    except KeyError:
        m = "Can't load tests config file, set '%s' accordingly." % portal_config_env
        pytest.exit(m)

    print ("Loading configuration from "
           "env PORTAL_TESTS_CONFIG: '%s' ..." % config_file)
    f = open(config_file, 'r')
    CONFIG = json.load(f)
    # for jenkins, the following variables are missing from the configuration
    # file, so figure them out and set into CONFIG accordingly
    config_vars = dict(extensions_dir="extensions", screenshots_dir="tests_results")
    for var_key, var_value in config_vars.items():
        if not CONFIG.get(var_key, None):
            # this will fail if not set up properly
            workspace = os.environ["WORKSPACE"]
            CONFIG[var_key] = os.path.join(workspace, var_value)

    print "CONFIG: test suite configuration:"
    pprint.pprint(CONFIG)
    return CONFIG


def set_env_variables():
    """
    Sets env variables according to values from the config file.

    """
    global CONFIG
    if "env_vars" in CONFIG:
        print "Setting env. variables ..."
        for var in CONFIG["env_vars"]:
            name = var["name"]
            old = os.environ.get(name, None)
            if var["extend_current"] and old:
                os.environ[name] = old + ":" + var["value"]
            else:
                os.environ[name] = var["value"]
            print ("Set env '%s' old value: '%s' current value: '%s'" %
                  (name, old, os.environ[name]))


def prepare_environment():
    """
    Load the test suite configuration and prepare
    the environment.
    Run executables before the test suite requires.

    """
    global CONFIG
    if CONFIG:
        print "\nCONFIG is initialized, not loading anything ..."
    else:
        print "\nCONFIG is NOT initialized, running preparation ..."
        load_configuration()
        set_env_variables()
        for command in CONFIG["executables"]:
            print "Current working directory: '%s'" % os.getcwd()
            print "Running '%s' ..." % command
            r = os.system(command)
            print "exit status: %s" % r


class TestBase(object):
    """
    Base unit test class, with all the utilities for starting Chrome.

    Test suite configuration is loaded once per test suite from here
    with this class initialization.

    """

    @classmethod
    def setup_class(cls):
        """
        Setup any state specific to the execution of the given module.

        """
        prepare_environment()

    @classmethod
    def get_config(cls):
        """
        Return the configuration object.
        If client module does just import, it will import the symbol
        before it is initialized, hence empty.

        Returns:
            configuration object

        """
        global CONFIG
        return CONFIG

    @classmethod
    def _get_extensions_options(cls, config_chrome_section):
        """
        Returns ChromeOptions object with extensions paths.

        The Chrome browser started by Selenium doesn't have any plugins loaded.
        We need plugins for kiosk and display, so we need to provide the list
        in the ChromeOptions object.

        The path can be relative, however it needs to load packed extensions.

        Args:
            config_chrome_section: corresponding chrome section from the
                configuration file describing the chrome configuration
                we want to load

        Returns:
            ChromeOptions object with extensions declarations

        """
        op = webdriver.ChromeOptions()
        # if the path doesn't exist, let it fail, being clever here
        # could lead to some unexpected surprises of picking up undesired browser
        op.binary_location = config_chrome_section["binary_path"]

        for chrome_argument in config_chrome_section["arguments"]:
            op.add_argument(chrome_argument)

        for ext_name in config_chrome_section["extensions"]:
            ext_dir = CONFIG["extensions_dir"]
            # be less verbose now
            print "Loading extension {} from {}".format(ext_name, ext_dir)
            op.add_extension('{}/{}.crx'.format(ext_dir, ext_name))
        return op

    @classmethod
    def run_browser(cls, config_chrome_section):
        """
        Runs browser with proper driver path and extensions.

        Args:
            config_chrome_section: corresponding chrome section from the
                configuration file describing the chrome configuration
                we want to load

        Returns:
            selenium browser driver handler

        """
        # print "Starting browser for configuration (from \"chromes\" section):"
        # pprint.pprint(config_chrome_section)
        capabilities = webdriver.DesiredCapabilities.CHROME.copy()
        options = cls._get_extensions_options(config_chrome_section)
        # remote or local chrome
        if config_chrome_section["remote"]:
            remote_capabilities = capabilities + options.to_capabilities()
            uri = config_chrome_section["uri"]
            browser = webdriver.chrome.webdriver.RemoteWebDriver(command_executor=uri,
                                                                 desired_capabilities=remote_capabilities)
            print("Remote webdriver connecting to %s") % uri
        else:
            driver = config_chrome_section["chrome_driver"]["path"]
            # Set environment variable for Chrome.
            # Chrome driver needs to have an environment variable set,
            # this must be set to the path to the webdriver file.
            os.environ["webdriver.chrome.driver"] = driver
            browser = webdriver.Chrome(executable_path=driver,
                                       chrome_options=options,
                                       desired_capabilities=capabilities)
            print("Remote webdriver connecting to %s") % uri
        return browser

    def setup_method(self, method):
        """
        Base method called before every test case method is called.

        """
        self.config = self.get_config()
        self.current_method = method.__name__
        print "current test: %s" % self.current_method

{'fraction_non_alphanumeric': 0.081537565521258, 'fraction_numerical': 0.00128130460104834, 'mean_word_length': 2.2459357277882797, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1447326', 'n_tokens_mistral': 4467, 'n_tokens_neox': 4306, 'n_words': 1511}