<commit_before>#!/usr/bin/env python3

import argparse
import logging
import os
import shutil
import subprocess
import sys

# constants
RED = "\033[31m"
GREEN = "\033[32m"
MAGENTA = "\033[95m"
RESET = "\033[0m"

LOGFILE = 'runtests.log'

logger = logging.getLogger(__file__)
logger.setLevel(logging.DEBUG)


class OsmaxxTestSuite:

    def main(self):
        self.run_tests()
        # FIXME: currently only works on development settings
        if os.environ.get('RUN_E2E') == 'true' or args.end_to_end_tests:
            self.run_e2e_tests()

    def run_e2e_tests(self):
        with TmpVirtualEnv() as tmp_venv:
            tmp_venv.run_python_script('e2e/e2e_tests.py')

    def run_tests(self):
        self.log_header('=== Development mode ===')

        self.WEBAPP_CONTAINER = "webappdev"
        self.DB_CONTAINER = "databasedev"
        self.COMPOSE_FILE = "compose-development.yml"

        self.setup()
        if args.webapp_checks:
            self.application_checks()
        if args.webapp_tests:
            self.application_tests()

        self.reset()  # FIXME: Don't always reset, only when necessary.

        if args.docker_composition_tests:
            self.reset()

            self.persisting_database_data_tests()

        self.tear_down()

    def setup(self):
        self.docker_compose(['pull'])
        self.reset_containers()

    def reset(self):
        self.reset_containers()

    def tear_down(self):
        self.reset_containers()

    def reset_containers(self):
        self.log_docker_compose('stop -t 0'.split())
        self.log_docker_compose('rm -vf'.split())
        self.log_docker_compose(['build'])
        self.docker_compose(['up', '-d', self.DB_CONTAINER])
        subprocess.check_call('sleep 10'.split())

    def reset_container(self, container_to_be_resetted):
        self.log_docker_compose(['stop', '-t', '0', container_to_be_resetted])
        self.log_docker_compose(['rm', '-vf', container_to_be_resetted])
        self.log_docker_compose(['build', container_to_be_resetted])

    def log_docker_compose(self, arg_list):
        logger.debug(self.docker_compose(arg_list).decode())

    def docker_compose(self, arg_list):
        command_line_list = ['docker-compose', '-f', self.COMPOSE_FILE] + arg_list
        return subprocess.check_output(command_line_list)

    def _log_colored(self, message, color):
        logger.info(''.join([color, message, RESET]))

    def log_failure(self, message):
        self._log_colored(message, RED)

    def log_success(self, message):
        self._log_colored(message, GREEN)

    def log_header(self, title):
        dashed_line = '-' * len(title)
        header = '\n'.join(['', dashed_line, title, dashed_line])
        self._log_colored(header, MAGENTA)

    # ################### CONCRETE TEST IMPLEMENTATIONS ####################

    def application_checks(self):
        # application tests
        self.log_header('Application checks:')

        try:
            self.log_docker_compose(['run', self.WEBAPP_CONTAINER, '/bin/bash', '-c', 'python3 manage.py check'])
            self.log_success("Checks passed successfully.")
        except subprocess.CalledProcessError as e:
            logger.info(e.output.decode())
            self.log_failure("Checks failed. Please have a look at the {logfile}!".format(logfile=LOGFILE))

    def application_tests(self):
        self.log_header('Application tests:')

        try:
            self.log_docker_compose(['run', self.WEBAPP_CONTAINER, '/bin/bash', '-c',
                                     'DJANGO_SETTINGS_MODULE=config.settings.test python3 manage.py test'])
            self.log_success("Tests passed successfully")
        except subprocess.CalledProcessError as e:
            logger.info(e.output.decode())
            self.log_failure("Tests failed. Please have a look at the {logfile};!".format(logfile=LOGFILE))

    def persisting_database_data_tests(self):
        try:
            migration_stdout = self.docker_compose(['run', self.WEBAPP_CONTAINER, 'bash',  '-c',
                                                    './manage.py migrate'])
        except subprocess.CalledProcessError as e:
            logger.info(e.output.decode())
            raise

        if b'Applying excerptexport' in migration_stdout:
            self.log_success("Migrations applied successfully.")
        else:
            self.log_failure("Migrations could not be applied!")<commit_msg>run application tests locally and inform, if env or requirements are missing<commit_after>#!/usr/bin/env python3

import argparse
import logging
import os
import shutil
import subprocess
import sys

# constants
from django.core.management import ManagementUtility

RED = "\033[31m"
GREEN = "\033[32m"
MAGENTA = "\033[95m"
RESET = "\033[0m"

LOGFILE = 'runtests.log'

logger = logging.getLogger(__file__)
logger.setLevel(logging.DEBUG)


class OsmaxxTestSuite:

    def main(self):
        self.run_tests()
        # FIXME: currently only works on development settings
        if os.environ.get('RUN_E2E') == 'true' or args.end_to_end_tests:
            self.run_e2e_tests()

    def run_e2e_tests(self):
        with TmpVirtualEnv() as tmp_venv:
            tmp_venv.run_python_script('e2e/e2e_tests.py')

    def run_tests(self):
        self.log_header('=== Development mode ===')

        self.WEBAPP_CONTAINER = "webappdev"
        self.DB_CONTAINER = "databasedev"
        self.COMPOSE_FILE = "compose-development.yml"

        if args.webapp_checks:
            self.setup()
            self.application_checks()
            self.reset()  # FIXME: Don't always reset, only when necessary.
            self.tear_down()

        if args.webapp_tests:
            self.application_tests()

        if args.docker_composition_tests:
            self.reset()
            self.persisting_database_data_tests()
            self.tear_down()

    def setup(self):
        self.docker_compose(['pull'])
        self.reset_containers()

    def reset(self):
        self.reset_containers()

    def tear_down(self):
        self.reset_containers()

    def reset_containers(self):
        self.log_docker_compose('stop -t 0'.split())
        self.log_docker_compose('rm -vf'.split())
        self.log_docker_compose(['build'])
        self.docker_compose(['up', '-d', self.DB_CONTAINER])
        subprocess.check_call('sleep 10'.split())

    def reset_container(self, container_to_be_resetted):
        self.log_docker_compose(['stop', '-t', '0', container_to_be_resetted])
        self.log_docker_compose(['rm', '-vf', container_to_be_resetted])
        self.log_docker_compose(['build', container_to_be_resetted])

    def log_docker_compose(self, arg_list):
        logger.debug(self.docker_compose(arg_list).decode())

    def docker_compose(self, arg_list):
        command_line_list = ['docker-compose', '-f', self.COMPOSE_FILE] + arg_list
        return subprocess.check_output(command_line_list)

    def _log_colored(self, message, color):
        logger.info(''.join([color, message, RESET]))

    def log_failure(self, message):
        self._log_colored(message, RED)

    def log_success(self, message):
        self._log_colored(message, GREEN)

    def log_header(self, title):
        dashed_line = '-' * len(title)
        header = '\n'.join(['', dashed_line, title, dashed_line])
        self._log_colored(header, MAGENTA)

    # ################### CONCRETE TEST IMPLEMENTATIONS ####################

    def application_checks(self):
        # application tests
        self.log_header('Application checks:')

        try:
            self.log_docker_compose(['run', self.WEBAPP_CONTAINER, '/bin/bash', '-c', 'python3 manage.py check'])
            self.log_success("Checks passed successfully.")
        except subprocess.CalledProcessError as e:
            logger.info(e.output.decode())
            self.log_failure("Checks failed. Please have a look at the {logfile}!".format(logfile=LOGFILE))

    def application_tests(self):
        self.log_header('Application tests:')

        try:
            import django  # noqa
        except ImportError:
            print('Are you in a activated virtualenv and have installed the settings?')
            print('virtualenv --python=/usr/bin/python3 tmp;source ./tmp/bin/activate;\
                pip install -r osmaxx-py/requirements/local.txt')
            return

        try:
            osmaxx_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'osmaxx-py')
            sys.path.append(osmaxx_path)
            os.chdir(osmaxx_path)
            os.environ['DJANGO_SETTINGS_MODULE'] = 'config.settings.test'
            utility = ManagementUtility(['manage.py', 'test'])
            utility.execute()
        except subprocess.CalledProcessError as e:
            logger.info(e.output.decode())
            self.log_failure("Tests failed. Please have a look at the {logfile};!".format(logfile=LOGFILE))

    def persisting_database_data_tests(self):
        try:
            migration_stdout = self.docker_compose(['run', self.WEBAPP_CONTAINER, 'bash',  '-c',
                                                    './manage.py migrate'])
        except subprocess.CalledProcessError as e:
            logger.info(e.output.decode())
            raise

        if b'Applying excerptexport' in migration_stdout:
            self.log_success("Migrations applied successfully.")
        else:
            self.log_failure("Migrations could not be applied!")
{'fraction_non_alphanumeric': 0.12416991672815432, 'fraction_numerical': 0.006535258775166017, 'mean_word_length': 2.696143358005454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5950018', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2839, 'n_words': 553}