<commit_before>#
# Paasmaker - Platform as a Service
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import unittest
import os
import signal
import logging
import tempfile
import uuid
import shutil
import hashlib
import logging
import subprocess

import tornado
import yaml

import configuration
import paasmaker

from pubsub import pub

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class ConfigurationStub(configuration.Configuration):
    """
    A test version of the configuration object, for unit tests.

    This class can handle generating a default configuration, using
    temporary paths, and behaves just like the normal Configuration object -
    except that it can also clean up after itself once the unit test is complete.
    """

    default_config = """
# The port to this test instance is the master port, for testing purposes.
http_port: %(master_port)d
node_token: %(node_token)s
scratch_directory: %(scratch_dir)s
master:
  host: localhost
  port: %(master_port)d

misc_ports:
  minimum: 42700
  maximum: 42799

redis:
  table:
    host: 0.0.0.0
    port: %(router_table_port)d
    managed: true
  # slaveof:
  #   enabled: true
  #   host: localhost
  #   port: 1234
  stats:
    host: 0.0.0.0
    port: %(router_stats_port)d
    managed: true
  jobs:
    host: 0.0.0.0
    port: %(jobs_port)d
    managed: true

plugins:
  - name: paasmaker.service.parameters
    class: paasmaker.pacemaker.service.parameters.ParametersService
    title: Parameters Service
  - name: paasmaker.scm.zip
    class: paasmaker.pacemaker.scm.zip.ZipSCM
    title: Zip file SCM
  - name: paasmaker.prepare.shell
    class: paasmaker.pacemaker.prepare.shell.ShellPrepare
    title: Shell preparer
  - name: paasmaker.runtime.shell
    class: paasmaker.heart.runtime.ShellRuntime
    title: Shell Runtime
  - name: paasmaker.auth.internal
    class: paasmaker.pacemaker.auth.internal.InternalAuth
    title: Internal Authentication

  - name: paasmaker.runtime.php
    class: paasmaker.heart.runtime.PHPRuntime
    title: PHP
  - name: paasmaker.startup.shell
    class: paasmaker.pacemaker.prepare.shell.ShellPrepare
    title: Shell startup
  - name: paasmaker.runtime.shell
    class: paasmaker.heart.runtime.ShellRuntime
    title: Shell Runtime
  - name: paasmaker.scmlist.dummy
    class: paasmaker.pacemaker.controller.scmlist.DummySCMList
    title: Dummy SCM lister
"""

    pacemaker_config = """
pacemaker:
  enabled: true
  super_token: %(super_token)s
  allow_supertoken: true
  dsn: "sqlite:///:memory:"
  cluster_hostname: local.paasmaker.net
  scmlisters:
    - for: paasmaker.scm.zip
      plugins:
        - paasmaker.scmlist.dummy
"""

    heart_config = """
heart:
  enabled: true
"""

    router_config = """
router:
  enabled: true
  stats_log: %(stats_log)s
"""

    def __init__(self, port=42600, modules=[], io_loop=None):
        # Choose filenames and set up example configuration.
        configfile = tempfile.mkstemp()
        self.params = {}

        allocator = paasmaker.util.port.FreePortFinder()

        self.params['node_token'] = <KEY>
        self.params['super_token'] = str(<KEY>
        self.params['scratch_dir'] = tempfile.mkdtemp()
        self.params['master_port'] = port
        self.params['router_table_port'] = allocator.free_in_range(42710, 42799)
        self.params['router_stats_port'] = allocator.free_in_range(42710, 42799)
        self.params['jobs_port'] = allocator.free_in_range(42710, 42799)
        self.params['broker_port'] = allocator.free_in_range(42710, 42799)
        self.params['stats_log'] = "%s/access.log.paasmaker" % self.params['scratch_dir']

        # Create the configuration file.
        configuration = self.default_config % self.params

        if 'pacemaker' in modules:<commit_msg>Set the configuration stub correctly so it shuts down Redis and Nginx processes at the end of unit tests.<commit_after>#
# Paasmaker - Platform as a Service
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import unittest
import os
import signal
import logging
import tempfile
import uuid
import shutil
import hashlib
import logging
import subprocess

import tornado
import yaml

import configuration
import paasmaker

from pubsub import pub

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class ConfigurationStub(configuration.Configuration):
    """
    A test version of the configuration object, for unit tests.

    This class can handle generating a default configuration, using
    temporary paths, and behaves just like the normal Configuration object -
    except that it can also clean up after itself once the unit test is complete.
    """

    default_config = """
# The port to this test instance is the master port, for testing purposes.
http_port: %(master_port)d
node_token: %(node_token)s
scratch_directory: %(scratch_dir)s
master:
  host: localhost
  port: %(master_port)d

misc_ports:
  minimum: 42700
  maximum: 42799

redis:
  table:
    host: 0.0.0.0
    port: %(router_table_port)d
    managed: true
    shutdown: true
  # slaveof:
  #   enabled: true
  #   host: localhost
  #   port: 1234
  stats:
    host: 0.0.0.0
    port: %(router_stats_port)d
    managed: true
    shutdown: true
  jobs:
    host: 0.0.0.0
    port: %(jobs_port)d
    managed: true
    shutdown: true

plugins:
  - name: paasmaker.service.parameters
    class: paasmaker.pacemaker.service.parameters.ParametersService
    title: Parameters Service
  - name: paasmaker.scm.zip
    class: paasmaker.pacemaker.scm.zip.ZipSCM
    title: Zip file SCM
  - name: paasmaker.prepare.shell
    class: paasmaker.pacemaker.prepare.shell.ShellPrepare
    title: Shell preparer
  - name: paasmaker.runtime.shell
    class: paasmaker.heart.runtime.ShellRuntime
    title: Shell Runtime
  - name: paasmaker.auth.internal
    class: paasmaker.pacemaker.auth.internal.InternalAuth
    title: Internal Authentication

  - name: paasmaker.runtime.php
    class: paasmaker.heart.runtime.PHPRuntime
    title: PHP
  - name: paasmaker.startup.shell
    class: paasmaker.pacemaker.prepare.shell.ShellPrepare
    title: Shell startup
  - name: paasmaker.runtime.shell
    class: paasmaker.heart.runtime.ShellRuntime
    title: Shell Runtime
  - name: paasmaker.scmlist.dummy
    class: paasmaker.pacemaker.controller.scmlist.DummySCMList
    title: Dummy SCM lister
"""

    pacemaker_config = """
pacemaker:
  enabled: true
  super_token: %(super_token)s
  allow_supertoken: true
  dsn: "sqlite:///:memory:"
  cluster_hostname: local.paasmaker.net
  scmlisters:
    - for: paasmaker.scm.zip
      plugins:
        - paasmaker.scmlist.dummy
"""

    heart_config = """
heart:
  enabled: true
"""

    router_config = """
router:
  enabled: true
  stats_log: %(stats_log)s
  nginx:
    managed: true
    shutdown: true
"""

    def __init__(self, port=42600, modules=[], io_loop=None):
        # Choose filenames and set up example configuration.
        configfile = tempfile.mkstemp()
        self.params = {}

        allocator = paasmaker.util.port.FreePortFinder()

        self.params['node_token'] = str(<KEY>
        self.params['super_token'] = str(<KEY>
        self.params['scratch_dir'] = tempfile.mkdtemp()
        self.params['master_port'] = port
        self.params['router_table_port'] = allocator.free_in_range(42710, 42799)
        self.params['router_stats_port'] = allocator.free_in_range(42710, 42799)
        self.params['jobs_port'] = allocator.free_in_range(42710, 42799)
        self.params['broker_port'] = allocator.free_in_range(42710, 42799)
        self.params['stats_log'] = "%s/access.log.paasmaker" % self.params['scratch_dir']

        # Create the configuration file.
        configuration = self.default_config % self.params

        if 'pacemaker' in modules:
{'fraction_non_alphanumeric': 0.09764517322155097, 'fraction_numerical': 0.01849340401923314, 'mean_word_length': 3.4352104975396394, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5211558', 'n_tokens_mistral': 2734, 'n_tokens_neox': 2566, 'n_words': 674}