<commit_before># Copyright 2015 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import json
import logging
import os
import re
import urlparse
from time import sleep

import chronos
import dateutil
import isodate
import monitoring_tools
import service_configuration_lib
from tron import command_context

from paasta_tools.utils import get_config_hash
from paasta_tools.utils import get_docker_url
from paasta_tools.utils import get_paasta_branch
from paasta_tools.utils import get_service_instance_list
from paasta_tools.utils import get_services_for_cluster
from paasta_tools.utils import InstanceConfig
from paasta_tools.utils import InvalidJobNameError
from paasta_tools.utils import load_deployments_json
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PATH_TO_SYSTEM_PAASTA_CONFIG_DIR
from paasta_tools.utils import timeout


# In Marathon spaces are not allowed, in Chronos periods are not allowed.
# In the Chronos docs a space is suggested as the natural separator
SPACER = " "
# Until Chronos supports dots in the job name, we use this separator internally
INTERNAL_SPACER = '.'
# Chronos creates Mesos tasks with an id composed of some arbitrary strings,
# the app's full name, a spacer, and a timestamp. This variable is that
# spacer. Note that we don't control this spacer, i.e. you can't change it
# here and expect the world to change with you. We need to know what it is so
# we can decompose Mesos task ids.
MESOS_TASK_SPACER = ':'

# this is the identifier prepended to a job name for use
# with temporary jobs, like those created by ``chronos_rerun.py``
TMP_JOB_IDENTIFIER = "tmp"

VALID_BOUNCE_METHODS = ['graceful']
PATH_TO_CHRONOS_CONFIG = os.path.join(PATH_TO_SYSTEM_PAASTA_CONFIG_DIR, 'chronos.json')
DEFAULT_SOA_DIR = service_configuration_lib.DEFAULT_SOA_DIR
log = logging.getLogger('__main__')


class LastRunState:
    """Cheap enum to represent the state of the last run"""
    Success, Fail, NotRun = range(0, 3)


class JobType:
    """ An enum to represent job types """
    Dependent, Scheduled = range(0, 2)


class ChronosNotConfigured(Exception):
    pass


class InvalidParentError(Exception):
    pass


class ChronosConfig(dict):

    def __init__(self, config, path):
        self.path = path
        super(ChronosConfig, self).__init__(config)

    def get_url(self):
        """:returns: The Chronos API endpoint"""
        try:
            return self['url']
        except KeyError:
            raise ChronosNotConfigured('Could not find chronos url in system chronos config: %s' % self.path)

    def get_username(self):
        """:returns: The Chronos API username"""
        try:
            return self['user']
        except KeyError:
            raise ChronosNotConfigured('Could not find chronos user in system chronos config: %s' % self.path)

    def get_password(self):
        """:returns: The Chronos API password"""
        try:
            return self['password']
        except KeyError:
            raise ChronosNotConfigured('Could not find chronos password in system chronos config: %s' % self.path)


def load_chronos_config(path=PATH_TO_CHRONOS_CONFIG):
    try:
        with open(path) as f:
            return ChronosConfig(json.load(f), path)
    except IOError as e:
        raise ChronosNotConfigured("Could not load chronos config file %s: %s" % (e.filename, e.strerror))


def get_chronos_client(config):
    """Returns a chronos client object for interacting with the API"""
    chronos_hosts = config.get_url()
    chronos_hostnames = [urlparse.urlsplit(hostname).netloc for hostname in chronos_hosts]
    log.info("Attempting to connect to Chronos servers: %s" % chronos_hosts)
    return chronos.connect(servers=chronos_hostnames,
                           username=config.get_username(),
                           password=config.get_password())


def compose_job_id(service, instance):
    """In chronos, a job is just service{SPACER}instance """
    return "%s%s%s" % (service, SPACER, instance)


def decompose_job_id(job_id):
    """ A custom implementation of utils.decompose_job_id, accounting for the
    possiblity of TMP_JOB_IDENTIFIER being prepended to the job name """
    decomposed = job_id.split(SPACER)
    if len(decomposed) == 3:
        if decomposed[0] != TMP_JOB_IDENTIFIER:
            raise InvalidJobNameError('invalid job id %s' % job_id)
        else:
            return (decomposed[1], decomposed[2])
    elif len(decomposed) == 2:
        return (decomposed[0], decomposed[1])
    else:
        raise InvalidJobNameError('invalid job id %s' % job_id)


class InvalidChronosConfigError(Exception):
    pass


class UnknownChronosJobError(Exception):
    pass


def read_chronos_jobs_for_service(service, cluster, soa_dir=DEFAULT_SOA_DIR):
    chronos_conf_file = 'chronos-%s' % cluster
    log.info("Reading Chronos configuration file: %s/%s/chronos-%s.yaml" % (soa_dir, service, cluster))

    return service_configuration_lib.read_extra_service_information(
        service,
        chronos_conf_file,
        soa_dir=soa_dir<commit_msg>decompose_job_id asserts that a job startswith the right identifier, rather than exact match<commit_after># Copyright 2015 Yelp Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import json
import logging
import os
import re
import urlparse
from time import sleep

import chronos
import dateutil
import isodate
import monitoring_tools
import service_configuration_lib
from tron import command_context

from paasta_tools.utils import get_config_hash
from paasta_tools.utils import get_docker_url
from paasta_tools.utils import get_paasta_branch
from paasta_tools.utils import get_service_instance_list
from paasta_tools.utils import get_services_for_cluster
from paasta_tools.utils import InstanceConfig
from paasta_tools.utils import InvalidJobNameError
from paasta_tools.utils import load_deployments_json
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PATH_TO_SYSTEM_PAASTA_CONFIG_DIR
from paasta_tools.utils import timeout


# In Marathon spaces are not allowed, in Chronos periods are not allowed.
# In the Chronos docs a space is suggested as the natural separator
SPACER = " "
# Until Chronos supports dots in the job name, we use this separator internally
INTERNAL_SPACER = '.'
# Chronos creates Mesos tasks with an id composed of some arbitrary strings,
# the app's full name, a spacer, and a timestamp. This variable is that
# spacer. Note that we don't control this spacer, i.e. you can't change it
# here and expect the world to change with you. We need to know what it is so
# we can decompose Mesos task ids.
MESOS_TASK_SPACER = ':'

# this is the identifier prepended to a job name for use
# with temporary jobs, like those created by ``chronos_rerun.py``
TMP_JOB_IDENTIFIER = "tmp"

VALID_BOUNCE_METHODS = ['graceful']
PATH_TO_CHRONOS_CONFIG = os.path.join(PATH_TO_SYSTEM_PAASTA_CONFIG_DIR, 'chronos.json')
DEFAULT_SOA_DIR = service_configuration_lib.DEFAULT_SOA_DIR
log = logging.getLogger('__main__')


class LastRunState:
    """Cheap enum to represent the state of the last run"""
    Success, Fail, NotRun = range(0, 3)


class JobType:
    """ An enum to represent job types """
    Dependent, Scheduled = range(0, 2)


class ChronosNotConfigured(Exception):
    pass


class InvalidParentError(Exception):
    pass


class ChronosConfig(dict):

    def __init__(self, config, path):
        self.path = path
        super(ChronosConfig, self).__init__(config)

    def get_url(self):
        """:returns: The Chronos API endpoint"""
        try:
            return self['url']
        except KeyError:
            raise ChronosNotConfigured('Could not find chronos url in system chronos config: %s' % self.path)

    def get_username(self):
        """:returns: The Chronos API username"""
        try:
            return self['user']
        except KeyError:
            raise ChronosNotConfigured('Could not find chronos user in system chronos config: %s' % self.path)

    def get_password(self):
        """:returns: The Chronos API password"""
        try:
            return self['password']
        except KeyError:
            raise ChronosNotConfigured('Could not find chronos password in system chronos config: %s' % self.path)


def load_chronos_config(path=PATH_TO_CHRONOS_CONFIG):
    try:
        with open(path) as f:
            return ChronosConfig(json.load(f), path)
    except IOError as e:
        raise ChronosNotConfigured("Could not load chronos config file %s: %s" % (e.filename, e.strerror))


def get_chronos_client(config):
    """Returns a chronos client object for interacting with the API"""
    chronos_hosts = config.get_url()
    chronos_hostnames = [urlparse.urlsplit(hostname).netloc for hostname in chronos_hosts]
    log.info("Attempting to connect to Chronos servers: %s" % chronos_hosts)
    return chronos.connect(servers=chronos_hostnames,
                           username=config.get_username(),
                           password=config.get_password())


def compose_job_id(service, instance):
    """In chronos, a job is just service{SPACER}instance """
    return "%s%s%s" % (service, SPACER, instance)


def decompose_job_id(job_id):
    """ A custom implementation of utils.decompose_job_id, accounting for the
    possiblity of TMP_JOB_IDENTIFIER being prepended to the job name """
    decomposed = job_id.split(SPACER)
    if len(decomposed) == 3:
        if not decomposed[0].startswith(TMP_JOB_IDENTIFIER):
            raise InvalidJobNameError('invalid job id %s' % job_id)
        else:
            return (decomposed[1], decomposed[2])
    elif len(decomposed) == 2:
        return (decomposed[0], decomposed[1])
    else:
        raise InvalidJobNameError('invalid job id %s' % job_id)


class InvalidChronosConfigError(Exception):
    pass


class UnknownChronosJobError(Exception):
    pass


def read_chronos_jobs_for_service(service, cluster, soa_dir=DEFAULT_SOA_DIR):
    chronos_conf_file = 'chronos-%s' % cluster
    log.info("Reading Chronos configuration file: %s/%s/chronos-%s.yaml" % (soa_dir, service, cluster))

    return service_configuration_lib.read_extra_service_information(
        service,
        chronos_conf_file,
        soa_dir=soa_dir
{'fraction_non_alphanumeric': 0.07650464993858572, 'fraction_numerical': 0.0033339182312686435, 'mean_word_length': 3.89017589017589, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1658590', 'n_tokens_mistral': 3424, 'n_tokens_neox': 3315, 'n_words': 1125}