<commit_before># Copyright 2016 Pinterest, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#  
#     http://www.apache.org/licenses/LICENSE-2.0
#    
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
from random import randrange
import time
import traceback
import daemon
import logging

from deployd.client.client import Client
from deployd.client.serverless_client import ServerlessClient
from deployd.common.config import Config
from deployd.common.exceptions import AgentException
from deployd.common.helper import Helper
from deployd.common.single_instance import SingleInstance
from deployd.common.env_status import EnvStatus
from deployd.common import utils
from deployd.common.executor import Executor
from deployd.common.types import DeployReport, PingStatus, DeployStatus, OpCode, \
    DeployStage, AgentStatus
from deployd import IS_PINTEREST

log = logging.getLogger(__name__)


class PingServer(object):
    def __init__(self, ag):
        self._agent = ag

    def __call__(self, deploy_report):
        return self._agent.update_deploy_status(deploy_report=deploy_report)

class AgentRunMode(object):
    SERVERLESS = "serverless"

    @staticmethod
    def is_serverless(mode):
        return AgentRunMode.SERVERLESS == mode

class DeployAgent(object):
    _STATUS_FILE = None
    _curr_report = None
    _config = None
    _env_status = None

    def __init__(self, client, estatus=None, conf=None, executor=None, helper=None):
        self._response = None
        # a map maintains env_name -> deploy_status
        self._envs = {}
        self._config = conf or Config()
        self._executor = executor
        self._helper = helper or Helper(self._config)
        self._STATUS_FILE = self._config.get_env_status_fn()
        self._client = client
        self._env_status = estatus or EnvStatus(self._STATUS_FILE)
        # load environment deploy status file from local disk
        self.load_status_file()

    def load_status_file(self):
        self._envs = self._env_status.load_envs()
        if not self._envs:
            self._envs = {}
            self._curr_report = None
            return

        self._curr_report = self._envs.values()[0]
        self._config.update_variables(self._curr_report)

    def serve_build(self):
        """This is the main function of the ``DeployAgent``.
        """
        log.info('The deploy agent is starting.')
        if not self._executor:
            self._executor = Executor(callback=PingServer(self), config=self._config)

        # start to ping server to get the latest deploy goal
        response = self._client.send_reports(self._envs)

        self._response = response
        if self._response:
            report = self._update_internal_deploy_goal(self._response)
            # failed to update
            if report.status_code != AgentStatus.SUCCEEDED:
                self._update_ping_reports(deploy_report=report)
                self._client.send_reports(self._envs)
                return

        while self._response and self._response.opCode and self._response.opCode != OpCode.NOOP:
            try:
                # update the current deploy goal
                if self._response.deployGoal:
                    deploy_report = self.process_deploy(self._response)
                else:
                    log.info('No new deploy goal to get updated')
                    deploy_report = DeployReport(AgentStatus.SUCCEEDED)

                if deploy_report.status_code == AgentStatus.ABORTED_BY_SERVER:
                    log.info('switch to the new deploy goal: {}'.format(self._response.deployGoal))
                    continue

            except Exception:
                # anything catch-up here should be treated as agent failure
                deploy_report = DeployReport(status_code=AgentStatus.AGENT_FAILED,
                                             error_code=1,
                                             output_msg=traceback.format_exc(),
                                             retry_times=1)

            self.update_deploy_status(deploy_report)
            if deploy_report.status_code in [AgentStatus.AGENT_FAILED,
                                             AgentStatus.TOO_MANY_RETRY,
                                             AgentStatus.SCRIPT_TIMEOUT]:
                log.error('Unexpeted exceptions: {}, error message {}'.format(
                    deploy_report.status_code, deploy_report.output_msg))
                return

        self.clean_stale_builds()
        if self._response and self._response.deployGoal:
            self._update_internal_deploy_goal(self._response)

        if self._response:
            log.info('Complete the current deploy with response: {}.'.format(self._response))
        else:
            log.info('Failed to get response from server, exit.')

    def serve_forever(self):
        log.info("Running deploy agent in daemon mode")
        while True:
            try:
                self.serve_build()
            except:
                log.exception("Deploy Agent got exception: {}".format(traceback.format_exc()))
            finally:
                time.sleep(self._config.get_daemon_sleep_time())
                self.load_status_file()


    def serve_once(self):
        log.info("Running deploy agent in non daemon mode")
        try:
            # randomly sleep some time before pinging server
            sleep_secs = randrange(10)
            log.info("Randomly sleep {} seconds before starting.".format(sleep_secs))
            time.sleep(sleep_secs)
            self.serve_build()
        except Exception:
            log.exception("Deploy Agent got exceptions: {}".format(traceback.format_exc()))

    def process_deploy(self, response):
        op_code = response.opCode
        deploy_goal = response.deployGoal
        if op_code == OpCode.TERMINATE or op_code == OpCode.DELETE:
            if deploy_goal.envName in self._envs:
                del self._envs[deploy_goal.envName]
            else:
                log.info('Cannot find env {} in the ping report'.format(deploy_goal.env_name))

            if self._curr_report.report.envName == deploy_goal.envName:
                self._curr_report = None

            return DeployReport(AgentStatus.SUCCEEDED, retry_times=1)
        else:
            curr_stage = deploy_goal.deployStage
            '''
            DOWNLOADING and STAGING are two reserved deploy stages owned by agent:
            DOWNLOADING: download the tarball from pinrepo
            STAGING: In this step, deploy agent will chmod and change the symlink pointing to
              new service code, and etc.
            '''
            if curr_stage == DeployStage.DOWNLOADING:<commit_msg>fix infinitely loop when server return op code DELETE (#261)<commit_after># Copyright 2016 Pinterest, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
from random import randrange
import time
import traceback
import daemon
import logging

from deployd.client.client import Client
from deployd.client.serverless_client import ServerlessClient
from deployd.common.config import Config
from deployd.common.exceptions import AgentException
from deployd.common.helper import Helper
from deployd.common.single_instance import SingleInstance
from deployd.common.env_status import EnvStatus
from deployd.common import utils
from deployd.common.executor import Executor
from deployd.common.types import DeployReport, PingStatus, DeployStatus, OpCode, \
    DeployStage, AgentStatus
from deployd import IS_PINTEREST

log = logging.getLogger(__name__)


class PingServer(object):
    def __init__(self, ag):
        self._agent = ag

    def __call__(self, deploy_report):
        return self._agent.update_deploy_status(deploy_report=deploy_report)

class AgentRunMode(object):
    SERVERLESS = "serverless"

    @staticmethod
    def is_serverless(mode):
        return AgentRunMode.SERVERLESS == mode

class DeployAgent(object):
    _STATUS_FILE = None
    _curr_report = None
    _config = None
    _env_status = None

    def __init__(self, client, estatus=None, conf=None, executor=None, helper=None):
        self._response = None
        # a map maintains env_name -> deploy_status
        self._envs = {}
        self._config = conf or Config()
        self._executor = executor
        self._helper = helper or Helper(self._config)
        self._STATUS_FILE = self._config.get_env_status_fn()
        self._client = client
        self._env_status = estatus or EnvStatus(self._STATUS_FILE)
        # load environment deploy status file from local disk
        self.load_status_file()

    def load_status_file(self):
        self._envs = self._env_status.load_envs()
        if not self._envs:
            self._envs = {}
            self._curr_report = None
            return

        self._curr_report = self._envs.values()[0]
        self._config.update_variables(self._curr_report)

    def serve_build(self):
        """This is the main function of the ``DeployAgent``.
        """
        log.info('The deploy agent is starting.')
        if not self._executor:
            self._executor = Executor(callback=PingServer(self), config=self._config)

        # start to ping server to get the latest deploy goal
        response = self._client.send_reports(self._envs)

        self._response = response
        if self._response:
            report = self._update_internal_deploy_goal(self._response)
            # failed to update
            if report.status_code != AgentStatus.SUCCEEDED:
                self._update_ping_reports(deploy_report=report)
                self._client.send_reports(self._envs)
                return

        while self._response and self._response.opCode and self._response.opCode != OpCode.NOOP:
            if self._response.opCode == OpCode.DELETE:
                log.info('Could not handle DELETE opreation currently, bail out.')
                break

            try:
                # update the current deploy goal
                if self._response.deployGoal:
                    deploy_report = self.process_deploy(self._response)
                else:
                    log.info('No new deploy goal to get updated')
                    deploy_report = DeployReport(AgentStatus.SUCCEEDED)

                if deploy_report.status_code == AgentStatus.ABORTED_BY_SERVER:
                    log.info('switch to the new deploy goal: {}'.format(self._response.deployGoal))
                    continue

            except Exception:
                # anything catch-up here should be treated as agent failure
                deploy_report = DeployReport(status_code=AgentStatus.AGENT_FAILED,
                                             error_code=1,
                                             output_msg=traceback.format_exc(),
                                             retry_times=1)

            self.update_deploy_status(deploy_report)
            if deploy_report.status_code in [AgentStatus.AGENT_FAILED,
                                             AgentStatus.TOO_MANY_RETRY,
                                             AgentStatus.SCRIPT_TIMEOUT]:
                log.error('Unexpeted exceptions: {}, error message {}'.format(
                    deploy_report.status_code, deploy_report.output_msg))
                return

        self.clean_stale_builds()
        if self._response and self._response.deployGoal:
            self._update_internal_deploy_goal(self._response)

        if self._response:
            log.info('Complete the current deploy with response: {}.'.format(self._response))
        else:
            log.info('Failed to get response from server, exit.')

    def serve_forever(self):
        log.info("Running deploy agent in daemon mode")
        while True:
            try:
                self.serve_build()
            except:
                log.exception("Deploy Agent got exception: {}".format(traceback.format_exc()))
            finally:
                time.sleep(self._config.get_daemon_sleep_time())
                self.load_status_file()


    def serve_once(self):
        log.info("Running deploy agent in non daemon mode")
        try:
            # randomly sleep some time before pinging server
            sleep_secs = randrange(10)
            log.info("Randomly sleep {} seconds before starting.".format(sleep_secs))
            time.sleep(sleep_secs)
            self.serve_build()
        except Exception:
            log.exception("Deploy Agent got exceptions: {}".format(traceback.format_exc()))

    def process_deploy(self, response):
        op_code = response.opCode
        deploy_goal = response.deployGoal
        if op_code == OpCode.TERMINATE or op_code == OpCode.DELETE:
            if deploy_goal.envName in self._envs:
                # TODO FIXME this does not work since envName could be None
                del self._envs[deploy_goal.envName]
            else:
                log.info('Cannot find env {} in the ping report'.format(deploy_goal.env_name))

            if self._curr_report.report.envName == deploy_goal.envName:
                self._curr_report = None

            return DeployReport(AgentStatus.SUCCEEDED, retry_times=1)
        else:
            curr_stage = deploy_goal.deployStage
            '''
            DOWNLOADING and STAGING are two reserved deploy stages owned by agent:
            DOWNLOADING: download the tarball from pinrepo
            STAGING: In this step, deploy agent will chmod and change the symlink pointing to
              new service code, and etc.
            '''
            if curr_stage == DeployStage.DOWNLOADING:
{'fraction_non_alphanumeric': 0.07374125394429963, 'fraction_numerical': 0.00212649197420771, 'mean_word_length': 2.2204550474928206, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '97137', 'n_tokens_mistral': 3877, 'n_tokens_neox': 3709, 'n_words': 1049}