<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2014, GEM Foundation.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program. If not, see
# <https://www.gnu.org/licenses/agpl.html>.

import sys
import logging
import traceback
import collections
import shutil
import tempfile
import psycopg2
import urllib
import urllib2

from openquake.engine import engine
from openquake.engine.db import models as oqe_models

from openquake.server.dbsettings import PLATFORM_DATABASES as DATABASES
from openquake.server.settings import DEBUG

# all the logs are sent to the platform; one would need a different logger
# to discriminate between progress logs and debug logs written in the file
DEFAULT_LOG_LEVEL = 'debug' if DEBUG else 'progress'


# FIXME. Configure logging by using the configuration stored in settings
logger = logging.getLogger(__name__)


class ProgressHandler(logging.Handler):
    """
    A logging handler to update the status of the job as seen
    from the platform.
    """
    def __init__(self, callback_url, calc):
        logging.Handler.__init__(self)
        self.callback_url = callback_url
        self.calc = calc
        self.description = calc.get_param('description')

    def emit(self, record):
        """
        Update the status field on icebox_calculation with the percentage
        """
        update_calculation(
            self.callback_url,
            status=record.getMessage(),
            description=self.description)


def safely_call(func, *args):
    """
    Call the given procedure with the given arguments safely, i.e.
    by trapping the exceptions and logging them.
    """
    try:
        func(*args)
    except Exception as e:
        logger.error(str(e), exc_info=True)


def run_calc(job_id, calc_dir,
             callback_url=None, foreign_calc_id=None,
             dbname="platform", log_file=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_id:
        the ID of the job on the engine
    :param calc_dir:
        the directory with the input files
    :param callback_url:
        the URL to call at the end of the calculation
    :param foreign_calc_id:
        the calculation id on the platform
    :param dbname:
        the platform database name
    """
    job = oqe_models.OqJob.objects.get(pk=job_id)
    update_calculation(callback_url, status="started", engine_id=job_id)

    progress_handler = ProgressHandler(callback_url, job)
    logging.root.addHandler(progress_handler)
    try:
        engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, 'xml,geojson,csv')
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        update_calculation(callback_url, status="failed", einfo=einfo)
        raise
    finally:
        logging.root.removeHandler(progress_handler)
        shutil.rmtree(calc_dir)

    # If requested to, signal job completion and trigger a migration of
    # results.
    if not None in (callback_url, foreign_calc_id):
        _trigger_migration(job, callback_url, foreign_calc_id, dbname)


def _trigger_migration(job, callback_url, foreign_calc_id, dbname="platform"):
    """
    Helper function to initiate a post-calculation migration of results.

    :param OqJob job:
        The job with which the calculation has been run
    :param str callback_url:
        A URL to POST a request to for pulling results out of the
        oq-engine-server.
    :param str foreign_calc_id:
        The id of the foreign calculation
    :param str dbname:
        a key to extract database connection settings from settings.DATABASES
        in order to get a cursor from the platform database<commit_msg>Don't export from the datastore<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2014, GEM Foundation.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program. If not, see
# <https://www.gnu.org/licenses/agpl.html>.

import sys
import logging
import traceback
import collections
import shutil
import tempfile
import psycopg2
import urllib
import urllib2

from openquake.engine import engine
from openquake.engine.db import models as oqe_models

from openquake.server.dbsettings import PLATFORM_DATABASES as DATABASES
from openquake.server.settings import DEBUG

# all the logs are sent to the platform; one would need a different logger
# to discriminate between progress logs and debug logs written in the file
DEFAULT_LOG_LEVEL = 'debug' if DEBUG else 'progress'


# FIXME. Configure logging by using the configuration stored in settings
logger = logging.getLogger(__name__)


class ProgressHandler(logging.Handler):
    """
    A logging handler to update the status of the job as seen
    from the platform.
    """
    def __init__(self, callback_url, calc):
        logging.Handler.__init__(self)
        self.callback_url = callback_url
        self.calc = calc
        self.description = calc.get_param('description')

    def emit(self, record):
        """
        Update the status field on icebox_calculation with the percentage
        """
        update_calculation(
            self.callback_url,
            status=record.getMessage(),
            description=self.description)


def safely_call(func, *args):
    """
    Call the given procedure with the given arguments safely, i.e.
    by trapping the exceptions and logging them.
    """
    try:
        func(*args)
    except Exception as e:
        logger.error(str(e), exc_info=True)


def run_calc(job_id, calc_dir,
             callback_url=None, foreign_calc_id=None,
             dbname="platform", log_file=None):
    """
    Run a calculation given the calculation ID. It is assumed that the
    entire calculation profile is already loaded into the oq-engine database
    and is ready to execute. This function never fails; errors are trapped
    but not logged since the engine already logs them.

    :param job_id:
        the ID of the job on the engine
    :param calc_dir:
        the directory with the input files
    :param callback_url:
        the URL to call at the end of the calculation
    :param foreign_calc_id:
        the calculation id on the platform
    :param dbname:
        the platform database name
    """
    job = oqe_models.OqJob.objects.get(pk=job_id)
    update_calculation(callback_url, status="started", engine_id=job_id)

    progress_handler = ProgressHandler(callback_url, job)
    logging.root.addHandler(progress_handler)
    try:
        engine.run_calc(job, DEFAULT_LOG_LEVEL, log_file, exports='')
    except:  # catch the errors before task spawning
        # do not log the errors, since the engine already does that
        exctype, exc, tb = sys.exc_info()
        einfo = ''.join(traceback.format_tb(tb))
        einfo += '%s: %s' % (exctype.__name__, exc)
        update_calculation(callback_url, status="failed", einfo=einfo)
        raise
    finally:
        logging.root.removeHandler(progress_handler)
        shutil.rmtree(calc_dir)

    # If requested to, signal job completion and trigger a migration of
    # results.
    if not None in (callback_url, foreign_calc_id):
        _trigger_migration(job, callback_url, foreign_calc_id, dbname)


def _trigger_migration(job, callback_url, foreign_calc_id, dbname="platform"):
    """
    Helper function to initiate a post-calculation migration of results.

    :param OqJob job:
        The job with which the calculation has been run
    :param str callback_url:
        A URL to POST a request to for pulling results out of the
        oq-engine-server.
    :param str foreign_calc_id:
        The id of the foreign calculation
    :param str dbname:
        a key to extract database connection settings from settings.DATABASES
        in order to get a cursor from the platform database
{'fraction_non_alphanumeric': 0.06738688629799978, 'fraction_numerical': 0.002353192854850786, 'mean_word_length': 3.213609734114466, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1972095', 'n_tokens_mistral': 2584, 'n_tokens_neox': 2534, 'n_words': 1019}