<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""Mixin proxy for risk jobs, and associated Risk Job Mixin decorators."""

from collections import defaultdict
import json
import os

from scipy.stats import norm

from openquake import job
from openquake import kvs
from openquake import logs
from openquake import shapes
from openquake.job import config as job_config
from openquake.job import mixins
from openquake.output import curve
from openquake.output import risk as risk_output
from openquake.parser import exposure
from openquake.parser import vulnerability
from openquake.utils.tasks import check_job_status

from celery.decorators import task

LOG = logs.LOG
BLOCK_SIZE = 100


def preload(fn):
    """Preload decorator."""

    def preloader(self, *args, **kwargs):
        """Define some preliminary steps needed before starting
        the risk processing. The decorator:

        * reads and stores in KVS the assets
        * reads and stores in KVS the vulnerability model
        * splits into blocks and stores in KVS the exposure sites
        """

        self.store_exposure_assets()
        self.store_vulnerability_model()
        self.partition()

        return fn(self, *args, **kwargs)
    return preloader


def output(fn):
    """ Decorator for output """

    def output_writer(self, *args, **kwargs):
        """ Write the output of a block to kvs. """
        fn(self, *args, **kwargs)
        conditional_loss_poes = [float(x) for x in self.params.get(
                    'CONDITIONAL_LOSS_POE', "0.01").split()]

        for block_id in self.blocks_keys:
            #pylint: disable=W0212
            self._write_output_for_block(self.job_id, block_id)

        for loss_poe in conditional_loss_poes:
            path = os.path.join(self.base_path,
                                self['OUTPUT_DIR'],
                                "losses_at-%s.xml" % loss_poe)
            writer = risk_output.create_loss_map_writer(
                self.job_id, self.serialize_results_to, path, False)

            if writer:
                metadata = {
                    "deterministic": False,
                    "poe": loss_poe,
                }

                writer.serialize(
                    [metadata]
                    + self.asset_losses_per_site(
                        loss_poe,
                        self.grid_assets_iterator(self.region.grid)))

    return output_writer


@task
def compute_risk(job_id, block_id, **kwargs):
    """ A task for computing risk, calls the mixed in compute_risk method """
    check_job_status(job_id)
    engine = job.Job.from_kvs(job_id)
    with mixins.Mixin(engine, RiskJobMixin) as mixed:
        return mixed.compute_risk(block_id, **kwargs)


def read_sites_from_exposure(a_job):
    """
    Given the exposure model specified in the job config, read all sites which
    are located within the region of interest.

    :param a_job: a Job object with an EXPOSURE parameter defined
    :type a_job: :py:class:`openquake.job.Job`

    :returns: a list of :py:class:`openquake.shapes.Site` objects
    """

    sites = []
    path = os.path.join(a_job.base_path, a_job.params[job_config.EXPOSURE])

    reader = exposure.ExposurePortfolioFile(path)
    constraint = a_job.region

    a_job.logger.debug("Constraining exposure parsing to %s", constraint)

    for site, _asset_data in reader.filter(constraint):

        # we don't want duplicates (bug 812395):
        if not site in sites:
            sites.append(site)
<commit_msg>Revert "risk/job/general.py uses job-specific logger"<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""Mixin proxy for risk jobs, and associated Risk Job Mixin decorators."""

from collections import defaultdict
import json
import os

from scipy.stats import norm

from openquake import job
from openquake import kvs
from openquake import logs
from openquake import shapes
from openquake.job import config as job_config
from openquake.job import mixins
from openquake.output import curve
from openquake.output import risk as risk_output
from openquake.parser import exposure
from openquake.parser import vulnerability
from openquake.utils.tasks import check_job_status

from celery.decorators import task

LOG = logs.LOG
BLOCK_SIZE = 100


def preload(fn):
    """Preload decorator."""

    def preloader(self, *args, **kwargs):
        """Define some preliminary steps needed before starting
        the risk processing. The decorator:

        * reads and stores in KVS the assets
        * reads and stores in KVS the vulnerability model
        * splits into blocks and stores in KVS the exposure sites
        """

        self.store_exposure_assets()
        self.store_vulnerability_model()
        self.partition()

        return fn(self, *args, **kwargs)
    return preloader


def output(fn):
    """ Decorator for output """

    def output_writer(self, *args, **kwargs):
        """ Write the output of a block to kvs. """
        fn(self, *args, **kwargs)
        conditional_loss_poes = [float(x) for x in self.params.get(
                    'CONDITIONAL_LOSS_POE', "0.01").split()]

        for block_id in self.blocks_keys:
            #pylint: disable=W0212
            self._write_output_for_block(self.job_id, block_id)

        for loss_poe in conditional_loss_poes:
            path = os.path.join(self.base_path,
                                self['OUTPUT_DIR'],
                                "losses_at-%s.xml" % loss_poe)
            writer = risk_output.create_loss_map_writer(
                self.job_id, self.serialize_results_to, path, False)

            if writer:
                metadata = {
                    "deterministic": False,
                    "poe": loss_poe,
                }

                writer.serialize(
                    [metadata]
                    + self.asset_losses_per_site(
                        loss_poe,
                        self.grid_assets_iterator(self.region.grid)))

    return output_writer


@task
def compute_risk(job_id, block_id, **kwargs):
    """ A task for computing risk, calls the mixed in compute_risk method """
    check_job_status(job_id)
    engine = job.Job.from_kvs(job_id)
    with mixins.Mixin(engine, RiskJobMixin) as mixed:
        return mixed.compute_risk(block_id, **kwargs)


def read_sites_from_exposure(a_job):
    """
    Given the exposure model specified in the job config, read all sites which
    are located within the region of interest.

    :param a_job: a Job object with an EXPOSURE parameter defined
    :type a_job: :py:class:`openquake.job.Job`

    :returns: a list of :py:class:`openquake.shapes.Site` objects
    """

    sites = []
    path = os.path.join(a_job.base_path, a_job.params[job_config.EXPOSURE])

    reader = exposure.ExposurePortfolioFile(path)
    constraint = a_job.region

    LOG.debug(
        "Constraining exposure parsing to %s" % constraint)

    for site, _asset_data in reader.filter(constraint):

        # we don't want duplicates (bug 812395):
        if not site in sites:
            sites.append(site)

{'fraction_non_alphanumeric': 0.08026483912184923, 'fraction_numerical': 0.007201765594145661, 'mean_word_length': 2.6794871794871793, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '750469', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2402, 'n_words': 839}