<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.


import unittest
import os

from openquake.shapes import Site
from openquake.input.exposure import ExposureDBWriter
from openquake.output.hazard import GmfDBWriter
from openquake.output.hazard import HazardCurveDBWriter
from openquake.parser.exposure import ExposurePortfolioFile
from openquake.calculators.risk.classical.core import ClassicalRiskCalculator
from openquake.calculators.risk.event_based.core import (
    EventBasedRiskCalculator)
from tests.utils import helpers

TEST_FILE = 'exposure-portfolio.xml'


# See data in output_hazard_unittest.py
def HAZARD_CURVE_DATA():
    return [
        (Site(-122.2, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.354, 0.114, 0.023, 0.002],
          'IMT': 'PGA',
          'statistics': 'mean'}),
        (Site(-122.1, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.454, 0.214, 0.123, 0.102],
          'IMT': 'PGA',
          'statistics': 'mean'}),
        (Site(-122.2, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.354, 0.114, 0.023, 0.002],
          'IMT': 'PGA',
          'statistics': 'quantile',
          'quantileValue': 0.25}),
        (Site(-122.1, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.454, 0.214, 0.123, 0.102],
          'IMT': 'PGA',
          'statistics': 'quantile',
          'quantileValue': 0.25}),
        (Site(-122.2, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.354, 0.114, 0.023, 0.002],
          'IMT': 'PGA',
          'endBranchLabel': '1'}),
        (Site(-122.1, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.454, 0.214, 0.123, 0.102],
          'IMT': 'PGA',
          'endBranchLabel': '1'}),
    ]


def GMF_DATA():
    return [
        {
            Site(-117, 40): {'groundMotion': 0.1},
            Site(-116, 40): {'groundMotion': 0.2},
            Site(-116, 41): {'groundMotion': 0.3},
            Site(-117, 41): {'groundMotion': 0.4},
        },
        {
            Site(-117, 40): {'groundMotion': 0.5},
            Site(-116, 40): {'groundMotion': 0.6},
            Site(-116, 41): {'groundMotion': 0.7},
            Site(-117, 41): {'groundMotion': 0.8},
        },
        {
            Site(-117, 42): {'groundMotion': 1.0},
            Site(-116, 42): {'groundMotion': 1.1},
            Site(-116, 41): {'groundMotion': 1.2},
            Site(-117, 41): {'groundMotion': 1.3},
        },
    ]


class HazardCurveDBReadTestCase(unittest.TestCase, helpers.DbTestCase):
    """
    Test the code to read hazard curves from DB.
    """
    def setUp(self):
        self.job = self.setup_classic_job()
        output_path = self.generate_output_path(self.job)
        hcw = HazardCurveDBWriter(output_path, self.job.id)
        hcw.serialize(HAZARD_CURVE_DATA())

    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def test_read_curve(self):
        """Verify _get_db_curve."""
        the_job = helpers.create_job({}, job_id=self.job.id)
        calculator = ClassicalRiskCalculator(the_job)

        curve1 = calculator._get_db_curve(Site(-122.2, 37.5))
        self.assertEquals(list(curve1.abscissae),
                          [0.005, 0.007, 0.0098, 0.0137])
        self.assertEquals(list(curve1.ordinates),
                          [0.354, 0.114, 0.023, 0.002])

        curve2 = calculator._get_db_curve(Site(-122.1, 37.5))
        self.assertEquals(list(curve2.abscissae),
                          [0.005, 0.007, 0.0098, 0.0137])
        self.assertEquals(list(curve2.ordinates),
                          [0.454, 0.214, 0.123, 0.102])


class GmfDBReadTestCase(unittest.TestCase, helpers.DbTestCase):
    """
    Test the code to read the ground motion fields from DB.
    """
    def setUp(self):
        self.job = self.setup_classic_job()
        for gmf in GMF_DATA():
            output_path = self.generate_output_path(self.job)
            hcw = GmfDBWriter(output_path, self.job.id)
            hcw.serialize(gmf)

    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def test_site_keys(self):
        """Verify _sites_to_gmf_keys"""
        params = {
            'REGION_VERTEX': '40,-117, 42,-117, 42,-116, 40,-116',
            'REGION_GRID_SPACING': '1.0'}

        the_job = helpers.create_job(params, job_id=self.job.id)
        calculator = EventBasedRiskCalculator(the_job)

        keys = calculator._sites_to_gmf_keys([Site(-117, 40), Site(-116, 42)])

        self.assertEquals(["0!0", "2!1"], keys)

    def test_read_gmfs(self):
        """Verify _get_db_gmfs."""
        params = {
            'REGION_VERTEX': '40,-117, 42,-117, 42,-116, 40,-116',
            'REGION_GRID_SPACING': '1.0'}

        the_job = helpers.create_job(params, job_id=self.job.id)
        calculator = EventBasedRiskCalculator(the_job)

        self.assertEquals(3, len(calculator._gmf_db_list(self.job.id)))

        # only the keys in gmfs are used
        gmfs = calculator._get_db_gmfs([], self.job.id)
        self.assertEquals({}, gmfs)

        # only the keys in gmfs are used
        sites = [Site(lon, lat)
                        for lon in xrange(-117, -115)
                        for lat in xrange(40, 43)]
        gmfs = calculator._get_db_gmfs(sites, self.job.id)
        # avoid rounding errors
        for k, v in gmfs.items():
            gmfs[k] = [round(i, 1) for i in v]

        self.assertEquals({
                '0!0': [0.1, 0.5, 0.0],
                '0!1': [0.2, 0.6, 0.0],
                '1!0': [0.4, 0.8, 1.3],
                '1!1': [0.3, 0.7, 1.2],
                '2!0': [0.0, 0.0, 1.0],
                '2!1': [0.0, 0.0, 1.1],
                }, gmfs)


class ExposureDBWriterTestCase(unittest.TestCase, helpers.DbTestCase):
    """
    Test the code to serialize exposure model to DB.
    """
    job = None
    path = os.path.join(helpers.SCHEMA_EXAMPLES_DIR, TEST_FILE)

    @classmethod
    def setUpClass(cls):
        inputs = [("exposure", cls.path)]
        cls.job = cls.setup_classic_job(inputs=inputs)

    @classmethod
    def tearDownClass(cls):
        cls.teardown_job(cls.job)

    def setUp(self):
        self.writer = ExposureDBWriter(self.job.oq_job_profile.input_set,
                                       self.path)

    def test_read_exposure(self):
        parser = ExposurePortfolioFile(self.path)

        # call tested function
        self.writer.serialize(parser)

        # test results
        model = self.writer.model

        self.assertFalse(model is None)

        # check model fields
        self.assertEquals("Collection of existing building in downtown Pavia",
                          model.description)
        self.assertEquals("buildings", model.category)
        self.assertEquals("CHF", model.coco_unit)
        self.assertEquals("EUR", model.reco_unit)
        self.assertEquals("USD", model.stco_unit)

        # check asset instances
        assets = sorted(model.exposuredata_set.all(), key=lambda e: e.value)

        def _to_site(pg_point):
            return Site(pg_point.x, pg_point.y)

        self.assertEquals("asset_01", assets[0].asset_ref)
        self.assertEquals(120, assets[0].area)
        self.assertEquals(12.95, assets[0].coco)
        self.assertEquals(55, assets[0].coco_deductible)
        self.assertEquals(999, assets[0].coco_limit)
        self.assertEquals(7, assets[0].number_of_units)
        self.assertEquals(109876, assets[0].reco)
        self.assertEquals(150000, assets[0].stco)
        self.assertEquals(150000, assets[0].value)
        self.assertEquals("RC/DMRF-D/LR", assets[0].taxonomy)
        self.assertEquals(Site(9.15000, 45.16667), _to_site(assets[0].site))

        self.assertEquals("asset_02", assets[1].asset_ref)
        self.assertEquals(250000, assets[1].value)
        self.assertEquals("RC/DMRF-D/HR", assets[1].taxonomy)
        self.assertEquals(Site(9.15333, 45.12200), _to_site(assets[1].site))

        self.assertEquals("asset_03", assets[2].asset_ref)
        self.assertEquals(500000, assets[2].value)
        self.assertEquals("RC/DMRF-D/LR", assets[2].taxonomy)
        self.assertEquals(Site(9.14777, 45.17999), _to_site(assets[2].site))<commit_msg>even more complete test<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.


import unittest
import os

from openquake.shapes import Site
from openquake.input.exposure import ExposureDBWriter
from openquake.output.hazard import GmfDBWriter
from openquake.output.hazard import HazardCurveDBWriter
from openquake.parser.exposure import ExposurePortfolioFile
from openquake.calculators.risk.classical.core import ClassicalRiskCalculator
from openquake.calculators.risk.event_based.core import (
    EventBasedRiskCalculator)
from tests.utils import helpers

TEST_FILE = 'exposure-portfolio.xml'


# See data in output_hazard_unittest.py
def HAZARD_CURVE_DATA():
    return [
        (Site(-122.2, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.354, 0.114, 0.023, 0.002],
          'IMT': 'PGA',
          'statistics': 'mean'}),
        (Site(-122.1, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.454, 0.214, 0.123, 0.102],
          'IMT': 'PGA',
          'statistics': 'mean'}),
        (Site(-122.2, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.354, 0.114, 0.023, 0.002],
          'IMT': 'PGA',
          'statistics': 'quantile',
          'quantileValue': 0.25}),
        (Site(-122.1, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.454, 0.214, 0.123, 0.102],
          'IMT': 'PGA',
          'statistics': 'quantile',
          'quantileValue': 0.25}),
        (Site(-122.2, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.354, 0.114, 0.023, 0.002],
          'IMT': 'PGA',
          'endBranchLabel': '1'}),
        (Site(-122.1, 37.5),
         {'investigationTimeSpan': '50.0',
          'IMLValues': [0.778, 1.09, 1.52, 2.13],
          'PoEValues': [0.454, 0.214, 0.123, 0.102],
          'IMT': 'PGA',
          'endBranchLabel': '1'}),
    ]


def GMF_DATA():
    return [
        {
            Site(-117, 40): {'groundMotion': 0.1},
            Site(-116, 40): {'groundMotion': 0.2},
            Site(-116, 41): {'groundMotion': 0.3},
            Site(-117, 41): {'groundMotion': 0.4},
        },
        {
            Site(-117, 40): {'groundMotion': 0.5},
            Site(-116, 40): {'groundMotion': 0.6},
            Site(-116, 41): {'groundMotion': 0.7},
            Site(-117, 41): {'groundMotion': 0.8},
        },
        {
            Site(-117, 42): {'groundMotion': 1.0},
            Site(-116, 42): {'groundMotion': 1.1},
            Site(-116, 41): {'groundMotion': 1.2},
            Site(-117, 41): {'groundMotion': 1.3},
        },
    ]


class HazardCurveDBReadTestCase(unittest.TestCase, helpers.DbTestCase):
    """
    Test the code to read hazard curves from DB.
    """
    def setUp(self):
        self.job = self.setup_classic_job()
        output_path = self.generate_output_path(self.job)
        hcw = HazardCurveDBWriter(output_path, self.job.id)
        hcw.serialize(HAZARD_CURVE_DATA())

    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def test_read_curve(self):
        """Verify _get_db_curve."""
        the_job = helpers.create_job({}, job_id=self.job.id)
        calculator = ClassicalRiskCalculator(the_job)

        curve1 = calculator._get_db_curve(Site(-122.2, 37.5))
        self.assertEquals(list(curve1.abscissae),
                          [0.005, 0.007, 0.0098, 0.0137])
        self.assertEquals(list(curve1.ordinates),
                          [0.354, 0.114, 0.023, 0.002])

        curve2 = calculator._get_db_curve(Site(-122.1, 37.5))
        self.assertEquals(list(curve2.abscissae),
                          [0.005, 0.007, 0.0098, 0.0137])
        self.assertEquals(list(curve2.ordinates),
                          [0.454, 0.214, 0.123, 0.102])


class GmfDBReadTestCase(unittest.TestCase, helpers.DbTestCase):
    """
    Test the code to read the ground motion fields from DB.
    """
    def setUp(self):
        self.job = self.setup_classic_job()
        for gmf in GMF_DATA():
            output_path = self.generate_output_path(self.job)
            hcw = GmfDBWriter(output_path, self.job.id)
            hcw.serialize(gmf)

    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def test_site_keys(self):
        """Verify _sites_to_gmf_keys"""
        params = {
            'REGION_VERTEX': '40,-117, 42,-117, 42,-116, 40,-116',
            'REGION_GRID_SPACING': '1.0'}

        the_job = helpers.create_job(params, job_id=self.job.id)
        calculator = EventBasedRiskCalculator(the_job)

        keys = calculator._sites_to_gmf_keys([Site(-117, 40), Site(-116, 42)])

        self.assertEquals(["0!0", "2!1"], keys)

    def test_read_gmfs(self):
        """Verify _get_db_gmfs."""
        params = {
            'REGION_VERTEX': '40,-117, 42,-117, 42,-116, 40,-116',
            'REGION_GRID_SPACING': '1.0'}

        the_job = helpers.create_job(params, job_id=self.job.id)
        calculator = EventBasedRiskCalculator(the_job)

        self.assertEquals(3, len(calculator._gmf_db_list(self.job.id)))

        # only the keys in gmfs are used
        gmfs = calculator._get_db_gmfs([], self.job.id)
        self.assertEquals({}, gmfs)

        # only the keys in gmfs are used
        sites = [Site(lon, lat)
                        for lon in xrange(-117, -115)
                        for lat in xrange(40, 43)]
        gmfs = calculator._get_db_gmfs(sites, self.job.id)
        # avoid rounding errors
        for k, v in gmfs.items():
            gmfs[k] = [round(i, 1) for i in v]

        self.assertEquals({
                '0!0': [0.1, 0.5, 0.0],
                '0!1': [0.2, 0.6, 0.0],
                '1!0': [0.4, 0.8, 1.3],
                '1!1': [0.3, 0.7, 1.2],
                '2!0': [0.0, 0.0, 1.0],
                '2!1': [0.0, 0.0, 1.1],
                }, gmfs)


class ExposureDBWriterTestCase(unittest.TestCase, helpers.DbTestCase):
    """
    Test the code to serialize exposure model to DB.
    """
    job = None
    path = os.path.join(helpers.SCHEMA_EXAMPLES_DIR, TEST_FILE)

    @classmethod
    def setUpClass(cls):
        inputs = [("exposure", cls.path)]
        cls.job = cls.setup_classic_job(inputs=inputs)

    @classmethod
    def tearDownClass(cls):
        cls.teardown_job(cls.job)

    def setUp(self):
        self.writer = ExposureDBWriter(self.job.oq_job_profile.input_set,
                                       self.path)

    def test_read_exposure(self):
        parser = ExposurePortfolioFile(self.path)

        # call tested function
        self.writer.serialize(parser)

        # test results
        model = self.writer.model

        self.assertFalse(model is None)

        # check model fields
        self.assertEquals("Collection of existing building in downtown Pavia",
                          model.description)
        self.assertEquals("buildings", model.category)
        self.assertEquals("CHF", model.coco_unit)
        self.assertEquals("EUR", model.reco_unit)
        self.assertEquals("USD", model.stco_unit)

        # check asset instances
        assets = sorted(model.exposuredata_set.all(), key=lambda e: e.value)

        def _to_site(pg_point):
            return Site(pg_point.x, pg_point.y)

        self.assertEquals("asset_01", assets[0].asset_ref)
        self.assertEquals(120, assets[0].area)
        self.assertEquals(12.95, assets[0].coco)
        self.assertEquals(55, assets[0].coco_deductible)
        self.assertEquals(999, assets[0].coco_limit)
        self.assertEquals(7, assets[0].number_of_units)
        self.assertEquals(109876, assets[0].reco)
        self.assertEquals(150000, assets[0].stco)
        self.assertEquals(150000, assets[0].value)
        self.assertEquals("RC/DMRF-D/LR", assets[0].taxonomy)
        self.assertEquals(Site(9.15000, 45.16667), _to_site(assets[0].site))

        self.assertEquals("asset_02", assets[1].asset_ref)
        self.assertEquals(119, assets[1].area)
        self.assertEquals(21.95, assets[1].coco)
        self.assertEquals(66, assets[1].coco_deductible)
        self.assertEquals(1999, assets[1].coco_limit)
        self.assertEquals(6, assets[1].number_of_units)
        self.assertEquals(205432, assets[1].reco)
        self.assertEquals(250000, assets[1].stco)
        self.assertEquals(250000, assets[1].value)
        self.assertEquals("RC/DMRF-D/HR", assets[1].taxonomy)
        self.assertEquals(Site(9.15333, 45.12200), _to_site(assets[1].site))

        self.assertEquals("asset_03", assets[2].asset_ref)
        self.assertEquals(118, assets[2].area)
        self.assertEquals(30.95, assets[2].coco)
        self.assertEquals(77, assets[2].coco_deductible)
        self.assertEquals(2888, assets[2].coco_limit)
        self.assertEquals(5, assets[2].number_of_units)
        self.assertEquals(495432, assets[2].reco)
        self.assertEquals(500000, assets[2].stco)
        self.assertEquals(500000, assets[2].value)
        self.assertEquals("RC/DMRF-D/LR", assets[2].taxonomy)
        self.assertEquals(Site(9.14777, 45.17999), _to_site(assets[2].site))
{'fraction_non_alphanumeric': 0.14460589171974522, 'fraction_numerical': 0.0686703821656051, 'mean_word_length': 2.32842000662471, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1436875', 'n_tokens_mistral': 7690, 'n_tokens_neox': 6667, 'n_words': 1404}