<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

# 2015.06.26 13:21:50 CEST
"""
Utilities to build a report writer generating a .rst report for a calculation
"""
from __future__ import print_function, unicode_literals
from openquake.baselib.python3compat import decode
import os
import sys
import mock
import time
import logging


from openquake.baselib.general import humansize
from openquake.commonlib import readinput, datastore, source, parallel


def indent(text):
    return '  ' + '\n  '.join(text.splitlines())


class ReportWriter(object):
    """
    A particularly smart view over the datastore
    """
    title = dict(
        params='Parameters',
        inputs='Input files',
        csm_info='Composite source model',
        required_params_per_trt='Required parameters per tectonic region type',
        ruptures_per_trt='Number of ruptures per tectonic region type',
        ruptures_events='Specific information for event based',
        rlzs_assoc='Realizations per (TRT, GSIM)',
        job_info='Informational data',
        biggest_ebr_gmf='Maximum memory allocated for the GMFs',
        avglosses_data_transfer='Estimated data transfer for the avglosses',
        exposure_info='Exposure model',
        short_source_info='Slowest sources',
        task_info='Information about the tasks',
        times_by_source_class='Computation times by source typology',
        performance='Slowest operations',
    )

    def __init__(self, dstore):
        self.dstore = dstore
        self.oq = oq = dstore['oqparam']
        self.text = (decode(oq.description) + '\n' + '=' * len(oq.description))
        info = dstore['job_info']
        dpath = dstore.hdf5path
        mtime = os.path.getmtime(dpath)
        self.text += '\n\n%s:%s updated %s' % (
            info.hostname, decode(dpath), time.ctime(mtime))
        # NB: in the future, the sitecol could be transferred as
        # an array by leveraging the HDF5 serialization protocol in
        # litetask decorator; for the moment however the size of the
        # data to transfer is given by the usual pickle
        sitecol_size = humansize(len(parallel.Pickled(dstore['sitecol'])))
        self.text += '\n\nnum_sites = %d, sitecol = %s' % (
            len(dstore['sitecol']), sitecol_size)

    def add(self, name, obj=None):
        """Add the view named `name` to the report text"""
        title = self.title[name]
        line = '-' * len(title)
        if obj:
            text = '\n::\n\n' + indent(str(obj))
        else:
            text = datastore.view(name, self.dstore)
        self.text + '\n'.join(['\n\n' + title, line, text])

    def make_report(self):
        """Build the report and return a restructed text string"""
        oq, ds = self.oq, self.dstore
        for name in ('params', 'inputs'):
            self.add(name)
        if 'composite_source_model' in ds:
            self.add('csm_info')
            self.add('required_params_per_trt')
        self.add('rlzs_assoc', ds['csm_info'].get_rlzs_assoc())
        if 'composite_source_model' in ds:
            self.add('ruptures_per_trt')
        if 'scenario' not in oq.calculation_mode:
            self.add('job_info')
        if oq.calculation_mode in ('event_based_rupture', 'event_based',
                                   'event_based_risk'):
            self.add('ruptures_events')
        if oq.calculation_mode in ('event_based_risk',):
            self.add('biggest_ebr_gmf')<commit_msg>Restored a += accidentally removed<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

# 2015.06.26 13:21:50 CEST
"""
Utilities to build a report writer generating a .rst report for a calculation
"""
from __future__ import print_function, unicode_literals
from openquake.baselib.python3compat import decode
import os
import sys
import mock
import time
import logging


from openquake.baselib.general import humansize
from openquake.commonlib import readinput, datastore, source, parallel


def indent(text):
    return '  ' + '\n  '.join(text.splitlines())


class ReportWriter(object):
    """
    A particularly smart view over the datastore
    """
    title = dict(
        params='Parameters',
        inputs='Input files',
        csm_info='Composite source model',
        required_params_per_trt='Required parameters per tectonic region type',
        ruptures_per_trt='Number of ruptures per tectonic region type',
        ruptures_events='Specific information for event based',
        rlzs_assoc='Realizations per (TRT, GSIM)',
        job_info='Informational data',
        biggest_ebr_gmf='Maximum memory allocated for the GMFs',
        avglosses_data_transfer='Estimated data transfer for the avglosses',
        exposure_info='Exposure model',
        short_source_info='Slowest sources',
        task_info='Information about the tasks',
        times_by_source_class='Computation times by source typology',
        performance='Slowest operations',
    )

    def __init__(self, dstore):
        self.dstore = dstore
        self.oq = oq = dstore['oqparam']
        self.text = (decode(oq.description) + '\n' + '=' * len(oq.description))
        info = dstore['job_info']
        dpath = dstore.hdf5path
        mtime = os.path.getmtime(dpath)
        self.text += '\n\n%s:%s updated %s' % (
            info.hostname, decode(dpath), time.ctime(mtime))
        # NB: in the future, the sitecol could be transferred as
        # an array by leveraging the HDF5 serialization protocol in
        # litetask decorator; for the moment however the size of the
        # data to transfer is given by the usual pickle
        sitecol_size = humansize(len(parallel.Pickled(dstore['sitecol'])))
        self.text += '\n\nnum_sites = %d, sitecol = %s' % (
            len(dstore['sitecol']), sitecol_size)

    def add(self, name, obj=None):
        """Add the view named `name` to the report text"""
        title = self.title[name]
        line = '-' * len(title)
        if obj:
            text = '\n::\n\n' + indent(str(obj))
        else:
            text = datastore.view(name, self.dstore)
        self.text += '\n'.join(['\n\n' + title, line, text])

    def make_report(self):
        """Build the report and return a restructed text string"""
        oq, ds = self.oq, self.dstore
        for name in ('params', 'inputs'):
            self.add(name)
        if 'composite_source_model' in ds:
            self.add('csm_info')
            self.add('required_params_per_trt')
        self.add('rlzs_assoc', ds['csm_info'].get_rlzs_assoc())
        if 'composite_source_model' in ds:
            self.add('ruptures_per_trt')
        if 'scenario' not in oq.calculation_mode:
            self.add('job_info')
        if oq.calculation_mode in ('event_based_rupture', 'event_based',
                                   'event_based_risk'):
            self.add('ruptures_events')
        if oq.calculation_mode in ('event_based_risk',):
            self.add('biggest_ebr_gmf')
{'fraction_non_alphanumeric': 0.10424112461281868, 'fraction_numerical': 0.007147962830593281, 'mean_word_length': 2.881183541377716, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2214207', 'n_tokens_mistral': 2566, 'n_tokens_neox': 2481, 'n_words': 772}