<commit_before># Copyright (C) 2013-2016 <NAME>, UC Denver
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR
# A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from fmda.fuel_moisture_da import execute_da_step, retrieve_mesowest_observations
from fmda.fuel_moisture_model import FuelMoistureModel
from ingest.grib_file import GribFile, GribMessage
from ingest.rtma_source import RTMA
from utils import Dict, ensure_dir, utc_to_esmf, delete
from vis.postprocessor import scalar_field_to_raster
from ssh_shuttle import send_product_to_server


import netCDF4
import numpy as np
import json
import sys
import logging
import os
import os.path as osp

from datetime import datetime, timedelta
import pytz


def postprocess_cycle(cycle, region_cfg, wksp_path):
    """
    Build rasters from the computed fuel moisture.

    :param cycle: the UTC cycle time
    :param region_cfg: the region configuration
    :param wksp_path: the workspace path
    :return: the postprocessing path
    """
    model_path = compute_model_path(cycle, region_cfg.code, wksp_path)
    year_month = '%04d%02d' % (cycle.year, cycle.month)
    cycle_dir = 'fmda-%s-%04d%02d%02d-%02d' %  (region_cfg.code, cycle.year, cycle.month, cycle.day, cycle.hour)
    postproc_path = osp.join(wksp_path, year_month, cycle_dir)

    # read in the longitudes and latitudes
    geo_path = osp.join(wksp_path, '%s-geo.nc' % region_cfg.code)
    logging.info('CYCLER reading longitudes and latitudes from NetCDF file %s' % geo_path )
    d = netCDF4.Dataset(geo_path)
    lats = d.variables['XLAT'][:,:]
    lons = d.variables['XLONG'][:,:]
    d.close()

    var_wisdom = {
        'fm' : {
            'native_unit' : '-',
            'colorbar' : '-',
            'colormap' : 'jet_r',
            'scale' : [0.0, 0.4]
        },
        'RH' : {
            'name' : 'Relative humidity',
            'native_unit' : '%',
            'colorbar' : '%',
            'colormap' : 'jet_r',
            'scale' : [0.0, 100.0]
        },
        'TD' : {
            'name' : 'Dew point temperature at 2m',
            'native_unit' : 'K',
            'colorbar' : 'F',
            'colormap' : 'jet',
            'scale' : [270.0, 320.0]
        },
        'T2' : {
            'name' : 'Temperature at 2m',
            'native_unit' : 'K',
            'colorbar' : 'F',
            'colormap' : 'jet',
            'scale' : [270.0, 320.0]
        },
        'PRECIPA' : {
            'name' : 'RTMA precipa',
            'native_unit' : 'kg/m^2/h',
            'colorbar' : 'kg/m^2/h',
            'colormap' : 'jet_r',
            'scale' : [0.0, 10.0]
        },
        'RAIN' : {
            'name' : 'Rain',
            'native_unit' : 'mm/h',
            'colorbar' : 'kg/m^2/h',
            'colormap' : 'jet_r',
            'scale' : [0.0, 10.0]
        },
        'HGT' : {
            'name' : 'Terrain height',
            'native_unit' : 'm',
            'colorbar' : 'm',
            'colormap' : 'jet_r',
            'scale' : [0.0, 5000.0]
        },
    }

    esmf_cycle = utc_to_esmf(cycle) 
    mf = { "1" : {esmf_cycle : {}}}
    manifest_name = 'fmda-%s-%04d%02d%02d-%02d.json' %  (region_cfg.code, cycle.year, cycle.month, cycle.day, cycle.hour)
    ensure_dir(osp.join(postproc_path, manifest_name))
    
    # read and process model variables
    with netCDF4.Dataset(model_path) as d:
        for i,name in [(0, '1-hr FM'), (1, '10-hr FM'), (2, '100-hr FM')]:
            fm_wisdom = var_wisdom['fm']
            fm_wisdom['name'] = '%s fuel moisture' % name
            raster_png, coords, cb_png = scalar_field_to_raster(d.variables['FMC_GC'][:,:,i], lats, lons, fm_wisdom)
            raster_name = 'fmda-%s-raster.png' % name
            cb_name = 'fmda-%s-raster-cb.png' % name
            with open(osp.join(postproc_path, raster_name), 'w') as f:
                f.write(raster_png)
            with open(osp.join(postproc_path, cb_name), 'w') as f:
                f.write(cb_png) <commit_msg>rain scale 2mm<commit_after># Copyright (C) 2013-2016 <NAME>, UC Denver
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR
# A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from fmda.fuel_moisture_da import execute_da_step, retrieve_mesowest_observations
from fmda.fuel_moisture_model import FuelMoistureModel
from ingest.grib_file import GribFile, GribMessage
from ingest.rtma_source import RTMA
from utils import Dict, ensure_dir, utc_to_esmf, delete
from vis.postprocessor import scalar_field_to_raster
from ssh_shuttle import send_product_to_server


import netCDF4
import numpy as np
import json
import sys
import logging
import os
import os.path as osp

from datetime import datetime, timedelta
import pytz


def postprocess_cycle(cycle, region_cfg, wksp_path):
    """
    Build rasters from the computed fuel moisture.

    :param cycle: the UTC cycle time
    :param region_cfg: the region configuration
    :param wksp_path: the workspace path
    :return: the postprocessing path
    """
    model_path = compute_model_path(cycle, region_cfg.code, wksp_path)
    year_month = '%04d%02d' % (cycle.year, cycle.month)
    cycle_dir = 'fmda-%s-%04d%02d%02d-%02d' %  (region_cfg.code, cycle.year, cycle.month, cycle.day, cycle.hour)
    postproc_path = osp.join(wksp_path, year_month, cycle_dir)

    # read in the longitudes and latitudes
    geo_path = osp.join(wksp_path, '%s-geo.nc' % region_cfg.code)
    logging.info('CYCLER reading longitudes and latitudes from NetCDF file %s' % geo_path )
    d = netCDF4.Dataset(geo_path)
    lats = d.variables['XLAT'][:,:]
    lons = d.variables['XLONG'][:,:]
    d.close()

    var_wisdom = {
        'fm' : {
            'native_unit' : '-',
            'colorbar' : '-',
            'colormap' : 'jet_r',
            'scale' : [0.0, 0.4]
        },
        'RH' : {
            'name' : 'Relative humidity',
            'native_unit' : '%',
            'colorbar' : '%',
            'colormap' : 'jet_r',
            'scale' : [0.0, 100.0]
        },
        'TD' : {
            'name' : 'Dew point temperature at 2m',
            'native_unit' : 'K',
            'colorbar' : 'F',
            'colormap' : 'jet',
            'scale' : [270.0, 320.0]
        },
        'T2' : {
            'name' : 'Temperature at 2m',
            'native_unit' : 'K',
            'colorbar' : 'F',
            'colormap' : 'jet',
            'scale' : [270.0, 320.0]
        },
        'PRECIPA' : {
            'name' : 'RTMA precipa',
            'native_unit' : 'kg/m^2/h',
            'colorbar' : 'kg/m^2/h',
            'colormap' : 'jet_r',
            'scale' : [0.0, 2.0]
        },
        'RAIN' : {
            'name' : 'Rain',
            'native_unit' : 'mm/h',
            'colorbar' : 'kg/m^2/h',
            'colormap' : 'jet_r',
            'scale' : [0.0, 2.0]
        },
        'HGT' : {
            'name' : 'Terrain height',
            'native_unit' : 'm',
            'colorbar' : 'm',
            'colormap' : 'jet_r',
            'scale' : [0.0, 5000.0]
        },
    }

    esmf_cycle = utc_to_esmf(cycle) 
    mf = { "1" : {esmf_cycle : {}}}
    manifest_name = 'fmda-%s-%04d%02d%02d-%02d.json' %  (region_cfg.code, cycle.year, cycle.month, cycle.day, cycle.hour)
    ensure_dir(osp.join(postproc_path, manifest_name))
    
    # read and process model variables
    with netCDF4.Dataset(model_path) as d:
        for i,name in [(0, '1-hr FM'), (1, '10-hr FM'), (2, '100-hr FM')]:
            fm_wisdom = var_wisdom['fm']
            fm_wisdom['name'] = '%s fuel moisture' % name
            raster_png, coords, cb_png = scalar_field_to_raster(d.variables['FMC_GC'][:,:,i], lats, lons, fm_wisdom)
            raster_name = 'fmda-%s-raster.png' % name
            cb_name = 'fmda-%s-raster-cb.png' % name
            with open(osp.join(postproc_path, raster_name), 'w') as f:
                f.write(raster_png)
            with open(osp.join(postproc_path, cb_name), 'w') as f:
                f.write(cb_png) 
{'fraction_non_alphanumeric': 0.12417704851615517, 'fraction_numerical': 0.018130254228704547, 'mean_word_length': 2.4706502636203864, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4779988', 'n_tokens_mistral': 3433, 'n_tokens_neox': 3260, 'n_words': 924}