<commit_before>'''
Processed BCSD-orig 20-year average country-level data by model

Aggregated to impact regions using area weights. Produced from BCSD-originals
using the GCP climate toolbox at https://github.com/ClimateImpactLab/pipelines
'''

from __future__ import absolute_import
import os
import itertools
import logging
from functools import reduce
import xarray as xr
import pandas as pd

from climate.toolbox import (
    load_climate_data,
    weighted_aggregate_grid_to_regions)


__author__ = '<NAME>'
__contact__ = '<EMAIL>'
__version__ = '0.1.0'

BCSD_orig_files = os.path.join(
    '/shares/gcp/sources/BCSD-original/{scenario}/day/atmos/{variable}',
    'r1i1p1/v1.0/{variable}_day_BCSD_{scenario}_r1i1p1_{model}_{year}.nc')

WRITE_PATH = os.path.join(
    '/shares/gcp/outputs/temps/web/gcp/climate/{scenario}/{agglev}/{variable}',
    '{variable}_{agglev}_{aggwt}_{model}_{pername}.nc')

ADDITIONAL_METADATA = dict(
    description=__file__.__doc__,
    author=__author__,
    contact=__contact__,
    version=__version__,
    repo='https://github.com/ClimateImpactLab/pipelines',
    file='/climate/jobs/impactlab-website/bcsd-orig.py',
    execute='climate.jobs.impactlab_website.bcsd_orig.main',
    project='gcp', 
    team='climate',
    geography='hierid',
    weighting='areawt',
    frequency='20yr')


def tasmin_under_32F(ds):
    '''
    Count of days with tasmin under 32F/0C
    '''
    return ds.tasmin.where((ds.tasmin- 273.15) < 0).count(dim='time')


def tasmax_over_95F(ds):
    '''
    Count of days with tasmax over 95F/35C
    '''
    return ds.tasmax.where((ds.tasmax- 273.15) > 35).count(dim='time')


def average_seasonal_temp(ds):
    '''
    Average seasonal tas
    '''
    return ds.tas.groupby('time.season').mean(dim='time')

JOBS = [
    # dict(variable='tasmax', transformation=tasmax_over_95F),
    # dict(variable='tasmin', transformation=tasmin_under_32F),
    dict(variable='tas', transformation=average_seasonal_temp)]

PERIODS = [
    dict(scenario='historical', pername='1986', years=list(range(1986, 2006))),
    dict(scenario='rcp85', pername='2020', years=list(range(2020, 2040))),
    dict(scenario='rcp85', pername='2040', years=list(range(2040, 2060))),
    dict(scenario='rcp85', pername='2080', years=list(range(2080, 2100)))]

MODELS = list(map(lambda x: dict(model=x), [
    'ACCESS1-0',
    'bcc-csm1-1',
    'BNU-ESM',
    'CanESM2',
    'CCSM4',
    'CESM1-BGC',
    'CNRM-CM5',
    'CSIRO-Mk3-6-0',
    'GFDL-CM3',
    'GFDL-ESM2G',
    'GFDL-ESM2M',
    'IPSL-CM5A-LR',
    'IPSL-CM5A-MR',
    'MIROC-ESM-CHEM',
    'MIROC-ESM',
    'MIROC5',
    'MPI-ESM-LR',
    'MPI-ESM-MR',
    'MRI-CGCM3',
    'inmcm4',
    'NorESM1-M']))

AGGREGATIONS = [{'agglev': 'ISO', 'aggwt': 'areawt'}]

ITERATION_COMPONENTS = (JOBS, PERIODS, MODELS, AGGREGATIONS)


def run_job(variable, transformation, years, **kwargs):

    aggwt, agglev = kwargs.get(aggwt, 'popwt'), kwargs.get(agglev, 'hierid')

    # Build job metadata
    metadata = {k: v for k, v in ADDITIONAL_METADATA.items()}
    metadata.update({
        'variable': variable,
        'transformation': transformation,
        'years': '{}-{}'.format(sorted(years)[0], sorted(years)[-1])},
        'aggwt': aggwt,
        'agglev': agglev)
<commit_msg>bug fix<commit_after>'''
Processed BCSD-orig 20-year average country-level data by model

Aggregated to impact regions using area weights. Produced from BCSD-originals
using the GCP climate toolbox at https://github.com/ClimateImpactLab/pipelines
'''

from __future__ import absolute_import
import os
import itertools
import logging
from functools import reduce
import xarray as xr
import pandas as pd

from climate.toolbox import (
    load_climate_data,
    weighted_aggregate_grid_to_regions)


__author__ = '<NAME>'
__contact__ = '<EMAIL>'
__version__ = '0.1.0'

BCSD_orig_files = os.path.join(
    '/shares/gcp/sources/BCSD-original/{scenario}/day/atmos/{variable}',
    'r1i1p1/v1.0/{variable}_day_BCSD_{scenario}_r1i1p1_{model}_{year}.nc')

WRITE_PATH = os.path.join(
    '/shares/gcp/outputs/temps/web/gcp/climate/{scenario}/{agglev}/{variable}',
    '{variable}_{agglev}_{aggwt}_{model}_{pername}.nc')

ADDITIONAL_METADATA = dict(
    description=__file__.__doc__,
    author=__author__,
    contact=__contact__,
    version=__version__,
    repo='https://github.com/ClimateImpactLab/pipelines',
    file='/climate/jobs/impactlab-website/bcsd-orig.py',
    execute='climate.jobs.impactlab_website.bcsd_orig.main',
    project='gcp', 
    team='climate',
    geography='hierid',
    weighting='areawt',
    frequency='20yr')


def tasmin_under_32F(ds):
    '''
    Count of days with tasmin under 32F/0C
    '''
    return ds.tasmin.where((ds.tasmin- 273.15) < 0).count(dim='time')


def tasmax_over_95F(ds):
    '''
    Count of days with tasmax over 95F/35C
    '''
    return ds.tasmax.where((ds.tasmax- 273.15) > 35).count(dim='time')


def average_seasonal_temp(ds):
    '''
    Average seasonal tas
    '''
    return ds.tas.groupby('time.season').mean(dim='time')

JOBS = [
    # dict(variable='tasmax', transformation=tasmax_over_95F),
    # dict(variable='tasmin', transformation=tasmin_under_32F),
    dict(variable='tas', transformation=average_seasonal_temp)]

PERIODS = [
    dict(scenario='historical', pername='1986', years=list(range(1986, 2006))),
    dict(scenario='rcp85', pername='2020', years=list(range(2020, 2040))),
    dict(scenario='rcp85', pername='2040', years=list(range(2040, 2060))),
    dict(scenario='rcp85', pername='2080', years=list(range(2080, 2100)))]

MODELS = list(map(lambda x: dict(model=x), [
    'ACCESS1-0',
    'bcc-csm1-1',
    'BNU-ESM',
    'CanESM2',
    'CCSM4',
    'CESM1-BGC',
    'CNRM-CM5',
    'CSIRO-Mk3-6-0',
    'GFDL-CM3',
    'GFDL-ESM2G',
    'GFDL-ESM2M',
    'IPSL-CM5A-LR',
    'IPSL-CM5A-MR',
    'MIROC-ESM-CHEM',
    'MIROC-ESM',
    'MIROC5',
    'MPI-ESM-LR',
    'MPI-ESM-MR',
    'MRI-CGCM3',
    'inmcm4',
    'NorESM1-M']))

AGGREGATIONS = [{'agglev': 'ISO', 'aggwt': 'areawt'}]

ITERATION_COMPONENTS = (JOBS, PERIODS, MODELS, AGGREGATIONS)


def run_job(variable, transformation, years, **kwargs):

    aggwt, agglev = kwargs.get(aggwt, 'popwt'), kwargs.get(agglev, 'hierid')

    # Build job metadata
    metadata = {k: v for k, v in ADDITIONAL_METADATA.items()}
    metadata.update({
        'variable': variable,
        'transformation': transformation,
        'years': '{}-{}'.format(sorted(years)[0], sorted(years)[-1]),
        'aggwt': aggwt,
        'agglev': agglev})

{'fraction_non_alphanumeric': 0.1653398652786283, 'fraction_numerical': 0.03643600734843846, 'mean_word_length': 3.971841704718417, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '409912', 'n_tokens_mistral': 2711, 'n_tokens_neox': 2504, 'n_words': 415}