<commit_before># ===============================================================================
# Copyright 2017 ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
# ============= standard library imports ========================
# ============= local library imports  ==========================
import os

from affine import Affine

from app.paths import paths, PathsNotSetExecption
from recharge.dynamic_raster_finder import get_prisms, get_geo, get_individ_ndvi
from recharge.raster import Raster
from recharge.raster_tools import get_tiff_transform_func, get_tiff_transform
from recharge.tools import day_generator


def extract_prism(day, out):
    if not os.path.isdir(out):
        os.makedirs(out)

    keys = ('min_temp', 'max_temp', 'temp', 'precip')

    geo = get_geo(day)
    mask_arr = Raster(paths.mask).as_bool_array
    # nr,nc = mask_arr.shape
    startr, endr = None, None
    for i, ri in enumerate(mask_arr):
        if ri.any():
            if startr is None:
                startr = i
        elif startr is not None:
            endr = i
            break

    geo['rows'] = endr - startr
    startc, endc = None, None
    for i, ri in enumerate(mask_arr.T):
        if ri.any():
            if startc is None:
                startc = i
        elif startc is not None:
            endc = i
            break

    geo['cols'] = endc - startc

    transform = get_tiff_transform(paths.mask)
    transform *= Affine.translation(startc, startr)
    geo['geotransform'] = transform.to_gdal()

    for k, arr in zip(keys, get_prisms(day)):

        # skip temp
        if k == 'temp':
            continue

        raster = Raster.fromarray(arr)
        marr = raster.unmasked()
        marr = marr[slice(startr, endr), slice(startc, endc)]
        marr = marr * arr
        timestamp = day.strftime('%m_%d_%Y')
        if k == 'precip':
            subpath = os.path.join('PRISM','precip','800m_std_all')
        elif k == 'max_temp':
            subpath = os.path.join('PRISM','Temp','Maximum_standard')
        elif k == 'min_temp':
            subpath = os.path.join('PRISM','Temp','Minimum_standard')

        p = os.path.join(out, subpath, '{}_{}.tif'.format(k, timestamp))

        raster.save(p, marr, geo)


def extract_penman(day, out):
    pass


def extract_ndvi(day, out):
    if not os.path.isdir(out):
        os.makedirs(out)

    geo = get_geo(day)
    mask_arr = Raster(paths.mask).as_bool_array
    # nr,nc = mask_arr.shape
    startr, endr = None, None
    for i, ri in enumerate(mask_arr):
        if ri.any():
            if startr is None:
                startr = i
        elif startr is not None:
            endr = i
            break

    geo['rows'] = endr - startr
    startc, endc = None, None
    for i, ri in enumerate(mask_arr.T):
        if ri.any():
            if startc is None:
                startc = i
        elif startc is not None:
            endc = i
            break

    geo['cols'] = endc - startc

    transform = get_tiff_transform(paths.mask)
    transform *= Affine.translation(startc, startr)
    geo['geotransform'] = transform.to_gdal()

    arr = get_individ_ndvi(day)
    raster = Raster.fromarray(arr)
    marr = raster.unmasked()
    marr = marr[slice(startr, endr), slice(startc, endc)]
    marr = marr * arr
    print marr
    timestamp = day.strftime('%m_%d_%Y')
    year = str(day.year)
    p = os.path.join(out, 'NDVI', 'NDVI', year, '{}_{}.tif'.format('ndvi', timestamp))

    raster.save(p, marr, geo)

def generate_dataset(daterange, out):
    if not paths.is_set():
        raise PathsNotSetExecption

    # extract_initial
    # extract_static

    for day in day_generator(*daterange):
        extract_prism(day, out)
        extract_ndvi(day, out)
        # extract_penman(day, out)


if __name__ == '__main__':<commit_msg>Add extract_PM & extract_initial capability to generate_dataset, used for clipping down rasters<commit_after># ===============================================================================
# Copyright 2017 ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
# ============= standard library imports ========================
# ============= local library imports  ==========================
import os

from affine import Affine

from app.paths import paths, PathsNotSetExecption
from recharge.dynamic_raster_finder import get_prisms, get_geo, get_individ_ndvi, get_penman
from recharge.raster import Raster
from recharge.raster_tools import get_tiff_transform_func, get_tiff_transform
from recharge.tools import day_generator
from recharge.dict_setup import make_pairs
from recharge import STATIC_KEYS, INITIAL_KEYS


def extract_prism(day, out):
    keys = ('min_temp', 'max_temp', 'temp', 'precip')

    geo = get_geo(day)
    mask_arr = Raster(paths.mask).as_bool_array
    # nr,nc = mask_arr.shape
    startr, endr = None, None
    for i, ri in enumerate(mask_arr):
        if ri.any():
            if startr is None:
                startr = i
        elif startr is not None:
            endr = i
            break

    geo['rows'] = endr - startr
    startc, endc = None, None
    for i, ri in enumerate(mask_arr.T):
        if ri.any():
            if startc is None:
                startc = i
        elif startc is not None:
            endc = i
            break

    geo['cols'] = endc - startc

    transform = get_tiff_transform(paths.mask)
    transform *= Affine.translation(startc, startr)
    geo['geotransform'] = transform.to_gdal()

    for k, arr in zip(keys, get_prisms(day)):

        # skip temp
        if k == 'temp':
            continue

        raster = Raster.fromarray(arr)
        marr = raster.unmasked()
        marr = marr[slice(startr, endr), slice(startc, endc)]
        marr = marr * arr
        timestamp = day.strftime('%m_%d_%Y')
        subpath = ''
        if k == 'precip':
            subpath = os.path.join('PRISM','precip','800m_std_all')
        elif k == 'max_temp':
            subpath = os.path.join('PRISM','Temp','Maximum_standard')
        elif k == 'min_temp':
            subpath = os.path.join('PRISM','Temp','Minimum_standard')

        p = os.path.join(out, subpath)
        if not os.path.isdir(p):
            os.makedirs(p)
        p = os.path.join(p, '{}_{}.tif'.format(k, timestamp))

        raster.save(p, marr, geo)


def extract_penman(day, out):
    keys = ('etrs', 'rg')

    geo = get_geo(day)
    mask_arr = Raster(paths.mask).as_bool_array
    # nr,nc = mask_arr.shape
    startr, endr = None, None
    for i, ri in enumerate(mask_arr):
        if ri.any():
            if startr is None:
                startr = i
        elif startr is not None:
            endr = i
            break

    geo['rows'] = endr - startr
    startc, endc = None, None
    for i, ri in enumerate(mask_arr.T):
        if ri.any():
            if startc is None:
                startc = i
        elif startc is not None:
            endc = i
            break

    geo['cols'] = endc - startc

    transform = get_tiff_transform(paths.mask)
    transform *= Affine.translation(startc, startr)
    geo['geotransform'] = transform.to_gdal()

    for k in keys:
        arr = get_penman(day, k)

        raster = Raster.fromarray(arr)
        marr = raster.unmasked()
        marr = marr[slice(startr, endr), slice(startc, endc)]
        marr = marr * arr

        year = str(day.year)
        yday = day.timetuple().tm_yday
        if k == 'etrs':
            p = os.path.join(out, 'PM_RAD', '{}{}'.format('PM', year))
            name = '{}_{}_{:03n}.tif'.format('PM_NM', year, yday)
        elif k == 'rg':
            p = os.path.join(out, 'PM_RAD', '{}{}'.format('rad', year))
            name = '{}_{}_{:03n}.tif'.format('RTOT', year, yday)

        if not os.path.isdir(p):
            os.makedirs(p)
        p = os.path.join(p, name)

        raster.save(p, marr, geo)


def extract_ndvi(day, out):
    geo = get_geo(day)
    mask_arr = Raster(paths.mask).as_bool_array
    # nr,nc = mask_arr.shape
    startr, endr = None, None
    for i, ri in enumerate(mask_arr):
        if ri.any():
            if startr is None:
                startr = i
        elif startr is not None:
            endr = i
            break

    geo['rows'] = endr - startr
    startc, endc = None, None
    for i, ri in enumerate(mask_arr.T):
        if ri.any():
            if startc is None:
                startc = i
        elif startc is not None:
            endc = i
            break

    geo['cols'] = endc - startc

    transform = get_tiff_transform(paths.mask)
    transform *= Affine.translation(startc, startr)
    geo['geotransform'] = transform.to_gdal()

    arr = get_individ_ndvi(day)
    raster = Raster.fromarray(arr)
    marr = raster.unmasked()
    marr = marr[slice(startr, endr), slice(startc, endc)]
    marr = marr * arr

    timestamp = day.strftime('%m_%d_%Y')
    year = str(day.year)
    p = os.path.join(out, 'NDVI', 'NDVI', year)
    if not os.path.isdir(p):
        os.makedirs(p)
    p = os.path.join(p, '{}_{}.tif'.format('ndvi', timestamp))

    raster.save(p, marr, geo)


def extract_initial(out):
    mask_arr = Raster(paths.mask).as_bool_array
    # nr,nc = mask_arr.shape
    startr, endr = None, None
    for i, ri in enumerate(mask_arr):
        if ri.any():
            if startr is None:
                startr = i
        elif startr is not None:
            endr = i
            break

    startc, endc = None, None
    for i, ri in enumerate(mask_arr.T):
        if ri.any():
            if startc is None:
                startc = i
        elif startc is not None:
            endc = i
            break

    transform = get_tiff_transform(paths.mask)
    transform *= Affine.translation(startc, startr)

    pairs = make_pairs(paths.initial_inputs, INITIAL_KEYS)
    for k, pair in pairs:
        print 'initial {} reduced'.format(k)
        raster = Raster(pair, root=paths.initial_inputs)
        geo = raster.geo
        geo['rows'] = endr - startr
        geo['cols'] = endc - startc
        geo['geotransform'] = transform.to_gdal()

        arr = raster.masked()

        raster = Raster.fromarray(arr)
        marr = raster.unmasked()
        marr = marr[slice(startr, endr), slice(startc, endc)]
        marr = marr * arr

        path = os.path.join(out, 'initialize')
        if not os.path.isdir(path):
            os.makedirs(path)
        path = os.path.join(path, '{}_{}.tif'.format(k, 'reduced'))

        raster.save(path, marr, geo)


def extract_static(day, out):
    pass


def generate_dataset(daterange, out):
    if not paths.is_set():
        raise PathsNotSetExecption

    extract_initial(out)
    # extract_static(out)

    for day in day_generator(*daterange):
        extract_prism(day, out)
        extract_ndvi(day, out)
        extract_penman(day, out)
        print 'day {}'.format(day.strftime('%m_%d_%Y'))


if __name__ == '__main__':
{'fraction_non_alphanumeric': 0.1539725577191685, 'fraction_numerical': 0.002136225453947909, 'mean_word_length': 2.4530496453900708, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182158', 'n_tokens_mistral': 4011, 'n_tokens_neox': 3800, 'n_words': 1038}