<commit_before>"""Fetch the NASA POWER Dataset.

For now, we just run each Monday for the current year RUN_2AM.sh
"""
import sys
import datetime
import subprocess

from tqdm import tqdm
import requests
import numpy as np
from pyiem import iemre
from pyiem.util import ncopen, logger, exponential_backoff

LOG = logger()


def main(argv):
    """Go Main Go."""
    year = int(argv[1])
    sts = datetime.date(year, 1, 1)
    ets = min([datetime.date(year, 12, 31), datetime.date.today()])
    current = {}
    now = ets
    while now >= sts:
        ds = iemre.get_grids(now, varnames="power_swdn")
        maxval = ds["power_swdn"].values.max()
        if np.isnan(maxval) or maxval < 0:
            LOG.debug("adding %s as currently empty", now)
            current[now] = {"data": ds, "dirty": False}
        now -= datetime.timedelta(days=1)
    sts = min(list(current.keys()))
    ets = max(list(current.keys()))
    LOG.debug("running between %s and %s", sts, ets)

    queue = []
    for x0 in np.arange(iemre.WEST, iemre.EAST, 5.0):
        for y0 in np.arange(iemre.SOUTH, iemre.NORTH, 5.0):
            queue.append([x0, y0])
    for x0, y0 in tqdm(queue, disable=not sys.stdout.isatty()):
        url = (
            "https://power.larc.nasa.gov/cgi-bin/v1/DataAccess.py?"
            "request=execute&identifier=Regional&"
            "parameters=ALLSKY_SFC_SW_DWN&"
            "startDate=%s&endDate=%s&userCommunity=SSE&"
            "tempAverage=DAILY&bbox=%s,%s,%s,%s&user=anonymous&"
            "outputList=NETCDF"
        ) % (
            sts.strftime("%Y%m%d"),
            ets.strftime("%Y%m%d"),
            y0,
            x0,
            min([y0 + 5.0, iemre.NORTH]) - 0.1,
            min([x0 + 5.0, iemre.EAST]) - 0.1,
        )
        req = exponential_backoff(requests.get, url, timeout=60)
        js = req.json()
        if "outputs" not in js:
            LOG.debug(url)
            LOG.debug(str(js))
            continue
        fn = js["outputs"]["netcdf"]
        req = exponential_backoff(requests.get, fn, timeout=60, stream=True)
        ncfn = "/tmp/power%s.nc" % (year,)
        with open(ncfn, "wb") as fh:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk:
                    fh.write(chunk)
            fh.close()
        with ncopen(ncfn) as nc:
            for day, _ in enumerate(nc.variables["time"][:]):
                date = sts + datetime.timedelta(days=day)
                if date not in current:
                    continue
                # kwh to MJ/d  3600 * 1000 / 1e6
                data = nc.variables["ALLSKY_SFC_SW_DWN"][day, :, :] * 3.6
                # Sometimes there are missing values?
                if np.ma.is_masked(data):
                    data[data.mask] = np.mean(data)
                i, j = iemre.find_ij(x0, y0)
                # resample data is 0.5, iemre is 0.125
                data = np.repeat(np.repeat(data, 4, axis=0), 4, axis=1)
                data = np.where(data < 0, np.nan, data)
                shp = np.shape(data)
                jslice = slice(j, j + shp[0])
                islice = slice(i, i + shp[1])
                # get currentdata
                present = current[date]["data"]["power_swdn"].values[
                    jslice, islice
                ]
                if present.mean() == data.mean():
                    continue
                current[date]["data"]["power_swdn"].values[
                    jslice, islice
                ] = data
                current[date]["dirty"] = True
    for date in current:<commit_msg>use new NASA Power API to fetch data<commit_after>"""Fetch the NASA POWER Dataset.

For now, we just run each Monday for the current year RUN_2AM.sh
"""
import sys
import time
import datetime
import subprocess

from tqdm import tqdm
import requests
import numpy as np
from pyiem import iemre
from pyiem.util import ncopen, logger, exponential_backoff

LOG = logger()


def main(argv):
    """Go Main Go."""
    year = int(argv[1])
    sts = datetime.date(year, 1, 1)
    ets = min([datetime.date(year, 12, 31), datetime.date.today()])
    current = {}
    now = ets
    while now >= sts:
        ds = iemre.get_grids(now, varnames="power_swdn")
        maxval = ds["power_swdn"].values.max()
        if np.isnan(maxval) or maxval < 0:
            LOG.debug("adding %s as currently empty", now)
            current[now] = {"data": ds, "dirty": False}
        now -= datetime.timedelta(days=1)
    if not current:
        LOG.info("Nothing to be done...")
        return
    sts = min(list(current.keys()))
    ets = max(list(current.keys()))
    LOG.debug("running between %s and %s", sts, ets)

    queue = []
    # 10x10 degree chunk is the max request size...
    for x0 in np.arange(iemre.WEST, iemre.EAST, 10.0):
        for y0 in np.arange(iemre.SOUTH, iemre.NORTH, 10.0):
            queue.append([x0, y0])
    for x0, y0 in tqdm(queue, disable=not sys.stdout.isatty()):
        url = (
            "https://power.larc.nasa.gov/api/temporal/daily/regional?"
            "latitude-min=%s&latitude-max=%s&longitude-min=%s&"
            "longitude-max=%s&parameters=ALLSKY_SFC_SW_DWN&community=SB&"
            "start=%s&end=%s&format=NETCDF"
        ) % (
            y0,
            y0 + 9.9,
            x0,
            x0 + 9.9,
            sts.strftime("%Y%m%d"),
            ets.strftime("%Y%m%d"),
        )
        req = exponential_backoff(requests.get, url, timeout=60)
        # Can't find docs on how many requests/sec are allowed...
        if req is not None and req.status_code == 429:
            LOG.debug("Got 429 (too-many-requests), sleeping 60")
            time.sleep(60)
            req = exponential_backoff(requests.get, url, timeout=60)
        if req is None or req.status_code != 200:
            LOG.info(
                "failed to download %s with %s %s",
                url,
                "req is none" if req is None else req.status_code,
                "req is none" if req is None else req.text,
            )
            continue
        ncfn = f"/tmp/power{year}.nc"
        with open(ncfn, "wb") as fh:
            for chunk in req.iter_content(chunk_size=1024):
                if chunk:
                    fh.write(chunk)
            fh.close()
        with ncopen(ncfn) as nc:
            for day, _ in enumerate(nc.variables["time"][:]):
                date = sts + datetime.timedelta(days=day)
                if date not in current:
                    continue
                # W/m2 to MJ/d 86400 / 1e6
                data = nc.variables["ALLSKY_SFC_SW_DWN"][day, :, :] * 0.0864
                # Sometimes there are missing values?
                if np.ma.is_masked(data):
                    data[data.mask] = np.mean(data)
                i, j = iemre.find_ij(x0, y0)
                # resample data is 0.5, iemre is 0.125
                data = np.repeat(np.repeat(data, 4, axis=0), 4, axis=1)
                data = np.where(data < 0, np.nan, data)
                shp = np.shape(data)
                jslice = slice(j, min([j + shp[0], iemre.NY]))
                islice = slice(i, min([i + shp[1], iemre.NX]))
                # LOG.debug("islice %s jslice: %s", islice, jslice)
                # align grids
                data = data[
                    slice(0, jslice.stop - jslice.start),
                    slice(0, islice.stop - islice.start),
                ]
                # get currentdata
                present = current[date]["data"]["power_swdn"].values[
                    jslice, islice
                ]
                if present.mean() == data.mean():
                    continue
                current[date]["data"]["power_swdn"].values[
                    jslice, islice
                ] = data
                current[date]["dirty"] = True
    for date in current:
{'fraction_non_alphanumeric': 0.1310722937651409, 'fraction_numerical': 0.019507841387224275, 'mean_word_length': 1.6835443037974684, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7525692', 'n_tokens_mistral': 2706, 'n_tokens_neox': 2511, 'n_words': 611}