<commit_before>"""
 Monthly download of the MERRA data, website suggests that the previous
 month is available around the 2-3 week of the next month, so this is run
 from RUN_MIDNIGHT.sh, but only on the 21rst of each month

MERRA2 Variables we want to process
-----------------------------------
http://goldsmr4.sci.gsfc.nasa.gov/data/MERRA2_MONTHLY/M2TMNXLND.5.12.4/doc/
MERRA2.README.pdf

(M2T1NXRAD)
SWGDNCLR    surface incoming shortwave flux assuming clear sky
SWTDN       toa incoming shortwave flux
SWGDN       surface incoming shortwave flux
SWGDNCLR    surface incoming shortwave flux assuming clear sky
"""
import datetime
import urllib2
import os
import sys


def trans(now):
    """ Hacky hack hack """
    if now.year < 1993:
        return '100'
    if now.year < 2001:
        return '200'
    if now.year < 2011:
        return '300'
    return '400'


def do_month(sts):
    """ Run for a given month """

    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    interval = datetime.timedelta(days=1)
    now = sts
    failures = 0
    while now < ets:
        uri = now.strftime(
            ("http://goldsmr4.gesdisc.eosdis.nasa.gov/daac-bin/OTF/"
             "HTTP_services.cgi?FILENAME=/data/s4pa/MERRA2"
             "/M2T1NXRAD.5.12.4/%Y/%m/MERRA2_"+trans(now)+".tavg1_2d_rad_"
             "Nx.%Y%m%d.nc4&FORMAT=bmM0Lw&"
             "BBOX=22.852,-129.727,52.383,-60.117"
             "&LABEL=MERRA2_"+trans(now)+".tavg1_2d_rad_Nx.%Y%m%d.SUB.nc"
             "&FLAGS=&SHORTNAME=M2T1NXRAD&SERVICE=SUBSET_MERRA2&"
             "LAYERS=&VERSION=1.02&VARIABLES=swgdn,swgdnclr,swtdn"))
        try:
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()
        except Exception as exp:
            print uri
            print exp
            return
        dirname = now.strftime("/mesonet/merra2/%Y")
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        if len(data) < 1000:
            print(("fetch_merra %s resulted in %s bytes, retry"
                   ) % (now.strftime("%Y%m%d"), len(data)))
            failures += 1
            if failures > 10:
                print("fetch_merra aborting with too many failures")
                return
            continue
        fp = open(now.strftime("/mesonet/merra2/%Y/%Y%m%d.nc"), 'w')
        fp.write(data)
        fp.close()<commit_msg>fix MERRA download to use URS auth<commit_after>"""
 Monthly download of the MERRA data, website suggests that the previous
 month is available around the 2-3 week of the next month, so this is run
 from RUN_MIDNIGHT.sh, but only on the 21rst of each month

MERRA2 Variables we want to process
-----------------------------------
http://goldsmr4.sci.gsfc.nasa.gov/data/MERRA2_MONTHLY/M2TMNXLND.5.12.4/doc/
MERRA2.README.pdf

(M2T1NXRAD)
SWGDNCLR    surface incoming shortwave flux assuming clear sky
SWTDN       toa incoming shortwave flux
SWGDN       surface incoming shortwave flux
SWGDNCLR    surface incoming shortwave flux assuming clear sky

NOTE: we need to have a ~/.netrc file to make this script happy.

"""
import datetime
import os
import sys
from pyiem.util import get_properties
import logging
import subprocess

logging.basicConfig(level=logging.DEBUG)

PROPS = get_properties()


def trans(now):
    """ Hacky hack hack """
    if now.year < 1993:
        return '100'
    if now.year < 2001:
        return '200'
    if now.year < 2011:
        return '300'
    return '400'


def do_month(sts):
    """ Run for a given month """

    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    interval = datetime.timedelta(days=1)
    now = sts
    while now < ets:
        uri = now.strftime(
            ("http://goldsmr4.gesdisc.eosdis.nasa.gov/daac-bin/OTF/"
             "HTTP_services.cgi?FILENAME=/data/s4pa/MERRA2"
             "/M2T1NXRAD.5.12.4/%Y/%m/MERRA2_"+trans(now)+".tavg1_2d_rad_"
             "Nx.%Y%m%d.nc4&FORMAT=bmM0Lw&"
             "BBOX=22.852,-129.727,52.383,-60.117"
             "&LABEL=MERRA2_"+trans(now)+".tavg1_2d_rad_Nx.%Y%m%d.SUB.nc"
             "&FLAGS=&SHORTNAME=M2T1NXRAD&SERVICE=SUBSET_MERRA2&"
             "LAYERS=&VERSION=1.02&VARIABLES=swgdn,swgdnclr,swtdn"))
        dirname = now.strftime("/mesonet/merra2/%Y")
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        localfn = now.strftime("/mesonet/merra2/%Y/%Y%m%d.nc")
        cmd = ("curl -n -c ~/.urscookies -b ~/.urscookies -L "
               "--url '%s' -o %s") % (uri, localfn)
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
                                stderr=subprocess.PIPE)
        _ = proc.stderr.read()
{'fraction_non_alphanumeric': 0.13542333120068245, 'fraction_numerical': 0.04030710172744722, 'mean_word_length': 2.2797202797202796, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2941583', 'n_tokens_mistral': 1798, 'n_tokens_neox': 1635, 'n_words': 369}