<commit_before>'''
 Monthly download of the MERRA data, website suggests that the previous 
 month is available around the 2-3 week of the next month, so this is run 
 from RUN_MIDNIGHT.sh, but only on the 21rst of each month
'''
import datetime
import urllib2
import os
import subprocess

def trans(now):
    if now.year < 1993:
        return '101'
    if now.year < 2001:
        return '201'
    return '300'

def do_month( sts ):
    ''' Run for a given month '''

    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    interval = datetime.timedelta(days=1)
    now = sts
    while now < ets:
    
        uri = now.strftime( ("http://goldsmr2.sci.gsfc.nasa.gov/daac-bin/OTF/"
            +"HTTP_services.cgi?FILENAME=%%2Fdata%%2Fs4pa%%2FMERRA%%2"
            +"FMAT1NXRAD.5.2.0%%2F%Y%%2F%m%%2FMERRA"+trans(now)+".prod.assim."
            +"tavg1_2d_rad_Nx.%Y%m%d.hdf&FORMAT=TmV0Q0RGL2d6aXAv&"
            +"BBOX=22.852,-129.727,52.383,-60.117&LABEL=MERRA"
            +trans(now)+".prod.assim.tavg1_2d_rad_Nx.%Y%m%d.SUB.nc.gz&"
            +"FLAGS=&SHORTNAME=MAT1NXRAD&SERVICE=SUBSET_LATS4D&LAYERS=&"
            +"VERSION=1.02&VARIABLES=swtdn,swgdn,swgdnclr,tauhgh,taulow,"
            +"taumid,tautot,cldhgh,cldlow,cldmid,cldtot") )
    
        try:
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()   
        except:
            uri = uri.replace("MERRA300", "MERRA301")
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()   
        dirname = now.strftime("/mnt/longterm2/merra/%Y")
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        fp = open( now.strftime("/mnt/longterm2/merra/%Y/%Y%m%d.nc.gz"), 'w')
        fp.write( data )
        fp.close()
        now += interval

    os.chdir(dirname)
    subprocess.call("gunzip *.gz", shell=True)

if __name__ == '__main__':
    ''' Run for last month month '''
    now = datetime.datetime.now()
    if now.day == 21:
        now = now - datetime.timedelta(days=35)
        now = now.replace(day=1)
        do_month( now )<commit_msg>allow running of old dates<commit_after>'''
 Monthly download of the MERRA data, website suggests that the previous 
 month is available around the 2-3 week of the next month, so this is run 
 from RUN_MIDNIGHT.sh, but only on the 21rst of each month
'''
import datetime
import urllib2
import os
import sys
import subprocess

def trans(now):
    if now.year < 1993:
        return '101'
    if now.year < 2001:
        return '201'
    return '300'

def do_month( sts ):
    ''' Run for a given month '''

    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    interval = datetime.timedelta(days=1)
    now = sts
    while now < ets:
    
        uri = now.strftime( ("http://goldsmr2.sci.gsfc.nasa.gov/daac-bin/OTF/"
            +"HTTP_services.cgi?FILENAME=%%2Fdata%%2Fs4pa%%2FMERRA%%2"
            +"FMAT1NXRAD.5.2.0%%2F%Y%%2F%m%%2FMERRA"+trans(now)+".prod.assim."
            +"tavg1_2d_rad_Nx.%Y%m%d.hdf&FORMAT=TmV0Q0RGL2d6aXAv&"
            +"BBOX=22.852,-129.727,52.383,-60.117&LABEL=MERRA"
            +trans(now)+".prod.assim.tavg1_2d_rad_Nx.%Y%m%d.SUB.nc.gz&"
            +"FLAGS=&SHORTNAME=MAT1NXRAD&SERVICE=SUBSET_LATS4D&LAYERS=&"
            +"VERSION=1.02&VARIABLES=swtdn,swgdn,swgdnclr,tauhgh,taulow,"
            +"taumid,tautot,cldhgh,cldlow,cldmid,cldtot") )
    
        try:
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()   
        except:
            uri = uri.replace("MERRA300", "MERRA301")
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()   
        dirname = now.strftime("/mnt/longterm2/merra/%Y")
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        fp = open( now.strftime("/mnt/longterm2/merra/%Y/%Y%m%d.nc.gz"), 'w')
        fp.write( data )
        fp.close()
        now += interval

    os.chdir(dirname)
    subprocess.call("gunzip *.gz", shell=True)

if __name__ == '__main__':
    ''' Run for last month month '''
    now = datetime.datetime.now()
    if len(sys.argv) == 3:
        now =  datetime.datetime(int(sys.argv[1]), int(sys.argv[2]), 1)
    else:
        now = now - datetime.timedelta(days=35)
        now = now.replace(day=1)
    do_month( now )
{'fraction_non_alphanumeric': 0.14931917839833833, 'fraction_numerical': 0.0420032310177706, 'mean_word_length': 2.475541299117883, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3800331', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1633, 'n_words': 295}