<commit_before>"""Download DayMet data and convert it to ATS format.

DayMet is downloaded in point mode based on lat-lon, then converted to
hdf5 files that ATS knows how to read.
"""

import requests
import datetime
import logging
import h5py
import sys, os
import numpy as np

def file_id(lat, lon):
    """Returns a lat-lon id for use in filenames"""
    return '{:.4f}_{:.4f}'.format(lat,lon).replace('.','p')

def daymet_rest_url(lat, lon, start, end, vars=None):
    """Generates the DayMet Rest API URL."""
    #
    # NOTE: it is unclear why this was implemented this way -- shouldn't it use
    # the standard requests dictionary-style syntax?  But if it ain't broke
    # don't fix it. --etc
    #    
    daymet_vars = ['dayl', 'prcp', 'srad', 'swe', 'tmax', 'tmin', 'vp']    

    # check variable names are valid
    if vars is None:
        vars = daymet_vars
    elif type(vars) is list:
        for v in vars:
            if v not in daymet_vars:
                raise RuntimeError('Requested DayMet variable "%s" is not a valid variable name.  Valid are "%r"'%(v, vald_vars))
    elif type(vars) is str:
        if vars not in daymet_vars:
            raise RuntimeError('Requested DayMet variable "%s" is not a valid variable name.  Valid are "%r"'%(vars, vald_vars))
        vars = [vars,]

    # generate the URL
    base = 'https://daymet.ornl.gov/single-pixel/api/data?'
    lat_str = 'lat={:.4f}'.format(lat)
    lon_str = 'lon={:.4f}'.format(lon)
    var_str = 'vars='+','.join(vars)
    start_str = 'start={}'.format(start)
    end_str = 'end={}'.format(end)

    return base + '&'.join([lat_str, lon_str, var_str, start_str, end_str])
        
def read_daymet(filename):
    """Reads a text file of the form provided by DayMet"""
    return np.genfromtxt(filename, skip_header=7, names=True, delimiter=',')



def download_daymet(outdir, lat, lon, start, end, vars=None):
    """Calls the DayMet Rest API to get data and save raw data."""

    url = daymet_rest_url(lat, lon, start, end, vars)
    logging.info('Querying: %s'%url)
    resp = requests.get(url)

    if resp.status_code != requests.codes.ok:
        logging.warning('  returned code: %r'%resp.status_code)    
        raise RuntimeError('Failed download on "%s" with error code "%r"'%(url, resp.status_code))
    else:
        logging.info('  returned code: %r'%resp.status_code)

    filename = os.path.join(outdir, 'daymet_raw_%s.dat'%file_id(lat, lon))
    logging.info('  writing to disk: %s'%filename)
    with open(filename, 'w') as fid:
        fid.write(resp.text)

    return read_daymet(filename)

def daymet_to_ats(dat):
    """Accepts a numpy named array of DayMet data and returns a dictionary ATS data."""
    dout = dict()
    logging.info('Converting to ATS')

    mean_air_temp_c = (dat['tmin_deg_c'] + dat['tmax_deg_c'])/2.0
    precip_ms = dat['prcp_mmday'] / 1.e3 / 86000.
    
    # Sat vap. press o/water Dingman D-7 (Bolton, 1980)
    sat_vp_Pa = 611.2 * np.exp(17.67 * mean_air_temp_c / (mean_air_temp_c + 243.5))

    dout['time [s]'] = np.arange(0, len(dat), 1)*86400.
    dout['air temperature [K]'] = 273.15 + mean_air_temp_c
    dout['incoming shortwave radiation [W m^-2]'] = dat['srad_Wm2']<commit_msg>bug fix in s per day, reported by <NAME><commit_after>"""Download DayMet data and convert it to ATS format.

DayMet is downloaded in point mode based on lat-lon, then converted to
hdf5 files that ATS knows how to read.
"""

import requests
import datetime
import logging
import h5py
import sys, os
import numpy as np

def file_id(lat, lon):
    """Returns a lat-lon id for use in filenames"""
    return '{:.4f}_{:.4f}'.format(lat,lon).replace('.','p')

def daymet_rest_url(lat, lon, start, end, vars=None):
    """Generates the DayMet Rest API URL."""
    #
    # NOTE: it is unclear why this was implemented this way -- shouldn't it use
    # the standard requests dictionary-style syntax?  But if it ain't broke
    # don't fix it. --etc
    #    
    daymet_vars = ['dayl', 'prcp', 'srad', 'swe', 'tmax', 'tmin', 'vp']    

    # check variable names are valid
    if vars is None:
        vars = daymet_vars
    elif type(vars) is list:
        for v in vars:
            if v not in daymet_vars:
                raise RuntimeError('Requested DayMet variable "%s" is not a valid variable name.  Valid are "%r"'%(v, vald_vars))
    elif type(vars) is str:
        if vars not in daymet_vars:
            raise RuntimeError('Requested DayMet variable "%s" is not a valid variable name.  Valid are "%r"'%(vars, vald_vars))
        vars = [vars,]

    # generate the URL
    base = 'https://daymet.ornl.gov/single-pixel/api/data?'
    lat_str = 'lat={:.4f}'.format(lat)
    lon_str = 'lon={:.4f}'.format(lon)
    var_str = 'vars='+','.join(vars)
    start_str = 'start={}'.format(start)
    end_str = 'end={}'.format(end)

    return base + '&'.join([lat_str, lon_str, var_str, start_str, end_str])
        
def read_daymet(filename):
    """Reads a text file of the form provided by DayMet"""
    return np.genfromtxt(filename, skip_header=7, names=True, delimiter=',')



def download_daymet(outdir, lat, lon, start, end, vars=None):
    """Calls the DayMet Rest API to get data and save raw data."""

    url = daymet_rest_url(lat, lon, start, end, vars)
    logging.info('Querying: %s'%url)
    resp = requests.get(url)

    if resp.status_code != requests.codes.ok:
        logging.warning('  returned code: %r'%resp.status_code)    
        raise RuntimeError('Failed download on "%s" with error code "%r"'%(url, resp.status_code))
    else:
        logging.info('  returned code: %r'%resp.status_code)

    filename = os.path.join(outdir, 'daymet_raw_%s.dat'%file_id(lat, lon))
    logging.info('  writing to disk: %s'%filename)
    with open(filename, 'w') as fid:
        fid.write(resp.text)

    return read_daymet(filename)

def daymet_to_ats(dat):
    """Accepts a numpy named array of DayMet data and returns a dictionary ATS data."""
    dout = dict()
    logging.info('Converting to ATS')

    mean_air_temp_c = (dat['tmin_deg_c'] + dat['tmax_deg_c'])/2.0
    precip_ms = dat['prcp_mmday'] / 1.e3 / 86400.
    
    # Sat vap. press o/water Dingman D-7 (Bolton, 1980)
    sat_vp_Pa = 611.2 * np.exp(17.67 * mean_air_temp_c / (mean_air_temp_c + 243.5))

    dout['time [s]'] = np.arange(0, len(dat), 1)*86400.
    dout['air temperature [K]'] = 273.15 + mean_air_temp_c
    dout['incoming shortwave radiation [W m^-2]'] = dat['srad_Wm2']
{'fraction_non_alphanumeric': 0.141003861003861, 'fraction_numerical': 0.014517374517374518, 'mean_word_length': 3.1861667744020683, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1181740', 'n_tokens_mistral': 2333, 'n_tokens_neox': 2231, 'n_words': 675}