<commit_before>import sys
sys.path.append('/home/jwalker/dynamics/python/atmos-tools')
sys.path.append('/home/jwalker/dynamics/python/atmos-read')

import numpy as np
import xray

import atmos as atm
import merra

# ----------------------------------------------------------------------
# Daily vertically integrated moisture transport cf. Fasullo and Webster 2003
# Combine daily data from individual years, months

datadir = atm.homedir() + 'datastore/merra/daily/'

years = range(1979, 2015)
months = [4, 5, 6, 7, 8, 9]

def datafile(datadir, year, mon):
    filn = datadir + 'merra_vimt_%d%02d.nc' % (year, mon)
    return filn

def savefile(datadir, years, months, pmin):
    mon1 = atm.month_str(months[0]).lower()
    mon2 = atm.month_str(months[-1]).lower()
    yr1 = years[0]
    yr2 = years[-1]
    filn = datadir + 'merra_vimt_ps-%.0fmb_%s-%s_%d-%d.nc'
    filn = filn % (pmin/100, mon1, mon2, yr1, yr2)
    return filn

# Read daily data from each year and month and concatenate together
for y, year in enumerate(years):
    for m, mon in enumerate(months):
        filn = datafile(datadir, year, mon)
        print('Loading ' + filn)
        with xray.open_dataset(filn) as ds:
            ds.load()
            if m == 0:
                dsyr = ds
            else:
                dsyr = xray.concat((dsyr, ds), dim='day')
    dsyr.coords['year'] = year
    if atm.isleap(year):
        # Standardize to non-leap year day numbers
        dsyr['day'] = dsyr['day'] - 1
    if y == 0:
        ds_all = dsyr
    else:
        ds_all = xray.concat((ds_all, dsyr), dim='year')

# Save to file
filn = savefile(datadir, years, months, ds_all['uq_int'].attrs['pmin'])
print('Saving to ' + filn)
ds_all.to_netcdf(filn)<commit_msg>Align days on leap and non-leap years<commit_after>import sys
sys.path.append('/home/jwalker/dynamics/python/atmos-tools')
sys.path.append('/home/jwalker/dynamics/python/atmos-read')

import numpy as np
import xray

import atmos as atm
import merra

# ----------------------------------------------------------------------
# Daily vertically integrated moisture transport cf. Fasullo and Webster 2003
# Combine daily data from individual years, months

datadir = atm.homedir() + 'datastore/merra/daily/'

years = range(1979, 2015)
months = [4, 5, 6, 7, 8, 9]

# Common set of days for leap and non-leap years
dmin, dmax = 91, 274
days = np.arange(dmin, dmax + 1)

def datafile(datadir, year, mon):
    filn = datadir + 'merra_vimt_%d%02d.nc' % (year, mon)
    return filn

def savefile(datadir, years, months, pmin):
    mon1 = atm.month_str(months[0]).lower()
    mon2 = atm.month_str(months[-1]).lower()
    yr1 = years[0]
    yr2 = years[-1]
    filn = datadir + 'merra_vimt_ps-%.0fmb_%s-%s_%d-%d.nc'
    filn = filn % (pmin/100, mon1, mon2, yr1, yr2)
    return filn

# Read daily data from each year and month and concatenate together
for y, year in enumerate(years):
    for m, mon in enumerate(months):
        filn = datafile(datadir, year, mon)
        print('Loading ' + filn)
        with xray.open_dataset(filn) as ds:
            ds.load()
            if m == 0:
                dsyr = ds
            else:
                dsyr = xray.concat((dsyr, ds), dim='day')
    dsyr.coords['year'] = year
    # Align leap and non-leap years
    dsyr = dsyr.reindex(day=days)
    if y == 0:
        ds_all = dsyr
    else:
        ds_all = xray.concat((ds_all, dsyr), dim='year')

# Save to file
filn = savefile(datadir, years, months, ds_all['uq_int'].attrs['pmin'])
print('Saving to ' + filn)
ds_all.to_netcdf(filn)
{'fraction_non_alphanumeric': 0.1690101237345332, 'fraction_numerical': 0.023340832395950507, 'mean_word_length': 2.521782178217822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '900394', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1259, 'n_words': 336}