<commit_before>#!/usr/bin/env python2

###############################################################################
# ------------------------- Description --------------------------------------- 
###############################################################################
# This script will be used to generate daily (and 6 hourly) fields of Air
# qualility variables that are only available in hourly time steps.

# TODO: Make this work for emissions also. 

# NOTE: Performance testing indicates that this script should take about 
# NOTE: ~100 minutes to average a decades worth of hourly data to daily. 

import sys
print 'Number of arguments:', len(sys.argv), 'arguments.'
print 'Argument List:', str(sys.argv)

hourlyVAR =  str(sys.argv[1]) # e.g. 'PSL'
scenario  =  str(sys.argv[2]) # e.g. '2000Base'
year      =  int(sys.argv[3]) # e.g. 2000 # NOTE: Not needed except for emissions

import cesm_nc_manager as cnm
import os
import numpy as np
from mpl_toolkits.basemap import Basemap, cm
from netCDF4 import Dataset 
import matplotlib.pyplot as plt
import numpy.ma as ma
from datetime import date
import matplotlib.ticker as tkr
import datetime
import time as timer

# Start the timer on this code
timeStart = timer.time()

# Load hourly PSL data
HourlyFile  = cnm.makeAQNCFile(hourlyVAR, scenario, 'hourly')
Hourly_nc   = Dataset(HourlyFile, 'r')    # This is an 11GB file. Soo, holy smokes
time_nc     = Hourly_nc.variables['time'] # days since YYYY-01-01 00:00:00 
lon         = Hourly_nc.variables['lon']
lat         = Hourly_nc.variables['lat']

# Divide by hours to get days yo
nDays = len(time_nc)/24 
nLat  = len(lat)
nLon  = len(lon)

# Create array to store daily averaged data
dailyVAR = np.zeros((nDays, nLat, nLon))

# Load the date file that defines the dates. 
DateFile  = cnm.makeAQNCFile('date', scenario, 'daily') 
date_nc   = Dataset(DateFile, 'r')
d         = date_nc.variables['date'][:]
t         = cnm.dateNumToDate(d)

# Close the date file. 
date_nc.close()

print 'Working on the large loop averaging hourly values for each day'
for i in np.arange(0, nDays): # nDays

	# We have to make a mask for each day of these hourly data.
	# Since the units are time are days from origin, all integers
	# are unique days at some hour (expressed by decimal). Se we 
	# need to mask each 24 hours. [0,1), [1,2), ...
	mask  = (time_nc[:] >= i) & (time_nc[:] < (i+1) ) 
	index = np.where(mask ==  True)[0]

	if len(index) != 24:
		raise ValueError('Day index does not have 24 hours. Code Broken')

	# Now that we have the index of a day, we need to take the 
	# average value from the data for those 24 hours 
	dayVAR            = Hourly_nc.variables[hourlyVAR][index, :, :]
	dayMeanVAR        = np.mean(dayVAR, axis=0)
	dailyVAR[i, :, :] = dayMeanVAR


meansCompleteTime = timer.time()
dt = (meansCompleteTime - timeStart) / 60. 
print '----------------------------------------------------------------------'
print 'It took ' + str(dt) + ' minutes to create daily averages.'
print '----------------------------------------------------------------------'

# Make sure the time array matches the daily mean array in length and
# date. This dimension needs to be described correctly. 
t_subset = d[0:nDays]

###############################################################################
# Write the new netcdf data with the exact same formatting as the
# data read here. 
# Create the Save name and make sure that this file does not exist
###############################################################################
outPutSavename = cnm.makeAQNCFile(hourlyVAR, scenario, 'daily')

# Will have to change where this is placed becuase I do not own the output
# This is temporary untill I can get permission to write in the scratch
outPutSavenameTail = outPutSavename[69:]
outPutDir = '/home/sbrey/projects/PMFutures/Python_output/' + '2000Base/'
outPutSavename =  outPutDir + outPutSavenameTail

fileCount = -1<commit_msg>updated to deal with different length strings of different RCP scenarios. Tested<commit_after>#!/usr/bin/env python2

###############################################################################
# ------------------------- Description --------------------------------------- 
###############################################################################
# This script will be used to generate daily (and 6 hourly) fields of Air
# qualility variables that are only available in hourly time steps.

# TODO: Make this work for emissions also. 

# NOTE: Performance testing indicates that this script should take about 
# NOTE: ~100 minutes to average a decades worth of hourly data to daily. 

import sys
print 'Number of arguments:', len(sys.argv), 'arguments.'
print 'Argument List:', str(sys.argv)

if len(sys.argv) != 1:
	hourlyVAR =  str(sys.argv[1]) # e.g. 'PSL'
	scenario  =  str(sys.argv[2]) # e.g. '2000Base'
	year      =  int(sys.argv[3]) # e.g. 2000 # NOTE: Not needed except for emissions
else: 
	# Development environment. Set variables
	hourlyVAR =  'PSL'
	scenario  =  '2050RCP45'
	year      =  2050 

import cesm_nc_manager as cnm
import os
import numpy as np
from mpl_toolkits.basemap import Basemap, cm
from netCDF4 import Dataset 
import matplotlib.pyplot as plt
import numpy.ma as ma
from datetime import date
import matplotlib.ticker as tkr
import datetime
import time as timer

# Start the timer on this code
timeStart = timer.time()

# Load hourly PSL data
HourlyFile  = cnm.makeAQNCFile(hourlyVAR, scenario, 'hourly')
Hourly_nc   = Dataset(HourlyFile, 'r')    # This is an 11GB file. Soo, holy smokes
time_nc     = Hourly_nc.variables['time'] # days since YYYY-01-01 00:00:00 
lon         = Hourly_nc.variables['lon']
lat         = Hourly_nc.variables['lat']

# Divide by hours to get days yo
nDays = len(time_nc)/24 
nLat  = len(lat)
nLon  = len(lon)

# Create array to store daily averaged data
dailyVAR = np.zeros((nDays, nLat, nLon))

# Load the date file that defines the dates. 
DateFile  = cnm.makeAQNCFile('date', scenario, 'daily') 
date_nc   = Dataset(DateFile, 'r')
d         = date_nc.variables['date'][:]
t         = cnm.dateNumToDate(d)

# Close the date file. 
date_nc.close()

print 'Working on the large loop averaging hourly values for each day'
for i in np.arange(0, nDays): # nDays

	# We have to make a mask for each day of these hourly data.
	# Since the units are time are days from origin, all integers
	# are unique days at some hour (expressed by decimal). Se we 
	# need to mask each 24 hours. [0,1), [1,2), ...
	mask  = (time_nc[:] >= i) & (time_nc[:] < (i+1) ) 
	index = np.where(mask ==  True)[0]

	if len(index) != 24:
		raise ValueError('Day index does not have 24 hours. Code Broken')

	# Now that we have the index of a day, we need to take the 
	# average value from the data for those 24 hours 
	dayVAR            = Hourly_nc.variables[hourlyVAR][index, :, :]
	dayMeanVAR        = np.mean(dayVAR, axis=0)
	dailyVAR[i, :, :] = dayMeanVAR


meansCompleteTime = timer.time()
dt = (meansCompleteTime - timeStart) / 60. 
print '----------------------------------------------------------------------'
print 'It took ' + str(dt) + ' minutes to create daily averages.'
print '----------------------------------------------------------------------'

# Make sure the time array matches the daily mean array in length and
# date. This dimension needs to be described correctly. 
t_subset = d[0:nDays]

###############################################################################
# Write the new netcdf data with the exact same formatting as the
# data read here. 
# Create the Save name and make sure that this file does not exist
###############################################################################
outPutSavename = cnm.makeAQNCFile(hourlyVAR, scenario, 'daily')

# Will have to change where this is placed becuase I do not own the output
# This is temporary untill I can get permission to write in the scratch

# Directory names have different charactor lengths between base and RCP 
if scenario == '2000Base':
	firstChar = 69
else:
	firstChar = 70
	
outPutSavenameTail = outPutSavename[firstChar:]
outPutDir = '/home/sbrey/projects/PMFutures/Python_output/' + scenario + '/'
outPutSavename =  outPutDir + outPutSavenameTail

fileCount = -1
{'fraction_non_alphanumeric': 0.21114476075105995, 'fraction_numerical': 0.015384615384615385, 'mean_word_length': 3.2821576763485476, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3682334', 'n_tokens_mistral': 2599, 'n_tokens_neox': 2403, 'n_words': 912}