<commit_before>"""
Filename:     parse_dates.py
Author:       <NAME>, <EMAIL>
Description:  Parse a list of dates and output a simple statistic

"""

import datetime
from dateutil.rrule import *
from matplotlib.dates import date2num
import calendar

import numpy
import argparse

import os
import sys
module_dir = os.path.join(os.environ['HOME'], 'phd', 'modules')
sys.path.insert(0, module_dir)
import general_io as gio


def bin_dates(date_list, timescale, start, end):
    """Take a list of dates and return totals in bins, according to 
    the requested timescale.
    
    Arguments:
      start  -- tuple: (year, month)
      end    -- tuple: (year, month) 
    
    """
        
    dt_list = map(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d'), date_list)
    num_list = map(date2num, dt_list)
    
    start_dt = datetime.datetime(start[0], start[1], 1)
    end_dt = datetime.datetime(end[0], start[1], 1) + datetime.timedelta(months=1)
    dt_bin_edges = rrule(MONTHLY, dtstart=start_dt, until=end_dt) #interval=1
    num_bin_edges = map(date2num, dt_bin_edges)
    
    hist_data, edges = numpy.histogram(num_list, bins=num_bin_edges)
    assert len(hist_data) == len(dt_bin_edges[:-1])

    histogram = {}
    for i in range(0, len(hist_data)):
        histogram[dt_bin_edges[i]] = hist_data[i]
    
    return histogram


def print_summary(histogram, dt_labels):
    """Print summary statistics to the screen"""

    monthly_totals = dict((month, 0) for month in range(1,13))
    for key, value in historgram.iteritems():
        monthly_totals[key.month] = monthly_totals[key.month] + value

    print 'Total days'
    for i in range(1,13):
        print calendar[i], monthly_totals[i]

     
def main(inargs):
    """Run the program"""

    # Read in the date list
    date_list = gio.read_dates(inargs.dates)
     
    # Put the data into monthly or seasonal bins
    histogram = bin_dates(date_list, inargs.totals, inargs.start, inargs.end)
    
    # Print summary stats to screen
    print_summary(histogram)
    

if __name__ == '__main__':

    extra_info =""" 
example:
    python parse_dates.py 
    hov-vrot-env-w567_Merra_250hPa_daily-anom-wrt-1979-2012_y181x360_np20-260_absolute14_lon225-335_dates.txt 

note:
    Assumes daily or higher timescale input data
    
author:
    <NAME>, <EMAIL>

"""

    description='Parse a list of dates and output various statistics'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info, 
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    # Required arguments
    parser.add_argument("dates", type=str, help="Name of input file containing the list of dates")
    
    # Time filters
    parser.add_argument("--start", type=int, nargs=2, metavar=('YEAR', 'MONTH'), default=(1979, 1), 
                        help="Time start filter (e.g. 1979 1)")
    parser.add_argument("--end", type=int, nargs=2, metavar=('YEAR', 'MONTHS'), default=(2012, 12),
                        help="Time end filter (e.g. 2012 12)")

    args = parser.parse_args()            
    main(args)
    <commit_msg>Added a function to parse_dates.py to write output file<commit_after>"""
Filename:     parse_dates.py
Author:       <NAME>, <EMAIL>imelb.edu.au
Description:  Parse a list of dates and output a simple statistic

"""

import datetime
from dateutil.rrule import *
from matplotlib.dates import date2num
import calendar

import numpy
import argparse
from collections import OrderedDict

import os
import sys
module_dir = os.path.join(os.environ['HOME'], 'phd', 'modules')
sys.path.insert(0, module_dir)
import general_io as gio


def bin_dates(date_list, timescale, start, end):
    """Take a list of dates and return totals in bins, according to 
    the requested timescale.
    
    Arguments:
      start  -- tuple: (year, month)
      end    -- tuple: (year, month) 
    
    Output:
      Returns an ordered dictionary
    
    """
        
    dt_list = map(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d'), date_list)
    num_list = map(date2num, dt_list)
    
    start_dt = datetime.datetime(start[0], start[1], 1)
    end_dt = datetime.datetime(end[0], start[1], 1) + datetime.timedelta(months=1)
    dt_bin_edges = rrule(MONTHLY, dtstart=start_dt, until=end_dt) #interval=1
    num_bin_edges = date2num(dt_bin_edges)
    
    hist_data, edges = numpy.histogram(num_list, bins=num_bin_edges)
    assert len(hist_data) == len(dt_bin_edges[:-1])

    histogram = {}
    for i in range(0, len(hist_data)):
        histogram[dt_bin_edges[i]] = hist_data[i]
    
    return OrderedDict(sorted(histogram.items(), key=lambda t: t[0])) 
    #t[1] would sort by value instead of key  


def print_summary(histogram):
    """Print summary statistics to the screen"""

    monthly_totals = dict((month, 0) for month in range(1,13))
    for key, value in historgram.iteritems():
        monthly_totals[key.month] = monthly_totals[key.month] + value

    print 'Total days'
    for i in range(1,13):
        print calendar[i], monthly_totals[i]


def write_totals(histogram, outfile):
    """Write the monthly totals to a netCDF file"""

    # Create time axis
    assert histogram.keys() == sorted(histogram.keys()), \
    'Time values in histogram dictionary are not monotonic'
    time_vals = date2num(histogram.keys()) - 1  #date2num converts to days since 0001-01-01 UTC + 1
    
    time_axis = cdms2.createAxis(time_vals)
    time_axis.id = 'time'
    time_axis.units = 'days since 0001-1-1'
    time_axis.axis = 'T'

    # Define variable attributes
    outvar_atts = {'id': 'total',
                   'long_name': 'total days for each month',
                   'units': 'days'}
    global_atts = {}

    # Write output file
    nio.write_netcdf(outfile, " ".join(sys.argv), 
                     global_atts, 
                     [histogram.values(),],
                     [outvar_atts,], 
                     [time_axis,])


def main(inargs):
    """Run the program"""

    # Read in the date list
    date_list = gio.read_dates(inargs.dates)
     
    # Put the data into monthly or seasonal bins
    histogram = bin_dates(date_list, inargs.totals, inargs.start, inargs.end)
    
    # Print summary stats to screen
    print_summary(histogram)
    
    # Write the output file
    if inargs.ofile:
        write_totals(histogram, inargs.ofile)
    

if __name__ == '__main__':

    extra_info =""" 
example:
    python parse_dates.py 
    hov-vrot-env-w567_Merra_250hPa_daily-anom-wrt-1979-2012_y181x360_np20-260_absolute14_lon225-335_dates.txt 

note:
    Assumes daily or higher timescale input data
    
author:
    <NAME>, <EMAIL>

"""

    description='Parse a list of dates and output various statistics'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info, 
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("dates", type=str, help="Name of input file containing the list of dates")
    
    parser.add_argument("--start", type=int, nargs=2, metavar=('YEAR', 'MONTH'), default=(1979, 1), 
                        help="Time start filter (e.g. 1979 1)")
    parser.add_argument("--end", type=int, nargs=2, metavar=('YEAR', 'MONTHS'), default=(2012, 12),
                        help="Time end filter (e.g. 2012 12)")
    parser.add_argument("--ofile", type=str, default=None,
                        help="Name of the netCDF output file")    


    args = parser.parse_args()            
    main(args)
    
{'fraction_non_alphanumeric': 0.11204121056020605, 'fraction_numerical': 0.022794591113972956, 'mean_word_length': 2.7516908212560387, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3284889', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2444, 'n_words': 629}