<commit_before>"""
Performs conversions of netCDF time coordinate data to/from datetime objects.
"""

from cpython.object cimport (PyObject_RichCompare, Py_LT, Py_LE, Py_EQ,
                             Py_NE, Py_GT, Py_GE)
from numpy cimport int64_t, int32_t
import cython
import numpy as np
import re
import sys
import time
from datetime import datetime as datetime_python
from datetime import timedelta, MINYEAR, MAXYEAR
import time                     # strftime
import warnings

microsec_units = ['microseconds','microsecond', 'microsec', 'microsecs']
millisec_units = ['milliseconds', 'millisecond', 'millisec', 'millisecs', 'msec', 'msecs', 'ms']
sec_units =      ['second', 'seconds', 'sec', 'secs', 's']
min_units =      ['minute', 'minutes', 'min', 'mins']
hr_units =       ['hour', 'hours', 'hr', 'hrs', 'h']
day_units =      ['day', 'days', 'd']
month_units =    ['month', 'months'] # only allowed for 360_day calendar
year_units =     ['common_year', 'common_years'] # only allowed for 365_day and noleap calendars
_units = microsec_units+millisec_units+sec_units+min_units+hr_units+day_units
# supported calendars. Includes synonyms ('standard'=='gregorian',
# '366_day'=='all_leap','365_day'=='noleap')
# see http://cfconventions.org/cf-conventions/cf-conventions#calendar
# for definitions.
_calendars = ['standard', 'gregorian', 'proleptic_gregorian',
              'noleap', 'julian', 'all_leap', '365_day', '366_day', '360_day']
_idealized_calendars= ['all_leap','noleap','366_day','365_day','360_day']
# Following are number of days per month
cdef int[12] _dayspermonth      = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
cdef int[12] _dayspermonth_leap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
# same as above, but including accumulated days of previous months.
cdef int[13] _cumdayspermonth = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365]
cdef int[13] _cumdayspermonth_leap = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366]

__version__ = '1.6.0'

# Adapted from http://delete.me.uk/2005/03/iso8601.html
# Note: This regex ensures that all ISO8601 timezone formats are accepted - but, due to legacy support for other timestrings, not all incorrect formats can be rejected.
#       For example, the TZ spec "+01:0" will still work even though the minutes value is only one character long.
ISO8601_REGEX = re.compile(r"(?P<year>[+-]?[0-9]+)(-(?P<month>[0-9]{1,2})(-(?P<day>[0-9]{1,2})"
                           r"(((?P<separator1>.)(?P<hour>[0-9]{1,2}):(?P<minute>[0-9]{1,2})(:(?P<second>[0-9]{1,2})(\.(?P<fraction>[0-9]+))?)?)?"
                           r"((?P<separator2>.?)(?P<timezone>Z|(([-+])([0-9]{2})((:([0-9]{2}))|([0-9]{2}))?)))?)?)?)?"
                           )
# Note: The re module apparently does not support branch reset groups that allow redefinition of the same group name in alternative branches as PCRE does.
#       Using two different group names is also somewhat ugly, but other solutions might hugely inflate the expression. feel free to contribute a better solution.
TIMEZONE_REGEX = re.compile(
    "(?P<prefix>[+-])(?P<hours>[0-9]{2})(?:(?::(?P<minutes1>[0-9]{2}))|(?P<minutes2>[0-9]{2}))?")

class real_datetime(datetime_python):
    """add dayofwk, dayofyr, daysinmonth attributes to python datetime instance"""
    @property
    def dayofwk(self):
        # 0=Monday, 6=Sunday
        return self.weekday()
    @property
    def dayofyr(self):
        return self.timetuple().tm_yday
    @property
    def daysinmonth(self):
        if _is_leap(self.year,'proleptic_gregorian'):
            return _dayspermonth_leap[self.month-1]
        else:
            return _dayspermonth[self.month-1]
    nanosecond = 0 # workaround for pandas bug (cftime issue #77)

def _datesplit(timestr):
    """split a time string into two components, units and the remainder
    after 'since'
    """
    try:
        (units, sincestring, remainder) = timestr.split(None,2)
    except ValueError as e:
        raise ValueError('Incorrectly formatted CF date-time unit_string')

    if sincestring.lower() != 'since':
        raise ValueError("no 'since' in unit_string")

    return units.lower(), remainder

def _dateparse(timestr,calendar,has_year_zero=None):
    """parse a string of the form time-units since yyyy-mm-dd hh:mm:ss,
    return a datetime instance"""
    # same as version in cftime, but returns a timezone naive
    # python datetime instance with the utc_offset included.

    calendar = calendar.lower()
    # set calendar-specific defaults for has_year_zero
    if has_year_zero is None:
        has_year_zero = _year_zero_defaults(calendar)
    (units, isostring) = _datesplit(timestr)
    if not ((units in month_units and calendar=='360_day') or (units in year_units and calendar in {'365_day', 'noleap'}) or units in _units):
        if units in month_units and calendar != '360_day':
            raise ValueError("'months since' units only allowed for '360_day' calendar")
        if units in year_units and calendar not in {'365_day', 'noleap'}:    
            raise ValueError("'%s' units only allowed for '365_day' and 'noleap' calendars" % units) 
        else:
            raise ValueError(
            "units must be one of 'seconds', 'minutes', 'hours' or 'days' (or singular version of these), got '%s'" % units)
    # parse the date string.
    year, month, day, hour, minute, second, microsecond, utc_offset =\
        _parse_date( isostring.strip() )
    if year == 0 and not has_year_zero and calendar in ['julian', 'standard', 'gregorian', 'proleptic_gregorian']:
        msg='zero not allowed as a reference year when has_year_zero=False'
        raise ValueError(msg)
    if calendar in ['noleap', '365_day'] and month == 2 and day == 29:
        raise ValueError(
            'cannot specify a leap day as the reference time with the noleap calendar')
    if calendar == '360_day' and day > 30:
        raise ValueError(
            'there are only 30 days in every month with the 360_day calendar')
    basedate = datetime(year, month, day, hour, minute, second,
                        microsecond,calendar=calendar,has_year_zero=has_year_zero)
    # subtract utc_offset from basedate time instance (which is timezone naive)
    if utc_offset:
        basedate -= timedelta(days=utc_offset/1440.)
    return basedate

def _can_use_python_datetime(date,calendar):
    gregorian = datetime(1582,10,15,calendar=calendar,has_year_zero=date.has_year_zero)
    return ((calendar == 'proleptic_gregorian' and date.year >= MINYEAR and date.year <= MAXYEAR) or \
           (calendar in ['gregorian','standard'] and date > gregorian and date.year <= MAXYEAR))

@cython.embedsignature(True)
def date2num(dates,units,calendar=None,has_year_zero=None):
    """
    Return numeric time values given datetime objects. The units
    of the numeric time values are described by the **units** argument
    and the **calendar** keyword. The datetime objects must
    be in UTC with no time-zone offset.  If there is a
    time-zone offset in **units**, it will be applied to the
    returned numeric values.

    **dates**: A datetime object or a sequence of datetime objects.
    The datetime objects should not include a time-zone offset. They
    can be either native python datetime instances (which use
    the proleptic gregorian calendar) or cftime.datetime instances.

    **units**: a string of the form **<time units> since <reference time>**
    describing the time units. **<time units>** can be days, hours, minutes,
    seconds, milliseconds or microseconds. **<reference time>** is the time
    origin. **months since** is allowed *only* for the **360_day** calendar
    and **common_years since** is allowed *only* for the **365_day** calendar.

    **calendar**: describes the calendar to be used in the time calculations.
    All the values currently defined in the
    `CF metadata convention <http://cfconventions.org/cf-conventions/cf-conventions#calendar>`__ are supported.
    Valid calendars **'standard', 'gregorian', 'proleptic_gregorian'
    'noleap', '365_day', '360_day', 'julian', 'all_leap', '366_day'**.
    Default is `None` which means the calendar associated with the first
    input datetime instance will be used.

    **has_year_zero**: If set to True, astronomical year numbering
    is used and the year zero exists.  If set to False for real-world
    calendars, then historical year numbering is used and the year 1 is
    preceded by year -1 and no year zero exists.
    The defaults are set to conform with
    CF version 1.9 conventions (False for 'julian', 'gregorian'/'standard', True
    for 'proleptic_gregorian' (ISO 8601) and True for the idealized
    calendars 'noleap'/'365_day', '360_day', 366_day'/'all_leap')
    Note that CF v1.9 does not specifically mention whether year zero
    is allowed in the proleptic_gregorian calendar, but ISO-8601 has
    a year zero so we have adopted this as the default.
    The defaults can only be over-ridden for the real-world calendars,
    for the the idealized calendars the year zero 
    always exists and the has_year_zero kwarg is ignored.
    This kwarg is not needed to define calendar systems allowed by CF
    (the calendar-specific defaults do this).

    returns a numeric time value, or an array of numeric time values
    with approximately 1 microsecond accuracy.
    """

    # input a scale or array-like?
    isscalar = False
    try:
        dates[0]
    except:
        isscalar = True

    # masked array input?
    ismasked = False
    if np.ma.isMA(dates) and np.ma.is_masked(dates):
        mask = dates.mask
        ismasked = True

    # are all input dates 'real' python datetime objects?
    dates = np.asanyarray(dates) # convert to numpy array
    shape = dates.shape # save shape of input
    all_python_datetimes = True
    for date in dates.flat:
        if not isinstance(date,datetime_python):
           all_python_datetimes = False
           break

    # if has_year_zero is None and calendar is None, use first input cftime.datetime instance
    # to determine if year zero is to be included.
    # If calendar is specified, use calendar specific defaults
    if has_year_zero is None:
        if calendar is None:
            d0 = dates.item(0)
            if isinstance(d0,datetime_python):
                has_year_zero = False
            else:
                try:
                    has_year_zero = d0.has_year_zero
                except AttributeError:
                    raise ValueError('has_year_zero not specified',type(d0))
        else:
            has_year_zero = _year_zero_defaults(calendar)

    # if calendar is None or '', use calendar of first input cftime.datetime instances.
    # if inputs are 'real' python datetime instances, use propleptic gregorian.
    if not calendar:
        if all_python_datetimes:
            calendar = 'proleptic_gregorian'
        else:
            d0 = dates.item(0)
            if isinstance(d0,datetime_python):
                calendar = 'proleptic_gregorian' 
            else:
                try:
                    calendar = d0.calendar
                except AttributeError:
                    raise ValueError('no calendar specified',type(d0))

    calendar = calendar.lower()
    basedate = _dateparse(units,calendar=calendar,has_year_zero=has_year_zero)
    (unit, isostring) = _datesplit(units)
    # real-world calendars cannot have zero as a reference year.
    if calendar in ['julian', 'standard', 'gregorian', 'proleptic_gregorian'] and not has_year_zero:
        if basedate.year == 0:
            msg='zero not allowed as a reference year, does not exist in Julian or Gregorian calendars'
            raise ValueError(msg)
    if unit not in UNIT_CONVERSION_FACTORS:
        raise ValueError("Unsupported time units provided, {!r}.".format(unit))
    if unit in ["months", "month"] and calendar != "360_day":
        raise ValueError("Units of months only valid for 360_day calendar.")
    unit_timedelta = timedelta(microseconds=UNIT_CONVERSION_FACTORS[unit])
    can_use_python_basedatetime = _can_use_python_datetime(basedate,calendar)

    if can_use_python_basedatetime and all_python_datetimes:
        use_python_datetime = True
        if not isinstance(basedate, datetime_python):
            basedate = real_datetime(basedate.year, basedate.month, basedate.day, 
                       basedate.hour, basedate.minute, basedate.second,
                       basedate.microsecond)
    else:
        use_python_datetime = False
        # convert basedate to specified calendar
        basedate =  to_calendar_specific_datetime(basedate, calendar, False, has_year_zero=has_year_zero)
    times = []
    for n, date in enumerate(dates.flat):
        if ismasked and mask.flat[n]:
            times.append(None)
            continue

        # use python datetime if possible.
        if use_python_datetime:
            # remove time zone offset
            if getattr(date, 'tzinfo',None) is not None:
                date = date.replace(tzinfo=None) - date.utcoffset()
        else: # convert date to same calendar specific cftime.datetime instance
            date = to_calendar_specific_datetime(date, calendar, False, has_year_zero=has_year_zero)

        td = date - basedate
        if td % unit_timedelta == timedelta(0):
            # Explicitly cast result to np.int64 for Windows compatibility
            quotient = np.int64(td // unit_timedelta)
            times.append(quotient)
        else:
            times.append(td / unit_timedelta)
                
    if ismasked: # convert to masked array if input was masked array
        times = np.array(times, dtype=float)  # None -> nan
        times = np.ma.masked_invalid(times)
        if isscalar:
            return times[0]
        else:
            return np.reshape(times, shape)
    if isscalar:
        return times[0]
    else:
        return np.reshape(np.array(times), shape)


@cython.embedsignature(True)
def num2pydate(times,units,calendar='standard'):
    """
    Always returns python datetime.datetime
    objects and raise an error if this is not possible.

    Same as
    num2date(times,units,calendar,only_use_cftime_datetimes=False,only_use_python_datetimes=True)
    """
    return num2date(times,units,calendar,only_use_cftime_datetimes=False,only_use_python_datetimes=True)


UNIT_CONVERSION_FACTORS = {
    "microseconds": 1,
    "microsecond": 1,
    "microsec": 1,
    "microsecs": 1,
    "milliseconds": 1000,
    "millisecond": 1000,
    "millisec": 1000,
    "millisecs": 1000,
    "msec": 1000,
    "msecs": 1000,
    "ms": 1000,
    "seconds": 1000000,
    "second": 1000000,
    "sec": 1000000,
    "secs": 1000000,
    "s": 1000000,
    "minutes": 60 * 1000000,
    "minute": 60 * 1000000,
    "min": 60 * 1000000,
    "mins": 60 * 1000000,
    "hours": 3600 * 1000000,
    "hour": 3600 * 1000000,
    "hr": 3600 * 1000000,
    "hrs": 3600 * 1000000,
    "h": 3600 * 1000000,
    "day": 86400 * 1000000,
    "days": 86400 * 1000000,
    "d": 86400 * 1000000,
    "month": 30 * 86400 * 1000000,  # Only allowed for 360_day calendar
    "months": 30 * 86400 * 1000000,
    "common_year": 365 * 86400 * 1000000, # Only allowed for 365_day and no_leap calendars
    "common_years": 365 * 86400 * 1000000 # Only allowed for 365_day and no_leap calendars
    
}

DATE_TYPES = {
     "proleptic_gregorian": DatetimeProlepticGregorian,
     "standard": DatetimeGregorian,
     "noleap": DatetimeNoLeap,
     "365_day": DatetimeNoLeap,
     "all_leap": DatetimeAllLeap,
     "366_day": DatetimeAllLeap,
     "julian": DatetimeJulian,
     "360_day": Datetime360Day,
     "gregorian": DatetimeGregorian
 }

def to_calendar_specific_datetime(dt, calendar, use_python_datetime,has_year_zero=None):
    if use_python_datetime:
        return real_datetime(
               dt.year,
               dt.month,
               dt.day,
               dt.hour,
               dt.minute,
               dt.second,
               dt.microsecond)
    else:
        date_type = DATE_TYPES[calendar]
        return date_type(
               dt.year,
               dt.month,
               dt.day,
               dt.hour,
               dt.minute,
               dt.second,
               dt.microsecond,
               calendar=calendar,has_year_zero=has_year_zero)

_MAX_INT64 = np.iinfo("int64").max
_MIN_INT64 = np.iinfo("int64").min


def cast_to_int(num, units=None):
    if num.dtype.kind in "iu":
        return num
    else:
        #if np.any(num < _MIN_INT64) or np.any(num > _MAX_INT64):
        if np.any(np.less(num,_MIN_INT64,casting='same_kind')) or np.any(np.greater(num,_MAX_INT64,casting='same_kind')):
            raise OverflowError('time values outside range of 64 bit signed integers')
        if isinstance(num, np.ma.core.MaskedArray):
            int_num = np.ma.masked_array(np.rint(num), dtype=np.int64)
            # use ceil instead of rint if 1 microsec less than a second
            # or floor if 1 microsec greater than a second (issue #187)
            if units not in microsec_units and units not in millisec_units:
                int_num = np.ma.where(int_num%1000000 == 1, \
                          np.ma.masked_array(np.floor(num),dtype=np.int64), int_num)
                int_num = np.ma.where(int_num%1000000 == 999999, \
                          np.ma.masked_array(np.ceil(num),dtype=np.int64), int_num)
        else:
            int_num = np.array(np.rint(num), dtype=np.int64)<commit_msg>revert change to see if windows tests fail<commit_after>"""
Performs conversions of netCDF time coordinate data to/from datetime objects.
"""

from cpython.object cimport (PyObject_RichCompare, Py_LT, Py_LE, Py_EQ,
                             Py_NE, Py_GT, Py_GE)
from numpy cimport int64_t, int32_t
import cython
import numpy as np
import re
import sys
import time
from datetime import datetime as datetime_python
from datetime import timedelta, MINYEAR, MAXYEAR
import time                     # strftime
import warnings

microsec_units = ['microseconds','microsecond', 'microsec', 'microsecs']
millisec_units = ['milliseconds', 'millisecond', 'millisec', 'millisecs', 'msec', 'msecs', 'ms']
sec_units =      ['second', 'seconds', 'sec', 'secs', 's']
min_units =      ['minute', 'minutes', 'min', 'mins']
hr_units =       ['hour', 'hours', 'hr', 'hrs', 'h']
day_units =      ['day', 'days', 'd']
month_units =    ['month', 'months'] # only allowed for 360_day calendar
year_units =     ['common_year', 'common_years'] # only allowed for 365_day and noleap calendars
_units = microsec_units+millisec_units+sec_units+min_units+hr_units+day_units
# supported calendars. Includes synonyms ('standard'=='gregorian',
# '366_day'=='all_leap','365_day'=='noleap')
# see http://cfconventions.org/cf-conventions/cf-conventions#calendar
# for definitions.
_calendars = ['standard', 'gregorian', 'proleptic_gregorian',
              'noleap', 'julian', 'all_leap', '365_day', '366_day', '360_day']
_idealized_calendars= ['all_leap','noleap','366_day','365_day','360_day']
# Following are number of days per month
cdef int[12] _dayspermonth      = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
cdef int[12] _dayspermonth_leap = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
# same as above, but including accumulated days of previous months.
cdef int[13] _cumdayspermonth = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365]
cdef int[13] _cumdayspermonth_leap = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366]

__version__ = '1.6.0'

# Adapted from http://delete.me.uk/2005/03/iso8601.html
# Note: This regex ensures that all ISO8601 timezone formats are accepted - but, due to legacy support for other timestrings, not all incorrect formats can be rejected.
#       For example, the TZ spec "+01:0" will still work even though the minutes value is only one character long.
ISO8601_REGEX = re.compile(r"(?P<year>[+-]?[0-9]+)(-(?P<month>[0-9]{1,2})(-(?P<day>[0-9]{1,2})"
                           r"(((?P<separator1>.)(?P<hour>[0-9]{1,2}):(?P<minute>[0-9]{1,2})(:(?P<second>[0-9]{1,2})(\.(?P<fraction>[0-9]+))?)?)?"
                           r"((?P<separator2>.?)(?P<timezone>Z|(([-+])([0-9]{2})((:([0-9]{2}))|([0-9]{2}))?)))?)?)?)?"
                           )
# Note: The re module apparently does not support branch reset groups that allow redefinition of the same group name in alternative branches as PCRE does.
#       Using two different group names is also somewhat ugly, but other solutions might hugely inflate the expression. feel free to contribute a better solution.
TIMEZONE_REGEX = re.compile(
    "(?P<prefix>[+-])(?P<hours>[0-9]{2})(?:(?::(?P<minutes1>[0-9]{2}))|(?P<minutes2>[0-9]{2}))?")

class real_datetime(datetime_python):
    """add dayofwk, dayofyr, daysinmonth attributes to python datetime instance"""
    @property
    def dayofwk(self):
        # 0=Monday, 6=Sunday
        return self.weekday()
    @property
    def dayofyr(self):
        return self.timetuple().tm_yday
    @property
    def daysinmonth(self):
        if _is_leap(self.year,'proleptic_gregorian'):
            return _dayspermonth_leap[self.month-1]
        else:
            return _dayspermonth[self.month-1]
    nanosecond = 0 # workaround for pandas bug (cftime issue #77)

def _datesplit(timestr):
    """split a time string into two components, units and the remainder
    after 'since'
    """
    try:
        (units, sincestring, remainder) = timestr.split(None,2)
    except ValueError as e:
        raise ValueError('Incorrectly formatted CF date-time unit_string')

    if sincestring.lower() != 'since':
        raise ValueError("no 'since' in unit_string")

    return units.lower(), remainder

def _dateparse(timestr,calendar,has_year_zero=None):
    """parse a string of the form time-units since yyyy-mm-dd hh:mm:ss,
    return a datetime instance"""
    # same as version in cftime, but returns a timezone naive
    # python datetime instance with the utc_offset included.

    calendar = calendar.lower()
    # set calendar-specific defaults for has_year_zero
    if has_year_zero is None:
        has_year_zero = _year_zero_defaults(calendar)
    (units, isostring) = _datesplit(timestr)
    if not ((units in month_units and calendar=='360_day') or (units in year_units and calendar in {'365_day', 'noleap'}) or units in _units):
        if units in month_units and calendar != '360_day':
            raise ValueError("'months since' units only allowed for '360_day' calendar")
        if units in year_units and calendar not in {'365_day', 'noleap'}:    
            raise ValueError("'%s' units only allowed for '365_day' and 'noleap' calendars" % units) 
        else:
            raise ValueError(
            "units must be one of 'seconds', 'minutes', 'hours' or 'days' (or singular version of these), got '%s'" % units)
    # parse the date string.
    year, month, day, hour, minute, second, microsecond, utc_offset =\
        _parse_date( isostring.strip() )
    if year == 0 and not has_year_zero and calendar in ['julian', 'standard', 'gregorian', 'proleptic_gregorian']:
        msg='zero not allowed as a reference year when has_year_zero=False'
        raise ValueError(msg)
    if calendar in ['noleap', '365_day'] and month == 2 and day == 29:
        raise ValueError(
            'cannot specify a leap day as the reference time with the noleap calendar')
    if calendar == '360_day' and day > 30:
        raise ValueError(
            'there are only 30 days in every month with the 360_day calendar')
    basedate = datetime(year, month, day, hour, minute, second,
                        microsecond,calendar=calendar,has_year_zero=has_year_zero)
    # subtract utc_offset from basedate time instance (which is timezone naive)
    if utc_offset:
        basedate -= timedelta(days=utc_offset/1440.)
    return basedate

def _can_use_python_datetime(date,calendar):
    gregorian = datetime(1582,10,15,calendar=calendar,has_year_zero=date.has_year_zero)
    return ((calendar == 'proleptic_gregorian' and date.year >= MINYEAR and date.year <= MAXYEAR) or \
           (calendar in ['gregorian','standard'] and date > gregorian and date.year <= MAXYEAR))

@cython.embedsignature(True)
def date2num(dates,units,calendar=None,has_year_zero=None):
    """
    Return numeric time values given datetime objects. The units
    of the numeric time values are described by the **units** argument
    and the **calendar** keyword. The datetime objects must
    be in UTC with no time-zone offset.  If there is a
    time-zone offset in **units**, it will be applied to the
    returned numeric values.

    **dates**: A datetime object or a sequence of datetime objects.
    The datetime objects should not include a time-zone offset. They
    can be either native python datetime instances (which use
    the proleptic gregorian calendar) or cftime.datetime instances.

    **units**: a string of the form **<time units> since <reference time>**
    describing the time units. **<time units>** can be days, hours, minutes,
    seconds, milliseconds or microseconds. **<reference time>** is the time
    origin. **months since** is allowed *only* for the **360_day** calendar
    and **common_years since** is allowed *only* for the **365_day** calendar.

    **calendar**: describes the calendar to be used in the time calculations.
    All the values currently defined in the
    `CF metadata convention <http://cfconventions.org/cf-conventions/cf-conventions#calendar>`__ are supported.
    Valid calendars **'standard', 'gregorian', 'proleptic_gregorian'
    'noleap', '365_day', '360_day', 'julian', 'all_leap', '366_day'**.
    Default is `None` which means the calendar associated with the first
    input datetime instance will be used.

    **has_year_zero**: If set to True, astronomical year numbering
    is used and the year zero exists.  If set to False for real-world
    calendars, then historical year numbering is used and the year 1 is
    preceded by year -1 and no year zero exists.
    The defaults are set to conform with
    CF version 1.9 conventions (False for 'julian', 'gregorian'/'standard', True
    for 'proleptic_gregorian' (ISO 8601) and True for the idealized
    calendars 'noleap'/'365_day', '360_day', 366_day'/'all_leap')
    Note that CF v1.9 does not specifically mention whether year zero
    is allowed in the proleptic_gregorian calendar, but ISO-8601 has
    a year zero so we have adopted this as the default.
    The defaults can only be over-ridden for the real-world calendars,
    for the the idealized calendars the year zero 
    always exists and the has_year_zero kwarg is ignored.
    This kwarg is not needed to define calendar systems allowed by CF
    (the calendar-specific defaults do this).

    returns a numeric time value, or an array of numeric time values
    with approximately 1 microsecond accuracy.
    """

    # input a scale or array-like?
    isscalar = False
    try:
        dates[0]
    except:
        isscalar = True

    # masked array input?
    ismasked = False
    if np.ma.isMA(dates) and np.ma.is_masked(dates):
        mask = dates.mask
        ismasked = True

    # are all input dates 'real' python datetime objects?
    dates = np.asanyarray(dates) # convert to numpy array
    shape = dates.shape # save shape of input
    all_python_datetimes = True
    for date in dates.flat:
        if not isinstance(date,datetime_python):
           all_python_datetimes = False
           break

    # if has_year_zero is None and calendar is None, use first input cftime.datetime instance
    # to determine if year zero is to be included.
    # If calendar is specified, use calendar specific defaults
    if has_year_zero is None:
        if calendar is None:
            d0 = dates.item(0)
            if isinstance(d0,datetime_python):
                has_year_zero = False
            else:
                try:
                    has_year_zero = d0.has_year_zero
                except AttributeError:
                    raise ValueError('has_year_zero not specified',type(d0))
        else:
            has_year_zero = _year_zero_defaults(calendar)

    # if calendar is None or '', use calendar of first input cftime.datetime instances.
    # if inputs are 'real' python datetime instances, use propleptic gregorian.
    if not calendar:
        if all_python_datetimes:
            calendar = 'proleptic_gregorian'
        else:
            d0 = dates.item(0)
            if isinstance(d0,datetime_python):
                calendar = 'proleptic_gregorian' 
            else:
                try:
                    calendar = d0.calendar
                except AttributeError:
                    raise ValueError('no calendar specified',type(d0))

    calendar = calendar.lower()
    basedate = _dateparse(units,calendar=calendar,has_year_zero=has_year_zero)
    (unit, isostring) = _datesplit(units)
    # real-world calendars cannot have zero as a reference year.
    if calendar in ['julian', 'standard', 'gregorian', 'proleptic_gregorian'] and not has_year_zero:
        if basedate.year == 0:
            msg='zero not allowed as a reference year, does not exist in Julian or Gregorian calendars'
            raise ValueError(msg)
    if unit not in UNIT_CONVERSION_FACTORS:
        raise ValueError("Unsupported time units provided, {!r}.".format(unit))
    if unit in ["months", "month"] and calendar != "360_day":
        raise ValueError("Units of months only valid for 360_day calendar.")
    unit_timedelta = timedelta(microseconds=UNIT_CONVERSION_FACTORS[unit])
    can_use_python_basedatetime = _can_use_python_datetime(basedate,calendar)

    if can_use_python_basedatetime and all_python_datetimes:
        use_python_datetime = True
        if not isinstance(basedate, datetime_python):
            basedate = real_datetime(basedate.year, basedate.month, basedate.day, 
                       basedate.hour, basedate.minute, basedate.second,
                       basedate.microsecond)
    else:
        use_python_datetime = False
        # convert basedate to specified calendar
        basedate =  to_calendar_specific_datetime(basedate, calendar, False, has_year_zero=has_year_zero)
    times = []
    for n, date in enumerate(dates.flat):
        if ismasked and mask.flat[n]:
            times.append(None)
            continue

        # use python datetime if possible.
        if use_python_datetime:
            # remove time zone offset
            if getattr(date, 'tzinfo',None) is not None:
                date = date.replace(tzinfo=None) - date.utcoffset()
        else: # convert date to same calendar specific cftime.datetime instance
            date = to_calendar_specific_datetime(date, calendar, False, has_year_zero=has_year_zero)

        td = date - basedate
        if td % unit_timedelta == timedelta(0):
            # Explicitly cast result to np.int64 for Windows compatibility
            quotient = np.int64(td // unit_timedelta)
            times.append(quotient)
        else:
            times.append(td / unit_timedelta)
                
    if ismasked: # convert to masked array if input was masked array
        times = np.array(times, dtype=float)  # None -> nan
        times = np.ma.masked_invalid(times)
        if isscalar:
            return times[0]
        else:
            return np.reshape(times, shape)
    if isscalar:
        return times[0]
    else:
        return np.reshape(np.array(times), shape)


@cython.embedsignature(True)
def num2pydate(times,units,calendar='standard'):
    """
    Always returns python datetime.datetime
    objects and raise an error if this is not possible.

    Same as
    num2date(times,units,calendar,only_use_cftime_datetimes=False,only_use_python_datetimes=True)
    """
    return num2date(times,units,calendar,only_use_cftime_datetimes=False,only_use_python_datetimes=True)


UNIT_CONVERSION_FACTORS = {
    "microseconds": 1,
    "microsecond": 1,
    "microsec": 1,
    "microsecs": 1,
    "milliseconds": 1000,
    "millisecond": 1000,
    "millisec": 1000,
    "millisecs": 1000,
    "msec": 1000,
    "msecs": 1000,
    "ms": 1000,
    "seconds": 1000000,
    "second": 1000000,
    "sec": 1000000,
    "secs": 1000000,
    "s": 1000000,
    "minutes": 60 * 1000000,
    "minute": 60 * 1000000,
    "min": 60 * 1000000,
    "mins": 60 * 1000000,
    "hours": 3600 * 1000000,
    "hour": 3600 * 1000000,
    "hr": 3600 * 1000000,
    "hrs": 3600 * 1000000,
    "h": 3600 * 1000000,
    "day": 86400 * 1000000,
    "days": 86400 * 1000000,
    "d": 86400 * 1000000,
    "month": 30 * 86400 * 1000000,  # Only allowed for 360_day calendar
    "months": 30 * 86400 * 1000000,
    "common_year": 365 * 86400 * 1000000, # Only allowed for 365_day and no_leap calendars
    "common_years": 365 * 86400 * 1000000 # Only allowed for 365_day and no_leap calendars
    
}

DATE_TYPES = {
     "proleptic_gregorian": DatetimeProlepticGregorian,
     "standard": DatetimeGregorian,
     "noleap": DatetimeNoLeap,
     "365_day": DatetimeNoLeap,
     "all_leap": DatetimeAllLeap,
     "366_day": DatetimeAllLeap,
     "julian": DatetimeJulian,
     "360_day": Datetime360Day,
     "gregorian": DatetimeGregorian
 }

def to_calendar_specific_datetime(dt, calendar, use_python_datetime,has_year_zero=None):
    if use_python_datetime:
        return real_datetime(
               dt.year,
               dt.month,
               dt.day,
               dt.hour,
               dt.minute,
               dt.second,
               dt.microsecond)
    else:
        date_type = DATE_TYPES[calendar]
        return date_type(
               dt.year,
               dt.month,
               dt.day,
               dt.hour,
               dt.minute,
               dt.second,
               dt.microsecond,
               calendar=calendar,has_year_zero=has_year_zero)

_MAX_INT64 = np.iinfo("int64").max
_MIN_INT64 = np.iinfo("int64").min


def cast_to_int(num, units=None):
    if num.dtype.kind in "iu":
        return num
    else:
        if np.any(num < _MIN_INT64) or np.any(num > _MAX_INT64):
        #if np.any(np.less(num,_MIN_INT64,casting='same_kind')) or np.any(np.greater(num,_MAX_INT64,casting='same_kind')):
            raise OverflowError('time values outside range of 64 bit signed integers')
        if isinstance(num, np.ma.core.MaskedArray):
            int_num = np.ma.masked_array(np.rint(num), dtype=np.int64)
            # use ceil instead of rint if 1 microsec less than a second
            # or floor if 1 microsec greater than a second (issue #187)
            if units not in microsec_units and units not in millisec_units:
                int_num = np.ma.where(int_num%1000000 == 1, \
                          np.ma.masked_array(np.floor(num),dtype=np.int64), int_num)
                int_num = np.ma.where(int_num%1000000 == 999999, \
                          np.ma.masked_array(np.ceil(num),dtype=np.int64), int_num)
        else:
            int_num = np.array(np.rint(num), dtype=np.int64)
{'fraction_non_alphanumeric': 0.10592894384455398, 'fraction_numerical': 0.038519615943474174, 'mean_word_length': 2.9252963542831583, 'pattern_counts': {'":': 86, '<': 47, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1742394', 'n_tokens_mistral': 11947, 'n_tokens_neox': 10619, 'n_words': 3296}