<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
from time import time, mktime
import datetime
from dateutil.relativedelta import relativedelta

search_re = re.compile(r'^([^-]+)-([^-/]+)(/(.*))?$')
only_int_re = re.compile(r'^\d+$')
any_int_re = re.compile(r'^\d+')
star_or_int_re = re.compile(r'^(\d+|\*)$')

__all__ = ('croniter',)


class croniter(object):
    MONTHS_IN_YEAR = 12
    RANGES = (
        (0, 59),
        (0, 23),
        (1, 31),
        (1, 12),
        (0, 6),
        (0, 59)
    )
    DAYS = (
        31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
    )

    ALPHACONV = (
        { },
        { },
        { },
        { 'jan':1, 'feb':2, 'mar':3, 'apr':4, 'may':5, 'jun':6,
          'jul':7, 'aug':8, 'sep':9, 'oct':10, 'nov':11, 'dec':12 },
        { 'sun':0, 'mon':1, 'tue':2, 'wed':3, 'thu':4, 'fri':5, 'sat':6 },
        { }
    )

    LOWMAP = (
        {},
        {},
        {0: 1},
        {0: 1},
        {7: 0},
        {},
    )

    bad_length = 'Exactly 5 or 6 columns has to be specified for iterator' \
                 'expression.'
    
    def __init__(self, expr_format, start_time=time()):
        if isinstance(start_time, datetime.datetime):
            start_time = mktime(start_time.timetuple())

        self.cur = start_time
        self.exprs = expr_format.split()

        if len(self.exprs) != 5 and len(self.exprs) != 6:
            raise ValueError(self.bad_length)

        expanded = []

        for i, expr in enumerate(self.exprs):
            e_list = expr.split(',')
            res = []

            while len(e_list) > 0:
                e = e_list.pop()
                t = re.sub(r'^\*(/.+)$', r'%d-%d\1' % (self.RANGES[i][0],
                                                       self.RANGES[i][1]),
                                                       str(e))
                m = search_re.search(t)

                if m:
                    (low, high, step) = m.group(1), m.group(2), m.group(4) or 1

                    if not any_int_re.search(low):
                        low = self.ALPHACONV[i][low.lower()]

                    if not any_int_re.search(high):
                        high = self.ALPHACONV[i][high.lower()]

                    if (not low or not high or int(low) > int(high)
                        or not only_int_re.search(str(step))):
                        raise ValueError("[%s] is not acceptable" %expr_format)

                    for j in xrange(int(low), int(high)+1):
                        if j % int(step) == 0:
                            e_list.append(j)
                else:
                    if not star_or_int_re.search(t):
                        t = self.ALPHACONV[i][t.lower()]
            
                    try:
                        t = int(t)<commit_msg>fixed range generator<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
from time import time, mktime
import datetime
from dateutil.relativedelta import relativedelta

search_re = re.compile(r'^([^-]+)-([^-/]+)(/(.*))?$')
only_int_re = re.compile(r'^\d+$')
any_int_re = re.compile(r'^\d+')
star_or_int_re = re.compile(r'^(\d+|\*)$')

__all__ = ('croniter',)


class croniter(object):
    MONTHS_IN_YEAR = 12
    RANGES = (
        (0, 59),
        (0, 23),
        (1, 31),
        (1, 12),
        (0, 6),
        (0, 59)
    )
    DAYS = (
        31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
    )

    ALPHACONV = (
        { },
        { },
        { },
        { 'jan':1, 'feb':2, 'mar':3, 'apr':4, 'may':5, 'jun':6,
          'jul':7, 'aug':8, 'sep':9, 'oct':10, 'nov':11, 'dec':12 },
        { 'sun':0, 'mon':1, 'tue':2, 'wed':3, 'thu':4, 'fri':5, 'sat':6 },
        { }
    )

    LOWMAP = (
        {},
        {},
        {0: 1},
        {0: 1},
        {7: 0},
        {},
    )

    bad_length = 'Exactly 5 or 6 columns has to be specified for iterator' \
                 'expression.'
    
    def __init__(self, expr_format, start_time=time()):
        if isinstance(start_time, datetime.datetime):
            start_time = mktime(start_time.timetuple())

        self.cur = start_time
        self.exprs = expr_format.split()

        if len(self.exprs) != 5 and len(self.exprs) != 6:
            raise ValueError(self.bad_length)

        expanded = []

        for i, expr in enumerate(self.exprs):
            e_list = expr.split(',')
            res = []

            while len(e_list) > 0:
                e = e_list.pop()
                t = re.sub(r'^\*(/.+)$', r'%d-%d\1' % (self.RANGES[i][0],
                                                       self.RANGES[i][1]),
                                                       str(e))
                m = search_re.search(t)

                if m:
                    (low, high, step) = m.group(1), m.group(2), m.group(4) or 1

                    if not any_int_re.search(low):
                        low = self.ALPHACONV[i][low.lower()]

                    if not any_int_re.search(high):
                        high = self.ALPHACONV[i][high.lower()]

                    if (not low or not high or int(low) > int(high)
                        or not only_int_re.search(str(step))):
                        raise ValueError("[%s] is not acceptable" %expr_format)

                    for j in xrange(int(low), int(high)+1, int(step)):
                        e_list.append(j)
                else:
                    if not star_or_int_re.search(t):
                        t = self.ALPHACONV[i][t.lower()]
            
                    try:
                        t = int(t)
{'fraction_non_alphanumeric': 0.16744269340974213, 'fraction_numerical': 0.030623209169054443, 'mean_word_length': 1.3407376362112322, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3472497', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1862, 'n_words': 390}