<commit_before>import datetime
from django.db.models.query import QuerySet
from django.template.defaultfilters import date
from django.utils.translation import ugettext, ugettext_lazy as _
from django.utils.dates import WEEKDAYS, WEEKDAYS_ABBR
from django.conf import settings
from schedule.models import Occurrence
from schedule.utils import OccurrenceReplacer

# Look for FIRST_DAY_OF_WEEK as a locale setting
first_day_of_week = ugettext('FIRST_DAY_OF_WEEK')
if first_day_of_week == 'FIRST_DAY_OF_WEEK':
    # FIRST_DAY_OF_WEEK was not set in the locale, try the settings
    first_day_of_week = getattr(settings, 'FIRST_DAY_OF_WEEK', "0")

try:
    first_day_of_week = int(first_day_of_week)
except:
    # default to Sunday
    first_day_of_week = 0

weekday_names = []
weekday_abbrs = []
if first_day_of_week == 1:
    # The calendar week starts on Monday
    for i in range(7):
        weekday_names.append( WEEKDAYS[i] )
        weekday_abbrs.append( WEEKDAYS_ABBR[i] )
else:
    # The calendar week starts on Sunday, not Monday
    weekday_names.append( WEEKDAYS[6] )
    weekday_abbrs.append( WEEKDAYS_ABBR[6] )
    for i in range(6):
        weekday_names.append( WEEKDAYS[i] )
        weekday_abbrs.append( WEEKDAYS_ABBR[i] )

class Period(object):
    '''
    This class represents a period of time. It can return a set of occurrences
    based on its events, and its time period (start and end).
    '''
    def __init__(self, events, start, end, parent_persisted_occurrences = None,
        occurrence_pool=None):
        self.start = start
        self.end = end
        self.events = events
        self.occurrence_pool = occurrence_pool
        if parent_persisted_occurrences is not None:
            self._persisted_occurrences = parent_persisted_occurrences

    def __eq__(self, period):
        return self.start==period.start and self.end==period.end and self.events==period.events

    def _get_sorted_occurrences(self):
        occurrences = []
        if hasattr(self, "occurrence_pool") and self.occurrence_pool is not None:
            for occurrence in self.occurrence_pool:
                if occurrence.start < self.end and occurrence.end >self.start:
                    occurrences.append(occurrence)
            return occurrences
        for event in self.events:
            event_occurrences = event.get_occurrences(self.start, self.end)
            occurrences += event_occurrences
        return sorted(occurrences)

    def cached_get_sorted_occurrences(self):
        if hasattr(self, '_occurrences'):
             return self._occurrences
        occs = self._get_sorted_occurrences()
        self._occurrences = occs
        return occs
    occurrences = property(cached_get_sorted_occurrences)

    def get_persisted_occurrences(self):
        if hasattr(self, '_persisted_occurrenes'):
            return self._persisted_occurrences
        else:
            self._persisted_occurrences = Occurrence.objects.filter(event__in = self.events)
            return self._persisted_occurrences

    def classify_occurrence(self, occurrence):
        if occurrence.cancelled:
            return None
        if occurrence.start > self.end or occurrence.end < self.start:
            return None
        started = False
        ended = False
        if occurrence.start >= self.start and occurrence.start < self.end:
            started = True
        if occurrence.end >=self.start and occurrence.end< self.end:
            ended = True<commit_msg>Revert "Skip cancelled occurrences while classifying"<commit_after>import datetime
from django.db.models.query import QuerySet
from django.template.defaultfilters import date
from django.utils.translation import ugettext, ugettext_lazy as _
from django.utils.dates import WEEKDAYS, WEEKDAYS_ABBR
from django.conf import settings
from schedule.models import Occurrence
from schedule.utils import OccurrenceReplacer

# Look for FIRST_DAY_OF_WEEK as a locale setting
first_day_of_week = ugettext('FIRST_DAY_OF_WEEK')
if first_day_of_week == 'FIRST_DAY_OF_WEEK':
    # FIRST_DAY_OF_WEEK was not set in the locale, try the settings
    first_day_of_week = getattr(settings, 'FIRST_DAY_OF_WEEK', "0")

try:
    first_day_of_week = int(first_day_of_week)
except:
    # default to Sunday
    first_day_of_week = 0

weekday_names = []
weekday_abbrs = []
if first_day_of_week == 1:
    # The calendar week starts on Monday
    for i in range(7):
        weekday_names.append( WEEKDAYS[i] )
        weekday_abbrs.append( WEEKDAYS_ABBR[i] )
else:
    # The calendar week starts on Sunday, not Monday
    weekday_names.append( WEEKDAYS[6] )
    weekday_abbrs.append( WEEKDAYS_ABBR[6] )
    for i in range(6):
        weekday_names.append( WEEKDAYS[i] )
        weekday_abbrs.append( WEEKDAYS_ABBR[i] )

class Period(object):
    '''
    This class represents a period of time. It can return a set of occurrences
    based on its events, and its time period (start and end).
    '''
    def __init__(self, events, start, end, parent_persisted_occurrences = None,
        occurrence_pool=None):
        self.start = start
        self.end = end
        self.events = events
        self.occurrence_pool = occurrence_pool
        if parent_persisted_occurrences is not None:
            self._persisted_occurrences = parent_persisted_occurrences

    def __eq__(self, period):
        return self.start==period.start and self.end==period.end and self.events==period.events

    def _get_sorted_occurrences(self):
        occurrences = []
        if hasattr(self, "occurrence_pool") and self.occurrence_pool is not None:
            for occurrence in self.occurrence_pool:
                if occurrence.start < self.end and occurrence.end >self.start:
                    occurrences.append(occurrence)
            return occurrences
        for event in self.events:
            event_occurrences = event.get_occurrences(self.start, self.end)
            occurrences += event_occurrences
        return sorted(occurrences)

    def cached_get_sorted_occurrences(self):
        if hasattr(self, '_occurrences'):
             return self._occurrences
        occs = self._get_sorted_occurrences()
        self._occurrences = occs
        return occs
    occurrences = property(cached_get_sorted_occurrences)

    def get_persisted_occurrences(self):
        if hasattr(self, '_persisted_occurrenes'):
            return self._persisted_occurrences
        else:
            self._persisted_occurrences = Occurrence.objects.filter(event__in = self.events)
            return self._persisted_occurrences

    def classify_occurrence(self, occurrence):
        if occurrence.start > self.end or occurrence.end < self.start:
            return None
        started = False
        ended = False
        if occurrence.start >= self.start and occurrence.start < self.end:
            started = True
        if occurrence.end >=self.start and occurrence.end< self.end:
            ended = True
{'fraction_non_alphanumeric': 0.07363727887242917, 'fraction_numerical': 0.002013519344167985, 'mean_word_length': 2.9851002865329512, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6158393', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2040, 'n_words': 530}