<commit_before>
import threading

from classtime.logging import logging
logging = logging.getLogger(__name__) # pylint: disable=C0103

from classtime.brain.remote_db import RemoteDatabaseFactory
from classtime.brain.local_db import LocalDatabaseFactory
from classtime.models.schedule import calculate_schedule_hash

class AcademicCalendar(object):
    """Manages academic calendar data for a particular institution

    Internally, uses a stack-based datatype access idiom. Usage::
     self.push_<datatype>()
     ... use self.cur_datatype() or self.cur_primary_keys() ...
     self.pop_<datatype>()

     where <datatype> is one of [terms, courses, sections, classtimes]
    """

    idle_workers = dict()

    def __init__(self, institution):
        """Create a calendar for a specific institution

        :param str institution: name of JSON configuration
            file which describes how to deal with this particular
            institution. See :py:class:`RemoteDatabaseFactory` for
            the exact file location. Use an existing config file
            as a template.
        """
        self._institution = institution
        self._term = None
        self._datatype_stack = ['terms']
        self._primary_keys_stack = [ ('term',) ]

        try:
            self._remote_db = RemoteDatabaseFactory.build(institution)
            self._remote_db.connect()
        except:
            raise

        try:
            self._local_db = LocalDatabaseFactory.build(institution)
            self._local_db.create()
        except:
            raise

    @classmethod
    def idly_fill(cls, institution, sleeptime=10, force_refresh=False):
        """Launches a new thread which idly fetches and saves
        course data from the given institution
        """
        # pylint: disable=W0212
        def _idly_download_courses(self, sleeptime, force_refresh):
            import time
            if self.doesnt_know_about(datatype='terms'):
                logging.info('[worker] Fetching all <{}> terms'.format(
                    institution))
                terms = self._fetch(datatype='terms')
                self._save(terms, datatype='terms')

            terms = [term.term
                     for term in self._local_db.query(datatype='terms').all()]

            terms.reverse()
            for termid in terms:
                if force_refresh or self.doesnt_know_about(datatype='courses', term=termid):
                    logging.info('[worker] Fetching courses - <{}> <term={}>'.format(institution, termid))
                    courses = self._fetch(datatype='courses', term=termid)
                    logging.info('[worker] Fetched {} courses'.format(len(courses)))
                    self._save(courses, datatype='courses')
                    msg = '[worker]...Saved courses - <{}> <term={}>'
                    logging.info(msg.format(len(courses), institution, termid))
                    for _ in range(sleeptime):
                        time.sleep(1)

        # pylint: enable=W0212 #pylint: disable=I0012
        if AcademicCalendar.idle_workers.get(institution) is None:
            idle_worker = threading.Thread(
                target=_idly_download_courses,
                args=(AcademicCalendar(institution), sleeptime, force_refresh))<commit_msg>Fix minor logging issue<commit_after>
import threading

from classtime.logging import logging
logging = logging.getLogger(__name__) # pylint: disable=C0103

from classtime.brain.remote_db import RemoteDatabaseFactory
from classtime.brain.local_db import LocalDatabaseFactory
from classtime.models.schedule import calculate_schedule_hash

class AcademicCalendar(object):
    """Manages academic calendar data for a particular institution

    Internally, uses a stack-based datatype access idiom. Usage::
     self.push_<datatype>()
     ... use self.cur_datatype() or self.cur_primary_keys() ...
     self.pop_<datatype>()

     where <datatype> is one of [terms, courses, sections, classtimes]
    """

    idle_workers = dict()

    def __init__(self, institution):
        """Create a calendar for a specific institution

        :param str institution: name of JSON configuration
            file which describes how to deal with this particular
            institution. See :py:class:`RemoteDatabaseFactory` for
            the exact file location. Use an existing config file
            as a template.
        """
        self._institution = institution
        self._term = None
        self._datatype_stack = ['terms']
        self._primary_keys_stack = [ ('term',) ]

        try:
            self._remote_db = RemoteDatabaseFactory.build(institution)
            self._remote_db.connect()
        except:
            raise

        try:
            self._local_db = LocalDatabaseFactory.build(institution)
            self._local_db.create()
        except:
            raise

    @classmethod
    def idly_fill(cls, institution, sleeptime=10, force_refresh=False):
        """Launches a new thread which idly fetches and saves
        course data from the given institution
        """
        # pylint: disable=W0212
        def _idly_download_courses(self, sleeptime, force_refresh):
            import time
            if self.doesnt_know_about(datatype='terms'):
                logging.info('[worker] Fetching all <{}> terms'.format(
                    institution))
                terms = self._fetch(datatype='terms')
                self._save(terms, datatype='terms')

            terms = [term.term
                     for term in self._local_db.query(datatype='terms').all()]

            terms.reverse()
            for termid in terms:
                if force_refresh or self.doesnt_know_about(datatype='courses', term=termid):
                    logging.info('[worker] Fetching courses - <{}> <term={}>'.format(institution, termid))
                    courses = self._fetch(datatype='courses', term=termid)
                    logging.info('[worker] Fetched {} courses'.format(len(courses)))
                    self._save(courses, datatype='courses')
                    msg = '[worker]...Saved courses - <{}> <term={}>'
                    logging.info(msg.format(institution, termid))
                    for _ in range(sleeptime):
                        time.sleep(1)

        # pylint: enable=W0212 #pylint: disable=I0012
        if AcademicCalendar.idle_workers.get(institution) is None:
            idle_worker = threading.Thread(
                target=_idly_download_courses,
                args=(AcademicCalendar(institution), sleeptime, force_refresh))
{'fraction_non_alphanumeric': 0.09593816307972113, 'fraction_numerical': 0.005759321006365566, 'mean_word_length': 2.2781917536015897, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4023879', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1725, 'n_words': 429}