<commit_before>"""
The schedule module is a basic cron for IceProd, executing tasks
at various intervals or fixed times.
"""

from __future__ import absolute_import, division, print_function

import time
from threading import Thread,Event,Condition
import logging
from functools import partial
from contextlib import contextmanager
from itertools import izip

import tornado.gen

import iceprod.server
from iceprod.server import module
from iceprod.server.schedule import Scheduler

logger = logging.getLogger('modules_schedule')

class schedule(module.module):
    """
    Run the schedule module, which handles periodic tasks.
    """

    def __init__(self,*args,**kwargs):
        # run default init
        super(schedule,self).__init__(*args,**kwargs)
        self.scheduler = None

    def start(self):
        """Start schedule"""
        super(schedule,self).start()
        if not self.scheduler:
            # make Scheduler
            self.scheduler = Scheduler(self.io_loop)
            def cb():
                try:
                    self._make_schedule()
                except Exception:
                    logger.error('error making schedule', exc_info=True)
            self.io_loop.add_callback(cb)

        self.scheduler.start()

    def stop(self):
        """Stop schedule"""
        if self.scheduler:
            self.scheduler = None
        super(schedule,self).stop()

    def kill(self):
        """Kill thread"""
        if self.scheduler:
            self.scheduler = None
        super(schedule,self).kill()

    def _make_schedule(self):
        """Make the default schedule"""

        # mark dataset complete
        self.scheduler.schedule('every 1 hours',
                self.modules['db']['cron_dataset_completion'])

        # collate node resources
        self.scheduler.schedule('every 1 hours',
                partial(self.modules['db']['node_collate_resources'],
                        site_id=self.cfg['site_id']))

        self.scheduler.schedule('every 6 hours',
                self.modules['db']['cron_remove_old_passkeys'])

        self.scheduler.schedule('every 10 minutes',
                self.modules['db']['cron_generate_web_graphs'])

        if ('master' in self.cfg and 'status' in self.cfg['master'] and
            self.cfg['master']['status']):
            self._master_schedule()

    def _master_schedule(self):
        # fake a grid, so we can do grid-like things
        from iceprod.server.grid import grid
        args = [None, self.cfg['queue']['*'], self.cfg, self.modules,
                self.io_loop, self.executor]
        master_grid = grid(*args)

        self.scheduler.schedule('every 1 minutes', master_grid.check_iceprod)

        # mark jobs complete
        self.scheduler.schedule('every 31 minutes',
                self.modules['db']['cron_job_completion'])
<commit_msg>setup gridftp proxy on master. closes #161<commit_after>"""
The schedule module is a basic cron for IceProd, executing tasks
at various intervals or fixed times.
"""

from __future__ import absolute_import, division, print_function

import time
from threading import Thread,Event,Condition
import logging
from functools import partial
from contextlib import contextmanager
from itertools import izip

import tornado.gen

import iceprod.server
from iceprod.server import module
from iceprod.server.schedule import Scheduler
from iceprod.server.globus import SiteGlobusProxy

logger = logging.getLogger('modules_schedule')

class schedule(module.module):
    """
    Run the schedule module, which handles periodic tasks.
    """

    def __init__(self,*args,**kwargs):
        # run default init
        super(schedule,self).__init__(*args,**kwargs)
        self.scheduler = None

    def start(self):
        """Start schedule"""
        super(schedule,self).start()
        if not self.scheduler:
            # make Scheduler
            self.scheduler = Scheduler(self.io_loop)
            def cb():
                try:
                    self._make_schedule()
                except Exception:
                    logger.error('error making schedule', exc_info=True)
            self.io_loop.add_callback(cb)

        self.scheduler.start()

    def stop(self):
        """Stop schedule"""
        if self.scheduler:
            self.scheduler = None
        super(schedule,self).stop()

    def kill(self):
        """Kill thread"""
        if self.scheduler:
            self.scheduler = None
        super(schedule,self).kill()

    def _make_schedule(self):
        """Make the default schedule"""

        # mark dataset complete
        self.scheduler.schedule('every 1 hours',
                self.modules['db']['cron_dataset_completion'])

        # collate node resources
        self.scheduler.schedule('every 1 hours',
                partial(self.modules['db']['node_collate_resources'],
                        site_id=self.cfg['site_id']))

        self.scheduler.schedule('every 6 hours',
                self.modules['db']['cron_remove_old_passkeys'])

        self.scheduler.schedule('every 10 minutes',
                self.modules['db']['cron_generate_web_graphs'])

        if ('master' in self.cfg and 'status' in self.cfg['master'] and
            self.cfg['master']['status']):
            self._master_schedule()

    def _master_schedule(self):
        # fake a grid, so we can do grid-like things
        from iceprod.server.grid import grid
        args = [None, self.cfg['queue']['*'], self.cfg, self.modules,
                self.io_loop, self.executor]
        master_grid = grid(*args)

        # make sure the gridftp proxy is set up
        proxy_kwargs = {}
        if 'gridftp_cfgfile' in self.cfg['queue']:
            proxy_kwargs['cfgfile'] = self.cfg['queue']['gridftp_cfgfile']
        proxy = SiteGlobusProxy(**proxy_kwargs)
        proxy.update_proxy()

        self.scheduler.schedule('every 1 minutes', master_grid.check_iceprod)

        # mark jobs complete
        self.scheduler.schedule('every 31 minutes',
                self.modules['db']['cron_job_completion'])

{'fraction_non_alphanumeric': 0.10849909584086799, 'fraction_numerical': 0.0031234588196613513, 'mean_word_length': 2.4509359047078845, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2117686', 'n_tokens_mistral': 1677, 'n_tokens_neox': 1585, 'n_words': 414}