<commit_before>import os
import re
import logging

from hashlib import md5
from configparser import SafeConfigParser
from datetime import timedelta, datetime

from crontab import CronTab
from send2trash import send2trash

class GarbageTruck:
    def __init__(self):
        self._logger = logging.getLogger('garbagetruck')
        self._cron = CronTab(user=True)
        self._config = SafeConfigParser()
        self._config_fn = os.path.join(os.path.expanduser('~'), '.garbagetruckrc')
        if os.path.exists(self._config_fn):
            self._config.read(self._config_fn)

    def set_job(self, run_command_format, name, dirs,
                files_older_than='90 days', check_every='week'):
        '''Set a job by adding or replacing based on the name.

        A call to `set_job` will either add a new job or replace a previously set job with the new
        parameters. The `name` is the identifier for a job and must be **unique** accross all jobs
        for the same user.

        :param run_command_format: the command called for running job (will interpolate `%s` with a
                                   job ID)
        :param name: a unique name for this job
        :param dirs: a list of directories to iterate looking for old files
        :param files_older_than: the period to use to determine what "old" is for each file
        :param check_every: the period to use for when to trigger looking for old files
        '''
        section_name = GarbageTruck._section_name_for(name)
        self._logger.debug('Setting job: %s (%s)', name, section_name)
        self.remove_job(name)
        job = self._cron.new(command=run_command_format % section_name,
                             comment=GarbageTruck._comment_for(name))
        job_period = GarbageTruck._cron_safe_period_from(check_every)
        period_method = getattr(job, job_period[1])
        period_method.every(job_period[0])
        # replace any pre-existing session
        self._config.remove_section(section_name)
        self._config.add_section(section_name)
        self._config.set(section_name, 'name', name)
        self._config.set(section_name, 'files_older_than', files_older_than)
        count = 0
        for dirname in dirs:
            count += 1
            optname = 'dir' + str(count)
            self._config.set(section_name, optname, dirname)

    def list_jobs(self):
        for section_name in self._config.sections():
            items = {'dirs': []}
            for name, value in self._config.items(section_name):
                if name.startswith('dir'):
                    items['dirs'].append(value)
                else:
                    items[name] = '"' + value + '"'
            items['dirs'] = '[' + ','.join('"%s"' % i for i in items['dirs']) + ']'
            self._logger.info('Job %s: %s', section_name,
                              ' '.join(['%s=%s' % (k,v) for k,v in items.iteritems()]))

    def remove_job(self, name):
        '''Remove a job.

        Removes a job previously added by `set_job`.

        :param name: the unique name used when the job was created
        '''
        section_name = GarbageTruck._section_name_for(name)
        self._logger.debug('Removing job: %s (%s)', name, section_name)
        self._cron.remove_all(comment=GarbageTruck._comment_for(name))
        self._config.remove_section(section_name)

    def save_changes(self):
        '''Save all changes made to the GarbageTruck.

        Writes out all the configuration and sets up the new schedule for the jobs changed before
        calling `save_changes`.
        '''
        self._logger.debug('Saving: %s', self._config_fn)
        self._cron.write()
        with open(self._config_fn, 'wb') as configfile:
            self._config.write(configfile)

    def run_job(self, id):
        '''Run a job.

        Runs a job added by `set_job`. Not normally called directly, this is what is used when run
        from the job scheduler on the configured interval.

        :param id: the unique identifier assigned to a job (this is **not** the name of the job).
        '''
        section_name = id
        if not self._config.has_section(section_name):
            self._logger.warn('Unable to run job %s: Does not exist', section_name)
            return
        name = self._config.get(section_name, 'name')
        self._logger.debug('Running: %s (%s)', name, section_name)
        files_older_than = self._config.get(section_name, 'files_older_than')
        period = GarbageTruck._period_from(files_older_than)
        lab = period[1] + 's'
        kwargs = {lab: period[0]}
        delta = timedelta(**kwargs)
        for dirname in self._get_dirs(section_name):
            self._run_job(delta, dirname)

    ######################################################################
    # private

    @staticmethod
    def _section_name_for(name):
        return md5(name).hexdigest()

    @staticmethod
    def _comment_for(name):
        return 'GarbageTruck: ' + name

    _PERIOD_RE = re.compile('^\s*(\d*)\s*(.+?)s?\s*$')
    @staticmethod
    def _period_from(str):
        match = GarbageTruck._PERIOD_RE.match(str)
        if not match:
            raise "Unable to parse period from " + str
        period = list(match.groups())
        period[0] = 1 if period[0] == '' else int(period[0])
        return period

    @staticmethod
    def _cron_safe_period_from(str):
        period = GarbageTruck._period_from(str)
        if period[1] == 'week':
            period[0] *= 7
            period[1] = 'day'
        elif period[1] == 'year':
            period[0] *= 12
            period[1] = 'month'
        return period

    def _get_dirs(self, section_name):
        dirs = []
        count = 0
        while True:
            count += 1
            optname = 'dir' + str(count)
            if not self._config.has_option(section_name, optname):
                return dirs
            dirs.append(self._config.get(section_name, optname))

    def _run_job(self, delta, dirname):
        if not os.path.exists(dirname):
            self._logger.warn('Ignoring %s: Does not exist', dirname)
            return<commit_msg>custom period exception; add debug log of job schedule when listing; prevent yearly checks<commit_after>import os
import re
import logging

from hashlib import md5
from configparser import SafeConfigParser
from datetime import timedelta, datetime

from crontab import CronTab
from send2trash import send2trash

class GarbageTruck:
    class InvalidPeriod(Exception):
        '''Indicates when an invalid period is provided'''
        pass

    def __init__(self):
        self._logger = logging.getLogger('garbagetruck')
        self._cron = CronTab(user=True)
        self._config = SafeConfigParser()
        self._config_fn = os.path.join(os.path.expanduser('~'), '.garbagetruckrc')
        if os.path.exists(self._config_fn):
            self._config.read(self._config_fn)

    def set_job(self, run_command_format, name, dirs,
                files_older_than='90 days', check_every='week'):
        '''Set a job by adding or replacing based on the name.

        A call to `set_job` will either add a new job or replace a previously set job with the new
        parameters. The `name` is the identifier for a job and must be **unique** accross all jobs
        for the same user.

        :param run_command_format: the command called for running job (will interpolate `%s` with a
                                   job ID)
        :param name: a unique name for this job
        :param dirs: a list of directories to iterate looking for old files
        :param files_older_than: the period to use to determine what "old" is for each file
        :param check_every: the period to use for when to trigger looking for old files
        '''
        section_name = GarbageTruck._section_name_for(name)
        self._logger.debug('Setting job: %s (%s)', name, section_name)
        self.remove_job(name)
        job = self._cron.new(command=run_command_format % section_name,
                             comment=GarbageTruck._comment_for(name))
        job_period = GarbageTruck._cron_safe_period_from(check_every)
        period_method = getattr(job, job_period[1])
        period_method.every(job_period[0])
        # replace any pre-existing session
        self._config.remove_section(section_name)
        self._config.add_section(section_name)
        self._config.set(section_name, 'name', name)
        self._config.set(section_name, 'files_older_than', files_older_than)
        self._config.set(section_name, 'check_every', check_every)
        count = 0
        for dirname in dirs:
            count += 1
            optname = 'dir' + str(count)
            self._config.set(section_name, optname, dirname)

    def list_jobs(self):
        for section_name in self._config.sections():
            name = self._config.get(section_name, 'name')
            items = {'dirs': []}
            for key, value in self._config.items(section_name):
                if key == 'name':
                    continue
                if key.startswith('dir'):
                    items['dirs'].append(value)
                else:
                    items[key] = '"' + value + '"'
            items['dirs'] = '[' + ','.join('"%s"' % i for i in items['dirs']) + ']'
            self._logger.info('Job %s: name="%s" %s', section_name, name,
                              ' '.join(['%s=%s' % (k,v) for k,v in items.iteritems()]))
            if self._logger.getEffectiveLevel() <= logging.DEBUG:
                job = next(self._cron.find_comment(GarbageTruck._comment_for(name)))
                self._logger.debug(str(job))

    def remove_job(self, name):
        '''Remove a job.

        Removes a job previously added by `set_job`.

        :param name: the unique name used when the job was created
        '''
        section_name = GarbageTruck._section_name_for(name)
        self._logger.debug('Removing job: %s (%s)', name, section_name)
        self._cron.remove_all(comment=GarbageTruck._comment_for(name))
        self._config.remove_section(section_name)

    def save_changes(self):
        '''Save all changes made to the GarbageTruck.

        Writes out all the configuration and sets up the new schedule for the jobs changed before
        calling `save_changes`.
        '''
        self._logger.debug('Saving: %s', self._config_fn)
        self._cron.write()
        with open(self._config_fn, 'wb') as configfile:
            self._config.write(configfile)

    def run_job(self, id):
        '''Run a job.

        Runs a job added by `set_job`. Not normally called directly, this is what is used when run
        from the job scheduler on the configured interval.

        :param id: the unique identifier assigned to a job (this is **not** the name of the job).
        '''
        section_name = id
        if not self._config.has_section(section_name):
            self._logger.warn('Unable to run job %s: Does not exist', section_name)
            return
        name = self._config.get(section_name, 'name')
        self._logger.debug('Running: %s (%s)', name, section_name)
        files_older_than = self._config.get(section_name, 'files_older_than')
        period = GarbageTruck._period_from(files_older_than)
        lab = period[1] + 's'
        kwargs = {lab: period[0]}
        delta = timedelta(**kwargs)
        for dirname in self._get_dirs(section_name):
            self._run_job(delta, dirname)

    ######################################################################
    # private

    @staticmethod
    def _section_name_for(name):
        return md5(name).hexdigest()

    @staticmethod
    def _comment_for(name):
        return 'GarbageTruck: ' + name

    _PERIOD_RE = re.compile('^\s*(\d*)\s*(.+?)s?\s*$')
    @staticmethod
    def _period_from(str):
        match = GarbageTruck._PERIOD_RE.match(str)
        if not match:
            raise GarbageTruck.InvalidPeriod("Unable to parse period from " + str)
        period = list(match.groups())
        period[0] = 1 if period[0] == '' else int(period[0])
        return period

    @staticmethod
    def _cron_safe_period_from(str):
        period = GarbageTruck._period_from(str)
        if period[1] == 'week':
            period[0] *= 7
            period[1] = 'day'
        elif period[1] == 'year':
            raise GarbageTruck.InvalidPeriod('Check schedule must be less than a year')
        return period

    def _get_dirs(self, section_name):
        dirs = []
        count = 0
        while True:
            count += 1
            optname = 'dir' + str(count)
            if not self._config.has_option(section_name, optname):
                return dirs
            dirs.append(self._config.get(section_name, optname))

    def _run_job(self, delta, dirname):
        if not os.path.exists(dirname):
            self._logger.warn('Ignoring %s: Does not exist', dirname)
            return
{'fraction_non_alphanumeric': 0.11420204978038068, 'fraction_numerical': 0.00385297064036372, 'mean_word_length': 2.358695652173913, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '215050', 'n_tokens_mistral': 3850, 'n_tokens_neox': 3711, 'n_words': 1047}