<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

from __future__ import print_function

"""BibSched - task management, scheduling and executing system for Invenio
"""

import os
import sys
import time
import re
import datetime
import marshal
import getopt
from itertools import chain
from socket import gethostname
from subprocess import Popen
import signal

from invenio.legacy.bibsched.bibtask_config import \
    CFG_BIBTASK_VALID_TASKS, \
    CFG_BIBTASK_MONOTASKS, \
    CFG_BIBTASK_FIXEDTIMETASKS
from invenio.config import \
    CFG_PREFIX, \
    CFG_TMPSHAREDDIR, \
    CFG_BIBSCHED_REFRESHTIME, \
    CFG_BINDIR, \
    CFG_LOGDIR, \
    CFG_RUNDIR, \
    CFG_BIBSCHED_GC_TASKS_OLDER_THAN, \
    CFG_BIBSCHED_GC_TASKS_TO_REMOVE, \
    CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE, \
    CFG_BIBSCHED_MAX_NUMBER_CONCURRENT_TASKS, \
    CFG_SITE_URL, \
    CFG_BIBSCHED_NODE_TASKS, \
    CFG_INSPIRE_SITE, \
    CFG_BIBSCHED_INCOMPATIBLE_TASKS, \
    CFG_BIBSCHED_NON_CONCURRENT_TASKS, \
    CFG_VERSION, \
    CFG_BIBSCHED_NEVER_STOPS
from invenio.legacy.dbquery import run_sql, real_escape_string
from invenio.ext.logging import register_exception
from invenio.utils.shell import run_shell_command

CFG_VALID_STATUS = ('WAITING', 'SCHEDULED', 'RUNNING', 'CONTINUING',
                    '% DELETED', 'ABOUT TO STOP', 'ABOUT TO SLEEP', 'STOPPED',
                    'SLEEPING', 'KILLED', 'NOW STOP', 'ERRORS REPORTED')

CFG_MOTD_PATH = os.path.join(CFG_TMPSHAREDDIR, "bibsched.motd")

ACTIVE_STATUS = ('SCHEDULED', 'ABOUT TO SLEEP', 'ABOUT TO STOP',
                 'CONTINUING', 'RUNNING')


SHIFT_RE = re.compile(r"([-\+]{0,1})([\d]+)([dhms])")


def register_emergency(msg, recipients=None):
    """Launch an emergency. This means to send email messages to each
    address in 'recipients'. By default recipients will be obtained via
    get_emergency_recipients() which loads settings from
    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES
    """
    from invenio.base.globals import cfg
    from invenio.ext.email import send_email
    if not recipients:
        recipients = get_emergency_recipients()
    recipients = set(recipients)
    recipients.add(cfg['CFG_SITE_ADMIN_EMAIL'])
    for address_str in recipients:
        send_email(
            cfg['CFG_SITE_SUPPORT_EMAIL'],
            address_str,
            "Emergency notification",
            msg
        )


def get_emergency_recipients(recipient_cfg=None, now=None):
    """
    Parse a list of appropriate emergency email recipients from
    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES, or from a provided dictionary
    comprised of 'time constraint' => 'comma separated list of addresses'

    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES format example:

    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES = {
       'Sunday 22:00-06:00': '0<EMAIL>',
       '06:00-18:00': '<EMAIL>,0<EMAIL>',
       '18:00-06:00': '<EMAIL>',
       '*': '<EMAIL>'}
    """
    from invenio.base.globals import cfg
    from invenio.utils.date import parse_runtime_limit

    if recipient_cfg is None:
        recipient_cfg = cfg['CFG_SITE_EMERGENCY_EMAIL_ADDRESSES']

    if now is None:
        now = datetime.datetime.now()

    recipients = set()
    for time_condition, address_str in recipient_cfg.items():
        if time_condition and time_condition is not '*':
            current_range, dummy_range = parse_runtime_limit(time_condition,
                                                             now=now)
            if not current_range[0] <= now <= current_range[1]:
                continue

        recipients.add(address_str)
    return list(recipients)


def get_pager():
    """
    Return the first available pager.
    """
    paths = (
        os.environ.get('PAGER', ''),
        cfg['CFG_BIBSCHED_LOG_PAGER'],
        '/usr/bin/less',
        '/bin/more'
    )
    for pager in paths:
        if os.path.exists(pager):
            return pager


def get_editor():
    """
    Return the first available editor.
    """
    paths = (
        os.environ.get('EDITOR', ''),
        CFG_BIBSCHED_EDITOR,
        '/usr/bin/vim',
        '/usr/bin/emacs',
        '/usr/bin/vi',
        '/usr/bin/nano',
    )
    for editor in paths:
        if os.path.exists(editor):
            return editor<commit_msg>legacy: absent cfg import in bibsched<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2014, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

from __future__ import print_function

"""BibSched - task management, scheduling and executing system for Invenio
"""

import os
import sys
import time
import re
import datetime
import marshal
import getopt
from itertools import chain
from socket import gethostname
from subprocess import Popen
import signal

from invenio.legacy.bibsched.bibtask_config import \
    CFG_BIBTASK_VALID_TASKS, \
    CFG_BIBTASK_MONOTASKS, \
    CFG_BIBTASK_FIXEDTIMETASKS
from invenio.config import \
    CFG_PREFIX, \
    CFG_TMPSHAREDDIR, \
    CFG_BIBSCHED_REFRESHTIME, \
    CFG_BINDIR, \
    CFG_LOGDIR, \
    CFG_RUNDIR, \
    CFG_BIBSCHED_GC_TASKS_OLDER_THAN, \
    CFG_BIBSCHED_GC_TASKS_TO_REMOVE, \
    CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE, \
    CFG_BIBSCHED_MAX_NUMBER_CONCURRENT_TASKS, \
    CFG_SITE_URL, \
    CFG_BIBSCHED_NODE_TASKS, \
    CFG_INSPIRE_SITE, \
    CFG_BIBSCHED_INCOMPATIBLE_TASKS, \
    CFG_BIBSCHED_NON_CONCURRENT_TASKS, \
    CFG_VERSION, \
    CFG_BIBSCHED_NEVER_STOPS
from invenio.base.globals import cfg
from invenio.legacy.dbquery import run_sql, real_escape_string
from invenio.ext.logging import register_exception
from invenio.utils.shell import run_shell_command

CFG_VALID_STATUS = ('WAITING', 'SCHEDULED', 'RUNNING', 'CONTINUING',
                    '% DELETED', 'ABOUT TO STOP', 'ABOUT TO SLEEP', 'STOPPED',
                    'SLEEPING', 'KILLED', 'NOW STOP', 'ERRORS REPORTED')

CFG_MOTD_PATH = os.path.join(CFG_TMPSHAREDDIR, "bibsched.motd")

ACTIVE_STATUS = ('SCHEDULED', 'ABOUT TO SLEEP', 'ABOUT TO STOP',
                 'CONTINUING', 'RUNNING')


SHIFT_RE = re.compile(r"([-\+]{0,1})([\d]+)([dhms])")


def register_emergency(msg, recipients=None):
    """Launch an emergency. This means to send email messages to each
    address in 'recipients'. By default recipients will be obtained via
    get_emergency_recipients() which loads settings from
    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES
    """
    from invenio.base.globals import cfg
    from invenio.ext.email import send_email
    if not recipients:
        recipients = get_emergency_recipients()
    recipients = set(recipients)
    recipients.add(cfg['CFG_SITE_ADMIN_EMAIL'])
    for address_str in recipients:
        send_email(
            cfg['CFG_SITE_SUPPORT_EMAIL'],
            address_str,
            "Emergency notification",
            msg
        )


def get_emergency_recipients(recipient_cfg=None, now=None):
    """
    Parse a list of appropriate emergency email recipients from
    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES, or from a provided dictionary
    comprised of 'time constraint' => 'comma separated list of addresses'

    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES format example:

    CFG_SITE_EMERGENCY_EMAIL_ADDRESSES = {
       'Sunday 22:00-06:00': '<EMAIL>76111<EMAIL>',
       '06:00-18:00': '<EMAIL>,<EMAIL>',
       '18:00-06:00': '<EMAIL>',
       '*': '<EMAIL>'}
    """
    from invenio.base.globals import cfg
    from invenio.utils.date import parse_runtime_limit

    if recipient_cfg is None:
        recipient_cfg = cfg['CFG_SITE_EMERGENCY_EMAIL_ADDRESSES']

    if now is None:
        now = datetime.datetime.now()

    recipients = set()
    for time_condition, address_str in recipient_cfg.items():
        if time_condition and time_condition is not '*':
            current_range, dummy_range = parse_runtime_limit(time_condition,
                                                             now=now)
            if not current_range[0] <= now <= current_range[1]:
                continue

        recipients.add(address_str)
    return list(recipients)


def get_pager():
    """
    Return the first available pager.
    """
    paths = (
        os.environ.get('PAGER', ''),
        cfg['CFG_BIBSCHED_LOG_PAGER'],
        '/usr/bin/less',
        '/bin/more'
    )
    for pager in paths:
        if os.path.exists(pager):
            return pager


def get_editor():
    """
    Return the first available editor.
    """
    paths = (
        os.environ.get('EDITOR', ''),
        cfg['CFG_BIBSCHED_EDITOR'],
        '/usr/bin/vim',
        '/usr/bin/emacs',
        '/usr/bin/vi',
        '/usr/bin/nano',
    )
    for editor in paths:
        if os.path.exists(editor):
            return editor
{'fraction_non_alphanumeric': 0.09383642828921375, 'fraction_numerical': 0.016100355590675623, 'mean_word_length': 2.997236478484011, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530559', 'n_tokens_mistral': 3555, 'n_tokens_neox': 3313, 'n_words': 863}