<commit_before># -*- coding: utf-8 -*-
'''
Main entry point for the hubble daemon
'''
from __future__ import print_function

# import lockfile
import argparse
import logging
import time
import pprint
import os
import random
import signal
import sys
import uuid
import json
import socket
import math

import salt.fileclient
import salt.fileserver
import salt.fileserver.gitfs
import salt.utils
import salt.utils.jid
import salt.utils.gitfs
import salt.log.setup
import hubblestack.splunklogging
from hubblestack import __version__
from croniter import croniter
from datetime import datetime

log = logging.getLogger(__name__)

__opts__ = {}
# This should work fine until we go to multiprocessing
SESSION_UUID = str(uuid.uuid4())


def run():
    '''
    Set up program, daemonize if needed
    '''
    # Don't put anything that needs config or logging above this line
    try:
        load_config()
    except Exception as e:
        print('An Error occurred while loading the config: {0}'.format(e))
        raise

    # Create cache directory if not present
    if not os.path.isdir(__opts__['cachedir']):
        os.makedirs(__opts__['cachedir'])

    if __opts__['version']:
        print(__version__)
        clean_up_process(None, None)
        sys.exit(0)

    if __opts__['daemonize']:
        # before becoming a daemon, check for other procs and possibly send
        # then a signal 15 (otherwise refuse to run)
        check_pidfile(kill_other=True)
        salt.utils.daemonize()
        create_pidfile()
    elif not __opts__['function']:
        # check the pidfile and possibly refuse to run
        # (assuming this isn't a single function call)
        check_pidfile(kill_other=False)

    signal.signal(signal.SIGTERM, clean_up_process)
    signal.signal(signal.SIGINT, clean_up_process)

    try:
        main()
    except KeyboardInterrupt:
        pass

    clean_up_process(None, None)


def main():
    '''
    Run the main hubble loop
    '''
    # Initial fileclient setup
    # Clear old locks
    if 'gitfs' in __opts__['fileserver_backend'] or 'git' in __opts__['fileserver_backend']:
        git_objects = [
            salt.utils.gitfs.GitFS(
                __opts__,
                __opts__['gitfs_remotes'],
                per_remote_overrides=salt.fileserver.gitfs.PER_REMOTE_OVERRIDES,
                per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY
            )
        ]
        ret = {}
        for obj in git_objects:
            lock_type = 'update'
            cleared, errors = salt.fileserver.clear_lock(obj.clear_lock,
                                                         'gitfs',
                                                         remote=None,
                                                         lock_type=lock_type)
            if cleared:
                ret.setdefault('cleared', []).extend(cleared)
            if errors:
                ret.setdefault('errors', []).extend(errors)
        if ret:
            log.info('One or more gitfs locks were removed: {0}'.format(ret))

    # Setup fileclient
    log.info('Setting up the fileclient/fileserver')

    # Set up fileclient
    retry_count = __opts__.get('fileserver_retry_count_on_startup', None)
    retry_time = __opts__.get('fileserver_retry_rate_on_startup', 30)
    count = 0
    while True:
        try:
            fc = salt.fileclient.get_file_client(__opts__)
            fc.channel.fs.update()
            last_fc_update = time.time()
            break
        except Exception as exc:
            if (retry_count is None or count < retry_count) and not __opts__['function']:
                log.exception('Exception thrown trying to setup fileclient. '
                              'Trying again in {0} seconds.'
                              .format(retry_time))
                count += 1
                time.sleep(retry_time)
                continue
            else:
                log.exception('Exception thrown trying to setup fileclient. Exiting.')
                sys.exit(1)

    # Check for single function run
    if __opts__['function']:
        run_function()
        sys.exit(0)

    last_grains_refresh = time.time() - __opts__['grains_refresh_frequency']

    log.info('Starting main loop')
    while True:
        # Check if fileserver needs update
        if time.time() - last_fc_update >= __opts__['fileserver_update_frequency']:
            try:
                fc.channel.fs.update()
                last_fc_update = time.time()
            except Exception as exc:
                retry = __opts__.get('fileserver_retry_rate', 900)
                last_fc_update += retry
                log.exception('Exception thrown trying to update fileclient. '
                              'Trying again in {0} seconds.'
                              .format(retry))

        if time.time() - last_grains_refresh >= __opts__['grains_refresh_frequency']:
            log.info('Refreshing grains')
            refresh_grains()
            last_grains_refresh = time.time()

        try:
            log.debug('Executing schedule')
            schedule()
        except Exception as e:
            log.exception('Error executing schedule')
        time.sleep(__opts__.get('scheduler_sleep_frequency', 0.5))

def getlastrunbycron(cron_exp):
    '''
    this function will use the cron_exp provided in the hubble config to
    execute the hubble processes as per the scheduled cron time
    '''
    base = datetime(2018, 1, 1, 0, 0)
    iter = croniter(cron_exp, base)
    next_datetime  = iter.get_next(datetime)
    log.error('next date is {0}'.format(next_datetime))
    epoch_datetime = time.mktime(next_datetime.timetuple())
    current_time = time.time()
    while epoch_datetime<current_time:
        next_datetime  = iter.get_next(datetime)
        epoch_datetime = time.mktime(next_datetime.timetuple())
    prev = iter.get_prev(datetime)
    log.error('prev is {0}'.format(prev))
    epoch_prev = time.mktime(prev.timetuple())
    last_run = epoch_prev
    log.error('last run is {0}'.format(last_run))
    return last_run

def getlastrunbybuckets(buckets, seconds):
    '''
    this function will use the host's ip to place the host in a bucket
    where each bucket executes hubble processes at a different time
    '''
    buckets = int(buckets) if int(buckets)!=0 else 256
    host_ip = socket.gethostbyname(socket.gethostname())
    ips = host_ip.split('.')
    sum = (int(ips[0])*256*256*256)+(int(ips[1])*256*256)+(int(ips[2])*256)+int(ips[3])
    bucket = sum%buckets
    current_time = time.time()
    base_time = seconds*(math.floor(current_time/seconds))
    splay = seconds/buckets
    seconds_between_buckets = splay
    random_int = random.randint(0,splay-1) if splay !=0 else 0
    bucket_execution_time = base_time+(seconds_between_buckets*bucket)+random_int
    if bucket_execution_time < current_time:
        last_run = bucket_execution_time<commit_msg>more debugging<commit_after># -*- coding: utf-8 -*-
'''
Main entry point for the hubble daemon
'''
from __future__ import print_function

# import lockfile
import argparse
import logging
import time
import pprint
import os
import random
import signal
import sys
import uuid
import json
import socket
import math

import salt.fileclient
import salt.fileserver
import salt.fileserver.gitfs
import salt.utils
import salt.utils.jid
import salt.utils.gitfs
import salt.log.setup
import hubblestack.splunklogging
from hubblestack import __version__
from croniter import croniter
from datetime import datetime

log = logging.getLogger(__name__)

__opts__ = {}
# This should work fine until we go to multiprocessing
SESSION_UUID = str(uuid.uuid4())


def run():
    '''
    Set up program, daemonize if needed
    '''
    # Don't put anything that needs config or logging above this line
    try:
        load_config()
    except Exception as e:
        print('An Error occurred while loading the config: {0}'.format(e))
        raise

    # Create cache directory if not present
    if not os.path.isdir(__opts__['cachedir']):
        os.makedirs(__opts__['cachedir'])

    if __opts__['version']:
        print(__version__)
        clean_up_process(None, None)
        sys.exit(0)

    if __opts__['daemonize']:
        # before becoming a daemon, check for other procs and possibly send
        # then a signal 15 (otherwise refuse to run)
        check_pidfile(kill_other=True)
        salt.utils.daemonize()
        create_pidfile()
    elif not __opts__['function']:
        # check the pidfile and possibly refuse to run
        # (assuming this isn't a single function call)
        check_pidfile(kill_other=False)

    signal.signal(signal.SIGTERM, clean_up_process)
    signal.signal(signal.SIGINT, clean_up_process)

    try:
        main()
    except KeyboardInterrupt:
        pass

    clean_up_process(None, None)


def main():
    '''
    Run the main hubble loop
    '''
    # Initial fileclient setup
    # Clear old locks
    if 'gitfs' in __opts__['fileserver_backend'] or 'git' in __opts__['fileserver_backend']:
        git_objects = [
            salt.utils.gitfs.GitFS(
                __opts__,
                __opts__['gitfs_remotes'],
                per_remote_overrides=salt.fileserver.gitfs.PER_REMOTE_OVERRIDES,
                per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY
            )
        ]
        ret = {}
        for obj in git_objects:
            lock_type = 'update'
            cleared, errors = salt.fileserver.clear_lock(obj.clear_lock,
                                                         'gitfs',
                                                         remote=None,
                                                         lock_type=lock_type)
            if cleared:
                ret.setdefault('cleared', []).extend(cleared)
            if errors:
                ret.setdefault('errors', []).extend(errors)
        if ret:
            log.info('One or more gitfs locks were removed: {0}'.format(ret))

    # Setup fileclient
    log.info('Setting up the fileclient/fileserver')

    # Set up fileclient
    retry_count = __opts__.get('fileserver_retry_count_on_startup', None)
    retry_time = __opts__.get('fileserver_retry_rate_on_startup', 30)
    count = 0
    while True:
        try:
            fc = salt.fileclient.get_file_client(__opts__)
            fc.channel.fs.update()
            last_fc_update = time.time()
            break
        except Exception as exc:
            if (retry_count is None or count < retry_count) and not __opts__['function']:
                log.exception('Exception thrown trying to setup fileclient. '
                              'Trying again in {0} seconds.'
                              .format(retry_time))
                count += 1
                time.sleep(retry_time)
                continue
            else:
                log.exception('Exception thrown trying to setup fileclient. Exiting.')
                sys.exit(1)

    # Check for single function run
    if __opts__['function']:
        run_function()
        sys.exit(0)

    last_grains_refresh = time.time() - __opts__['grains_refresh_frequency']

    log.info('Starting main loop')
    while True:
        # Check if fileserver needs update
        if time.time() - last_fc_update >= __opts__['fileserver_update_frequency']:
            try:
                fc.channel.fs.update()
                last_fc_update = time.time()
            except Exception as exc:
                retry = __opts__.get('fileserver_retry_rate', 900)
                last_fc_update += retry
                log.exception('Exception thrown trying to update fileclient. '
                              'Trying again in {0} seconds.'
                              .format(retry))

        if time.time() - last_grains_refresh >= __opts__['grains_refresh_frequency']:
            log.info('Refreshing grains')
            refresh_grains()
            last_grains_refresh = time.time()

        try:
            log.debug('Executing schedule')
            schedule()
        except Exception as e:
            log.exception('Error executing schedule')
        time.sleep(__opts__.get('scheduler_sleep_frequency', 0.5))

def getlastrunbycron(cron_exp):
    '''
    this function will use the cron_exp provided in the hubble config to
    execute the hubble processes as per the scheduled cron time
    '''
    base = datetime(2018, 1, 1, 0, 0)
    iter = croniter(cron_exp, base)
    next_datetime  = iter.get_next(datetime)
    log.error('next date is {0}'.format(next_datetime))
    epoch_datetime = time.mktime(next_datetime.timetuple())
    log.error('epoch date is {0}'.format(epoch_datetime))
    current_time = time.time()
    log.error('current time is {0}'.format(current_time))
    while epoch_datetime<current_time:
        next_datetime  = iter.get_next(datetime)
        epoch_datetime = time.mktime(next_datetime.timetuple())
    log.error('out of while loop')
    prev = iter.get_prev(datetime)
    log.error('prev is {0}'.format(prev))
    epoch_prev = time.mktime(prev.timetuple())
    last_run = epoch_prev
    log.error('last run is {0}'.format(last_run))
    return last_run

def getlastrunbybuckets(buckets, seconds):
    '''
    this function will use the host's ip to place the host in a bucket
    where each bucket executes hubble processes at a different time
    '''
    buckets = int(buckets) if int(buckets)!=0 else 256
    host_ip = socket.gethostbyname(socket.gethostname())
    ips = host_ip.split('.')
    sum = (int(ips[0])*256*256*256)+(int(ips[1])*256*256)+(int(ips[2])*256)+int(ips[3])
    bucket = sum%buckets
    current_time = time.time()
    base_time = seconds*(math.floor(current_time/seconds))
    splay = seconds/buckets
    seconds_between_buckets = splay
    random_int = random.randint(0,splay-1) if splay !=0 else 0
    bucket_execution_time = base_time+(seconds_between_buckets*bucket)+random_int
    if bucket_execution_time < current_time:
        last_run = bucket_execution_time
{'fraction_non_alphanumeric': 0.09278276993296249, 'fraction_numerical': 0.008843246327200113, 'mean_word_length': 2.325349774721366, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2446661', 'n_tokens_mistral': 4184, 'n_tokens_neox': 3992, 'n_words': 1030}