<commit_before>import sys
import os
import logging
import json
import time
import stat
import argparse

from pprint import pformat
from shutil import rmtree, copy

from configobj import ConfigObj
from YearSet import YearSet, SetStatus
from shutil import copyfile
from lib.filemanager import FileManager
from lib.runmanager import RunManager
from lib.mailer import Mailer
from jobs.JobStatus import JobStatus
from util import (setup_globus,
                  check_globus,
                  print_message,
                  print_debug,
                  print_line)


def parse_args(argv=None, print_help=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', help='Path to configuration file.')
    parser.add_argument(
        '-u',
        '--ui',
        help='Turn on the GUI.',
        action='store_true')
    parser.add_argument(
        '-l',
        '--log',
        help='Path to logging output file.')
    parser.add_argument(
        '-n',
        '--no-cleanup',
        help='Don\'t perform post run cleanup. This will leave all files in place.',
        action='store_true')
    parser.add_argument(
        '-m',
        '--no-monitor',
        help='Don\'t run the remote monitor or move any files over globus.',
        action='store_true')
    parser.add_argument(
        '-f',
        '--file-list',
        help='Turn on debug output of the internal file_list so you can see what the current state of the model files are',
        action='store_true')
    parser.add_argument(
        '-r',
        '--resource-dir',
        help='Path to custom resource directory')
    parser.add_argument(
        '-i',
        '--input-path',
        help='Custom input path')
    parser.add_argument(
        '-o',
        '--output-path',
        help='Custom output path')
    if print_help:
        parser.print_help()
        return
    return parser.parse_args(argv)


def initialize(argv, **kwargs):
    """
    Parse the commandline arguments, and setup the master config dict

    Parameters:
        parser (argparse.ArgumentParser): The parser object
        event_list (EventList): The main list of events
        thread_list (list): the main list of all running threads
        mutex (threading.Lock): A mutex to handle db access
        kill_event (threading.Event): An event used to kill all running threads
    """
    # Setup the parser
    args = parse_args(argv=argv)
    if not args.config:
        parse_args(print_help=True)
        return False, False, False
    event_list = kwargs['event_list']
    thread_list = kwargs['thread_list']
    mutex = kwargs['mutex']
    event = kwargs['kill_event']
    print_line(
        ui=False,
        line='Entering setup',
        event_list=event_list,
        current_state=True)

    # check if globus config is valid, else remove it
    globus_config = os.path.join(os.path.expanduser('~'), '.globus.cfg')
    if os.path.exists(globus_config):
        try:
            conf = ConfigObj(globus_config)
        except:
            os.remove(globus_config)

    if not os.path.exists(args.config):
        print "Invalid config, {} does not exist".format(args.config)
        return False, False, False

    # Check that there are no white space errors in the config file
    line_index = check_config_white_space(args.config)
    if line_index != 0:
        print '''
ERROR: line {num} does not have a space after the \'=\', white space is required.
Please add a space and run again.'''.format(num=line_index)
        return False, False, False

    # read the config file and setup the config dict
    try:
        config = ConfigObj(args.config)
    except Exception as e:
        print "Error parsing config file {}".format(args.config)
        parse_args(print_help=True)
        return False, False, False

    # run validator for config file
    if config.get('global'):
        if args.resource_dir:
            config['global']['resource_dir'] = args.resource_dir
        else:
            config['global']['resource_dir'] = os.path.join(
                sys.prefix,
                'share',
                'processflow',
                'resources')
    else:
        return False, False, False

    # Setup boolean config flags
    config['global']['ui'] = True if args.ui else False
    config['global']['no_cleanup'] = True if args.no_cleanup else False
    config['global']['no_monitor'] = True if args.no_monitor else False
    config['global']['print_file_list'] = True if args.file_list else False

    template_path = os.path.join(
        config['global']['resource_dir'],
        'config_template.json')

    with open(template_path, 'r') as template_file:
        template = json.load(template_file)

    valid, messages = verify_config(config, template)
    if not valid:
        for message in messages:
            print message
        return False, False, False

    if args.input_path:
        config['global']['input_path'] = args.input_path
    else:
        config['global']['input_path'] = os.path.join(
            config['global']['project_path'],
            'input')
    if args.output_path:
        config['global']['output_path'] = args.output_path
    else:
        config['global']['output_path'] = os.path.join(
            config['global']['project_path'],
            'output')

    # setup output and cache directories
    if not os.path.exists(config['global']['input_path']):
        os.makedirs(config['global']['input_path'])
    if not os.path.exists(config['global']['output_path']):
        os.makedirs(config['global']['output_path'])

    pp_path = os.path.join(config['global']['output_path'], 'pp')
    if not os.path.exists(pp_path):
        os.makedirs(pp_path)
    diags_path = os.path.join(config['global']['output_path'], 'diags')
    if not os.path.exists(diags_path):
        os.makedirs(diags_path)
    config['global']['pp_path'] = pp_path
    config['global']['diags_path'] = diags_path

    # Copy the config into the input directory for safe keeping
    input_config_path = os.path.join(
        config['global']['input_path'], 'run.cfg')
    try:
        copy(args.config, input_config_path)
    except:
        pass

    # setup logging
    if args.log:
        log_path = args.log
    else:
        log_path = os.path.join(
            config.get('global').get('output_path'),
            'workflow.log')
    config['global']['log_path'] = log_path
    logging.basicConfig(
        format='%(asctime)s:%(levelname)s: %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        filename=log_path,
        filemode='w',
        level=logging.INFO)
    logging.getLogger('globus_sdk').setLevel(logging.WARNING)

    # Make sure the set_frequency is a list of ints
    set_frequency = config['global']['set_frequency']
    if not isinstance(set_frequency, list):
        set_frequency = [int(set_frequency)]
    else:<commit_msg>Suppresses excess logging messages from globus_cli<commit_after>import sys
import os
import logging
import json
import time
import stat
import argparse

from pprint import pformat
from shutil import rmtree, copy

from configobj import ConfigObj
from YearSet import YearSet, SetStatus
from shutil import copyfile
from lib.filemanager import FileManager
from lib.runmanager import RunManager
from lib.mailer import Mailer
from jobs.JobStatus import JobStatus
from util import (setup_globus,
                  check_globus,
                  print_message,
                  print_debug,
                  print_line)


def parse_args(argv=None, print_help=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', help='Path to configuration file.')
    parser.add_argument(
        '-u',
        '--ui',
        help='Turn on the GUI.',
        action='store_true')
    parser.add_argument(
        '-l',
        '--log',
        help='Path to logging output file.')
    parser.add_argument(
        '-n',
        '--no-cleanup',
        help='Don\'t perform post run cleanup. This will leave all files in place.',
        action='store_true')
    parser.add_argument(
        '-m',
        '--no-monitor',
        help='Don\'t run the remote monitor or move any files over globus.',
        action='store_true')
    parser.add_argument(
        '-f',
        '--file-list',
        help='Turn on debug output of the internal file_list so you can see what the current state of the model files are',
        action='store_true')
    parser.add_argument(
        '-r',
        '--resource-dir',
        help='Path to custom resource directory')
    parser.add_argument(
        '-i',
        '--input-path',
        help='Custom input path')
    parser.add_argument(
        '-o',
        '--output-path',
        help='Custom output path')
    if print_help:
        parser.print_help()
        return
    return parser.parse_args(argv)


def initialize(argv, **kwargs):
    """
    Parse the commandline arguments, and setup the master config dict

    Parameters:
        parser (argparse.ArgumentParser): The parser object
        event_list (EventList): The main list of events
        thread_list (list): the main list of all running threads
        mutex (threading.Lock): A mutex to handle db access
        kill_event (threading.Event): An event used to kill all running threads
    """
    # Setup the parser
    args = parse_args(argv=argv)
    if not args.config:
        parse_args(print_help=True)
        return False, False, False
    event_list = kwargs['event_list']
    thread_list = kwargs['thread_list']
    mutex = kwargs['mutex']
    event = kwargs['kill_event']
    print_line(
        ui=False,
        line='Entering setup',
        event_list=event_list,
        current_state=True)

    # check if globus config is valid, else remove it
    globus_config = os.path.join(os.path.expanduser('~'), '.globus.cfg')
    if os.path.exists(globus_config):
        try:
            conf = ConfigObj(globus_config)
        except:
            os.remove(globus_config)

    if not os.path.exists(args.config):
        print "Invalid config, {} does not exist".format(args.config)
        return False, False, False

    # Check that there are no white space errors in the config file
    line_index = check_config_white_space(args.config)
    if line_index != 0:
        print '''
ERROR: line {num} does not have a space after the \'=\', white space is required.
Please add a space and run again.'''.format(num=line_index)
        return False, False, False

    # read the config file and setup the config dict
    try:
        config = ConfigObj(args.config)
    except Exception as e:
        print "Error parsing config file {}".format(args.config)
        parse_args(print_help=True)
        return False, False, False

    # run validator for config file
    if config.get('global'):
        if args.resource_dir:
            config['global']['resource_dir'] = args.resource_dir
        else:
            config['global']['resource_dir'] = os.path.join(
                sys.prefix,
                'share',
                'processflow',
                'resources')
    else:
        return False, False, False

    # Setup boolean config flags
    config['global']['ui'] = True if args.ui else False
    config['global']['no_cleanup'] = True if args.no_cleanup else False
    config['global']['no_monitor'] = True if args.no_monitor else False
    config['global']['print_file_list'] = True if args.file_list else False

    template_path = os.path.join(
        config['global']['resource_dir'],
        'config_template.json')

    with open(template_path, 'r') as template_file:
        template = json.load(template_file)

    valid, messages = verify_config(config, template)
    if not valid:
        for message in messages:
            print message
        return False, False, False

    if args.input_path:
        config['global']['input_path'] = args.input_path
    else:
        config['global']['input_path'] = os.path.join(
            config['global']['project_path'],
            'input')
    if args.output_path:
        config['global']['output_path'] = args.output_path
    else:
        config['global']['output_path'] = os.path.join(
            config['global']['project_path'],
            'output')

    # setup output and cache directories
    if not os.path.exists(config['global']['input_path']):
        os.makedirs(config['global']['input_path'])
    if not os.path.exists(config['global']['output_path']):
        os.makedirs(config['global']['output_path'])

    pp_path = os.path.join(config['global']['output_path'], 'pp')
    if not os.path.exists(pp_path):
        os.makedirs(pp_path)
    diags_path = os.path.join(config['global']['output_path'], 'diags')
    if not os.path.exists(diags_path):
        os.makedirs(diags_path)
    config['global']['pp_path'] = pp_path
    config['global']['diags_path'] = diags_path

    # Copy the config into the input directory for safe keeping
    input_config_path = os.path.join(
        config['global']['input_path'], 'run.cfg')
    try:
        copy(args.config, input_config_path)
    except:
        pass

    # setup logging
    if args.log:
        log_path = args.log
    else:
        log_path = os.path.join(
            config.get('global').get('output_path'),
            'workflow.log')
    config['global']['log_path'] = log_path
    logging.basicConfig(
        format='%(asctime)s:%(levelname)s: %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        filename=log_path,
        filemode='w',
        level=logging.INFO)
    logging.getLogger('globus_sdk').setLevel(logging.ERROR)
    logging.getLogger('globus_cli').setLevel(logging.ERROR)

    # Make sure the set_frequency is a list of ints
    set_frequency = config['global']['set_frequency']
    if not isinstance(set_frequency, list):
        set_frequency = [int(set_frequency)]
    else:
{'fraction_non_alphanumeric': 0.11774437171833417, 'fraction_numerical': 0.0001438538444939941, 'mean_word_length': 2.5835051546391754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4378970', 'n_tokens_mistral': 4025, 'n_tokens_neox': 3964, 'n_words': 1083}