<commit_before>#! /bin/env python
# -*- coding: utf-8 -*-
import sys
import logbook
from rq import Queue
from redis import Redis
from harvester.config import parse_env

JOB_TIMEOUT = 86400  # 24 hrs


def queue_deep_harvest(redis_host,
                       redis_port,
                       redis_password,
                       redis_timeout,
                       rq_queue,
                       collection_id,
                       timeout=JOB_TIMEOUT):
    '''Queue job onto RQ queue'''
    rQ = Queue(
        rq_queue,
        connection=Redis(
            host=redis_host,
            port=redis_port,
            password=<PASSWORD>,
            socket_connect_timeout=redis_timeout))
    job = rQ.enqueue_call(
        func='s3stash.stash_collection.main',
        kwargs=dict(registry_id=collection_id),
        timeout=timeout)
    return job


def main(collection_ids, log_handler=None, rq_queue='normal-stage'):
    ''' Queue a deep harvest of a nuxeo collection on a worker'''
    if not log_handler:
        log_handler = logbook.StderrHandler(level='DEBUG')
    log_handler.push_application()
    for cid in [x for x in collection_ids.split(';')]:
        queue_deep_harvest(
            config['redis_host'],
            config['redis_port'],
            config['redis_password'],
            config['redis_connect_timeout'],
            rq_queue=rq_queue,
            collection_id=cid)
    log_handler.pop_application()


def def_args():
    import argparse
    parser = argparse.ArgumentParser(
        description='Queue a Nuxeo deep harvesting job')
    parser.add_argument(
        'collection_ids', type=str, help='Collection ids, ";" delimited')
    return parser


if __name__ == '__main__':
    parser = def_args()
    args = parser.parse_args()
    config = parse_env(None)
    if not args.collection_ids:
        parser.print_help()
        sys.exit(27)
    main(args.collection_ids)

"""
Copyright © 2016, Regents of the University of California
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:<commit_msg>Make this take standard args list, email, queue, ids<commit_after>#! /bin/env python
# -*- coding: utf-8 -*-
import sys
import logbook
from rq import Queue
from redis import Redis
from harvester.config import parse_env

JOB_TIMEOUT = 86400  # 24 hrs


def queue_deep_harvest(redis_host,
                       redis_port,
                       redis_password,
                       redis_timeout,
                       rq_queue,
                       collection_id,
                       timeout=JOB_TIMEOUT):
    '''Queue job onto RQ queue'''
    rQ = Queue(
        rq_queue,
        connection=Redis(
            host=redis_host,
            port=redis_port,
            password=<PASSWORD>,
            socket_connect_timeout=redis_timeout))
    job = rQ.enqueue_call(
        func='s3stash.stash_collection.main',
        kwargs=dict(registry_id=collection_id),
        timeout=timeout)
    return job


def main(collection_ids, log_handler=None, rq_queue='normal-stage'):
    ''' Queue a deep harvest of a nuxeo collection on a worker'''
    if not log_handler:
        log_handler = logbook.StderrHandler(level='DEBUG')
    log_handler.push_application()
    for cid in [x for x in collection_ids.split(';')]:
        queue_deep_harvest(
            config['redis_host'],
            config['redis_port'],
            config['redis_password'],
            config['redis_connect_timeout'],
            rq_queue=rq_queue,
            collection_id=cid)
    log_handler.pop_application()


def def_args():
    import argparse
    parser = argparse.ArgumentParser(
        description='Queue a Nuxeo deep harvesting job')
    parser.add_argument('user_email', type=str, help='user email')
    parser.add_argument('rq_queue', type=str, help='RQ Queue to put job in')
    parser.add_argument(
        'collection_ids', type=str, help='Collection ids, ";" delimited')
    return parser


if __name__ == '__main__':
    parser = def_args()
    args = parser.parse_args()
    config = parse_env(None)
    if not args.collection_ids or not args.rq_queue:
        parser.print_help()
        sys.exit(27)
    main(args.collection_ids, rq_queue=args.rq_queue)

"""
Copyright © 2016, Regents of the University of California
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
{'fraction_non_alphanumeric': 0.09044894366197183, 'fraction_numerical': 0.006602112676056338, 'mean_word_length': 2.525989138867339, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2620172', 'n_tokens_mistral': 1360, 'n_tokens_neox': 1308, 'n_words': 339}