<commit_before>#!/usr/bin/env python

__author__ = '<NAME>'
__license__ = 'Apache 2.0'

import argparse

import yaml

from pyon.public import Container, GreenProcessSupervisor

from pyon.container.cc import IContainerAgent
from pyon.net.endpoint import RPCClient

version = "2.0"     # TODO: extract this from the code once versioning is automated again
description = '''
pyon (ION capability container) v%s
''' % (version)

def setup_ipython():
    from IPython.config.loader import Config
    cfg = Config()
    shell_config = cfg.InteractiveShellEmbed
    shell_config.prompt_in1 = '><> '
    shell_config.prompt_in2 = '... '
    shell_config.prompt_out = '--> '
    shell_config.confirm_exit = False

    # First import the embeddable shell class
    from IPython.frontend.terminal.embed import InteractiveShellEmbed

    # Now create an instance of the embeddable shell. The first argument is a
    # string with options exactly as you would type them if you were starting
    # IPython at the system command line. Any parameters you want to define for
    # configuration can thus be specified here.
    ipshell = InteractiveShellEmbed(config=cfg,
                           banner1 = 'Dropping into IPython',
                           exit_msg = 'Leaving Interpreter, back to program.')

    ipshell('Pyon shell (ION R2)')

# From http://stackoverflow.com/questions/6037503/python-unflatten-dict/6037657#6037657
def unflatten(dictionary):
    resultDict = dict()
    for key, value in dictionary.iteritems():
        parts = key.split(".")
        d = resultDict
        for part in parts[:-1]:
            if part not in d:
                d[part] = dict()
            d = d[part]
        d[parts[-1]] = value
    return resultDict
    
def main(opts, *args, **kwargs):
    print 'Starting ION CC with options: ', opts

    container = Container(*args, **kwargs)

    # start and wait for container to signal ready
    ready = container.start()
    ready.get()

    if opts.rel:
        client = RPCClient(node=container.node, name=container.name, iface=IContainerAgent)
        client.start_rel_from_url(opts.rel)

    if not opts.noshell:
        setup_ipython()
    else:
        container.serve_forever()

def parse_args(tokens):
    """ Exploit yaml's spectacular type inference (and ensure consistency with config files) """
    args, kwargs = [], {}
    for token in tokens:
        token = token.lstrip('-')
        if '=' in token:
            key,val = token.split('=', 1)
            cfg = unflatten({key: yaml.load(val)})
            kwargs.update(cfg)
        else:
            args.append(yaml.load(token))

    return args, kwargs

if __name__ == '__main__':
    proc_types = GreenProcessSupervisor.type_callables.keys()

    # NOTE: Resist the temptation to add manual parameters here! Most container config options
    # should be in the config file (pyon.yml), which can also be specified on the command-line via the extra args
    
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-d', '--daemon', action='store_true')
    parser.add_argument('-n', '--noshell', action='store_true')
    parser.add_argument('-r', '--rel', type=str, help='Path to a rel file to launch.')
    parser.add_argument('--version', action='version', version='pyon v%s' % (version))
    opts, extra = parser.parse_known_args()
    args, kwargs = parse_args(extra)

    if opts.daemon:
        # TODO: The daemonizing code may need to be moved inside the Container class (so it happens per-process)
        from daemon import DaemonContext
        from lockfile import FileLock

        # TODO: May need to generate a pidfile based on some parameter or cc name
        with DaemonContext(pidfile=FileLock('cc.pid')):
            main(opts, *args, **kwargs)
    else:
        main(opts, *args, **kwargs)
    <commit_msg>--pidfile support for cc script<commit_after>#!/usr/bin/env python

__author__ = '<NAME>'
__license__ = 'Apache 2.0'

import argparse

import yaml
from uuid import uuid4

from pyon.public import Container, GreenProcessSupervisor

from pyon.container.cc import IContainerAgent
from pyon.net.endpoint import RPCClient

version = "2.0"     # TODO: extract this from the code once versioning is automated again
description = '''
pyon (ION capability container) v%s
''' % (version)

def setup_ipython():
    from IPython.config.loader import Config
    cfg = Config()
    shell_config = cfg.InteractiveShellEmbed
    shell_config.prompt_in1 = '><> '
    shell_config.prompt_in2 = '... '
    shell_config.prompt_out = '--> '
    shell_config.confirm_exit = False

    # First import the embeddable shell class
    from IPython.frontend.terminal.embed import InteractiveShellEmbed

    # Now create an instance of the embeddable shell. The first argument is a
    # string with options exactly as you would type them if you were starting
    # IPython at the system command line. Any parameters you want to define for
    # configuration can thus be specified here.
    ipshell = InteractiveShellEmbed(config=cfg,
                           banner1 = 'Dropping into IPython',
                           exit_msg = 'Leaving Interpreter, back to program.')

    ipshell('Pyon shell (ION R2)')

# From http://stackoverflow.com/questions/6037503/python-unflatten-dict/6037657#6037657
def unflatten(dictionary):
    resultDict = dict()
    for key, value in dictionary.iteritems():
        parts = key.split(".")
        d = resultDict
        for part in parts[:-1]:
            if part not in d:
                d[part] = dict()
            d = d[part]
        d[parts[-1]] = value
    return resultDict
    
def main(opts, *args, **kwargs):
    print 'Starting ION CC with options: ', opts

    container = Container(*args, **kwargs)

    # start and wait for container to signal ready
    ready = container.start()
    ready.get()

    if opts.rel:
        client = RPCClient(node=container.node, name=container.name, iface=IContainerAgent)
        client.start_rel_from_url(opts.rel)

    if not opts.noshell:
        setup_ipython()
    else:
        container.serve_forever()

def parse_args(tokens):
    """ Exploit yaml's spectacular type inference (and ensure consistency with config files) """
    args, kwargs = [], {}
    for token in tokens:
        token = token.lstrip('-')
        if '=' in token:
            key,val = token.split('=', 1)
            cfg = unflatten({key: yaml.load(val)})
            kwargs.update(cfg)
        else:
            args.append(yaml.load(token))

    return args, kwargs

if __name__ == '__main__':
    proc_types = GreenProcessSupervisor.type_callables.keys()

    # NOTE: Resist the temptation to add manual parameters here! Most container config options
    # should be in the config file (pyon.yml), which can also be specified on the command-line via the extra args
    
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-d', '--daemon', action='store_true')
    parser.add_argument('-n', '--noshell', action='store_true')
    parser.add_argument('-r', '--rel', type=str, help='Path to a rel file to launch.')
    parser.add_argument('-p', '--pidfile', type=str, help='PID file to use when --daemon specified. Defaults to cc-<rand>.pid')
    parser.add_argument('--version', action='version', version='pyon v%s' % (version))
    opts, extra = parser.parse_known_args()
    args, kwargs = parse_args(extra)

    if opts.daemon:
        # TODO: The daemonizing code may need to be moved inside the Container class (so it happens per-process)
        from daemon import DaemonContext
        from lockfile import FileLock

        # TODO: May need to generate a pidfile based on some parameter or cc name
        pidfile = opts.pidfile or 'cc-%s.pid' % str(uuid4())[0:4]
        with DaemonContext(pidfile=FileLock(pidfile)):
            main(opts, *args, **kwargs)
    else:
        main(opts, *args, **kwargs)
    
{'fraction_non_alphanumeric': 0.10566037735849057, 'fraction_numerical': 0.008553459119496854, 'mean_word_length': 2.9013738959764477, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4174213', 'n_tokens_mistral': 2386, 'n_tokens_neox': 2300, 'n_words': 730}