<commit_before>import yaml
import itertools
import argh
import s
import os


@argh.arg('cmd', nargs='?', default=None)
def run(action_name, cmd, tty=False):
    """run an action"""
    build(action_name)
    tty = True if cmd == 'bash' else tty
    action = _get_action(action_name)

    stop(action_name)

    try:
        _start_deps(action)
        print('start main')
        s.shell.run(_run_cmd(action['main'], tty=tty, cmd=cmd), interactive=True)
    finally:
        stop(action_name)


def stop(action_name):
    action = _get_action(action_name)
    for container_name, data in action.items():
        with s.exceptions.ignore():
            s.shell.run('docker kill', data['tag'])
            s.shell.run('docker rm', data['tag'])



_services = './docker/state/services.yml'


def _exposed_port(tag):
    return int(s.shell.run('docker port', tag).split(':')[-1])


def _start_deps(action):
    s.shell.run('rm -f', _services)
    to_start = list(s.dicts.drop(action, 'main').items())
    started = []
    for i in itertools.count():
        container_name, data = to_start.pop()
        deps = data.get('depends', [])
        if not deps or all(x in started for x in deps):
            print('start dep:', container_name)
            s.shell.run(_run_cmd(data, bg=True))
            started.append(container_name)
            with open(_services, 'a') as f:
                f.write(yaml.dump({container_name: {'port': _exposed_port(data['tag']),
                                                    'host': _host_ip()}}))
        else:
            to_start.append([container_name, data])
        assert i < 500, 'never resolved dependency order. remaining: {}'.format(to_start)
        if not to_start:
            return


def build(action_name, container=None, nocache=False, pull=False, force=False):
    """build an actions containers"""
    action = _get_action(action_name)
    for container_name, data in action.items():
        if not container or container_name == container:
            if force or not s.shell.run('docker inspect', data['tag'], zero=True, stream=False):
                s.shell.run(_build_cmd(data, nocache, pull), stream=True)
                print(s.colors.green('built:'), data['tag'])


def show():
    """show available actions"""
    data = _get_actions()
    for action_name in sorted(data):
        print('')
        print(action_name)
        for container in data[action_name]:
            print('', container)


def main():
    assert os.path.isdir('docker'), 'must have a dir: ./docker'
    assert os.path.isfile('system.yml'), 'must have a file: ./system.yml'
    if s.shell.override('--stream'):
        with s.shell.set_stream():
            _main()
    else:
        _main()


def _main():
    s.shell.run('mkdir -p ./docker/state')
    s.shell.dispatch_commands(globals(), __name__)


def _tag_containers(action_name, action):
    for container_name, data in action.items():
        action[container_name]['tag'] = '_'.join([
            os.path.basename(os.getcwd()),
            action_name,
            container_name
        ])<commit_msg>add ports and allow no dependencies<commit_after>import yaml
import itertools
import argh
import s
import os


_services = './docker/state/services.yml'


@argh.arg('cmd', nargs='?', default=None)
def run(action_name, cmd, tty=False, no_clean_before=False, no_clean_after=False):
    """run an action"""
    build(action_name)
    tty = True if cmd == 'bash' else tty
    action = _get_action(action_name)
    if not no_clean_before:
        stop(action_name)
    try:
        _start_deps(action)
        print('start main')
        s.shell.run(_run_cmd(action['main'], tty=tty, cmd=cmd), interactive=True)
    finally:
        if not no_clean_after:
            stop(action_name)


def ports(action_name, container_name=None):
    action = _get_action(action_name)
    if container_name:
        print(_exposed_port(action[container_name]['tag']))
    else:
        for container_name, data in action.items():
            port = _exposed_port(data['tag'])
            if port:
                print('{container_name}:{port}'.format(**locals()))


def stop(action_name):
    action = _get_action(action_name)
    for container_name, data in action.items():
        with s.exceptions.ignore():
            s.shell.run('docker kill', data['tag'])
            s.shell.run('docker rm', data['tag'])


def build(action_name, container=None, nocache=False, pull=False, force=False):
    """build an actions containers"""
    action = _get_action(action_name)
    for container_name, data in action.items():
        if not container or container_name == container:
            if force or not s.shell.run('docker inspect', data['tag'], zero=True, stream=False):
                s.shell.run(_build_cmd(data, nocache, pull), stream=True)
                print(s.colors.green('built:'), data['tag'])


def show():
    """show available actions"""
    data = _get_actions()
    for action_name in sorted(data):
        print('')
        print(action_name)
        for container in data[action_name]:
            print('', container)


def main():
    assert os.path.isdir('docker'), 'must have a dir: ./docker'
    assert os.path.isfile('system.yml'), 'must have a file: ./system.yml'
    if s.shell.override('--stream'):
        with s.shell.set_stream():
            _main()
    else:
        _main()


def _main():
    s.shell.run('mkdir -p ./docker/state')
    s.shell.dispatch_commands(globals(), __name__)


def _exposed_port(tag):
    port = s.shell.run('docker port', tag).split(':')[-1]
    if port:
        return int(port)


def _start_deps(action):
    s.shell.run('rm -f', _services)
    to_start = list(s.dicts.drop(action, 'main').items())
    if to_start:
        started = []
        for i in itertools.count():
            container_name, data = to_start.pop()
            deps = data.get('depends', [])
            if not deps or all(x in started for x in deps):
                print('start dep:', container_name)
                s.shell.run(_run_cmd(data, bg=True))
                started.append(container_name)
                with open(_services, 'a') as f:
                    f.write(yaml.dump({container_name: {'port': _exposed_port(data['tag']),
                                                        'host': _host_ip()}}))
            else:
                to_start.append([container_name, data])
            assert i < 500, 'never resolved dependency order. remaining: {}'.format(to_start)
            if not to_start:
                return


def _tag_containers(action_name, action):
    for container_name, data in action.items():
        action[container_name]['tag'] = '_'.join([
            os.path.basename(os.getcwd()),
            action_name,
            container_name
        ])
{'fraction_non_alphanumeric': 0.1395007342143906, 'fraction_numerical': 0.0011747430249632893, 'mean_word_length': 2.505404014410705, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5126100', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2055, 'n_words': 492}