<commit_before>from __future__ import print_function

from functools import wraps
import json
import os
import sys
import traceback
import cStringIO
import re
import shutil
import socket

import mischief.actors.pipe as p
import mischief.actors.actor as a
import docker_utils

MAX_OUTPUT = 1000


def truncated_stdout(f):
    """Decorator to truncate stdout to final `MAX_OUTPUT` characters. """

    @wraps(f)
    def wrapper(*args, **kwargs):
        old_stdout = sys.stdout
        old_stdout.flush()
        sys.stdout = cStringIO.StringIO()
        out = ''
        try:
            result = f(*args, **kwargs)
            stdout = sys.stdout.getvalue()
            out = stdout + '\nSUCCESS' if stdout else ''
            return result
        except Exception:
            out = traceback.format_exc() + '\nERROR'
        finally:
            sys.stdout = old_stdout
            print(out[-MAX_OUTPUT:], end='')
    return wrapper


def with_payload(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        payload = json.loads(sys.stdin.read())
        kwargs.update(payload)
        return f(*args, **kwargs)
    return wrapper


def with_member_info(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        kwargs['members'] = list(member_info(sys.stdin.readlines()))
        return f(*args, **kwargs)
    return wrapper


def member_info(lines):
    for line in lines:
        member = {}
        parts = line.split()
        member['node'] = parts[0]
        member['ip'] = parts[1]
        member['role'] = parts[2]
        member['tags'] = dict(part.split('=') for part in parts[3].split(','))
        yield member


def serf_aware_spawn(actor, name, **kwargs):
    """Spawn actor and save address info in ``/actor_info``. """

    ip = os.environ.get('IP') or p.get_local_ip('8.8.8.8')
    kwargs['ip'] = ip
    proc = a.spawn(actor, **kwargs)
    try:
        os.makedirs('/actors')
    except OSError:
        pass
    with open('/actors/{}'.format(name), 'w') as f:
        identifier, ip, port = proc.address()
        json.dump({'name': identifier, 'ip': ip, 'port': port}, f)
    return proc


def read_etc_hosts():
    etc = [line.strip().split()
           for line in open('/etc/hosts').readlines()
           if not line.startswith('#')]
    return {host: line[0] for line in etc for host in line[1:]}


def write_etc_hosts(etc):
    shutil.copyfile('/etc/hosts', '/etc/hosts.orig')
    ip_hosts = {}
    for host, ip in etc.items():
        ip_hosts.setdefault(ip, []).append(host)
    with open('/etc/hosts', 'w') as f:
        f.writelines(
            ' '.join([ip] + hosts)+'\n' for ip, hosts in ip_hosts.items())


def get_ports():
    """Get the ports mapping for this node."""

    def _get_ports():
        cinfo = docker_utils.client.inspect_container(socket.gethostname())
        for port, host_ports in cinfo['NetworkSettings']['Ports'].items():
            if host_ports is not None:
                yield port, [host['HostPort'] for host in host_ports]

    return {
        'ports': dict(_get_ports()),
        'ip': os.environ.get('IP') or p.get_local_ip('8.8.8.8')
    }


def encode_ports(ports):
    """Dict[str, Any] -> str"""

    protocols = {'tcp': 't', 'udp': 'u'}

    def one_port():
        for internal, externals in ports.items():
            internal_port, protocol = internal.split('/')
            all_externals = ','.join(externals)
            short_proto = protocols[protocol]
            yield ('{internal_port}{short_proto}{all_externals}'.
                   format(**locals()))

    return '|'.join(one_port())


def decode_ports(port_string):
    """str -> Dict[str, Any]"""

    def split_one_map(map_str):
        internal, externals = re.split('[tu]', map_str)
        internal_port = '{}/{}'.format(
            internal, 'tcp' if 't' in map_str else 'u')
        return internal_port, externals.split(',')

    return dict(map(split_one_map,
                    [port for port in port_string.split('|') if port]))
<commit_msg>Fix some stray code not using config<commit_after>from __future__ import print_function

from functools import wraps
import json
import os
import sys
import traceback
import cStringIO
import re
import shutil
import socket

import mischief.actors.pipe as p
import mischief.actors.actor as a
import docker_utils
import config

MAX_OUTPUT = 1000


def truncated_stdout(f):
    """Decorator to truncate stdout to final `MAX_OUTPUT` characters. """

    @wraps(f)
    def wrapper(*args, **kwargs):
        old_stdout = sys.stdout
        old_stdout.flush()
        sys.stdout = cStringIO.StringIO()
        out = ''
        try:
            result = f(*args, **kwargs)
            stdout = sys.stdout.getvalue()
            out = stdout + '\nSUCCESS' if stdout else ''
            return result
        except Exception:
            out = traceback.format_exc() + '\nERROR'
        finally:
            sys.stdout = old_stdout
            print(out[-MAX_OUTPUT:], end='')
    return wrapper


def with_payload(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        payload = json.loads(sys.stdin.read())
        kwargs.update(payload)
        return f(*args, **kwargs)
    return wrapper


def with_member_info(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        kwargs['members'] = list(member_info(sys.stdin.readlines()))
        return f(*args, **kwargs)
    return wrapper


def member_info(lines):
    for line in lines:
        member = {}
        parts = line.split()
        member['node'] = parts[0]
        member['ip'] = parts[1]
        member['role'] = parts[2]
        member['tags'] = dict(part.split('=') for part in parts[3].split(','))
        yield member


def serf_aware_spawn(actor, name, **kwargs):
    """Spawn actor and save address info in ``/actor_info``. """

    ip = config.ip
    kwargs['ip'] = ip
    proc = a.spawn(actor, **kwargs)
    try:
        os.makedirs('/actors')
    except OSError:
        pass
    with open('/actors/{}'.format(name), 'w') as f:
        identifier, ip, port = proc.address()
        json.dump({'name': identifier, 'ip': ip, 'port': port}, f)
    return proc


def read_etc_hosts():
    etc = [line.strip().split()
           for line in open('/etc/hosts').readlines()
           if not line.startswith('#')]
    return {host: line[0] for line in etc for host in line[1:]}


def write_etc_hosts(etc):
    shutil.copyfile('/etc/hosts', '/etc/hosts.orig')
    ip_hosts = {}
    for host, ip in etc.items():
        ip_hosts.setdefault(ip, []).append(host)
    with open('/etc/hosts', 'w') as f:
        f.writelines(
            ' '.join([ip] + hosts)+'\n' for ip, hosts in ip_hosts.items())


def get_ports():
    """Get the ports mapping for this node."""

    def _get_ports():
        cinfo = docker_utils.client.inspect_container(socket.gethostname())
        for port, host_ports in cinfo['NetworkSettings']['Ports'].items():
            if host_ports is not None:
                yield port, [host['HostPort'] for host in host_ports]

    return {
        'ports': dict(_get_ports()),
        'ip': config.ip
    }


def encode_ports(ports):
    """Dict[str, Any] -> str"""

    protocols = {'tcp': 't', 'udp': 'u'}

    def one_port():
        for internal, externals in ports.items():
            internal_port, protocol = internal.split('/')
            all_externals = ','.join(externals)
            short_proto = protocols[protocol]
            yield ('{internal_port}{short_proto}{all_externals}'.
                   format(**locals()))

    return '|'.join(one_port())


def decode_ports(port_string):
    """str -> Dict[str, Any]"""

    def split_one_map(map_str):
        internal, externals = re.split('[tu]', map_str)
        internal_port = '{}/{}'.format(
            internal, 'tcp' if 't' in map_str else 'u')
        return internal_port, externals.split(',')

    return dict(map(split_one_map,
                    [port for port in port_string.split('|') if port]))

{'fraction_non_alphanumeric': 0.14797136038186157, 'fraction_numerical': 0.003517146087174978, 'mean_word_length': 2.5090348171000443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2612154', 'n_tokens_mistral': 2563, 'n_tokens_neox': 2475, 'n_words': 648}