<commit_before>import httplib
import os
import shutil
import subprocess

import requests
from retrying import retry

import util

DOCKER_NETWORK = 'couchdb2_cluster_network'
DOCKER_CREATE_NETWORK = 'docker network create --subnet=172.16.31.10/16 {}'.format(DOCKER_NETWORK)
DOCKER_START_NODE = 'docker run --net {cluster_network} --ip="{node_ip}" -v {node_data_dir}:/opt/couchdb/data -d ' \
                    '-v {node_config_dir}:/opt/couchdb/etc --name="{node_name}" docker-couchdb-2:1.0.0'
DOCKER_FIND_NODE = 'docker ps --filter "name={node_name}" -qa'
BASE_NODE_URL = 'http://{ip}:5984/{db}'
NODE_URL = 'http://{ip}:5984/{slug}'
SECURED_NODE_URL = 'http://{user}:{password}@{ip}:5984/{slug}'
COUCHDB_CLUSTER_SETUP = {
    'url': 'http://{user}:{password}@{ip}:5984/_cluster_setup',
    'payload': '',
    'finish_payload': '{"action":"finish_cluster"}'
}


def start(num_nodes, admin, password):
    try:
        print 'Network setup.'
        subprocess.check_output(DOCKER_CREATE_NETWORK, shell=True)
    except subprocess.CalledProcessError:
        print ('Network exists - ignoring.')

    nodes = []
    for node_num in range(2, int(num_nodes) + 2):
        node_ip = '173.19.0.{}'.format(node_num)
        node_dir = 'node{}'.format(node_num)
        container_name = 'couchdb' + node_dir
        nodes.append(util.node(node_dir, node_ip, container_name))

        try:
            container_id = subprocess.check_output(DOCKER_FIND_NODE.format(node_name=container_name), shell=True)
            if container_id:
                print ("removing container {} {}".format(container_name, container_id))
                subprocess.check_output('docker rm -f {}'.format(container_id), shell=True)
        except:
            pass  # Ignore exceptions for now.
    for node in nodes:
        data_dir_path, node_config_path = make_node_config(node.dir, node.ip, node.name)
        start_cmd = DOCKER_START_NODE.format(cluster_network=DOCKER_NETWORK,
                                             node_ip=node.ip,
                                             node_data_dir=data_dir_path,
                                             node_config_dir=node_config_path,
                                             node_name=node.name)
        print start_cmd
        subprocess.check_output(start_cmd, shell=True)

        print ("Initializing node")
        initial_configuration(node.ip)
        create_admin_user(node.name, node.ip, admin, "admin", password)
        advanced_configuration(node.name, node.ip, admin, password, "<PASSWORD>")

    master_node_ip = nodes[0].ip
    enable_cluster(master_node_ip, admin, password)
    add_nodes_to_cluster(master_node_ip, nodes, admin, password)

    response = requests.post(url=COUCHDB_CLUSTER_SETUP['url'].format(user=admin, password=password, ip=master_node_ip),
                             json={"action": "finish_cluster"})
    print('Cluster setup: complete {}'.format(response.text))
    print('Success')


def add_nodes_to_cluster(master_node_ip, node_ips, admin, password):
    for node in node_ips[1:]:
        url = 'http://{user}:{password}@{master_node_ip}:5984/_cluster_setup'.format(user=admin,
                                                                                     password=password,
                                                                                     master_node_ip=master_node_ip)
        print ("Adding node {} to cluster {}".format(node.ip, url))
        response = requests.post(url=url, json={"action": "add_node",
                                                "host": node.ip,
                                                "username": admin,
                                                "password": password
                                                })
        print (response.text)


def make_node_config(node_dir, node_ip, name):
    local_config_path = os.path.abspath(os.path.join(os.curdir, 'config'))
    data_dir_path = os.path.abspath(os.path.join(os.curdir, node_dir, 'data'))
    node_config_path = os.path.abspath(os.path.join(node_dir, 'local.d'))
    if os.path.exists(data_dir_path) or os.path.exists(node_config_path):
        cmd = 'docker run -v {}:/node_dir --entrypoint="/bin/sh" --rm alpine -c "rm -rf /node_dir/{}" '.format(
            os.path.abspath(os.curdir), node_dir)
        print('Removing existing node config. {}'.format(cmd))
        subprocess.check_output(cmd, shell=True)
    # import ipdb
    # ipdb.set_trace()
    shutil.copytree(local_config_path, node_config_path)
    with open(os.path.join(node_config_path, 'vm.args'), 'r+') as f:
        vm_config = f.read()
        vm_config = vm_config.replace('{{node_name}}', '-name {}@{}'.format(name, node_ip))
        f.seek(0)
        f.write(vm_config)
        f.truncate()

    return data_dir_path, node_config_path


def enable_cluster(master_node_ip, admin, password):
    cluster_url = COUCHDB_CLUSTER_SETUP['url'].format(user=admin, password=password, ip=master_node_ip)<commit_msg>Clean up.<commit_after>import httplib
import os
import shutil
import subprocess

import requests
from retrying import retry

import util

DOCKER_NETWORK = 'couchdb2_cluster_network'
DOCKER_CREATE_NETWORK = 'docker network create --subnet=172.16.31.10/16 {}'.format(DOCKER_NETWORK)
DOCKER_START_NODE = 'docker run --net {cluster_network} --ip="{node_ip}" -v {node_data_dir}:/opt/couchdb/data -d ' \
                    '-v {node_config_dir}:/opt/couchdb/etc --name="{node_name}" docker-couchdb-2:1.0.0'
DOCKER_FIND_NODE = 'docker ps --filter "name={node_name}" -qa'
BASE_NODE_URL = 'http://{ip}:5984/{db}'
NODE_URL = 'http://{ip}:5984/{slug}'
SECURED_NODE_URL = 'http://{user}:{password}@{ip}:5984/{slug}'
COUCHDB_CLUSTER_SETUP = {
    'url': 'http://{user}:{password}@{ip}:5984/_cluster_setup',
    'payload': '',
    'finish_payload': '{"action":"finish_cluster"}'
}


def start(num_nodes, admin, password):
    try:
        print('Network setup.')
        subprocess.check_output(DOCKER_CREATE_NETWORK, shell=True)
    except subprocess.CalledProcessError:
        print ('Network exists - ignoring.')

    nodes = []
    for node_num in range(2, int(num_nodes) + 2):
        node_ip = '173.19.0.{}'.format(node_num)
        node_dir = 'node{}'.format(node_num)
        container_name = 'couchdb' + node_dir
        nodes.append(util.node(node_dir, node_ip, container_name))

        try:
            container_id = subprocess.check_output(DOCKER_FIND_NODE.format(node_name=container_name), shell=True)
            if container_id:
                print ("removing container {} {}".format(container_name, container_id))
                subprocess.check_output('docker rm -f {}'.format(container_id), shell=True)
        except:
            pass  # Ignore exceptions for now.
    for node in nodes:
        data_dir_path, node_config_path = make_node_config(node.dir, node.ip, node.name)
        start_cmd = DOCKER_START_NODE.format(cluster_network=DOCKER_NETWORK,
                                             node_ip=node.ip,
                                             node_data_dir=data_dir_path,
                                             node_config_dir=node_config_path,
                                             node_name=node.name)
        print(start_cmd)
        subprocess.check_output(start_cmd, shell=True)

        print ("Initializing node")
        initial_configuration(node.ip)
        create_admin_user(node.name, node.ip, admin, "admin", password)
        advanced_configuration(node.name, node.ip, admin, password, "<PASSWORD>")

    master_node_ip = nodes[0].ip
    enable_cluster(master_node_ip, admin, password)
    add_nodes_to_cluster(master_node_ip, nodes, admin, password)

    response = requests.post(url=COUCHDB_CLUSTER_SETUP['url'].format(user=admin, password=password, ip=master_node_ip),
                             json={"action": "finish_cluster"})
    print('Cluster setup: complete {}'.format(response.text))
    print('Success')


def add_nodes_to_cluster(master_node_ip, node_ips, admin, password):
    for node in node_ips[1:]:
        url = 'http://{user}:{password}@{master_node_ip}:5984/_cluster_setup'.format(user=admin,
                                                                                     password=password,
                                                                                     master_node_ip=master_node_ip)
        print ("Adding node {} to cluster {}".format(node.ip, url))
        response = requests.post(url=url, json={"action": "add_node",
                                                "host": node.ip,
                                                "username": admin,
                                                "password": password
                                                })
        print (response.text)


def make_node_config(node_dir, node_ip, name):
    local_config_path = os.path.abspath(os.path.join(os.curdir, 'config'))
    data_dir_path = os.path.abspath(os.path.join(os.curdir, node_dir, 'data'))
    node_config_path = os.path.abspath(os.path.join(node_dir, 'local.d'))
    if os.path.exists(data_dir_path) or os.path.exists(node_config_path):
        cmd = 'docker run -v {}:/node_dir --entrypoint="/bin/sh" --rm alpine -c "rm -rf /node_dir/{}" '.format(
            os.path.abspath(os.curdir), node_dir)
        print('Removing existing node config. {}'.format(cmd))
        subprocess.check_output(cmd, shell=True)
    shutil.copytree(local_config_path, node_config_path)
    with open(os.path.join(node_config_path, 'vm.args'), 'r+') as f:
        vm_config = f.read()
        vm_config = vm_config.replace('{{node_name}}', '-name {}@{}'.format(name, node_ip))
        f.seek(0)
        f.write(vm_config)
        f.truncate()

    return data_dir_path, node_config_path


def enable_cluster(master_node_ip, admin, password):
    cluster_url = COUCHDB_CLUSTER_SETUP['url'].format(user=admin, password=password, ip=master_node_ip)
{'fraction_non_alphanumeric': 0.12722339463370516, 'fraction_numerical': 0.009446286805346196, 'mean_word_length': 2.409386776293251, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 44, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7092486', 'n_tokens_mistral': 3115, 'n_tokens_neox': 3027, 'n_words': 529}