<commit_before>#!/usr/bin/python2
# Copyright 2014 CloudFounders NV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Contains grid management functionality
"""

import ConfigParser
import base64
import getpass
import hashlib
import os
import sys
import urllib2
import re

from optparse import OptionParser
from random import choice
from string import lowercase
from subprocess import check_output


class Manager(object):
    """
    Contains grid management functionality
    """

    @staticmethod
    def install_node(ip, password, create_filesystems=True, clean=False):
        """
        Installs the Open vStorage software on a (remote) node.
        """

        # Load client, local or remote
        is_local = Client.is_local(ip)
        client = Client.load(ip, password, bypass_local=True)

        if clean:
            Manager._clean(client)
        if create_filesystems:
            Manager._create_filesystems(client)

        # Ask a bunch of questions and prepare HRD files for installation
        first_node = False
        if is_local:
            # @TODO: Try to figure out whether this is the first node or not.
            first_node = Helper.ask_yesno('Is this a first node installation?', default_value=True)
        configuration = {'openvstorage': {}}
        configuration['openvstorage']['ovs.host.hypervisor'] = 'VMWARE'
        configuration['openvstorage']['ovs.host.name'] = Helper.ask_string('Enter hypervisor hostname', default_value='esxi')
        ip, username, password = None, 'root', None
        while True:
            ip = Helper.ask_string('Enter hypervisor ip address', default_value=ip)
            username = Helper.ask_string('Enter hypervisor username', default_value=username)
            password = getpass.getpass()
            try:
                request = urllib2.Request('https://{0}/mob'.format(ip))
                auth = base64.encodestring('{0}:{1}'.format(username, password)).replace('\n', '')
                request.add_header("Authorization", "Basic %s" % auth)
                urllib2.urlopen(request).read()
                break
            except Exception as ex:
                print 'Could not connect to {0}: {1}'.format(ip, ex)
        configuration['openvstorage']['ovs.host.ip'] = ip
        configuration['openvstorage']['ovs.host.login'] = username
        configuration['openvstorage']['ovs.host.password'] = password

        configuration['openvstorage-core'] = {}
        ipaddresses = client.run("ip a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1").strip().split('\n')
        ipaddresses = [ip for ip in ipaddresses if ip != '127.0.0.1']
        configuration['openvstorage-core']['ovs.grid.ip'] = Helper.ask_choice(ipaddresses,
                                                                              question='Choose public ip address',
                                                                              default_value=Helper.find_in_list(ipaddresses, ip))
        mountpoints = client.run('mount -v').strip().split('\n')
        mountpoints = [p.split(' ')[2] for p in mountpoints if len(p.split(' ')) > 2 and ('/mnt/' in p.split(' ')[2] or '/var' in p.split(' ')[2])]
        unique_id = sorted(client.run("ip a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g'").strip().split('\n'))[0]
        configuration['openvstorage-core']['ovs.core.memcache.localnode.name'] = unique_id
        mountpoint = Helper.ask_choice(mountpoints,
                                       question='Select arakoon database mountpoint',
                                       default_value=Helper.find_in_list(mountpoints, 'db'))
        mountpoints.remove(mountpoint)
        configuration['openvstorage-core']['ovs.core.db.mountpoint'] = mountpoint
        configuration['openvstorage-core']['ovs.core.db.arakoon.node.name'] = unique_id
        configuration['openvstorage-core']['volumedriver.arakoon.node.name'] = unique_id
        configuration['openvstorage-core']['ovs.core.broker.localnode.name'] = unique_id

        configuration['openvstorage-webapps'] = {}
        configuration['openvstorage-webapps']['ovs.webapps.certificate.period'] = Helper.ask_integer('GUI certificate lifetime', min_value=1, max_value=365 * 10, default_value=365)

        if is_local:
            grid_id = Helper.ask_integer('Enter grid ID (needs to be unique): ', min_value=1, max_value=32767)
            es_cluster_name = Helper.ask_string('Enter elastic search cluster name', default_value='ovses')
            es_cluster_name = '{0}_{1}'.format(es_cluster_name, grid_id)
            osis_key = ''.join(choice(lowercase) for _ in range(25))
        else:
            from ovs.plugin.provider.configuration import Configuration
            es_cluster_name = Configuration.get('elasticsearch.cluster.name')
            grid_id = Configuration.get('grid.id')
            osis_key = Configuration.get('osis.key')

        configuration['grid'] = {}
        configuration['grid']['grid.id'] = grid_id
        configuration['grid']['grid.useavahi'] = 1
        configuration['grid']['grid.node.roles'] = 'node'
        configuration['grid']['grid.node.machineid'] = unique_id

        configuration['elasticsearch'] = {}
        configuration['elasticsearch']['elasticsearch.cluster.name'] = es_cluster_name

        if first_node:
            configuration['grid_master'] = {}
            configuration['grid_master']['gridmaster.grid.id'] = grid_id
            configuration['grid_master']['gridmaster.useavahi'] = 1
            configuration['grid_master']['gridmaster.superadminpasswd'] = hashlib.sha256(''.join(choice(lowercase) for _ in range(25))).hexdigest()
        else:
            configuration['grid']['grid.master.ip'] = ''

        configuration['osis'] = {}
        configuration['osis']['osis.key'] = osis_key

        client.dir_ensure('/opt/jumpscale/cfg/hrd', True)
        if is_local:
            for filename in configuration:
                with open('/opt/jumpscale/cfg/hrd/{0}.hrd'.format(filename), 'w') as hrd:
                    hrd.write('\n'.join(['%s=%s' % i for i in configuration[filename].iteritems()]))
        else:
            for filename in configuration:
                with open('/opt/jumpscale/cfg/hrd/{0}_{1}.hr_'.format(filename, ip), 'w') as hrd:
                    hrd.write('\n'.join(['%s=%s' % i for i in configuration[filename].iteritems()]))
                # The file_upload method has the destination first, then the source. Yeah, makes sense... not.
                client.file_upload('/opt/jumpscale/cfg/hrd/{0}.hrd'.format(filename),
                                   '/opt/jumpscale/cfg/hrd/{0}_{1}.hr_'.format(filename, ip))

        # Make sure all software is up-to-date
        client.run('apt-get update')
        client.file_append('/etc/security/limits.conf', '\nroot soft core  unlimited\novs  soft core  unlimited\n')

        # Install base framework, JumpScale in this case
        install_branch = Manager._prepare_jscore(client, is_local)
        Manager._install_jscore(client, install_branch)

        # Install Open vStorage
        print 'Installing Open vStorage...'
        client.run('apt-get -y -q install python-dev')
        client.run('jpackage_install -n openvstorage')

    @staticmethod
    def init_node(ip, password, join_masters=False):
        """
        Initializes a node, making sure all required services are up and running.
        Optionally, the node can also join the masters, also participating in the arakoon and memcache
        clusters. Joining the masters will result in the services being restarted on all master nodes.
        Please note that initializing a node, joining a grid with < 3 nodes, will result in joining the
        master nodes regardless of the given parameter.
        """

        from configobj import ConfigObj
        from ovs.dal.hybrids.pmachine import PMachine
        from ovs.dal.lists.pmachinelist import PMachineList
        from ovs.dal.hybrids.vmachine import VMachine
        from ovs.dal.lists.vmachinelist import VMachineList
        from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagement
        from ovs.plugin.provider.configuration import Configuration

        if ip == '127.0.0.1':
            print 'Do not use 127.0.0.1 as ip address, use the public grid ip instead.'
            sys.exit(1)

        # Make sure to ALWAYS reload the client when switching targets, as Fabric seems to be singleton-ish
        is_local = Client.is_local(ip)
        client = Client.load(ip, password)  # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish
        unique_id = sorted(client.run("ip a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g'").strip().split('\n'))[0].strip()

        arakoon_management = ArakoonManagement()
        nodes = Manager._get_cluster_nodes()  # All nodes, including the local and the new one
        arakoon_nodes = arakoon_management.getCluster('ovsdb').listNodes()
        client = Client.load(ip, password)
        new_node_hostname = client.run('hostname')
        if is_local and Configuration.get('grid.node.id') != '1':
            # If the script is executed local and there are multiple nodes, the script is executed on node 2+.
            # This is not allowed since a the existing configuration is requred to extend. On extra nodes there is no
            # current configuration to base all work on
            print 'This script can only be executed local on the first node. Subsequent nodes need to be installed from one of the master nodes.'
            sys.exit(1)
        if len(arakoon_nodes) < 3:
            print 'Insufficient master nodes'
            join_masters = True
        clusters = arakoon_management.listClusters()

        all_services = ['arakoon_ovsdb', 'arakoon_voldrv', 'memcached', 'rabbitmq', 'ovs_consumer_volumerouter',
                        'ovs_flower', 'ovs_scheduled_tasks', 'webapp_api', 'nginx', 'ovs_workers']
        arakoon_configfiles = ['/opt/OpenvStorage/config/arakoon/{0}/{0}_client.cfg'.format(cluster) for cluster in clusters]
        generic_configfiles = {'/opt/OpenvStorage/config/memcacheclient.cfg': 11211,
                               '/opt/OpenvStorage/config/rabbitmqclient.cfg': 5672}

        if join_masters:
            print 'Joining master nodes, services going down.'

            # Stop services (on all nodes)
            for node in nodes:
                node_client = Client.load(node, password)
                for service in all_services:
                    node_client.run('jsprocess disable -n {0}'.format(service))
                    node_client.run('jsprocess stop -n {0}'.format(service))

            # Fetch some information
            client = Client.load(ip, password)  # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish
            remote_ips = client.run("ip a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1").strip().split('\n')
            remote_ip = [ipa.strip() for ipa in nodes if ipa in remote_ips][0]

            # Configure arakoon
            for cluster in clusters:
                # The Arakoon extension is not used since the config file needs to be parsed/loaded anyway to be
                # able to update it
                cfg = ConfigObj('/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster))
                global_section = cfg.get('global')
                cluster_nodes = global_section['cluster'] if type(global_section['cluster']) == list else [global_section['cluster']]
                if unique_id not in cluster_nodes:
                    client = Client.load(ip, password)
                    remote_config = client.file_read('/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster))
                    with open('/tmp/arakoon_{0}_cfg'.format(unique_id), 'w') as the_file:
                        the_file.write(remote_config)
                    remote_cfg = ConfigObj('/tmp/arakoon_{0}_cfg'.format(unique_id))
                    cluster_nodes.append(unique_id)
                    global_section['cluster'] = cluster_nodes
                    cfg.update({'global': global_section})
                    cfg.update({unique_id: remote_cfg.get(unique_id)})
                    cfg.write()
                    for node in nodes:
                        node_client = Client.load(node, password)
                        node_client.file_upload('/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster),
                                                '/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster))
                client = Client.load(ip, password)
                arakoon_create_directories = """
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagement
arakoon_management = ArakoonManagement()
arakoon_cluster = arakoon_management.getCluster('%(cluster)s')
arakoon_cluster.createDirs(arakoon_cluster.listLocalNodes()[0])
""" % {'cluster': cluster}
                client.run('/opt/OpenvStorage/bin/python -c """{}"""'.format(arakoon_create_directories))

            # Update all nodes hosts file with new node and new node hosts file with all others
            for node in nodes:
                client_node = Client.load(node, password)
                update_hosts_file = """
from ovs.plugin.provider.net import Net
Net.updateHostsFile(hostsfile='/etc/hosts', ip='%(ip)s', hostname='%(host)s')
""" % {'ip': ip,
     'host': new_node_hostname}
                client_node.run('python -c """{}"""'.format(update_hosts_file))
                if node != ip:
                    client_node.run('jsprocess enable -n rabbitmq')
                    client_node.run('jsprocess start -n rabbitmq')
                else:
                    for subnode in nodes:
                        client_node = Client.load(subnode, password)
                        node_hostname = client_node.run('hostname')
                        update_hosts_file = """
from ovs.plugin.provider.net import Net
Net.updateHostsFile(hostsfile='/etc/hosts', ip='%(ip)s', hostname='%(host)s')
""" % {'ip': subnode,
     'host': node_hostname}
                        client = Client.load(ip, password)
                        client.run('python -c """{}"""'.format(update_hosts_file))

            # Update arakoon cluster configuration in voldrv configuration files
            for node in nodes:
                client_node = Client.load(node, password)
                update_voldrv = """
import os
from ovs.plugin.provider.configuration import Configuration
from ovs.extensions.storageserver.volumestoragerouter import VolumeStorageRouterConfiguration
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagement
arakoon_management = ArakoonManagement()
voldrv_arakoon_cluster_id = 'voldrv'
voldrv_arakoon_cluster = arakoon_management.getCluster(voldrv_arakoon_cluster_id)
voldrv_arakoon_client_config = voldrv_arakoon_cluster.getClientConfig()
configuration_dir = Configuration.get('ovs.core.cfgdir')
for json_file in os.listdir('{0}/voldrv_vpools'.format(configuration_dir)):
    if json_file.endswith('.json'):
        vsr_config = VolumeStorageRouterConfiguration(json_file.replace('.json', ''))
        vsr_config.configure_arakoon_cluster(voldrv_arakoon_cluster_id, voldrv_arakoon_client_config)
"""
                client_node.run('python -c """{}"""'.format(update_voldrv))

            # Join rabbitMQ clusters
            client = Client.load(ip, password)
            client.run('rabbitmq-server -detached; sleep 5; rabbitmqctl stop_app; sleep 5; rabbitmqctl reset; sleep 5; rabbitmqctl stop; sleep 5;')
            if not is_local:
                # Copy rabbitmq cookie
                rabbitmq_cookie_file = '/var/lib/rabbitmq/.erlang.cookie'
                client.dir_ensure(os.path.dirname(rabbitmq_cookie_file), True)
                client.file_upload(rabbitmq_cookie_file, rabbitmq_cookie_file)
                client.file_attribs(rabbitmq_cookie_file, mode=400)
                client.run('rabbitmq-server -detached; sleep 5; rabbitmqctl stop_app; sleep 5;')
                # If not local, a cluster needs to be joined.
                master_client = Client.load(Configuration.get('grid.master.ip'), password)
                master_hostname = master_client.run('hostname')
                client = Client.load(ip, password)
                client.run('rabbitmqctl join_cluster rabbit@{}; sleep 5;'.format(master_hostname))
                client.run('rabbitmqctl stop; sleep 5;')

            # Update local client configurations
            for config in arakoon_configfiles:
                cfg = ConfigObj(config)
                global_section = cfg.get('global')
                cluster_nodes = global_section['cluster'] if type(global_section['cluster']) == list else [global_section['cluster']]
                if unique_id not in cluster_nodes:
                    cluster_nodes.append(unique_id)<commit_msg>OVS-438 - Fixed issue with first node init<commit_after>#!/usr/bin/python2
# Copyright 2014 CloudFounders NV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Contains grid management functionality
"""

import ConfigParser
import base64
import getpass
import hashlib
import os
import sys
import urllib2
import re

from optparse import OptionParser
from random import choice
from string import lowercase
from subprocess import check_output


class Manager(object):
    """
    Contains grid management functionality
    """

    @staticmethod
    def install_node(ip, password, create_filesystems=True, clean=False):
        """
        Installs the Open vStorage software on a (remote) node.
        """

        # Load client, local or remote
        is_local = Client.is_local(ip)
        client = Client.load(ip, password, bypass_local=True)

        if clean:
            Manager._clean(client)
        if create_filesystems:
            Manager._create_filesystems(client)

        # Ask a bunch of questions and prepare HRD files for installation
        first_node = False
        if is_local:
            # @TODO: Try to figure out whether this is the first node or not.
            first_node = Helper.ask_yesno('Is this a first node installation?', default_value=True)
        configuration = {'openvstorage': {}}
        configuration['openvstorage']['ovs.host.hypervisor'] = 'VMWARE'
        configuration['openvstorage']['ovs.host.name'] = Helper.ask_string('Enter hypervisor hostname', default_value='esxi')
        ip, username, password = None, 'root', None
        while True:
            ip = Helper.ask_string('Enter hypervisor ip address', default_value=ip)
            username = Helper.ask_string('Enter hypervisor username', default_value=username)
            password = getpass.getpass()
            try:
                request = urllib2.Request('https://{0}/mob'.format(ip))
                auth = base64.encodestring('{0}:{1}'.format(username, password)).replace('\n', '')
                request.add_header("Authorization", "Basic %s" % auth)
                urllib2.urlopen(request).read()
                break
            except Exception as ex:
                print 'Could not connect to {0}: {1}'.format(ip, ex)
        configuration['openvstorage']['ovs.host.ip'] = ip
        configuration['openvstorage']['ovs.host.login'] = username
        configuration['openvstorage']['ovs.host.password'] = password

        configuration['openvstorage-core'] = {}
        ipaddresses = client.run("ip a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1").strip().split('\n')
        ipaddresses = [ip for ip in ipaddresses if ip != '127.0.0.1']
        configuration['openvstorage-core']['ovs.grid.ip'] = Helper.ask_choice(ipaddresses,
                                                                              question='Choose public ip address',
                                                                              default_value=Helper.find_in_list(ipaddresses, ip))
        mountpoints = client.run('mount -v').strip().split('\n')
        mountpoints = [p.split(' ')[2] for p in mountpoints if len(p.split(' ')) > 2 and ('/mnt/' in p.split(' ')[2] or '/var' in p.split(' ')[2])]
        unique_id = sorted(client.run("ip a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g'").strip().split('\n'))[0]
        configuration['openvstorage-core']['ovs.core.memcache.localnode.name'] = unique_id
        mountpoint = Helper.ask_choice(mountpoints,
                                       question='Select arakoon database mountpoint',
                                       default_value=Helper.find_in_list(mountpoints, 'db'))
        mountpoints.remove(mountpoint)
        configuration['openvstorage-core']['ovs.core.db.mountpoint'] = mountpoint
        configuration['openvstorage-core']['ovs.core.db.arakoon.node.name'] = unique_id
        configuration['openvstorage-core']['volumedriver.arakoon.node.name'] = unique_id
        configuration['openvstorage-core']['ovs.core.broker.localnode.name'] = unique_id

        configuration['openvstorage-webapps'] = {}
        configuration['openvstorage-webapps']['ovs.webapps.certificate.period'] = Helper.ask_integer('GUI certificate lifetime', min_value=1, max_value=365 * 10, default_value=365)

        if is_local:
            grid_id = Helper.ask_integer('Enter grid ID (needs to be unique): ', min_value=1, max_value=32767)
            es_cluster_name = Helper.ask_string('Enter elastic search cluster name', default_value='ovses')
            es_cluster_name = '{0}_{1}'.format(es_cluster_name, grid_id)
            osis_key = ''.join(choice(lowercase) for _ in range(25))
        else:
            from ovs.plugin.provider.configuration import Configuration
            es_cluster_name = Configuration.get('elasticsearch.cluster.name')
            grid_id = Configuration.get('grid.id')
            osis_key = Configuration.get('osis.key')

        configuration['grid'] = {}
        configuration['grid']['grid.id'] = grid_id
        configuration['grid']['grid.useavahi'] = 1
        configuration['grid']['grid.node.roles'] = 'node'
        configuration['grid']['grid.node.machineid'] = unique_id

        configuration['elasticsearch'] = {}
        configuration['elasticsearch']['elasticsearch.cluster.name'] = es_cluster_name

        if first_node:
            configuration['grid_master'] = {}
            configuration['grid_master']['gridmaster.grid.id'] = grid_id
            configuration['grid_master']['gridmaster.useavahi'] = 1
            configuration['grid_master']['gridmaster.superadminpasswd'] = hashlib.sha256(''.join(choice(lowercase) for _ in range(25))).hexdigest()
        else:
            configuration['grid']['grid.master.ip'] = ''

        configuration['osis'] = {}
        configuration['osis']['osis.key'] = osis_key

        client.dir_ensure('/opt/jumpscale/cfg/hrd', True)
        if is_local:
            for filename in configuration:
                with open('/opt/jumpscale/cfg/hrd/{0}.hrd'.format(filename), 'w') as hrd:
                    hrd.write('\n'.join(['%s=%s' % i for i in configuration[filename].iteritems()]))
        else:
            for filename in configuration:
                with open('/opt/jumpscale/cfg/hrd/{0}_{1}.hr_'.format(filename, ip), 'w') as hrd:
                    hrd.write('\n'.join(['%s=%s' % i for i in configuration[filename].iteritems()]))
                # The file_upload method has the destination first, then the source. Yeah, makes sense... not.
                client.file_upload('/opt/jumpscale/cfg/hrd/{0}.hrd'.format(filename),
                                   '/opt/jumpscale/cfg/hrd/{0}_{1}.hr_'.format(filename, ip))

        # Make sure all software is up-to-date
        client.run('apt-get update')
        client.file_append('/etc/security/limits.conf', '\nroot soft core  unlimited\novs  soft core  unlimited\n')

        # Install base framework, JumpScale in this case
        install_branch = Manager._prepare_jscore(client, is_local)
        Manager._install_jscore(client, install_branch)

        # Install Open vStorage
        print 'Installing Open vStorage...'
        client.run('apt-get -y -q install python-dev')
        client.run('jpackage_install -n openvstorage')

    @staticmethod
    def init_node(ip, password, join_masters=False):
        """
        Initializes a node, making sure all required services are up and running.
        Optionally, the node can also join the masters, also participating in the arakoon and memcache
        clusters. Joining the masters will result in the services being restarted on all master nodes.
        Please note that initializing a node, joining a grid with < 3 nodes, will result in joining the
        master nodes regardless of the given parameter.
        """

        from configobj import ConfigObj
        from ovs.dal.hybrids.pmachine import PMachine
        from ovs.dal.lists.pmachinelist import PMachineList
        from ovs.dal.hybrids.vmachine import VMachine
        from ovs.dal.lists.vmachinelist import VMachineList
        from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagement
        from ovs.plugin.provider.configuration import Configuration

        if ip == '127.0.0.1':
            print 'Do not use 127.0.0.1 as ip address, use the public grid ip instead.'
            sys.exit(1)

        # Make sure to ALWAYS reload the client when switching targets, as Fabric seems to be singleton-ish
        is_local = Client.is_local(ip)
        client = Client.load(ip, password)  # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish
        unique_id = sorted(client.run("ip a | grep link/ether | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | sed 's/://g'").strip().split('\n'))[0].strip()

        arakoon_management = ArakoonManagement()
        nodes = Manager._get_cluster_nodes()  # All nodes, including the local and the new one
        arakoon_nodes = arakoon_management.getCluster('ovsdb').listNodes()
        client = Client.load(ip, password)
        new_node_hostname = client.run('hostname')
        if is_local and Configuration.get('grid.node.id') != '1':
            # If the script is executed local and there are multiple nodes, the script is executed on node 2+.
            # This is not allowed since a the existing configuration is requred to extend. On extra nodes there is no
            # current configuration to base all work on
            print 'This script can only be executed local on the first node. Subsequent nodes need to be installed from one of the master nodes.'
            sys.exit(1)
        if len(arakoon_nodes) < 3:
            print 'Insufficient master nodes'
            join_masters = True
        clusters = arakoon_management.listClusters()

        all_services = ['arakoon_ovsdb', 'arakoon_voldrv', 'memcached', 'rabbitmq', 'ovs_consumer_volumerouter',
                        'ovs_flower', 'ovs_scheduled_tasks', 'webapp_api', 'nginx', 'ovs_workers']
        arakoon_configfiles = ['/opt/OpenvStorage/config/arakoon/{0}/{0}_client.cfg'.format(cluster) for cluster in clusters]
        generic_configfiles = {'/opt/OpenvStorage/config/memcacheclient.cfg': 11211,
                               '/opt/OpenvStorage/config/rabbitmqclient.cfg': 5672}

        if join_masters:
            print 'Joining master nodes, services going down.'

            # Stop services (on all nodes)
            for node in nodes:
                node_client = Client.load(node, password)
                for service in all_services:
                    node_client.run('jsprocess disable -n {0}'.format(service))
                    node_client.run('jsprocess stop -n {0}'.format(service))

            # Fetch some information
            client = Client.load(ip, password)  # Make sure to ALWAYS reload the client, as Fabric seems to be singleton-ish
            remote_ips = client.run("ip a | grep 'inet ' | sed 's/\s\s*/ /g' | cut -d ' ' -f 3 | cut -d '/' -f 1").strip().split('\n')
            remote_ip = [ipa.strip() for ipa in nodes if ipa in remote_ips][0]

            # Configure arakoon
            for cluster in clusters:
                # The Arakoon extension is not used since the config file needs to be parsed/loaded anyway to be
                # able to update it
                cfg = ConfigObj('/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster))
                global_section = cfg.get('global')
                cluster_nodes = global_section['cluster'] if type(global_section['cluster']) == list else [global_section['cluster']]
                if unique_id not in cluster_nodes:
                    client = Client.load(ip, password)
                    remote_config = client.file_read('/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster))
                    with open('/tmp/arakoon_{0}_cfg'.format(unique_id), 'w') as the_file:
                        the_file.write(remote_config)
                    remote_cfg = ConfigObj('/tmp/arakoon_{0}_cfg'.format(unique_id))
                    cluster_nodes.append(unique_id)
                    global_section['cluster'] = cluster_nodes
                    cfg.update({'global': global_section})
                    cfg.update({unique_id: remote_cfg.get(unique_id)})
                    cfg.write()
                    for node in nodes:
                        node_client = Client.load(node, password)
                        node_client.file_upload('/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster),
                                                '/opt/OpenvStorage/config/arakoon/{0}/{0}.cfg'.format(cluster))
                client = Client.load(ip, password)
                arakoon_create_directories = """
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagement
arakoon_management = ArakoonManagement()
arakoon_cluster = arakoon_management.getCluster('%(cluster)s')
arakoon_cluster.createDirs(arakoon_cluster.listLocalNodes()[0])
""" % {'cluster': cluster}
                client.run('/opt/OpenvStorage/bin/python -c """{}"""'.format(arakoon_create_directories))

            # Update all nodes hosts file with new node and new node hosts file with all others
            for node in nodes:
                client_node = Client.load(node, password)
                update_hosts_file = """
from ovs.plugin.provider.net import Net
Net.updateHostsFile(hostsfile='/etc/hosts', ip='%(ip)s', hostname='%(host)s')
""" % {'ip': ip,
     'host': new_node_hostname}
                client_node.run('python -c """{}"""'.format(update_hosts_file))
                if node != ip:
                    client_node.run('jsprocess enable -n rabbitmq')
                    client_node.run('jsprocess start -n rabbitmq')
                else:
                    for subnode in nodes:
                        client_node = Client.load(subnode, password)
                        node_hostname = client_node.run('hostname')
                        update_hosts_file = """
from ovs.plugin.provider.net import Net
Net.updateHostsFile(hostsfile='/etc/hosts', ip='%(ip)s', hostname='%(host)s')
""" % {'ip': subnode,
     'host': node_hostname}
                        client = Client.load(ip, password)
                        client.run('python -c """{}"""'.format(update_hosts_file))

            # Update arakoon cluster configuration in voldrv configuration files
            for node in nodes:
                client_node = Client.load(node, password)
                update_voldrv = """
import os
from ovs.plugin.provider.configuration import Configuration
from ovs.extensions.storageserver.volumestoragerouter import VolumeStorageRouterConfiguration
from ovs.extensions.db.arakoon.ArakoonManagement import ArakoonManagement
arakoon_management = ArakoonManagement()
voldrv_arakoon_cluster_id = 'voldrv'
voldrv_arakoon_cluster = arakoon_management.getCluster(voldrv_arakoon_cluster_id)
voldrv_arakoon_client_config = voldrv_arakoon_cluster.getClientConfig()
configuration_dir = Configuration.get('ovs.core.cfgdir')
if not os.path.exists('{0}/voldrv_vpools'.format(configuration_dir)):
    os.makedirs('{0}/voldrv_vpools'.format(configuration_dir))
for json_file in os.listdir('{0}/voldrv_vpools'.format(configuration_dir)):
    if json_file.endswith('.json'):
        vsr_config = VolumeStorageRouterConfiguration(json_file.replace('.json', ''))
        vsr_config.configure_arakoon_cluster(voldrv_arakoon_cluster_id, voldrv_arakoon_client_config)
"""
                client_node.run('python -c """{}"""'.format(update_voldrv))

            # Join rabbitMQ clusters
            client = Client.load(ip, password)
            client.run('rabbitmq-server -detached; sleep 5; rabbitmqctl stop_app; sleep 5; rabbitmqctl reset; sleep 5; rabbitmqctl stop; sleep 5;')
            if not is_local:
                # Copy rabbitmq cookie
                rabbitmq_cookie_file = '/var/lib/rabbitmq/.erlang.cookie'
                client.dir_ensure(os.path.dirname(rabbitmq_cookie_file), True)
                client.file_upload(rabbitmq_cookie_file, rabbitmq_cookie_file)
                client.file_attribs(rabbitmq_cookie_file, mode=400)
                client.run('rabbitmq-server -detached; sleep 5; rabbitmqctl stop_app; sleep 5;')
                # If not local, a cluster needs to be joined.
                master_client = Client.load(Configuration.get('grid.master.ip'), password)
                master_hostname = master_client.run('hostname')
                client = Client.load(ip, password)
                client.run('rabbitmqctl join_cluster rabbit@{}; sleep 5;'.format(master_hostname))
                client.run('rabbitmqctl stop; sleep 5;')

            # Update local client configurations
            for config in arakoon_configfiles:
                cfg = ConfigObj(config)
                global_section = cfg.get('global')
                cluster_nodes = global_section['cluster'] if type(global_section['cluster']) == list else [global_section['cluster']]
                if unique_id not in cluster_nodes:
                    cluster_nodes.append(unique_id)
{'fraction_non_alphanumeric': 0.11383915885194658, 'fraction_numerical': 0.007303211139528275, 'mean_word_length': 2.5012436573475276, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3078391', 'n_tokens_mistral': 10023, 'n_tokens_neox': 9518, 'n_words': 2411}