<commit_before># This file is part of VoltDB.
# Copyright (C) 2008-2016 VoltDB Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.

import os
from collections import defaultdict
import json
import traceback
from xml.etree.ElementTree import Element, SubElement, tostring, XML
import sys
from flask import jsonify
import HTTPListener
import DeploymentConfig
from Validation import ServerInputs, DatabaseInputs, JsonInputs, UserInputs, ConfigValidation
from logging.handlers import RotatingFileHandler
import logging
import ast
import itertools
from Validation import Validation


def convert_xml_to_json(config_path):
    """
    Method to get the json content from xml file
    :param config_path (string): path of xml file
    """
    with open(config_path) as config_file:
        xml = config_file.read()
    config_content = XML(xml)
    xml_final = etree_to_dict(config_content)

    D2 = {}
    for (k, v) in zip(xml_final.keys(), xml_final.values()):
        D2[k] = v

    # To get the list of servers in case of old members[] (for backward compatible)
    if 'members' in D2[k] and 'member' in D2[k]['members'] and D2[k]['members']['member']:
        if type(D2[k]['members']['member']) is dict:
            member_json = get_field_from_xml(D2[k]['members']['member'], 'dict')
            HTTPListener.Global.SERVERS[member_json[0]['id']] = member_json[0]
        else:
            member_json = get_field_from_xml(D2[k]['members']['member'], 'list')
            for member in member_json:
                HTTPListener.Global.SERVERS[member['id']] = member

    if type(D2[k]['databases']['database']) is dict:
        db_json = get_field_from_xml(D2[k]['databases']['database'],
                                     'dict', 'database')
        HTTPListener.Global.DATABASES[db_json[0]['id']] = db_json[0]
    else:
        db_json = get_field_from_xml(D2[k]['databases']['database'],
                                     'list', 'database')
        for database in db_json:
            HTTPListener.Global.DATABASES[database['id']] = database

    if type(D2[k]['deployments']['deployment']) is dict:
        deployment_json = get_deployment_from_xml(D2[k]['deployments']
                                                  ['deployment'], 'dict')
        HTTPListener.Global.DEPLOYMENT[deployment_json[0]['databaseid']] = deployment_json[0]
    else:
        deployment_json = get_deployment_from_xml(D2[k]['deployments']
                                                  ['deployment'], 'list')
        for deployment in deployment_json:
            HTTPListener.Global.DEPLOYMENT[deployment['databaseid']] = deployment


    if D2[k]['deployments'] and 'deployment' in D2[k]['deployments']:
        if type(D2[k]['deployments']['deployment']) is dict:
            set_deployment_users(D2[k]['deployments']['deployment'])
        else:
            for deployment in D2[k]['deployments']['deployment']:
                set_deployment_users(deployment)


def set_deployment_users(deployment):
    if 'users' in deployment and deployment['users'] is not None\
            and 'user' in deployment['users']:
        if type(deployment) is dict:
            user_json = get_users_from_xml(deployment,
                                           'dict')
            for user in user_json:
                HTTPListener.Global.DEPLOYMENT_USERS[int(user['userid'])] = user
        else:
            user_json = get_users_from_xml(deployment,
                                           'list')
            for deployment_user in user_json:
                HTTPListener.Global.DEPLOYMENT_USERS[int(deployment_user['userid'])] = deployment_user


def validate_and_convert_xml_to_json(config_path):
    """
    Method to get the json content from xml file
    :param config_path (string): path of xml file
    """
    log_file = os.path.join(HTTPListener.Global.DATA_PATH, 'voltdeploy.log')

    handler = RotatingFileHandler(log_file)
    handler.setFormatter(logging.Formatter(
        "%(asctime)s|%(levelname)s|%(message)s"))
    log = logging.getLogger('werkzeug')
    log.setLevel(logging.NOTSET)
    log.addHandler(handler)

    try:
        with open(config_path) as config_file:
            xml = config_file.read()
        config_content = XML(xml)
        xml_final = etree_to_dict(config_content)

        D2 = {}
        for (k, v) in zip(xml_final.keys(), xml_final.values()):
            D2[k] = v

        populate_database(D2[k]['databases']['database'], log)

        if 'members' in D2[k] and 'member' in D2[k]['members'] and D2[k]['members']['member']:
            if type(D2[k]['members']['member']) is dict:
                populate_server(D2[k]['members']['member'], D2[k]['databases']['database'], log)

        populate_deployment(D2[k]['deployments']['deployment'], log)

    except Exception as err:
        log.error("Error while reloading configuration: %s", "Invalid file content.")


def populate_database(databases, log):
    success = True
    if type(databases) is dict:
        db_json = get_database_from_xml(databases,
                                     'dict', log, 'database')
        req = HTTPListener.DictClass()
        req.json = {}
        req.json = db_json[0]
        inputs = DatabaseInputs(req)
        if not inputs.validate():
            success = False
            sys.stdout.write(str(inputs.errors))
            log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DATABASES = {db_json[0]['id']: db_json[0]}
    else:
        db_json = get_database_from_xml(databases,
                                     'list', log, 'database')
        success = True
        result = check_duplicate_database(db_json)
        if result != "":
            success = False
            log.error("Error while reloading configuration: %s", result)
        else:
            for database in db_json:
                req = HTTPListener.DictClass()
                req.json = {}
                req.json = database
                inputs = DatabaseInputs(req)
                if not inputs.validate():
                    success = False
                    sys.stdout.write(str(inputs.errors))
                    log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DATABASES = {}
            for database in db_json:
                HTTPListener.Global.DATABASES[database['id']] = database


def populate_server(servers, databases, log):
    success = True
    if type(servers) is dict:
        member_json = get_field_from_xml(servers, 'dict')
        req = HTTPListener.DictClass()
        req.json = {}
        req.json = member_json[0]
        inputs = ServerInputs(req)
        if not inputs.validate():
            success = False
            sys.stdout.write(str(inputs.errors))
            log.error("Error while reloading configuration: %s", str(inputs.errors))
        else:
            result = validate_server_ports_dict(member_json[0], databases, True)
            if result is not None:
                success = False
                log.error("Error while reloading configuration: %s", result)

        if success is True:
            HTTPListener.Global.SERVERS = {member_json[0]['id']: member_json[0]}
    else:
        member_json = get_field_from_xml(servers, 'list')
        for member in member_json:
            req = HTTPListener.DictClass()
            req.json = {}
            req.json = member
            inputs = ServerInputs(req)
            if not inputs.validate():
                success = False
                sys.stdout.write(str(inputs.errors))
                log.error("Error while reloading configuration: %s", str(inputs.errors))
            result = validate_server_ports_list(member_json, databases, False)
            if result is not None:
                success = False
                log.error("Error while reloading configuration: %s", result)

        if success is True:
            HTTPListener.Global.SERVERS = {}
            for member in member_json:
                HTTPListener.Global.SERVERS[member['id']] = member


def populate_deployment(deployments, log):
    success = True
    if type(deployments) is dict:
        deployment_json = get_deployment_from_xml(deployments, 'dict')
        req = HTTPListener.DictClass()
        req.json = {}
        req.json = deployment_json[0]
        inputs = JsonInputs(req)
        if not inputs.validate():
            success = False
            sys.stdout.write(str(inputs.errors))
            log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DEPLOYMENT = {deployment_json[0]['databaseid']: deployment_json[0]}
    else:
        deployment_json = get_deployment_from_xml(deployments, 'list')
        for deployment in deployment_json:
            req = HTTPListener.DictClass()
            req.json = {}
            req.json = deployment
            inputs = JsonInputs(req)
            if not inputs.validate():
                success = False
                sys.stdout.write(str(inputs.errors))
                log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DEPLOYMENT = {}
            for deployment in deployment_json:
                HTTPListener.Global.DEPLOYMENT[deployment['databaseid']] = deployment
    success = True
    if type(deployments) is list:
        users = []
        for deployment in deployments:
            if 'users' in deployment and deployment['users'] is not None \
                    and 'user' in deployment['users']:
                user_json = get_users_from_xml(deployment,
                                               'dict')
                if type(user_json) is dict:
                    req = HTTPListener.DictClass()
                    req.json = {}
                    user_json['plaintext'] = bool(user_json['plaintext'])
                    req.json = user_json
                    inputs = UserInputs(req)
                    if not inputs.validate():
                        success = False
                        sys.stdout.write(str(inputs.errors))
                        log.error("Error while reloading configuration: %s", str(inputs.errors))

                    if success is True:
                        users.append(user_json)

                elif type(user_json) is list:
                    for user in user_json:
                        req = HTTPListener.DictClass()
                        req.json = {}
                        user['plaintext'] = bool(user['plaintext'])
                        req.json = user
                        inputs = UserInputs(req)
                        if not inputs.validate():
                            success = False
                            sys.stdout.write(str(inputs.errors))
                            log.error("Error while reloading configuration: %s", str(inputs.errors))

                    if len(user_json)> 1:
                        result = check_duplicate_user(user_json)
                        if result != "":
                            success = False
                            log.error("Error while reloading configuration: %s", result)

                    if success is True:
                        for user in user_json:
                            users.append(user)
        if len(users) > 0:
            HTTPListener.Global.DEPLOYMENT_USERS = {}
            for user in users:
                HTTPListener.Global.DEPLOYMENT_USERS[int(user['userid'])] = user
    else:
        user_json = get_users_from_xml(deployments,
                                       'dict')
        if type(user_json) is dict:
            req = HTTPListener.DictClass()
            req.json = {}
            user_json['plaintext'] = bool(user_json['plaintext'])
            req.json = user_json
            inputs = UserInputs(req)
            if not inputs.validate():
                success = False
                sys.stdout.write(str(inputs.errors))
                log.error("Error while reloading configuration: %s", str(inputs.errors))

            if success is True:
                HTTPListener.Global.DEPLOYMENT_USERS = {int(user_json['userid']): user_json}
        elif type(user_json) is list:
            for user in user_json:
                req = HTTPListener.DictClass()
                req.json = {}
                user['plaintext'] = bool(user['plaintext'])
                req.json = user
                inputs = UserInputs(req)
                if not inputs.validate():
                    success = False
                    sys.stdout.write(str(inputs.errors))
                    log.error("Error while reloading configuration: %s", str(inputs.errors))

            if len(user_json)> 1:
                result = check_duplicate_user(user_json)
                if result != "":
                    success = False
                    log.error("Error while reloading configuration: %s", result)

            if success is True:
                HTTPListener.Global.DEPLOYMENT_USERS = {}
                for user in user_json:
                    HTTPListener.Global.DEPLOYMENT_USERS[int(user['userid'])] = user


def validate_server_ports_dict(member, databases, isDict):
    arr = ["http-listener", "admin-listener", "internal-listener", "replication-listener", "zookeeper-listener",
           "client-listener"]

    specified_port_values = {
        "http-listener": HTTPListener.get_port(member['http-listener']),
        "admin-listener": HTTPListener.get_port(member['admin-listener']),
        "replication-listener": HTTPListener.get_port(member['replication-listener']),
        "client-listener": HTTPListener.get_port(member['client-listener']),
        "zookeeper-listener": HTTPListener.get_port(member['zookeeper-listener']),
        "internal-listener": HTTPListener.get_port(member['internal-listener'])
    }

    for option in arr:
        value = specified_port_values[option]
        for port_key in specified_port_values.keys():
            if option != port_key and value is not None and specified_port_values[port_key] == value:
                return "Duplicate port"


def validate_server_ports_list(members, databases, isDict):
    arr = ["http-listener", "admin-listener", "internal-listener", "replication-listener", "zookeeper-listener",
           "client-listener"]

    for i in range(len(members)):
        specified_port_values = {
            "http-listener": HTTPListener.get_port(members[i]['http-listener']),
            "admin-listener": HTTPListener.get_port(members[i]['admin-listener']),
            "replication-listener": HTTPListener.get_port(members[i]['replication-listener']),
            "client-listener": HTTPListener.get_port(members[i]['client-listener']),
            "zookeeper-listener": HTTPListener.get_port(members[i]['zookeeper-listener']),
            "internal-listener": HTTPListener.get_port(members[i]['internal-listener'])
        }

        for option in arr:
            value = specified_port_values[option]
            for port_key in specified_port_values.keys():
                if option != port_key and value is not None and specified_port_values[port_key] == value:
                    return "Duplicate port"

    if type(databases) is dict:
        for option in arr:
            result = check_port_valid(option, databases['members']['member'])
            if result is not None:
                return result

    elif type(databases) is list:
        for database in databases:
            for option in arr:
                result = check_port_valid(option, database['members']['member'])
                if result is not None:
                    return result


def check_port_valid(port_option, servers):
    result = None
    for i in range(len(servers)):
        for j in range(i + 1, len(servers)):
            if servers[i]['hostname'] == servers[j]['hostname']:
                result = compare(port_option, servers[i], servers[j])
    if result is not None:
        return result


def check_duplicate_database(databases):
    for i in range(len(databases)):
        for j in range(i + 1, len(databases)):
            result = compare_database(databases[i], databases[j])
    if result is not None:
        return result


def check_duplicate_user(users):
    for i in range(len(users)):
        for j in range(i + 1, len(users)):
            result = compare_user(users[i], users[j])
    if result is not None:
        return result


def get_servers_from_database_id(database_id):
    servers = []
    database = HTTPListener.Global.DATABASES.get(int(database_id))
    if database is None:
        return 'No database found for id: %u' % int(database_id)
    else:
        members = database['members']

    for server_id in members:
        server = HTTPListener.Global.SERVERS.get(server_id)
        servers.append(server)
    return servers


def compare(port_option, first, second):
    if port_option == "http-listener":
        default_port = "8080"
    if port_option == "admin-listener":
        default_port = "21211"
    if port_option == "zookeeper-listener":
        default_port = "7181"
    if port_option == "replication-listener":
        default_port = "5555"
    if port_option == "client-listener":
        default_port = "21212"
    if port_option == "internal-listener":
        default_port = "3021"

    first_port = HTTPListener.get_port(first[port_option])
    if first_port is None or first_port == "":
        first_port = default_port

    second_port = HTTPListener.get_port(second[port_option])
    if second_port is None or second_port == "":
        second_port = default_port

    if first_port == second_port:
        return "Port %s for the same host is already used by server %s for %s" % \
                              (first_port, first['hostname'], port_option)


def compare_database(first, second):
    if first['name'] == second['name']:
        return 'Duplicate database name: %s' % first['name']
    else:
        return ""


def compare_user(first, second):
    if first['name'] == second['name']:
        return 'Duplicate user name: %s' % first['name']
    else:
        return ""


def get_deployment_from_xml(deployment_xml, is_list):
    """
    Function to get deployment json from the xml content
    :param deployment_xml: raw deployment object
    :param is_list: flag to know if it is list or dict
    :return: deployment object in required format
    """
    deployments = []
    if is_list is 'list':
        for deployment in deployment_xml:
            deployments.append(get_deployment(deployment))
    else:
        deployments.append(get_deployment(deployment_xml))
    return deployments


def get_deployment(deployment, is_upload=False):
    """
    Gets the required deployment object
    :param deployment: raw deployment object
    :param is_upload: flag to know if it is list or dict
    :return: deployment object in required format
    """
    new_deployment = {}
    for field in deployment:
        if field == 'export' or field == 'import':
            result = set_export_import_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'admin-mode':
            result = set_admin_mode_filed(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'cluster':
            result = set_cluster_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'commandlog':
            result = set_command_log_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'heartbeat':
            result = set_heartbeat_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'httpd':
            result = set_httpd_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'partition-detection':
            result = set_partition_detection_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'security':
            result = set_security_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'snapshot':
            result = set_snapshot_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'systemsettings':
            result = set_system_setting_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'dr':
            result = set_dr_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'users':
            result = set_users_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        else:
            new_deployment[field] = convert_field_required_format(deployment, field)
    return new_deployment


def set_export_import_field(deployment, field, new_deployment):
    result = 'success'
    try:
        if deployment[field] is not None:
            new_deployment[field] = {}
            new_deployment[field]['configuration'] = {}
            if type(deployment[field]['configuration']) is list:
                new_deployment[field]['configuration'] = get_field_from_xml(
                    deployment[field]['configuration'], 'list', 'export')
            else:
                new_deployment[field]['configuration'] = get_field_from_xml(
                    deployment[field]['configuration'], 'dict', 'export')
        else:
            new_deployment[field] = None
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_admin_mode_filed(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['adminstartup'] = parse_bool_string(deployment[field]
                                                                  ['adminstartup'])
        new_deployment[field]['port'] = int(deployment[field]['port'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_cluster_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['hostcount'] = int(deployment[field]['hostcount'])
        new_deployment[field]['kfactor'] = int(deployment[field]['kfactor'])
        new_deployment[field]['sitesperhost'] = int(deployment[field]
                                                    ['sitesperhost'])
        new_deployment[field]['elastic'] = str(deployment[field]['elastic'])
        new_deployment[field]['schema'] = str(deployment[field]['schema'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_command_log_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['synchronous'] = parse_bool_string(deployment[field]
                                                                 ['synchronous'])
        new_deployment[field]['logsize'] = int(deployment[field]['logsize'])
        new_deployment[field]['frequency'] = {}
        new_deployment[field]['frequency']['transactions'] = int(
            deployment[field]['frequency']['transactions'])
        new_deployment[field]['frequency']['time'] = int(deployment[field]
                                                         ['frequency']
                                                         ['time'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_heartbeat_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['timeout'] = int(deployment[field]['timeout'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_partition_detection_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['snapshot'] = {}
        new_deployment[field]['snapshot']['prefix'] = deployment[field]['snapshot']['prefix']
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_httpd_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['port'] = int(deployment[field]['port'])
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['jsonapi'] = {}
        new_deployment[field]['jsonapi']['enabled'] = parse_bool_string(
            deployment[field]['jsonapi']['enabled'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_security_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['provider'] = str(deployment[field]['provider'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_snapshot_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['frequency'] = str(deployment[field]['frequency'])<commit_msg>VDM-156 Modified code to upload deployment file with snmp configuration.<commit_after># This file is part of VoltDB.
# Copyright (C) 2008-2016 VoltDB Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.

import os
from collections import defaultdict
import json
import traceback
from xml.etree.ElementTree import Element, SubElement, tostring, XML
import sys
from flask import jsonify
import HTTPListener
import DeploymentConfig
from Validation import ServerInputs, DatabaseInputs, JsonInputs, UserInputs, ConfigValidation
from logging.handlers import RotatingFileHandler
import logging
import ast
import itertools
from Validation import Validation


def convert_xml_to_json(config_path):
    """
    Method to get the json content from xml file
    :param config_path (string): path of xml file
    """
    with open(config_path) as config_file:
        xml = config_file.read()
    config_content = XML(xml)
    xml_final = etree_to_dict(config_content)

    D2 = {}
    for (k, v) in zip(xml_final.keys(), xml_final.values()):
        D2[k] = v

    # To get the list of servers in case of old members[] (for backward compatible)
    if 'members' in D2[k] and 'member' in D2[k]['members'] and D2[k]['members']['member']:
        if type(D2[k]['members']['member']) is dict:
            member_json = get_field_from_xml(D2[k]['members']['member'], 'dict')
            HTTPListener.Global.SERVERS[member_json[0]['id']] = member_json[0]
        else:
            member_json = get_field_from_xml(D2[k]['members']['member'], 'list')
            for member in member_json:
                HTTPListener.Global.SERVERS[member['id']] = member

    if type(D2[k]['databases']['database']) is dict:
        db_json = get_field_from_xml(D2[k]['databases']['database'],
                                     'dict', 'database')
        HTTPListener.Global.DATABASES[db_json[0]['id']] = db_json[0]
    else:
        db_json = get_field_from_xml(D2[k]['databases']['database'],
                                     'list', 'database')
        for database in db_json:
            HTTPListener.Global.DATABASES[database['id']] = database

    if type(D2[k]['deployments']['deployment']) is dict:
        deployment_json = get_deployment_from_xml(D2[k]['deployments']
                                                  ['deployment'], 'dict')
        HTTPListener.Global.DEPLOYMENT[deployment_json[0]['databaseid']] = deployment_json[0]
    else:
        deployment_json = get_deployment_from_xml(D2[k]['deployments']
                                                  ['deployment'], 'list')
        for deployment in deployment_json:
            HTTPListener.Global.DEPLOYMENT[deployment['databaseid']] = deployment


    if D2[k]['deployments'] and 'deployment' in D2[k]['deployments']:
        if type(D2[k]['deployments']['deployment']) is dict:
            set_deployment_users(D2[k]['deployments']['deployment'])
        else:
            for deployment in D2[k]['deployments']['deployment']:
                set_deployment_users(deployment)


def set_deployment_users(deployment):
    if 'users' in deployment and deployment['users'] is not None\
            and 'user' in deployment['users']:
        if type(deployment) is dict:
            user_json = get_users_from_xml(deployment,
                                           'dict')
            for user in user_json:
                HTTPListener.Global.DEPLOYMENT_USERS[int(user['userid'])] = user
        else:
            user_json = get_users_from_xml(deployment,
                                           'list')
            for deployment_user in user_json:
                HTTPListener.Global.DEPLOYMENT_USERS[int(deployment_user['userid'])] = deployment_user


def validate_and_convert_xml_to_json(config_path):
    """
    Method to get the json content from xml file
    :param config_path (string): path of xml file
    """
    log_file = os.path.join(HTTPListener.Global.DATA_PATH, 'voltdeploy.log')

    handler = RotatingFileHandler(log_file)
    handler.setFormatter(logging.Formatter(
        "%(asctime)s|%(levelname)s|%(message)s"))
    log = logging.getLogger('werkzeug')
    log.setLevel(logging.NOTSET)
    log.addHandler(handler)

    try:
        with open(config_path) as config_file:
            xml = config_file.read()
        config_content = XML(xml)
        xml_final = etree_to_dict(config_content)

        D2 = {}
        for (k, v) in zip(xml_final.keys(), xml_final.values()):
            D2[k] = v

        populate_database(D2[k]['databases']['database'], log)

        if 'members' in D2[k] and 'member' in D2[k]['members'] and D2[k]['members']['member']:
            if type(D2[k]['members']['member']) is dict:
                populate_server(D2[k]['members']['member'], D2[k]['databases']['database'], log)

        populate_deployment(D2[k]['deployments']['deployment'], log)

    except Exception as err:
        log.error("Error while reloading configuration: %s", "Invalid file content.")


def populate_database(databases, log):
    success = True
    if type(databases) is dict:
        db_json = get_database_from_xml(databases,
                                     'dict', log, 'database')
        req = HTTPListener.DictClass()
        req.json = {}
        req.json = db_json[0]
        inputs = DatabaseInputs(req)
        if not inputs.validate():
            success = False
            sys.stdout.write(str(inputs.errors))
            log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DATABASES = {db_json[0]['id']: db_json[0]}
    else:
        db_json = get_database_from_xml(databases,
                                     'list', log, 'database')
        success = True
        result = check_duplicate_database(db_json)
        if result != "":
            success = False
            log.error("Error while reloading configuration: %s", result)
        else:
            for database in db_json:
                req = HTTPListener.DictClass()
                req.json = {}
                req.json = database
                inputs = DatabaseInputs(req)
                if not inputs.validate():
                    success = False
                    sys.stdout.write(str(inputs.errors))
                    log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DATABASES = {}
            for database in db_json:
                HTTPListener.Global.DATABASES[database['id']] = database


def populate_server(servers, databases, log):
    success = True
    if type(servers) is dict:
        member_json = get_field_from_xml(servers, 'dict')
        req = HTTPListener.DictClass()
        req.json = {}
        req.json = member_json[0]
        inputs = ServerInputs(req)
        if not inputs.validate():
            success = False
            sys.stdout.write(str(inputs.errors))
            log.error("Error while reloading configuration: %s", str(inputs.errors))
        else:
            result = validate_server_ports_dict(member_json[0], databases, True)
            if result is not None:
                success = False
                log.error("Error while reloading configuration: %s", result)

        if success is True:
            HTTPListener.Global.SERVERS = {member_json[0]['id']: member_json[0]}
    else:
        member_json = get_field_from_xml(servers, 'list')
        for member in member_json:
            req = HTTPListener.DictClass()
            req.json = {}
            req.json = member
            inputs = ServerInputs(req)
            if not inputs.validate():
                success = False
                sys.stdout.write(str(inputs.errors))
                log.error("Error while reloading configuration: %s", str(inputs.errors))
            result = validate_server_ports_list(member_json, databases, False)
            if result is not None:
                success = False
                log.error("Error while reloading configuration: %s", result)

        if success is True:
            HTTPListener.Global.SERVERS = {}
            for member in member_json:
                HTTPListener.Global.SERVERS[member['id']] = member


def populate_deployment(deployments, log):
    success = True
    if type(deployments) is dict:
        deployment_json = get_deployment_from_xml(deployments, 'dict')
        req = HTTPListener.DictClass()
        req.json = {}
        req.json = deployment_json[0]
        inputs = JsonInputs(req)
        if not inputs.validate():
            success = False
            sys.stdout.write(str(inputs.errors))
            log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DEPLOYMENT = {deployment_json[0]['databaseid']: deployment_json[0]}
    else:
        deployment_json = get_deployment_from_xml(deployments, 'list')
        for deployment in deployment_json:
            req = HTTPListener.DictClass()
            req.json = {}
            req.json = deployment
            inputs = JsonInputs(req)
            if not inputs.validate():
                success = False
                sys.stdout.write(str(inputs.errors))
                log.error("Error while reloading configuration: %s", str(inputs.errors))

        if success is True:
            HTTPListener.Global.DEPLOYMENT = {}
            for deployment in deployment_json:
                HTTPListener.Global.DEPLOYMENT[deployment['databaseid']] = deployment
    success = True
    if type(deployments) is list:
        users = []
        for deployment in deployments:
            if 'users' in deployment and deployment['users'] is not None \
                    and 'user' in deployment['users']:
                user_json = get_users_from_xml(deployment,
                                               'dict')
                if type(user_json) is dict:
                    req = HTTPListener.DictClass()
                    req.json = {}
                    user_json['plaintext'] = bool(user_json['plaintext'])
                    req.json = user_json
                    inputs = UserInputs(req)
                    if not inputs.validate():
                        success = False
                        sys.stdout.write(str(inputs.errors))
                        log.error("Error while reloading configuration: %s", str(inputs.errors))

                    if success is True:
                        users.append(user_json)

                elif type(user_json) is list:
                    for user in user_json:
                        req = HTTPListener.DictClass()
                        req.json = {}
                        user['plaintext'] = bool(user['plaintext'])
                        req.json = user
                        inputs = UserInputs(req)
                        if not inputs.validate():
                            success = False
                            sys.stdout.write(str(inputs.errors))
                            log.error("Error while reloading configuration: %s", str(inputs.errors))

                    if len(user_json)> 1:
                        result = check_duplicate_user(user_json)
                        if result != "":
                            success = False
                            log.error("Error while reloading configuration: %s", result)

                    if success is True:
                        for user in user_json:
                            users.append(user)
        if len(users) > 0:
            HTTPListener.Global.DEPLOYMENT_USERS = {}
            for user in users:
                HTTPListener.Global.DEPLOYMENT_USERS[int(user['userid'])] = user
    else:
        user_json = get_users_from_xml(deployments,
                                       'dict')
        if type(user_json) is dict:
            req = HTTPListener.DictClass()
            req.json = {}
            user_json['plaintext'] = bool(user_json['plaintext'])
            req.json = user_json
            inputs = UserInputs(req)
            if not inputs.validate():
                success = False
                sys.stdout.write(str(inputs.errors))
                log.error("Error while reloading configuration: %s", str(inputs.errors))

            if success is True:
                HTTPListener.Global.DEPLOYMENT_USERS = {int(user_json['userid']): user_json}
        elif type(user_json) is list:
            for user in user_json:
                req = HTTPListener.DictClass()
                req.json = {}
                user['plaintext'] = bool(user['plaintext'])
                req.json = user
                inputs = UserInputs(req)
                if not inputs.validate():
                    success = False
                    sys.stdout.write(str(inputs.errors))
                    log.error("Error while reloading configuration: %s", str(inputs.errors))

            if len(user_json)> 1:
                result = check_duplicate_user(user_json)
                if result != "":
                    success = False
                    log.error("Error while reloading configuration: %s", result)

            if success is True:
                HTTPListener.Global.DEPLOYMENT_USERS = {}
                for user in user_json:
                    HTTPListener.Global.DEPLOYMENT_USERS[int(user['userid'])] = user


def validate_server_ports_dict(member, databases, isDict):
    arr = ["http-listener", "admin-listener", "internal-listener", "replication-listener", "zookeeper-listener",
           "client-listener"]

    specified_port_values = {
        "http-listener": HTTPListener.get_port(member['http-listener']),
        "admin-listener": HTTPListener.get_port(member['admin-listener']),
        "replication-listener": HTTPListener.get_port(member['replication-listener']),
        "client-listener": HTTPListener.get_port(member['client-listener']),
        "zookeeper-listener": HTTPListener.get_port(member['zookeeper-listener']),
        "internal-listener": HTTPListener.get_port(member['internal-listener'])
    }

    for option in arr:
        value = specified_port_values[option]
        for port_key in specified_port_values.keys():
            if option != port_key and value is not None and specified_port_values[port_key] == value:
                return "Duplicate port"


def validate_server_ports_list(members, databases, isDict):
    arr = ["http-listener", "admin-listener", "internal-listener", "replication-listener", "zookeeper-listener",
           "client-listener"]

    for i in range(len(members)):
        specified_port_values = {
            "http-listener": HTTPListener.get_port(members[i]['http-listener']),
            "admin-listener": HTTPListener.get_port(members[i]['admin-listener']),
            "replication-listener": HTTPListener.get_port(members[i]['replication-listener']),
            "client-listener": HTTPListener.get_port(members[i]['client-listener']),
            "zookeeper-listener": HTTPListener.get_port(members[i]['zookeeper-listener']),
            "internal-listener": HTTPListener.get_port(members[i]['internal-listener'])
        }

        for option in arr:
            value = specified_port_values[option]
            for port_key in specified_port_values.keys():
                if option != port_key and value is not None and specified_port_values[port_key] == value:
                    return "Duplicate port"

    if type(databases) is dict:
        for option in arr:
            result = check_port_valid(option, databases['members']['member'])
            if result is not None:
                return result

    elif type(databases) is list:
        for database in databases:
            for option in arr:
                result = check_port_valid(option, database['members']['member'])
                if result is not None:
                    return result


def check_port_valid(port_option, servers):
    result = None
    for i in range(len(servers)):
        for j in range(i + 1, len(servers)):
            if servers[i]['hostname'] == servers[j]['hostname']:
                result = compare(port_option, servers[i], servers[j])
    if result is not None:
        return result


def check_duplicate_database(databases):
    for i in range(len(databases)):
        for j in range(i + 1, len(databases)):
            result = compare_database(databases[i], databases[j])
    if result is not None:
        return result


def check_duplicate_user(users):
    for i in range(len(users)):
        for j in range(i + 1, len(users)):
            result = compare_user(users[i], users[j])
    if result is not None:
        return result


def get_servers_from_database_id(database_id):
    servers = []
    database = HTTPListener.Global.DATABASES.get(int(database_id))
    if database is None:
        return 'No database found for id: %u' % int(database_id)
    else:
        members = database['members']

    for server_id in members:
        server = HTTPListener.Global.SERVERS.get(server_id)
        servers.append(server)
    return servers


def compare(port_option, first, second):
    if port_option == "http-listener":
        default_port = "8080"
    if port_option == "admin-listener":
        default_port = "21211"
    if port_option == "zookeeper-listener":
        default_port = "7181"
    if port_option == "replication-listener":
        default_port = "5555"
    if port_option == "client-listener":
        default_port = "21212"
    if port_option == "internal-listener":
        default_port = "3021"

    first_port = HTTPListener.get_port(first[port_option])
    if first_port is None or first_port == "":
        first_port = default_port

    second_port = HTTPListener.get_port(second[port_option])
    if second_port is None or second_port == "":
        second_port = default_port

    if first_port == second_port:
        return "Port %s for the same host is already used by server %s for %s" % \
                              (first_port, first['hostname'], port_option)


def compare_database(first, second):
    if first['name'] == second['name']:
        return 'Duplicate database name: %s' % first['name']
    else:
        return ""


def compare_user(first, second):
    if first['name'] == second['name']:
        return 'Duplicate user name: %s' % first['name']
    else:
        return ""


def get_deployment_from_xml(deployment_xml, is_list):
    """
    Function to get deployment json from the xml content
    :param deployment_xml: raw deployment object
    :param is_list: flag to know if it is list or dict
    :return: deployment object in required format
    """
    deployments = []
    if is_list is 'list':
        for deployment in deployment_xml:
            deployments.append(get_deployment(deployment))
    else:
        deployments.append(get_deployment(deployment_xml))
    return deployments


def get_deployment(deployment, is_upload=False):
    """
    Gets the required deployment object
    :param deployment: raw deployment object
    :param is_upload: flag to know if it is list or dict
    :return: deployment object in required format
    """
    new_deployment = {}
    for field in deployment:
        if field == 'export' or field == 'import':
            result = set_export_import_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'admin-mode':
            result = set_admin_mode_filed(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'cluster':
            result = set_cluster_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'commandlog':
            result = set_command_log_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'heartbeat':
            result = set_heartbeat_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'httpd':
            result = set_httpd_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'partition-detection':
            result = set_partition_detection_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'security':
            result = set_security_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'snapshot':
            result = set_snapshot_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'systemsettings':
            result = set_system_setting_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'dr':
            result = set_dr_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'users':
            result = set_users_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        elif field == 'snmp':
            result = set_snmp_field(deployment, field, new_deployment)
            if is_upload and 'success' not in result:
                return handle_errors(field, result)
        else:
            new_deployment[field] = convert_field_required_format(deployment, field)
    return new_deployment


def set_export_import_field(deployment, field, new_deployment):
    result = 'success'
    try:
        if deployment[field] is not None:
            new_deployment[field] = {}
            new_deployment[field]['configuration'] = {}
            if type(deployment[field]['configuration']) is list:
                new_deployment[field]['configuration'] = get_field_from_xml(
                    deployment[field]['configuration'], 'list', 'export')
            else:
                new_deployment[field]['configuration'] = get_field_from_xml(
                    deployment[field]['configuration'], 'dict', 'export')
        else:
            new_deployment[field] = None
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_admin_mode_filed(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['adminstartup'] = parse_bool_string(deployment[field]
                                                                  ['adminstartup'])
        new_deployment[field]['port'] = int(deployment[field]['port'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_cluster_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['hostcount'] = int(deployment[field]['hostcount'])
        new_deployment[field]['kfactor'] = int(deployment[field]['kfactor'])
        new_deployment[field]['sitesperhost'] = int(deployment[field]
                                                    ['sitesperhost'])
        new_deployment[field]['elastic'] = str(deployment[field]['elastic'])
        new_deployment[field]['schema'] = str(deployment[field]['schema'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_command_log_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['synchronous'] = parse_bool_string(deployment[field]
                                                                 ['synchronous'])
        new_deployment[field]['logsize'] = int(deployment[field]['logsize'])
        new_deployment[field]['frequency'] = {}
        new_deployment[field]['frequency']['transactions'] = int(
            deployment[field]['frequency']['transactions'])
        new_deployment[field]['frequency']['time'] = int(deployment[field]
                                                         ['frequency']
                                                         ['time'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_heartbeat_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['timeout'] = int(deployment[field]['timeout'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_partition_detection_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['snapshot'] = {}
        new_deployment[field]['snapshot']['prefix'] = deployment[field]['snapshot']['prefix']
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_httpd_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['port'] = int(deployment[field]['port'])
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['jsonapi'] = {}
        new_deployment[field]['jsonapi']['enabled'] = parse_bool_string(
            deployment[field]['jsonapi']['enabled'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_security_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['provider'] = str(deployment[field]['provider'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_snmp_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]['enabled'])
        new_deployment[field]['target'] = str(deployment[field]['target'])
        new_deployment[field]['community'] = str(deployment[field]['community'])
        new_deployment[field]['username'] = str(deployment[field]['username'])
        new_deployment[field]['authprotocol'] = str(deployment[field]['authprotocol'])
        new_deployment[field]['authkey'] = str(deployment[field]['authkey'])
        new_deployment[field]['privacyprotocol'] = str(deployment[field]['privacyprotocol'])
        new_deployment[field]['privacykey'] = str(deployment[field]['privacykey'])
    except Exception, err:
        result = str(err)
        print_errors(field, result)
    finally:
        return result


def set_snapshot_field(deployment, field, new_deployment):
    result = 'success'
    try:
        new_deployment[field] = {}
        new_deployment[field]['enabled'] = parse_bool_string(deployment[field]
                                                             ['enabled'])
        new_deployment[field]['frequency'] = str(deployment[field]['frequency'])
{'fraction_non_alphanumeric': 0.10546697038724373, 'fraction_numerical': 0.0030664096723322237, 'mean_word_length': 2.2208928269089676, 'pattern_counts': {'":': 46, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 82}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3417621', 'n_tokens_mistral': 15395, 'n_tokens_neox': 14992, 'n_words': 3641}