<commit_before>import getpass
import configparser
import logging
import os
import sys
import time
from os.path import join, expanduser
import traceback
import threading
import urllib
import urllib.request
import urllib.parse

import maccli.service.auth
import maccli.service.instance
import maccli.service.provider
import maccli.service.macfile
import maccli.service.resource
import maccli.service.infrastructure
import maccli.facade.macfile
import maccli.service.configuration
import maccli.helper.macfile
import maccli.helper.cmd
import maccli.dao.inheritance
from maccli.view.view_generic import show_error, show
import maccli.view.view_location
import maccli.view.view_instance
import maccli.view.view_cookbook
import maccli.view.view_generic
import maccli.view.view_resource
import maccli.view.view_infrastructure
import maccli.view.view_hardware
from maccli.view.view_generic import GREEN, RED
from maccli.config import AUTH_SECTION, USER_OPTION, APIKEY_OPTION, MAC_FILE, EXCEPTION_EXIT_CODE, CONFIGURATION_FAILED, \
    CREATION_FAILED
from maccli.helper.exception import MacParseEnvException, MacErrorCreatingTier, MacParseParamException, \
    MacResourceException, MacJsonException


def help():
    maccli.view.view_generic.general_help()


def login():
    try:
        maccli.view.view_generic.show("")
        maccli.view.view_generic.show(
            "If you are using Manageacloud SaaS and you don't have credentials, please register at https://manageacloud.com")
        maccli.view.view_generic.show("")
        maccli.view.view_generic.show("If you are using Manageacloud Community please set the following variable:")
        maccli.view.view_generic.show("    export MAC=http://my_community_server/api/v1/")
        maccli.view.view_generic.show("")
        maccli.view.view_generic.show("More information at http://manageacloud/docs/getting-started/install")
        maccli.view.view_generic.show("")
        username = input("Username or email: ")
        password = <PASSWORD>()

        user, api_key = maccli.service.auth.authenticate(username, password)
        if api_key is not None:
            config = configparser.ConfigParser()
            config.add_section(AUTH_SECTION)
            config.set(AUTH_SECTION, USER_OPTION, user)
            config.set(AUTH_SECTION, APIKEY_OPTION, api_key)
            with open(join(expanduser('~'), MAC_FILE), 'w') as cfgfile:
                config.write(cfgfile)
            print("Login succeeded!")

    except KeyboardInterrupt as e:
        show_error("")
        show_error("Authentication cancelled.")

    except Exception as e:
        show_error(e)
        sys.exit(EXCEPTION_EXIT_CODE)


def no_credentials():
    show("You need to login into Manageacloud.com")
    show()
    show("    mac login")
    show()
    show("If you do not have an account, register at https://manageacloud.com/register")


def instance_list():
    # try:
    json = maccli.service.instance.list_instances()
    maccli.view.view_instance.show_instances(json)

    # except Exception as e:
    #     show_error(e)
    #     sys.exit(EXCEPTION_EXIT_CODE)


def instance_ssh(raw_ids, command):
    # TODO move to parameter
    active_job_limit = 50

    if raw_ids == ["all"]:  # run in all instances
        ids = maccli.service.instance.list_instances()
    else:
        ids = maccli.service.instance.list_instances(name_or_ids=raw_ids)

    try:
        # array for job list
        jobs = {}

        maccli.logger.debug("Starting threads for command %s" % command)

        started_count = 0
        completed_count = 0
        total_count = len(ids)
        for id in ids:
            server_name = id["servername"]
            raw_id = id["id"]
            if command is None:
                maccli.service.instance.ssh_interactive_instance(id['id'])
            else:
                # define and add job
                t = threading.Thread(target=_run_cmd_simple, args=(server_name, raw_id, command))

                maccli.logger.debug("Starting job for %s. Active Jobs: %s." % (server_name, len(jobs)))
                if not t.is_alive():
                    started_count += 1
                    t.start()

                jobs[raw_id] = t

                # check if we are above the limit
                need_to_sleep = True
                while len(jobs) >= active_job_limit:
                    maccli.logger.debug("Job limit reached. Active Jobs: %s." % len(jobs))

                    keys = list(jobs.keys())
                    for key in keys:
                        if not jobs[key].is_alive():
                            completed_count += 1
                            show("Status: %s/%s/%s" % (completed_count, started_count, total_count))
                            jobs[key].join()
                            del jobs[key]
                            need_to_sleep = False
                        maccli.logger.debug("Job finished. Active Jobs: %s." % len(jobs))

                    if need_to_sleep:
                        time.sleep(1)

        # waiting for the existing thread to finish.
        # if a thread is too slow, we will report it
        slow_threads = {}
        while len(jobs) > 0:

            keys = list(jobs.keys())
            for key in keys:

                if not jobs[key].is_alive():
                    completed_count += 1
                    show("Status: %s/%s/%s" % (completed_count, started_count, total_count))
                    jobs[key].join()
                    del jobs[key]
                    if key in slow_threads:
                        del slow_threads[key]
                else:
                    if key in slow_threads:
                        if slow_threads[key] > 10:
                            show("Server %s is running the query slow: %s secs" % (key, slow_threads[key]))
                        slow_threads[key] += 1
                    else:
                        slow_threads[key] = 1

            time.sleep(1)

    except KeyboardInterrupt:
        show_error("Aborting")
    except Exception as e:
        tb = traceback.format_exc()
        show_error(e)
        show_error(tb)
        sys.exit(EXCEPTION_EXIT_CODE)


def _run_cmd_simple(server_name, raw_id, command):
    rc, stdout, stderr = maccli.service.instance.ssh_command_instance(raw_id, command)
    maccli.view.view_generic.showc("[%s]" % server_name, GREEN)
    if stdout:
        maccli.view.view_generic.show(stdout)
    if stderr:
        maccli.view.view_generic.showc(raw_id, RED)
        maccli.view.view_generic.showc(stderr, RED)
    #maccli.view.view_generic.show()


def instance_create(cookbook_tag, bootstrap_raw, deployment, location, servername, provider, release_raw,
                    release_version_raw,
                    branch, hardware, lifespan, environments, hd, port, net):
    # allow format release:release_version, ie ubuntu:trusty
    if ":" in release_raw:
        release, release_version = release_raw.split(":", 1)<commit_msg>When SSH multiple servers, ensure one line per server in the response.<commit_after>import getpass
import configparser
import logging
import os
import sys
import time
from os.path import join, expanduser
import traceback
import threading
import urllib
import urllib.request
import urllib.parse

# TODO implement progress bar
#from progressbar import ReverseBar, Percentage, ETA, RotatingMarker, Timer, ProgressBar

import maccli.service.auth
import maccli.service.instance
import maccli.service.provider
import maccli.service.macfile
import maccli.service.resource
import maccli.service.infrastructure
import maccli.facade.macfile
import maccli.service.configuration
import maccli.helper.macfile
import maccli.helper.cmd
import maccli.dao.inheritance
from maccli.view.view_generic import show_error, show
import maccli.view.view_location
import maccli.view.view_instance
import maccli.view.view_cookbook
import maccli.view.view_generic
import maccli.view.view_resource
import maccli.view.view_infrastructure
import maccli.view.view_hardware
from maccli.view.view_generic import GREEN, RED
from maccli.config import AUTH_SECTION, USER_OPTION, APIKEY_OPTION, MAC_FILE, EXCEPTION_EXIT_CODE, CONFIGURATION_FAILED, \
    CREATION_FAILED
from maccli.helper.exception import MacParseEnvException, MacErrorCreatingTier, MacParseParamException, \
    MacResourceException, MacJsonException


def help():
    maccli.view.view_generic.general_help()


def login():
    try:
        maccli.view.view_generic.show("")
        maccli.view.view_generic.show(
            "If you are using Manageacloud SaaS and you don't have credentials, please register at https://manageacloud.com")
        maccli.view.view_generic.show("")
        maccli.view.view_generic.show("If you are using Manageacloud Community please set the following variable:")
        maccli.view.view_generic.show("    export MAC=http://my_community_server/api/v1/")
        maccli.view.view_generic.show("")
        maccli.view.view_generic.show("More information at http://manageacloud/docs/getting-started/install")
        maccli.view.view_generic.show("")
        username = input("Username or email: ")
        password = <PASSWORD>()

        user, api_key = maccli.service.auth.authenticate(username, password)
        if api_key is not None:
            config = configparser.ConfigParser()
            config.add_section(AUTH_SECTION)
            config.set(AUTH_SECTION, USER_OPTION, user)
            config.set(AUTH_SECTION, APIKEY_OPTION, api_key)
            with open(join(expanduser('~'), MAC_FILE), 'w') as cfgfile:
                config.write(cfgfile)
            print("Login succeeded!")

    except KeyboardInterrupt as e:
        show_error("")
        show_error("Authentication cancelled.")

    except Exception as e:
        show_error(e)
        sys.exit(EXCEPTION_EXIT_CODE)


def no_credentials():
    show("You need to login into Manageacloud.com")
    show()
    show("    mac login")
    show()
    show("If you do not have an account, register at https://manageacloud.com/register")


def instance_list():
    # try:
    json = maccli.service.instance.list_instances()
    maccli.view.view_instance.show_instances(json)

    # except Exception as e:
    #     show_error(e)
    #     sys.exit(EXCEPTION_EXIT_CODE)


def instance_ssh(raw_ids, command):
    # TODO move to parameter
    active_job_limit = 50

    if raw_ids == ["all"]:  # run in all instances
        ids = maccli.service.instance.list_instances()
    else:
        ids = maccli.service.instance.list_instances(name_or_ids=raw_ids)

    try:
        # array for job list
        jobs = {}

        maccli.logger.debug("Starting threads for command %s" % command)

        started_count = 0
        completed_count = 0
        total_count = len(ids)

        # bar = ProgressBar(maxval=total_count,widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()

        for id in ids:
            server_name = id["servername"]
            raw_id = id["id"]
            if command is None:
                maccli.service.instance.ssh_interactive_instance(id['id'])
            else:
                # define and add job
                t = threading.Thread(target=_run_cmd_simple, args=(server_name, raw_id, command))

                maccli.logger.debug("Starting job for %s. Active Jobs: %s." % (server_name, len(jobs)))
                if not t.is_alive():
                    started_count += 1
                    t.start()

                jobs[raw_id] = t

                # check if we are above the limit
                need_to_sleep = True
                while len(jobs) >= active_job_limit:
                    maccli.logger.debug("Job limit reached. Active Jobs: %s." % len(jobs))

                    keys = list(jobs.keys())
                    for key in keys:
                        if not jobs[key].is_alive():
                            completed_count += 1
                            # bar.update(completed_count)
                            show("Status: %s/%s/%s" % (completed_count, started_count, total_count))
                            jobs[key].join()
                            del jobs[key]
                            need_to_sleep = False
                        maccli.logger.debug("Job finished. Active Jobs: %s." % len(jobs))

                    if need_to_sleep:
                        time.sleep(1)

        # waiting for the existing thread to finish.
        # if a thread is too slow, we will report it
        slow_threads = {}
        while len(jobs) > 0:

            keys = list(jobs.keys())
            for key in keys:

                if not jobs[key].is_alive():
                    completed_count += 1
                    # bar.update(completed_count)
                    show("Status: %s/%s/%s" % (completed_count, started_count, total_count))
                    jobs[key].join()
                    del jobs[key]
                    if key in slow_threads:
                        del slow_threads[key]
                else:
                    if key in slow_threads:
                        if slow_threads[key] > 10:
                            show("Server %s is running the query slow: %s secs" % (key, slow_threads[key]))
                        slow_threads[key] += 1
                    else:
                        slow_threads[key] = 1

            time.sleep(1)

    except KeyboardInterrupt:
        show_error("Aborting")
    except Exception as e:
        tb = traceback.format_exc()
        show_error(e)
        show_error(tb)
        sys.exit(EXCEPTION_EXIT_CODE)


def _run_cmd_simple(server_name, raw_id, command):
    rc, stdout, stderr = maccli.service.instance.ssh_command_instance(raw_id, command)
    maccli.view.view_generic.showc("[%s]" % server_name, GREEN)
    if stdout:
        maccli.view.view_generic.show(stdout)
    elif stderr:
        maccli.view.view_generic.showc(raw_id, RED)
        maccli.view.view_generic.showc(stderr, RED)
    else:
        maccli.view.view_generic.show("\n")
    #maccli.view.view_generic.show()


def instance_create(cookbook_tag, bootstrap_raw, deployment, location, servername, provider, release_raw,
                    release_version_raw,
                    branch, hardware, lifespan, environments, hd, port, net):
    # allow format release:release_version, ie ubuntu:trusty
    if ":" in release_raw:
        release, release_version = release_raw.split(":", 1)
{'fraction_non_alphanumeric': 0.09297177642501384, 'fraction_numerical': 0.002213613724405091, 'mean_word_length': 2.2306145251396647, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1510719', 'n_tokens_mistral': 4266, 'n_tokens_neox': 4051, 'n_words': 987}