<commit_before>import logging
import errno
import stat
import socket
from plumbum.machines.base import PopenAddons
from plumbum.machines.remote import BaseRemoteMachine
from plumbum.machines.session import ShellSession
from plumbum.lib import _setdoc, six
from plumbum.path.local import LocalPath
from plumbum.path.remote import RemotePath, StatRes
from plumbum.commands.processes import iter_lines
try:
    # Sigh... we need to gracefully-import paramiko for Sphinx builds, etc
    import paramiko
except ImportError:
    class paramiko(object):
        def __nonzero__(self):
            return False
        __bool__ = __nonzero__
        def __getattr__(self, name):
            raise ImportError("No module named paramiko")
    paramiko = paramiko()


logger = logging.getLogger("plumbum.paramiko")

class ParamikoPopen(PopenAddons):
    def __init__(self, argv, stdin, stdout, stderr, encoding, stdin_file = None,
            stdout_file = None, stderr_file = None):
        self.argv = argv
        self.channel = stdout.channel
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.custom_encoding = encoding
        self.returncode = None
        self.pid = None
        self.stdin_file = stdin_file
        self.stdout_file = stdout_file
        self.stderr_file = stderr_file
    def poll(self):
        if self.returncode is None:
            if self.channel.exit_status_ready():
                return self.wait()
        return self.returncode
    def wait(self):
        if self.returncode is None:
            self.channel.recv_exit_status()
            self.returncode = self.channel.exit_status
            self.close()
        return self.returncode
    def close(self):
        self.channel.shutdown_read()
        self.channel.shutdown_write()
        self.channel.close()
    def kill(self):
        # possible way to obtain pid:
        # "(cmd ; echo $?) & echo ?!"
        # and then client.exec_command("kill -9 %s" % (pid,))
        raise EnvironmentError("Cannot kill remote processes, we don't have their PIDs")
    terminate = kill
    def send_signal(self, sig):
        raise NotImplementedError()
    def communicate(self):
        stdout = []
        stderr = []
        infile = self.stdin_file
        sources = [("1", stdout, self.stdout, self.stdout_file), ("2", stderr, self.stderr, self.stderr_file)]
        i = 0
        while sources:
            if infile:
                try:
                    line = infile.readline()
                except (ValueError, IOError):
                    line = None
                logger.debug("communicate: %r", line)
                if not line:
                    infile.close()
                    infile = None
                    self.stdin.close()
                else:
                    self.stdin.write(line)
                    self.stdin.flush()

            i = (i + 1) % len(sources)
            name, coll, pipe, outfile = sources[i]
            line = pipe.readline()
            # logger.debug("%s> %r", name, line)
            if not line:
                del sources[i]
            elif outfile:
                outfile.write(line)
                outfile.flush()
            else:
                coll.append(line)
        self.wait()
        stdout = "".join(s for s in stdout).encode(self.custom_encoding)
        stderr = "".join(s for s in stderr).encode(self.custom_encoding)
        return stdout, stderr

    def iter_lines(self, timeout=None, **kwargs):
        if timeout is not None:
            raise NotImplementedError("The 'timeout' parameter is not supported with ParamikoMachine")
        return iter_lines(self, _iter_lines=_iter_lines, **kwargs)

    __iter__ = iter_lines

class ParamikoMachine(BaseRemoteMachine):
    """
    An implementation of :class:`remote machine <plumbum.machines.remote.BaseRemoteMachine>`
    over Paramiko (a Python implementation of openSSH2 client/server). Invoking a remote command
    translates to invoking it over SSH ::

        with ParamikoMachine("yourhostname") as rem:
            r_ls = rem["ls"]
            # r_ls is the remote `ls`
            # executing r_ls() is equivalent to `ssh yourhostname ls`, only without
            # spawning a new ssh client

    :param host: the host name to connect to (SSH server)

    :param user: the user to connect as (if ``None``, the default will be used)

    :param port: the server's port (if ``None``, the default will be used)

    :param password: the user's password (if a password-based authentication is to be performed)
                     (if ``None``, key-based authentication will be used)

    :param keyfile: the path to the identity file (if ``None``, the default will be used)

    :param load_system_host_keys: whether or not to load the system's host keys (from ``/etc/ssh``
                                  and ``~/.ssh``). The default is ``True``, which means Paramiko
                                  behaves much like the ``ssh`` command-line client

    :param missing_host_policy: the value passed to the underlying ``set_missing_host_key_policy``
                                of the client. The default is ``None``, which means
                                ``set_missing_host_key_policy`` is not invoked and paramiko's
                                default behavior (reject) is employed

    :param encoding: the remote machine's encoding (defaults to UTF8)

    :param look_for_keys: set to False to disable searching for discoverable
                          private key files in ``~/.ssh``

    :param connect_timeout: timeout for TCP connection
    """

    class RemoteCommand(BaseRemoteMachine.RemoteCommand):
        def __or__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __gt__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __rshift__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __ge__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __lt__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __lshift__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")

    def __init__(self, host, user = None, port = None, password = None, keyfile = None,
            load_system_host_keys = True, missing_host_policy = None, encoding = "utf8",
            look_for_keys = None, connect_timeout = None, keep_alive = 0):
        self.host = host
        kwargs = {}
        if user:
            self._fqhost = "%s@%s" % (user, host)
            kwargs['username'] = user
        else:
            self._fqhost = host
        self._client = paramiko.SSHClient()
        if load_system_host_keys:
            self._client.load_system_host_keys()
        if port is not None:
            kwargs["port"] = port
        if keyfile is not None:
            kwargs["key_filename"] = keyfile
        if password is not None:
            kwargs["password"] = password
        if missing_host_policy is not None:
            self._client.set_missing_host_key_policy(missing_host_policy)
        if look_for_keys is not None:
            kwargs["look_for_keys"] = look_for_keys
        if connect_timeout is not None:
            kwargs["timeout"] = connect_timeout
        self._client.connect(host, **kwargs)
        self._keep_alive = keep_alive
        self._sftp = None
        BaseRemoteMachine.__init__(self, encoding, connect_timeout)

    def __str__(self):
        return "paramiko://%s" % (self._fqhost,)

    def close(self):
        BaseRemoteMachine.close(self)
        self._client.close()

    @property
    def sftp(self):
        """
        Returns an SFTP client on top of the current SSH connection; it can be used to manipulate
        files directly, much like an interactive FTP/SFTP session
        """
        if not self._sftp:
            self._sftp = self._client.open_sftp()
        return self._sftp

    @_setdoc(BaseRemoteMachine)
    def session(self, isatty = False, term = "vt100", width = 80, height = 24, new_session = False):
        # new_session is ignored for ParamikoMachine
        trans = self._client.get_transport()
        trans.set_keepalive(self._keep_alive)
        chan = trans.open_session()<commit_msg>Adding support for GSS-API (Paramiko 1.15 and above needed)<commit_after>import logging
import errno
import stat
import socket
from plumbum.machines.base import PopenAddons
from plumbum.machines.remote import BaseRemoteMachine
from plumbum.machines.session import ShellSession
from plumbum.lib import _setdoc, six
from plumbum.path.local import LocalPath
from plumbum.path.remote import RemotePath, StatRes
from plumbum.commands.processes import iter_lines
try:
    # Sigh... we need to gracefully-import paramiko for Sphinx builds, etc
    import paramiko
except ImportError:
    class paramiko(object):
        def __nonzero__(self):
            return False
        __bool__ = __nonzero__
        def __getattr__(self, name):
            raise ImportError("No module named paramiko")
    paramiko = paramiko()


logger = logging.getLogger("plumbum.paramiko")

class ParamikoPopen(PopenAddons):
    def __init__(self, argv, stdin, stdout, stderr, encoding, stdin_file = None,
            stdout_file = None, stderr_file = None):
        self.argv = argv
        self.channel = stdout.channel
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.custom_encoding = encoding
        self.returncode = None
        self.pid = None
        self.stdin_file = stdin_file
        self.stdout_file = stdout_file
        self.stderr_file = stderr_file
    def poll(self):
        if self.returncode is None:
            if self.channel.exit_status_ready():
                return self.wait()
        return self.returncode
    def wait(self):
        if self.returncode is None:
            self.channel.recv_exit_status()
            self.returncode = self.channel.exit_status
            self.close()
        return self.returncode
    def close(self):
        self.channel.shutdown_read()
        self.channel.shutdown_write()
        self.channel.close()
    def kill(self):
        # possible way to obtain pid:
        # "(cmd ; echo $?) & echo ?!"
        # and then client.exec_command("kill -9 %s" % (pid,))
        raise EnvironmentError("Cannot kill remote processes, we don't have their PIDs")
    terminate = kill
    def send_signal(self, sig):
        raise NotImplementedError()
    def communicate(self):
        stdout = []
        stderr = []
        infile = self.stdin_file
        sources = [("1", stdout, self.stdout, self.stdout_file), ("2", stderr, self.stderr, self.stderr_file)]
        i = 0
        while sources:
            if infile:
                try:
                    line = infile.readline()
                except (ValueError, IOError):
                    line = None
                logger.debug("communicate: %r", line)
                if not line:
                    infile.close()
                    infile = None
                    self.stdin.close()
                else:
                    self.stdin.write(line)
                    self.stdin.flush()

            i = (i + 1) % len(sources)
            name, coll, pipe, outfile = sources[i]
            line = pipe.readline()
            # logger.debug("%s> %r", name, line)
            if not line:
                del sources[i]
            elif outfile:
                outfile.write(line)
                outfile.flush()
            else:
                coll.append(line)
        self.wait()
        stdout = "".join(s for s in stdout).encode(self.custom_encoding)
        stderr = "".join(s for s in stderr).encode(self.custom_encoding)
        return stdout, stderr

    def iter_lines(self, timeout=None, **kwargs):
        if timeout is not None:
            raise NotImplementedError("The 'timeout' parameter is not supported with ParamikoMachine")
        return iter_lines(self, _iter_lines=_iter_lines, **kwargs)

    __iter__ = iter_lines

class ParamikoMachine(BaseRemoteMachine):
    """
    An implementation of :class:`remote machine <plumbum.machines.remote.BaseRemoteMachine>`
    over Paramiko (a Python implementation of openSSH2 client/server). Invoking a remote command
    translates to invoking it over SSH ::

        with ParamikoMachine("yourhostname") as rem:
            r_ls = rem["ls"]
            # r_ls is the remote `ls`
            # executing r_ls() is equivalent to `ssh yourhostname ls`, only without
            # spawning a new ssh client

    :param host: the host name to connect to (SSH server)

    :param user: the user to connect as (if ``None``, the default will be used)

    :param port: the server's port (if ``None``, the default will be used)

    :param password: the user's password (if a password-based authentication is to be performed)
                     (if ``None``, key-based authentication will be used)

    :param keyfile: the path to the identity file (if ``None``, the default will be used)

    :param load_system_host_keys: whether or not to load the system's host keys (from ``/etc/ssh``
                                  and ``~/.ssh``). The default is ``True``, which means Paramiko
                                  behaves much like the ``ssh`` command-line client

    :param missing_host_policy: the value passed to the underlying ``set_missing_host_key_policy``
                                of the client. The default is ``None``, which means
                                ``set_missing_host_key_policy`` is not invoked and paramiko's
                                default behavior (reject) is employed

    :param encoding: the remote machine's encoding (defaults to UTF8)

    :param look_for_keys: set to False to disable searching for discoverable
                          private key files in ``~/.ssh``

    :param connect_timeout: timeout for TCP connection

    .. note:: If Paramiko 1.15 or above is installed, can use GSS_API authentication

    :param bool gss_auth: ``True`` if you want to use GSS-API authentication

    :param bool gss_kex: Perform GSS-API Key Exchange and user authentication

    :param bool gss_deleg_creds: Delegate GSS-API client credentials or not

    :param str gss_host: The targets name in the kerberos database. default: hostname
    """

    class RemoteCommand(BaseRemoteMachine.RemoteCommand):
        def __or__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __gt__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __rshift__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __ge__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __lt__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")
        def __lshift__(self, *_):
            raise NotImplementedError("Not supported with ParamikoMachine")

    def __init__(self, host, user = None, port = None, password = <PASSWORD>, keyfile = None,
            load_system_host_keys = True, missing_host_policy = None, encoding = "utf8",
            look_for_keys = None, connect_timeout = None, keep_alive = 0, gss_auth = False,
            gss_kex = None, gss_deleg_creds = None, gss_host = None):
        self.host = host
        kwargs = {}
        if user:
            self._fqhost = "%s@%s" % (user, host)
            kwargs['username'] = user
        else:
            self._fqhost = host
        self._client = paramiko.SSHClient()
        if load_system_host_keys:
            self._client.load_system_host_keys()
        if port is not None:
            kwargs["port"] = port
        if keyfile is not None:
            kwargs["key_filename"] = keyfile
        if password is not None:
            kwargs["password"] = password
        if missing_host_policy is not None:
            self._client.set_missing_host_key_policy(missing_host_policy)
        if look_for_keys is not None:
            kwargs["look_for_keys"] = look_for_keys
        if connect_timeout is not None:
            kwargs["timeout"] = connect_timeout
        if gss_auth:
            kwargs['gss_auth'] = gss_auth
            kwargs['gss_kex'] = gss_kex
            kwargs['gss_deleg_creds'] = gss_deleg_creds
            if not gss_host:
                gss_host = host
            kwargs['gss_host'] = gss_host
        self._client.connect(host, **kwargs)
        self._keep_alive = keep_alive
        self._sftp = None
        BaseRemoteMachine.__init__(self, encoding, connect_timeout)

    def __str__(self):
        return "paramiko://%s" % (self._fqhost,)

    def close(self):
        BaseRemoteMachine.close(self)
        self._client.close()

    @property
    def sftp(self):
        """
        Returns an SFTP client on top of the current SSH connection; it can be used to manipulate
        files directly, much like an interactive FTP/SFTP session
        """
        if not self._sftp:
            self._sftp = self._client.open_sftp()
        return self._sftp

    @_setdoc(BaseRemoteMachine)
    def session(self, isatty = False, term = "vt100", width = 80, height = 24, new_session = False):
        # new_session is ignored for ParamikoMachine
        trans = self._client.get_transport()
        trans.set_keepalive(self._keep_alive)
        chan = trans.open_session()
{'fraction_non_alphanumeric': 0.08733574988672406, 'fraction_numerical': 0.002152242863615768, 'mean_word_length': 2.18488455988456, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7133072', 'n_tokens_mistral': 4955, 'n_tokens_neox': 4788, 'n_words': 1472}