<commit_before>#  Copyright 2008-2013 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from __future__ import with_statement
from fnmatch import fnmatchcase
from robot.utils import unic
import os
import re
import stat
import time
import glob

from .config import (Configuration, IntegerEntry, NewlineEntry, StringEntry,
                     TimeEntry)


class SSHClientException(RuntimeError):
    pass


class _ClientConfiguration(Configuration):

    def __init__(self, host, alias, port, timeout, newline, prompt, term_type,
                 width, height, path_separator, encoding):
        super(_ClientConfiguration, self).__init__(
            index=IntegerEntry(None),
            host=StringEntry(host),
            alias=StringEntry(alias),
            port=IntegerEntry(port),
            timeout=TimeEntry(timeout),
            newline=NewlineEntry(newline),
            prompt=StringEntry(prompt),
            term_type=StringEntry(term_type),
            width=IntegerEntry(width),
            height=IntegerEntry(height),
            path_separator=StringEntry(path_separator),
            encoding=StringEntry(encoding)
        )


class AbstractSSHClient(object):
    """A base class for SSH client implementations, defines the public API.

    Subclasses  provide the tool/language specific concrete implementations.
    """
    def __init__(self, host, alias=None, port=22, timeout=3, newline='LF',
                 prompt=None, term_type='vt100', width=80, height=24,
                 path_separator='/', encoding='utf8'):
        self.config = _ClientConfiguration(host, alias, port, timeout, newline,
                                           prompt, term_type, width, height,
                                           path_separator, encoding)
        self._sftp_client = None
        self._shell = None
        self._started_commands = []

    @staticmethod
    def enable_logging(path):
        """Log SSH events to file.

        :param path: A filename where the log events are written
        :returns: Whether logging was successfully enabled.
        """
        raise NotImplementedError

    @property
    def sftp_client(self):
        if not self._sftp_client:
            self._sftp_client = self._create_sftp_client()
        return self._sftp_client

    @property
    def shell(self):
        if not self._shell:
            self._shell = self._create_shell()
        return self._shell

    def _create_sftp_client(self):
        raise NotImplementedError

    def _create_shell(self):
        raise NotImplementedError

    def close(self):
        """Closes the connection.
        """
        self.client.close()

    def login(self, username, password, delay=None):
        """Login using given credentials.

        :param str username: username to log in with
        :param str password: password for `<PASSWORD>`
        :returns: If prompt is defined, read and return output until prompt.
            Otherwise all output is read and returned.
        """
        try:
            self._login(username, password)
        except SSHClientException:
            raise SSHClientException("Authentication failed for user '%s'."
                                     % username)
        return self._read_login_output(delay)

    def _login(self, username, password):
        raise NotImplementedError

    def login_with_public_key(self, username, keyfile, password, delay=None):
        """Login using given credentials.

        :param str username: username to log in with
        :param str keyfile: path to a valid OpenSSH keyfile
        :param str password: password used in unlocking the keyfile
        :returns: If prompt is defined, read and return output until prompt.
            Otherwise all output is read and returned.
        """
        self._verify_key_file(keyfile)
        try:
            self._login_with_public_key(username, keyfile, password)
        except SSHClientException:
            raise SSHClientException("Login with public key failed for user "
                                     "'%s'." % username)
        return self._read_login_output(delay)

    def _login_with_public_key(self, username, keyfile, password):
        raise NotImplementedError

    def _read_login_output(self, delay):
        if self.config.prompt:
            return self.read_until_prompt()
        return self.read(delay)

    def _verify_key_file(self, keyfile):
        if not os.path.exists(keyfile):
            raise SSHClientException("Given key file '%s' does not exist." %
                                     keyfile)
        try:
            open(keyfile).close()
        except IOError:
            raise SSHClientException("Could not read key file '%s'." % keyfile)

    def execute_command(self, command):
        """Execute given command over existing connection.

        :returns: 3-tuple (stdout, stderr, return_code)
        """
        self.start_command(command)
        return self.read_command_output()

    def start_command(self, command):
        """Execute given command over existing connection."""
        command = self._encode(command)
        self._started_commands.append(self._start_command(command))

    def _encode(self, text):
        if isinstance(text, str):
            return text
        if not isinstance(text, basestring):
            text = unicode(text)
        return text.encode(self.config.encoding)

    def _start_command(self, command):
        raise NotImplementedError

    def read_command_output(self):
        """Read output of a previously started command.

        :returns: 3-tuple (stdout, stderr, return_code)
        """
        try:
            return self._started_commands.pop().read_outputs()
        except IndexError:
            raise SSHClientException('No started commands to read output from.')

    def write(self, text, add_newline=False):
        """Write `text` in shell session.

        :param str text: the text to be written
        :param bool add_newline: if True, a newline will be added to `text`.
        """
        text = self._encode(text)
        if add_newline:
            text += self.config.newline
        self.shell.write(text)

    def read(self, delay=None):
        """Read and return currently available output."""
        output = self.shell.read()
        if delay:
            output += self._delayed_read(delay)
        return self._decode(output)

    def _decode(self, output):
        return output.decode(self.config.encoding)

    def _delayed_read(self, delay):
        delay = TimeEntry(delay).value
        max_time = time.time() + self.config.get('timeout').value
        output = ''
        while time.time() < max_time:
            time.sleep(delay)
            read = self.shell.read()
            if not read:
                break
            output += read
        return output

    def read_char(self):
        """Read and return a single char from the current session."""
        server_output = ''
        while True:
            try:
                server_output += self.shell.read_byte()
                return self._decode(server_output)
            except UnicodeDecodeError:
                pass

    def read_until(self, expected):
        """Read and return from the output until expected.

        :param str expected: text to look for in the output
        :raises SSHClientException: if expected is not found in output when
            timeout expires.

        timeout is defined with :py:meth:`open_connection()`
        """
        return self._read_until(lambda s: expected in s, expected)

    def read_until_newline(self):
        """Read and return from the output up to the first newline character.

        :raises SSHClientException: if newline is not found in output when
            timeout expires.

        timeout is defined with :py:meth:`open_connection()`
        """
        return self.read_until(self.config.newline)

    def read_until_prompt(self):
        """Read and return from the output until prompt.

        :raises SSHClientException: if prompt is not set or it is not found
            in output when timeout expires.

        prompt and timeout are defined with :py:meth:`open_connection()`
        """
        if not self.config.prompt:
            raise SSHClientException('Prompt is not set.')
        return self.read_until(self.config.prompt)

    def read_until_regexp(self, regexp):
        """Read and return from the output until regexp matches.

        :param regexp: a pattern or a compiled regexp obect used for matching
        :raises SSHClientException: if match is not found in output when
            timeout expires.

        timeout is defined with :py:meth:`open_connection()`
        """
        if isinstance(regexp, basestring):
            regexp = re.compile(regexp)
        return self._read_until(lambda s: regexp.search(s), regexp.pattern)

    def write_until_expected(self, text, expected, timeout, interval):
        """Write text until expected output appears or timeout expires.

        :param str text: Text to be written using #write_bare().
        :param str expected: Text to look for in the output.
        :param int timeout: The timeout during which `expected` must appear
            in the output. Can be defined either as seconds or as a Robot
            Framework time string, e.g. 1 minute 20 seconds.
        :param int interval: Time to wait between repeated writings. Can be
            defined similarly as `timeout`.
        """
        interval = TimeEntry(interval)
        timeout = TimeEntry(timeout)
        max_time = time.time() + timeout.value
        while time.time() < max_time:
            self.write(text)
            try:
                return self._read_until(lambda s: expected in s, expected,
                                        timeout=interval.value)
            except SSHClientException:
                pass
        raise SSHClientException("No match found for '%s' in %s."
                                 % (expected, timeout))

    def _read_until(self, matcher, expected, timeout=None):
        server_output = ''
        timeout = TimeEntry(timeout) if timeout else self.config.get('timeout')
        max_time = time.time() + timeout.value
        while time.time() < max_time:
            try:
                server_output += self.shell.read_byte()
                decoded_output = self._decode(server_output)
                if matcher(decoded_output):
                    return decoded_output
            except UnicodeDecodeError:
                pass
        raise SSHClientException("No match found for '%s' in %s\nOutput:\n%s."
                                 % (expected, timeout, decoded_output))

    def put_file(self, source, destination='.', mode='0744', newline='',
                 path_separator=''):
        """Put file(s) from localhost to remote host.

        :param source: Local file path. May be a simple pattern containing
            '*' and '?', in which case all matching files are tranferred
        :param destination: Remote path. If many files are transferred,
            must be a directory. Defaults to users home directory.
        :param mode: File permissions for the remote file. Defined as a
            Unix file format string, e.g. '0600'
        :param newline: Newline character to be used in the remote file.
            Default is 'LF', i.e. the line feed character.
        """
        # Path separator was deprecated in SSHLibrary 1.2
        path_separator = path_separator or self.config.path_separator
        return self.sftp_client.put_file(source, destination, mode, newline,
                                         path_separator)

    def put_directory(self, source, destination='.', mode='0744', newline='',
                      recursive=False):
        return self.sftp_client.put_directory(source, destination, mode,
                                              newline,
                                              self.config.path_separator,
                                              recursive)

    def get_file(self, source, destination='.', path_separator=''):
        """Get file(s) from the remote host to localhost.

        :param source: Remote file path. May be a simple pattern containing
            '*' and '?', in which case all matching files are tranferred
            :param destintation: Local path. If many files are transferred,
            must be a directory. Defaults to current working directory.
        """
        # Path separator was deprecated in SSHLibrary 1.2
        path_separator = path_separator or self.config.path_separator
        return self.sftp_client.get_file(source, destination, path_separator)

    def get_directory(self, source, destination='.', recursive=False):
        return self.sftp_client.get_directory(source, destination,
                                              self.config.path_separator,
                                              recursive)

    def list_dir(self, path, pattern=None, absolute=False):
        items = self.sftp_client.list(path, pattern, absolute)
        return sorted(items)

    def list_files_in_dir(self, path, pattern=None, absolute=False):
        files = self.sftp_client.list_files(path, pattern, absolute)
        return sorted(files)

    def list_dirs_in_dir(self, path, pattern=None, absolute=False):
        dirs = self.sftp_client.list_dirs(path, pattern, absolute)
        return sorted(dirs)

    def dir_exists(self, path):
        return self.sftp_client.is_dir(path)

    def file_exists(self, path):
        return self.sftp_client.is_file(path)


class AbstractShell(object):

    def read(self):
        raise NotImplementedError

    def read_byte(self):
        raise NotImplementedError

    def write(self):
        raise NotImplementedError


class SFTPFileInfo(object):

    def __init__(self, name, mode):
        self.name = unic(name)
        self.mode = mode

    def is_regular(self):
        return stat.S_ISREG(self.mode)

    def is_directory(self):
        return stat.S_ISDIR(self.mode)


class AbstractSFTPClient(object):

    def __init__(self):
        self._homedir = self._absolute_path('.')

    def _absolute_path(self, path):
        raise NotImplementedError

    def is_file(self, path):
        try:
            item = self._stat(path)
        except IOError:
            return False
        return item.is_regular()

    def _stat(self):
        raise NotImplementedError

    def is_dir(self, path):
        try:
            item = self._stat(path)
        except IOError:
            return False
        return item.is_directory()

    def list(self, path, pattern=None, absolute=False):
        return self._list_filtered(path, self._get_item_names, pattern,
                                   absolute)

    def _list_filtered(self, path, filter_method, pattern=None, absolute=False):
        self._verify_remote_dir_exists(path)
        items = filter_method(path)
        if pattern:
            items = self._filter_by_pattern(items, pattern)
        if absolute:
            items = self._include_absolute_path(items, path)
        return items

    def _verify_remote_dir_exists(self, path):
        if not self.is_dir(path):
            raise SSHClientException("There was no directory matching '%s'." %
                                     path)

    def _get_item_names(self, path):
        return [item.name for item in self._list(path)]

    def _list(self, path):
        raise NotImplementedError

    def _filter_by_pattern(self, items, pattern):
        return [name for name in items if fnmatchcase(name, pattern)]

    def _include_absolute_path(self, items, path):
        absolute_path = self._absolute_path(path)
        if absolute_path[1:3] == ':\\':
            absolute_path += '\\'
        else:
            absolute_path += '/'
        return [absolute_path + name for name in items]

    def list_files(self, path, pattern=None, absolute=False):
        return self._list_filtered(path, self._get_file_names, pattern,
                                   absolute)

    def _get_file_names(self, path):
        return [item.name for item in self._list(path) if item.is_regular()]

    def list_dirs(self, path, pattern=None, absolute=False):
        return self._list_filtered(path, self._get_directory_names, pattern,
                                   absolute)

    def _get_directory_names(self, path):
        return [item.name for item in self._list(path) if item.is_directory()]

    def get_directory(self, source, destination, path_separator='/',
                      recursive=False):
        source = self._remove_ending_path_separator(path_separator, source)
        self._verify_remote_dir_exists(source)
        files = []
        for child in self.list(source):
            remote = source + path_separator + child
            local = os.path.join(destination, child)
            if self.is_file(remote):
                files += self.get_file(remote, local)
            elif recursive:
                files += self.get_directory(remote, local, path_separator,
                                            recursive)
        return files

    def _remove_ending_path_separator(self, path_separator, source):
        if source.endswith(path_separator):
            source = source[:-len(path_separator)]
        return source

    def get_file(self, source, destination, path_separator='/'):
        remote_files = self._get_get_file_sources(source, path_separator)
        if not remote_files:
            msg = "There were no source files matching '%s'." % source
            raise SSHClientException(msg)
        local_files = self._get_get_file_destinations(remote_files, destination)
        files = zip(remote_files, local_files)
        for src, dst in files:
            self._get_file(src, dst)
        return files

    def _get_get_file_sources(self, source, path_separator):
        if path_separator in source:
            path, pattern = source.rsplit(path_separator, 1)
        else:
            path, pattern = '', source
        if not path:
            path = '.'
        return [filename for filename in
                self.list_files(path, pattern, absolute=True)]

    def _get_get_file_destinations(self, sourcefiles, destination):
        target_is_dir = destination.endswith(os.sep) or destination == '.'
        if not target_is_dir and len(sourcefiles) > 1:
            raise SSHClientException('Cannot copy multiple source files to one '
                                     'destination file.')
        destination = os.path.abspath(destination.replace('/', os.sep))
        self._create_missing_local_dirs(destination, target_is_dir)
        if target_is_dir:
            return [os.path.join(destination, os.path.basename(name))
                    for name in sourcefiles]
        return [destination]

    def _create_missing_local_dirs(self, destination, target_is_dir):
        if not target_is_dir:
            destination = os.path.dirname(destination)
        if not os.path.exists(destination):
            os.makedirs(destination)

    def _get_file(self, src, dst):
        raise NotImplementedError

    def put_directory(self, source, destination, mode, newline,
                      path_separator='/', recursive=False):
        self._verify_local_dir_exists(source)
        destination = self._remove_ending_path_separator(path_separator,
                                                         destination)
        files = []
        parent = os.path.basename(os.path.abspath(source))
        remote_destination_exists = self.is_dir(destination)
        os.chdir(os.path.dirname(source))
        for subdirectory, _, filenames in os.walk(parent):
            for filename in filenames:
                local_path = os.path.join(subdirectory, filename)
                if destination.endswith('.'):
                    remote_path = path_separator.join([subdirectory, filename])
                else:
                    remote_path = path_separator.join([destination,
                                                       subdirectory,
                                                       filename])
                    if not remote_destination_exists:
                        remote_path = remote_path.replace(parent +
                                                          path_separator, '')
                files += self.put_file(local_path, remote_path, mode, newline,
                                     path_separator)
            if not recursive:
                break
        return files

    def _verify_local_dir_exists(self, path):
        if not os.path.isdir(path):
            raise SSHClientException("There was no source path matching '%s'."
                                     % path)

    def put_file(self, sources, destination, mode, newline, path_separator='/'):
        mode = int(mode, 8)
        newline = {'CRLF': '\r\n', 'LF': '\n'}.get(newline.upper(), None)
        localfiles = self._get_put_file_sources(sources)
        remotefiles, remotedir = self._get_put_file_destinations(localfiles,
                                                                 destination,
                                                                 path_separator)
        self._create_missing_remote_path(remotedir)
        files = zip(localfiles, remotefiles)
        for src, dst in files:
            self._put_file(src, dst, mode, newline)
        return files

    def _get_put_file_sources(self, source):
        sources = [f for f in glob.glob(source.replace('/', os.sep))
                   if os.path.isfile(f)]
        if not sources:<commit_msg>Refactored `Put Directory` implementation<commit_after>#  Copyright 2008-2013 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from __future__ import with_statement
from fnmatch import fnmatchcase
from robot.utils import unic
import os
import re
import stat
import time
import glob

from .config import (Configuration, IntegerEntry, NewlineEntry, StringEntry,
                     TimeEntry)


class SSHClientException(RuntimeError):
    pass


class _ClientConfiguration(Configuration):

    def __init__(self, host, alias, port, timeout, newline, prompt, term_type,
                 width, height, path_separator, encoding):
        super(_ClientConfiguration, self).__init__(
            index=IntegerEntry(None),
            host=StringEntry(host),
            alias=StringEntry(alias),
            port=IntegerEntry(port),
            timeout=TimeEntry(timeout),
            newline=NewlineEntry(newline),
            prompt=StringEntry(prompt),
            term_type=StringEntry(term_type),
            width=IntegerEntry(width),
            height=IntegerEntry(height),
            path_separator=StringEntry(path_separator),
            encoding=StringEntry(encoding)
        )


class AbstractSSHClient(object):
    """A base class for SSH client implementations, defines the public API.

    Subclasses  provide the tool/language specific concrete implementations.
    """
    def __init__(self, host, alias=None, port=22, timeout=3, newline='LF',
                 prompt=None, term_type='vt100', width=80, height=24,
                 path_separator='/', encoding='utf8'):
        self.config = _ClientConfiguration(host, alias, port, timeout, newline,
                                           prompt, term_type, width, height,
                                           path_separator, encoding)
        self._sftp_client = None
        self._shell = None
        self._started_commands = []

    @staticmethod
    def enable_logging(path):
        """Log SSH events to file.

        :param path: A filename where the log events are written
        :returns: Whether logging was successfully enabled.
        """
        raise NotImplementedError

    @property
    def sftp_client(self):
        if not self._sftp_client:
            self._sftp_client = self._create_sftp_client()
        return self._sftp_client

    @property
    def shell(self):
        if not self._shell:
            self._shell = self._create_shell()
        return self._shell

    def _create_sftp_client(self):
        raise NotImplementedError

    def _create_shell(self):
        raise NotImplementedError

    def close(self):
        """Closes the connection.
        """
        self.client.close()

    def login(self, username, password, delay=None):
        """Login using given credentials.

        :param str username: username to log in with
        :param str password: <PASSWORD> `<PASSWORD>`
        :returns: If prompt is defined, read and return output until prompt.
            Otherwise all output is read and returned.
        """
        try:
            self._login(username, password)
        except SSHClientException:
            raise SSHClientException("Authentication failed for user '%s'."
                                     % username)
        return self._read_login_output(delay)

    def _login(self, username, password):
        raise NotImplementedError

    def login_with_public_key(self, username, keyfile, password, delay=None):
        """Login using given credentials.

        :param str username: username to log in with
        :param str keyfile: path to a valid OpenSSH keyfile
        :param str password: password used in unlocking the keyfile
        :returns: If prompt is defined, read and return output until prompt.
            Otherwise all output is read and returned.
        """
        self._verify_key_file(keyfile)
        try:
            self._login_with_public_key(username, keyfile, password)
        except SSHClientException:
            raise SSHClientException("Login with public key failed for user "
                                     "'%s'." % username)
        return self._read_login_output(delay)

    def _login_with_public_key(self, username, keyfile, password):
        raise NotImplementedError

    def _read_login_output(self, delay):
        if self.config.prompt:
            return self.read_until_prompt()
        return self.read(delay)

    def _verify_key_file(self, keyfile):
        if not os.path.exists(keyfile):
            raise SSHClientException("Given key file '%s' does not exist." %
                                     keyfile)
        try:
            open(keyfile).close()
        except IOError:
            raise SSHClientException("Could not read key file '%s'." % keyfile)

    def execute_command(self, command):
        """Execute given command over existing connection.

        :returns: 3-tuple (stdout, stderr, return_code)
        """
        self.start_command(command)
        return self.read_command_output()

    def start_command(self, command):
        """Execute given command over existing connection."""
        command = self._encode(command)
        self._started_commands.append(self._start_command(command))

    def _encode(self, text):
        if isinstance(text, str):
            return text
        if not isinstance(text, basestring):
            text = unicode(text)
        return text.encode(self.config.encoding)

    def _start_command(self, command):
        raise NotImplementedError

    def read_command_output(self):
        """Read output of a previously started command.

        :returns: 3-tuple (stdout, stderr, return_code)
        """
        try:
            return self._started_commands.pop().read_outputs()
        except IndexError:
            raise SSHClientException('No started commands to read output from.')

    def write(self, text, add_newline=False):
        """Write `text` in shell session.

        :param str text: the text to be written
        :param bool add_newline: if True, a newline will be added to `text`.
        """
        text = self._encode(text)
        if add_newline:
            text += self.config.newline
        self.shell.write(text)

    def read(self, delay=None):
        """Read and return currently available output."""
        output = self.shell.read()
        if delay:
            output += self._delayed_read(delay)
        return self._decode(output)

    def _decode(self, output):
        return output.decode(self.config.encoding)

    def _delayed_read(self, delay):
        delay = TimeEntry(delay).value
        max_time = time.time() + self.config.get('timeout').value
        output = ''
        while time.time() < max_time:
            time.sleep(delay)
            read = self.shell.read()
            if not read:
                break
            output += read
        return output

    def read_char(self):
        """Read and return a single char from the current session."""
        server_output = ''
        while True:
            try:
                server_output += self.shell.read_byte()
                return self._decode(server_output)
            except UnicodeDecodeError:
                pass

    def read_until(self, expected):
        """Read and return from the output until expected.

        :param str expected: text to look for in the output
        :raises SSHClientException: if expected is not found in output when
            timeout expires.

        timeout is defined with :py:meth:`open_connection()`
        """
        return self._read_until(lambda s: expected in s, expected)

    def read_until_newline(self):
        """Read and return from the output up to the first newline character.

        :raises SSHClientException: if newline is not found in output when
            timeout expires.

        timeout is defined with :py:meth:`open_connection()`
        """
        return self.read_until(self.config.newline)

    def read_until_prompt(self):
        """Read and return from the output until prompt.

        :raises SSHClientException: if prompt is not set or it is not found
            in output when timeout expires.

        prompt and timeout are defined with :py:meth:`open_connection()`
        """
        if not self.config.prompt:
            raise SSHClientException('Prompt is not set.')
        return self.read_until(self.config.prompt)

    def read_until_regexp(self, regexp):
        """Read and return from the output until regexp matches.

        :param regexp: a pattern or a compiled regexp obect used for matching
        :raises SSHClientException: if match is not found in output when
            timeout expires.

        timeout is defined with :py:meth:`open_connection()`
        """
        if isinstance(regexp, basestring):
            regexp = re.compile(regexp)
        return self._read_until(lambda s: regexp.search(s), regexp.pattern)

    def write_until_expected(self, text, expected, timeout, interval):
        """Write text until expected output appears or timeout expires.

        :param str text: Text to be written using #write_bare().
        :param str expected: Text to look for in the output.
        :param int timeout: The timeout during which `expected` must appear
            in the output. Can be defined either as seconds or as a Robot
            Framework time string, e.g. 1 minute 20 seconds.
        :param int interval: Time to wait between repeated writings. Can be
            defined similarly as `timeout`.
        """
        interval = TimeEntry(interval)
        timeout = TimeEntry(timeout)
        max_time = time.time() + timeout.value
        while time.time() < max_time:
            self.write(text)
            try:
                return self._read_until(lambda s: expected in s, expected,
                                        timeout=interval.value)
            except SSHClientException:
                pass
        raise SSHClientException("No match found for '%s' in %s."
                                 % (expected, timeout))

    def _read_until(self, matcher, expected, timeout=None):
        server_output = ''
        timeout = TimeEntry(timeout) if timeout else self.config.get('timeout')
        max_time = time.time() + timeout.value
        while time.time() < max_time:
            try:
                server_output += self.shell.read_byte()
                decoded_output = self._decode(server_output)
                if matcher(decoded_output):
                    return decoded_output
            except UnicodeDecodeError:
                pass
        raise SSHClientException("No match found for '%s' in %s\nOutput:\n%s."
                                 % (expected, timeout, decoded_output))

    def put_file(self, source, destination='.', mode='0744', newline='',
                 path_separator=''):
        """Put file(s) from localhost to remote host.

        :param source: Local file path. May be a simple pattern containing
            '*' and '?', in which case all matching files are tranferred
        :param destination: Remote path. If many files are transferred,
            must be a directory. Defaults to users home directory.
        :param mode: File permissions for the remote file. Defined as a
            Unix file format string, e.g. '0600'
        :param newline: Newline character to be used in the remote file.
            Default is 'LF', i.e. the line feed character.
        """
        # Path separator was deprecated in SSHLibrary 1.2
        path_separator = path_separator or self.config.path_separator
        return self.sftp_client.put_file(source, destination, mode, newline,
                                         path_separator)

    def put_directory(self, source, destination='.', mode='0744', newline='',
                      recursive=False):
        return self.sftp_client.put_directory(source, destination, mode,
                                              newline,
                                              self.config.path_separator,
                                              recursive)

    def get_file(self, source, destination='.', path_separator=''):
        """Get file(s) from the remote host to localhost.

        :param source: Remote file path. May be a simple pattern containing
            '*' and '?', in which case all matching files are tranferred
            :param destintation: Local path. If many files are transferred,
            must be a directory. Defaults to current working directory.
        """
        # Path separator was deprecated in SSHLibrary 1.2
        path_separator = path_separator or self.config.path_separator
        return self.sftp_client.get_file(source, destination, path_separator)

    def get_directory(self, source, destination='.', recursive=False):
        return self.sftp_client.get_directory(source, destination,
                                              self.config.path_separator,
                                              recursive)

    def list_dir(self, path, pattern=None, absolute=False):
        items = self.sftp_client.list(path, pattern, absolute)
        return sorted(items)

    def list_files_in_dir(self, path, pattern=None, absolute=False):
        files = self.sftp_client.list_files(path, pattern, absolute)
        return sorted(files)

    def list_dirs_in_dir(self, path, pattern=None, absolute=False):
        dirs = self.sftp_client.list_dirs(path, pattern, absolute)
        return sorted(dirs)

    def dir_exists(self, path):
        return self.sftp_client.is_dir(path)

    def file_exists(self, path):
        return self.sftp_client.is_file(path)


class AbstractShell(object):

    def read(self):
        raise NotImplementedError

    def read_byte(self):
        raise NotImplementedError

    def write(self):
        raise NotImplementedError


class SFTPFileInfo(object):

    def __init__(self, name, mode):
        self.name = unic(name)
        self.mode = mode

    def is_regular(self):
        return stat.S_ISREG(self.mode)

    def is_directory(self):
        return stat.S_ISDIR(self.mode)


class AbstractSFTPClient(object):

    def __init__(self):
        self._homedir = self._absolute_path('.')

    def _absolute_path(self, path):
        raise NotImplementedError

    def is_file(self, path):
        try:
            item = self._stat(path)
        except IOError:
            return False
        return item.is_regular()

    def _stat(self):
        raise NotImplementedError

    def is_dir(self, path):
        try:
            item = self._stat(path)
        except IOError:
            return False
        return item.is_directory()

    def list(self, path, pattern=None, absolute=False):
        return self._list_filtered(path, self._get_item_names, pattern,
                                   absolute)

    def _list_filtered(self, path, filter_method, pattern=None, absolute=False):
        self._verify_remote_dir_exists(path)
        items = filter_method(path)
        if pattern:
            items = self._filter_by_pattern(items, pattern)
        if absolute:
            items = self._include_absolute_path(items, path)
        return items

    def _verify_remote_dir_exists(self, path):
        if not self.is_dir(path):
            raise SSHClientException("There was no directory matching '%s'." %
                                     path)

    def _get_item_names(self, path):
        return [item.name for item in self._list(path)]

    def _list(self, path):
        raise NotImplementedError

    def _filter_by_pattern(self, items, pattern):
        return [name for name in items if fnmatchcase(name, pattern)]

    def _include_absolute_path(self, items, path):
        absolute_path = self._absolute_path(path)
        if absolute_path[1:3] == ':\\':
            absolute_path += '\\'
        else:
            absolute_path += '/'
        return [absolute_path + name for name in items]

    def list_files(self, path, pattern=None, absolute=False):
        return self._list_filtered(path, self._get_file_names, pattern,
                                   absolute)

    def _get_file_names(self, path):
        return [item.name for item in self._list(path) if item.is_regular()]

    def list_dirs(self, path, pattern=None, absolute=False):
        return self._list_filtered(path, self._get_directory_names, pattern,
                                   absolute)

    def _get_directory_names(self, path):
        return [item.name for item in self._list(path) if item.is_directory()]

    def get_directory(self, source, destination, path_separator='/',
                      recursive=False):
        source = self._remove_ending_path_separator(path_separator, source)
        self._verify_remote_dir_exists(source)
        files = []
        for child in self.list(source):
            remote = source + path_separator + child
            local = os.path.join(destination, child)
            if self.is_file(remote):
                files += self.get_file(remote, local)
            elif recursive:
                files += self.get_directory(remote, local, path_separator,
                                            recursive)
        return files

    def _remove_ending_path_separator(self, path_separator, source):
        if source.endswith(path_separator):
            source = source[:-len(path_separator)]
        return source

    def get_file(self, source, destination, path_separator='/'):
        remote_files = self._get_get_file_sources(source, path_separator)
        if not remote_files:
            msg = "There were no source files matching '%s'." % source
            raise SSHClientException(msg)
        local_files = self._get_get_file_destinations(remote_files, destination)
        files = zip(remote_files, local_files)
        for src, dst in files:
            self._get_file(src, dst)
        return files

    def _get_get_file_sources(self, source, path_separator):
        if path_separator in source:
            path, pattern = source.rsplit(path_separator, 1)
        else:
            path, pattern = '', source
        if not path:
            path = '.'
        return [filename for filename in
                self.list_files(path, pattern, absolute=True)]

    def _get_get_file_destinations(self, sourcefiles, destination):
        target_is_dir = destination.endswith(os.sep) or destination == '.'
        if not target_is_dir and len(sourcefiles) > 1:
            raise SSHClientException('Cannot copy multiple source files to one '
                                     'destination file.')
        destination = os.path.abspath(destination.replace('/', os.sep))
        self._create_missing_local_dirs(destination, target_is_dir)
        if target_is_dir:
            return [os.path.join(destination, os.path.basename(name))
                    for name in sourcefiles]
        return [destination]

    def _create_missing_local_dirs(self, destination, target_is_dir):
        if not target_is_dir:
            destination = os.path.dirname(destination)
        if not os.path.exists(destination):
            os.makedirs(destination)

    def _get_file(self, src, dst):
        raise NotImplementedError

    def put_directory(self, source, destination, mode, newline,
                      path_separator='/', recursive=False):
        self._verify_local_dir_exists(source)
        destination = self._remove_ending_path_separator(path_separator, destination)
        if self.is_dir(destination):
            destination = destination + path_separator +\
                                 source.rsplit(os.path.sep)[-1]
        return self._put_directory(source, destination, mode, newline,
                      path_separator, recursive)

    def _put_directory(self, source, destination, mode, newline,
                      path_separator, recursive):
        files = []
        for item in os.listdir(source):
            local_path = os.path.join(source, item)
            remote_path = destination + path_separator + item
            if os.path.isfile(local_path):
                files += self.put_file(local_path, remote_path, mode, newline,
                                       path_separator)
            elif recursive and os.path.isdir(local_path):
                files += self._put_directory(local_path, remote_path, mode,
                                            newline, path_separator, recursive)
        return files

    def _verify_local_dir_exists(self, path):
        if not os.path.isdir(path):
            raise SSHClientException("There was no source path matching '%s'."
                                     % path)

    def put_file(self, sources, destination, mode, newline, path_separator='/'):
        mode = int(mode, 8)
        newline = {'CRLF': '\r\n', 'LF': '\n'}.get(newline.upper(), None)
        localfiles = self._get_put_file_sources(sources)
        remotefiles, remotedir = self._get_put_file_destinations(localfiles,
                                                                 destination,
                                                                 path_separator)
        self._create_missing_remote_path(remotedir)
        files = zip(localfiles, remotefiles)
        for src, dst in files:
            self._put_file(src, dst, mode, newline)
        return files

    def _get_put_file_sources(self, source):
        sources = [f for f in glob.glob(source.replace('/', os.sep))
                   if os.path.isfile(f)]
        if not sources:
{'fraction_non_alphanumeric': 0.08328255128986391, 'fraction_numerical': 0.0022344099126548854, 'mean_word_length': 2.1106430777871386, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 81, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2625584', 'n_tokens_mistral': 11742, 'n_tokens_neox': 11142, 'n_words': 3367}