<commit_before>import os
import stat
import inspect

import paramiko
from confiture.schema.containers import Value
from confiture.schema.types import String

from marty.remotemethods import DefaultRemoteMethodSchema, RemoteMethod
from marty.datastructures import Tree, Blob


# Monkey-patch Paramiko in order to provide a listdir function that works
# with bytes instead of unicode strings:

def listdir_attr_b(self, path=b'.'):
    path = self._adjust_cwd(path)
    self._log(paramiko.common.DEBUG, 'listdir(%r)' % path)
    t, msg = self._request(paramiko.sftp.CMD_OPENDIR, path)
    if t != paramiko.sftp.CMD_HANDLE:
        raise paramiko.sftp.SFTPError('Expected handle')
    handle = msg.get_binary()
    filelist = []
    while True:
        try:
            t, msg = self._request(paramiko.sftp.CMD_READDIR, handle)
        except EOFError:
            # done with handle
            break
        if t != paramiko.sftp.CMD_NAME:
            raise paramiko.sftp.SFTPError('Expected name response')
        count = msg.get_int()
        for i in range(count):
            filename = msg.get_string()
            longname = msg.get_string()
            attr = paramiko.sftp_attr.SFTPAttributes._from_msg(msg, filename, longname)
            if (filename != b'.') and (filename != b'..'):
                filelist.append(attr)
    self._request(paramiko.sftp.CMD_CLOSE, handle)
    return filelist
paramiko.SFTPClient.listdir_attr_b = listdir_attr_b

# End of monkey-path

CHECKSUM_LOOP = 'sh -c \'while read filename; do sha1sum "$filename" || echo "failed"; done;\''


class SSHRemoteMethodSchema(DefaultRemoteMethodSchema):

    root = Value(String(), default='/')
    server = Value(String())
    login = Value(String(), default='root')
    password = Value(String())


class SSH(RemoteMethod):

    """ SSH remote.
    """

    config_schema = SSHRemoteMethodSchema()

    def initialize(self):
        self._ssh = paramiko.client.SSHClient()
        self._ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
        self._ssh.connect(self.config.get('server'),
                          username=self.config.get('login'),
                          password=self.config.get('password'),
                          allow_agent=False,
                          compress=False,
                          look_for_keys=False)
        transport = self._ssh.get_transport()
        # transport.window_size = 2147483647
        # transport.packetizer.REKEY_BYTES = pow(2, 40)
        # transport.packetizer.REKEY_PACKETS = pow(2, 40)
        self._sftp = paramiko.SFTPClient.from_transport(transport)

        # Launch the checksum computing loop:
        self._checksum_stdin, self._checksum_stdout, _ = self._ssh.exec_command(CHECKSUM_LOOP)
        # Workaround because Paramiko open stdout as text mode and not binary:
        self._checksum_stdout._set_mode('rb')

    @property
    def root(self):
        return self.config.get('root').encode('utf-8')

    def get_tree(self, path):
        path = path.lstrip(os.sep.encode('utf-8'))
        directory = os.path.join(self.root, path)
        tree = Tree()
        directory_items = self._sftp.listdir_attr_b(directory)<commit_msg>Removed useless comment from SSH remote class<commit_after>import os
import stat
import inspect

import paramiko
from confiture.schema.containers import Value
from confiture.schema.types import String

from marty.remotemethods import DefaultRemoteMethodSchema, RemoteMethod
from marty.datastructures import Tree, Blob


# Monkey-patch Paramiko in order to provide a listdir function that works
# with bytes instead of unicode strings:

def listdir_attr_b(self, path=b'.'):
    path = self._adjust_cwd(path)
    self._log(paramiko.common.DEBUG, 'listdir(%r)' % path)
    t, msg = self._request(paramiko.sftp.CMD_OPENDIR, path)
    if t != paramiko.sftp.CMD_HANDLE:
        raise paramiko.sftp.SFTPError('Expected handle')
    handle = msg.get_binary()
    filelist = []
    while True:
        try:
            t, msg = self._request(paramiko.sftp.CMD_READDIR, handle)
        except EOFError:
            # done with handle
            break
        if t != paramiko.sftp.CMD_NAME:
            raise paramiko.sftp.SFTPError('Expected name response')
        count = msg.get_int()
        for i in range(count):
            filename = msg.get_string()
            longname = msg.get_string()
            attr = paramiko.sftp_attr.SFTPAttributes._from_msg(msg, filename, longname)
            if (filename != b'.') and (filename != b'..'):
                filelist.append(attr)
    self._request(paramiko.sftp.CMD_CLOSE, handle)
    return filelist
paramiko.SFTPClient.listdir_attr_b = listdir_attr_b

# End of monkey-path

CHECKSUM_LOOP = 'sh -c \'while read filename; do sha1sum "$filename" || echo "failed"; done;\''


class SSHRemoteMethodSchema(DefaultRemoteMethodSchema):

    root = Value(String(), default='/')
    server = Value(String())
    login = Value(String(), default='root')
    password = Value(String())


class SSH(RemoteMethod):

    """ SSH remote.
    """

    config_schema = SSHRemoteMethodSchema()

    def initialize(self):
        self._ssh = paramiko.client.SSHClient()
        self._ssh.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
        self._ssh.connect(self.config.get('server'),
                          username=self.config.get('login'),
                          password=self.config.get('password'),
                          allow_agent=False,
                          compress=False,
                          look_for_keys=False)
        transport = self._ssh.get_transport()
        self._sftp = paramiko.SFTPClient.from_transport(transport)

        # Launch the checksum computing loop:
        self._checksum_stdin, self._checksum_stdout, _ = self._ssh.exec_command(CHECKSUM_LOOP)
        # Workaround because Paramiko open stdout as text mode and not binary:
        self._checksum_stdout._set_mode('rb')

    @property
    def root(self):
        return self.config.get('root').encode('utf-8')

    def get_tree(self, path):
        path = path.lstrip(os.sep.encode('utf-8'))
        directory = os.path.join(self.root, path)
        tree = Tree()
        directory_items = self._sftp.listdir_attr_b(directory)
{'fraction_non_alphanumeric': 0.10756972111553785, 'fraction_numerical': 0.00350597609561753, 'mean_word_length': 2.704840613931523, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3365287', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1858, 'n_words': 405}