<commit_before>import os
import errno
import getpass
import filecmp
import posixpath

try:
    import paramiko
except ImportError:
    paramiko = None

from dvc.logger import Logger
from dvc.utils import copyfile, file_md5
from dvc.progress import progress
from dvc.remote.base import RemoteBase
from dvc.config import Config
from dvc.exceptions import DvcException


def sizeof_fmt(num, suffix='B'):
    """ Convert number of bytes to human-readable string """
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)


def percent_cb(name, complete, total):
    """ Callback for updating target progress """
    Logger.debug('{}: {} transferred out of {}'.format(name,
                                                       sizeof_fmt(complete),
                                                       sizeof_fmt(total)))
    progress.update_target(name, complete, total)


def create_cb(name):
    """ Create callback function for multipart object """
    return (lambda cur, tot: percent_cb(name, cur, tot))


class RemoteSSH(RemoteBase):
    scheme = 'ssh'

    #NOTE: temporarily only absolute paths are allowed
    REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'

    REQUIRES = {'paramiko': paramiko}
    PARAM_MD5 = 'md5'

    def __init__(self, project, config):
        self.project = project
        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
        self.host = self.group('host')
        self.user = self.group('user')
        if not self.user:
            self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser())
        self.prefix = self.group('path')

    def md5s_to_path_infos(self, md5s):
        return [{'scheme': 'ssh',
                 'host': self.host,
                 'user': self.user,
                 'path': posixpath.join(self.prefix, md5[0:2], md5[2:])} for md5 in md5s]

    def ssh(self, host=None, user=None):
        Logger.debug("Establishing ssh connection with '{}' as user '{}'".format(host, user))

        ssh = paramiko.SSHClient()

        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh.connect(host, username=user)

        return ssh

    def _exists(self, ssh, path_info):
        try:
            ssh.open_sftp().open(path_info['path'], 'r')
            return True
        except IOError as exc:
            if exc.errno != errno.ENOENT:
                raise
        return False

    def exists(self, path_infos):
        ret = []
        ssh = self.ssh(host=self.host, user=self.user)
        stdout = self._exec(ssh, 'find {} -type f -follow -print'.format(self.prefix))
        plist = stdout.split()
        ssh.close()

        for path_info in path_infos:
            exists = False
            if path_info['path'] in plist:<commit_msg>remote: ssh: remove unused code<commit_after>import os
import errno
import getpass
import filecmp
import posixpath

try:
    import paramiko
except ImportError:
    paramiko = None

from dvc.logger import Logger
from dvc.utils import copyfile, file_md5
from dvc.progress import progress
from dvc.remote.base import RemoteBase
from dvc.config import Config
from dvc.exceptions import DvcException


def sizeof_fmt(num, suffix='B'):
    """ Convert number of bytes to human-readable string """
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)


def percent_cb(name, complete, total):
    """ Callback for updating target progress """
    Logger.debug('{}: {} transferred out of {}'.format(name,
                                                       sizeof_fmt(complete),
                                                       sizeof_fmt(total)))
    progress.update_target(name, complete, total)


def create_cb(name):
    """ Create callback function for multipart object """
    return (lambda cur, tot: percent_cb(name, cur, tot))


class RemoteSSH(RemoteBase):
    scheme = 'ssh'

    #NOTE: temporarily only absolute paths are allowed
    REGEX = r'^ssh://((?P<user>.*)@)?(?P<host>[^/]*):(?P<path>/.*)$'

    REQUIRES = {'paramiko': paramiko}
    PARAM_MD5 = 'md5'

    def __init__(self, project, config):
        self.project = project
        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
        self.host = self.group('host')
        self.user = self.group('user')
        if not self.user:
            self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser())
        self.prefix = self.group('path')

    def md5s_to_path_infos(self, md5s):
        return [{'scheme': 'ssh',
                 'host': self.host,
                 'user': self.user,
                 'path': posixpath.join(self.prefix, md5[0:2], md5[2:])} for md5 in md5s]

    def ssh(self, host=None, user=None):
        Logger.debug("Establishing ssh connection with '{}' as user '{}'".format(host, user))

        ssh = paramiko.SSHClient()

        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh.connect(host, username=user)

        return ssh

    def exists(self, path_infos):
        ret = []
        ssh = self.ssh(host=self.host, user=self.user)
        stdout = self._exec(ssh, 'find {} -type f -follow -print'.format(self.prefix))
        plist = stdout.split()
        ssh.close()

        for path_info in path_infos:
            exists = False
            if path_info['path'] in plist:
{'fraction_non_alphanumeric': 0.13626723223753975, 'fraction_numerical': 0.008837044892188053, 'mean_word_length': 2.388622754491018, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302638', 'n_tokens_mistral': 1839, 'n_tokens_neox': 1727, 'n_words': 450}