<commit_before>from __future__ import with_statement

import hashlib
import os
import stat
import tempfile
from fnmatch import filter as fnfilter

from fabric.state import output, connections, env
from fabric.utils import warn


class SFTP(object):
    """
    SFTP helper class, which is also a facade for paramiko.SFTPClient.
    """
    def __init__(self, host_string):
        self.ftp = connections[host_string].open_sftp()


    # Recall that __getattr__ is the "fallback" attribute getter, and is thus
    # pretty safe to use for facade-like behavior as we're doing here.
    def __getattr__(self, attr):
        return getattr(self.ftp, attr)


    def isdir(self, path):
        try:
            return stat.S_ISDIR(self.ftp.lstat(path).st_mode)
        except IOError:
            return False


    def islink(self, path):
        try:
            return stat.S_ISLNK(self.ftp.lstat(path).st_mode)
        except IOError:
            return False


    def exists(self, path):
        try:
            self.ftp.lstat(path).st_mode
        except IOError:
            return False
        return True


    def glob(self, path):
        dirpart, pattern = os.path.split(path)
        rlist = self.ftp.listdir(dirpart)

        names = fnfilter([f for f in rlist if not f[0] == '.'], pattern)
        if len(names):
            return [os.path.join(dirpart, name) for name in names]
        else:
            return [path]


    def walk(self, top, topdown=True, onerror=None, followlinks=False):
        from os.path import join, isdir, islink

        # We may not have read permission for top, in which case we can't get a
        # list of the files the directory contains. os.path.walk always
        # suppressed the exception then, rather than blow up for a minor reason
        # when (say) a thousand readable directories are still left to visit.
        # That logic is copied here.
        try:
            # Note that listdir and error are globals in this module due to
            # earlier import-*.
            names = self.ftp.listdir(top)
        except Exception, err:
            if onerror is not None:
                onerror(err)
            return

        dirs, nondirs = [], []
        for name in names:
            if self.isdir(join(top, name)):
                dirs.append(name)
            else:
                nondirs.append(name)

        if topdown:
            yield top, dirs, nondirs

        for name in dirs:
            path = join(top, name)
            if followlinks or not self.islink(path):
                for x in self.walk(path, topdown, onerror, followlinks):
                    yield x
        if not topdown:
            yield top, dirs, nondirs


    def mkdir(self, path, use_sudo):
        from fabric.api import sudo, hide
        if use_sudo:
            with hide('everything'):
                sudo('mkdir %s' % path)
        else:
            self.ftp.mkdir(path)


    def get(self, remote_path, local_path, local_is_path):
        # Handle format string interpolation (e.g. %(dirname)s)
        path_vars = {
            'host': env.host_string.replace(':', '-'),
            'basename': os.path.basename(remote_path),
            'dirname': os.path.dirname(remote_path),
            'path': remote_path
        }
        if local_is_path:
            # Interpolate, then abspath (to make sure any /// are compressed)
            local_path = os.path.abspath(local_path % path_vars)
            # Ensure we give Paramiko a file by prepending and/or creating
            # local directories as appropriate.
            dirpath, filepath = os.path.split(local_path)
            if dirpath and not os.path.exists(dirpath):
                os.makedirs(dirpath)
            if os.path.isdir(local_path):
                local_path = os.path.join(local_path, path_vars['basename'])
        if output.running:
            print("[%s] download: %s <- %s" % (
                env.host_string,
                local_path if local_is_path else "<file obj>",
                remote_path
            ))
        # Warn about overwrites, but keep going
        if local_is_path and os.path.exists(local_path):
            msg = "Local file %s already exists and is being overwritten."
            warn(msg % local_path)
        # Have to bounce off FS if doing file-like objects
        fd, real_local_path = None, local_path
        if not local_is_path:
            fd, real_local_path = tempfile.mkstemp()
        self.ftp.get(remote_path, real_local_path)
        # Return file contents (if it needs stuffing into a file-like obj)
        # or the final local file path (otherwise)
        result = None
        if not local_is_path:
            file_obj = os.fdopen(fd)
            result = file_obj.read()
            # Clean up temporary file
            file_obj.close()
            os.remove(real_local_path)
        else:
            result = real_local_path
        return result


    def get_dir(self, remote_path, local_path):
        if os.path.basename(remote_path):
            strip = os.path.dirname(remote_path)
        else:
            strip = os.path.dirname(os.path.dirname(remote_path))

        result = []
        for context, dirs, files in self.walk(remote_path):
            lcontext = context.replace(strip,'')
            lcontext = lcontext.lstrip('/')
            lcontext = os.path.join(local_path, lcontext)

            if not os.path.exists(lcontext):
                os.mkdir(lcontext)
            for d in dirs:
                n = os.path.join(lcontext, d)
                if not os.path.exists(n):
                    os.mkdir(n)
            for f in files:
                remote_path = os.path.join(context, f)
                n = os.path.join(lcontext, f)
                result.append(self.get(remote_path, n, True))
        return result

<commit_msg>Update get_dir to work with format strings<commit_after>from __future__ import with_statement

import hashlib
import os
import stat
import tempfile
from fnmatch import filter as fnfilter

from fabric.state import output, connections, env
from fabric.utils import warn


class SFTP(object):
    """
    SFTP helper class, which is also a facade for paramiko.SFTPClient.
    """
    def __init__(self, host_string):
        self.ftp = connections[host_string].open_sftp()


    # Recall that __getattr__ is the "fallback" attribute getter, and is thus
    # pretty safe to use for facade-like behavior as we're doing here.
    def __getattr__(self, attr):
        return getattr(self.ftp, attr)


    def isdir(self, path):
        try:
            return stat.S_ISDIR(self.ftp.lstat(path).st_mode)
        except IOError:
            return False


    def islink(self, path):
        try:
            return stat.S_ISLNK(self.ftp.lstat(path).st_mode)
        except IOError:
            return False


    def exists(self, path):
        try:
            self.ftp.lstat(path).st_mode
        except IOError:
            return False
        return True


    def glob(self, path):
        dirpart, pattern = os.path.split(path)
        rlist = self.ftp.listdir(dirpart)

        names = fnfilter([f for f in rlist if not f[0] == '.'], pattern)
        if len(names):
            return [os.path.join(dirpart, name) for name in names]
        else:
            return [path]


    def walk(self, top, topdown=True, onerror=None, followlinks=False):
        from os.path import join, isdir, islink

        # We may not have read permission for top, in which case we can't get a
        # list of the files the directory contains. os.path.walk always
        # suppressed the exception then, rather than blow up for a minor reason
        # when (say) a thousand readable directories are still left to visit.
        # That logic is copied here.
        try:
            # Note that listdir and error are globals in this module due to
            # earlier import-*.
            names = self.ftp.listdir(top)
        except Exception, err:
            if onerror is not None:
                onerror(err)
            return

        dirs, nondirs = [], []
        for name in names:
            if self.isdir(join(top, name)):
                dirs.append(name)
            else:
                nondirs.append(name)

        if topdown:
            yield top, dirs, nondirs

        for name in dirs:
            path = join(top, name)
            if followlinks or not self.islink(path):
                for x in self.walk(path, topdown, onerror, followlinks):
                    yield x
        if not topdown:
            yield top, dirs, nondirs


    def mkdir(self, path, use_sudo):
        from fabric.api import sudo, hide
        if use_sudo:
            with hide('everything'):
                sudo('mkdir %s' % path)
        else:
            self.ftp.mkdir(path)


    def get(self, remote_path, local_path, local_is_path):
        # Handle format string interpolation (e.g. %(dirname)s)
        path_vars = {
            'host': env.host_string.replace(':', '-'),
            'basename': os.path.basename(remote_path),
            'dirname': os.path.dirname(remote_path),
            'path': remote_path
        }
        if local_is_path:
            # Interpolate, then abspath (to make sure any /// are compressed)
            local_path = os.path.abspath(local_path % path_vars)
            # Ensure we give Paramiko a file by prepending and/or creating
            # local directories as appropriate.
            dirpath, filepath = os.path.split(local_path)
            if dirpath and not os.path.exists(dirpath):
                os.makedirs(dirpath)
            if os.path.isdir(local_path):
                local_path = os.path.join(local_path, path_vars['basename'])
        if output.running:
            print("[%s] download: %s <- %s" % (
                env.host_string,
                local_path if local_is_path else "<file obj>",
                remote_path
            ))
        # Warn about overwrites, but keep going
        if local_is_path and os.path.exists(local_path):
            msg = "Local file %s already exists and is being overwritten."
            warn(msg % local_path)
        # Have to bounce off FS if doing file-like objects
        fd, real_local_path = None, local_path
        if not local_is_path:
            fd, real_local_path = tempfile.mkstemp()
        self.ftp.get(remote_path, real_local_path)
        # Return file contents (if it needs stuffing into a file-like obj)
        # or the final local file path (otherwise)
        result = None
        if not local_is_path:
            file_obj = os.fdopen(fd)
            result = file_obj.read()
            # Clean up temporary file
            file_obj.close()
            os.remove(real_local_path)
        else:
            result = real_local_path
        return result


    def get_dir(self, remote_path, local_path):
        # Decide what needs to be stripped from remote paths so they're all
        # relative to the given remote_path
        if os.path.basename(remote_path):
            strip = os.path.dirname(remote_path)
        else:
            strip = os.path.dirname(os.path.dirname(remote_path))

        # Store all paths gotten so we can return them when done
        result = []
        # Use our facsimile of os.walk to find all files within remote_path
        for context, dirs, files in self.walk(remote_path):
            # Normalize current directory to be relative
            # E.g. remote_path of /var/log and current dir of /var/log/apache2
            # would be turned into just 'apache2'
            lcontext = context.replace(strip, '').lstrip('/')
            # Prepend local path to that to arrive at the local mirrored
            # version of this directory. So if local_path was 'mylogs', we'd
            # end up with 'mylogs/apache2'
            lcontext = os.path.join(local_path, lcontext)

            # Download any files in current directory
            for f in files:
                # Construct full remote path to this file
                remote_path = os.path.join(context, f)
                # If local_path isn't using a format string that expands to
                # include its remote path, we need to add it here.
                if "%(path)s" not in local_path \
                    and "%(dirname)s" not in local_path:
                    lpath = os.path.join(lcontext, f)
                # Otherwise, just passthrough local_path to self.get()
                else:
                    lpath = local_path
                # Now we can make a call to self.get() with specific file paths
                # on both ends.
                result.append(self.get(remote_path, lpath, True))
        return result


{'fraction_non_alphanumeric': 0.08375853677682707, 'fraction_numerical': 0.0003924954863019075, 'mean_word_length': 1.8105007721155968, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3340212', 'n_tokens_mistral': 3683, 'n_tokens_neox': 3528, 'n_words': 1144}