<commit_before>from deploy import lazy_file, pipe
import ftplib
import os
import paramiko
import tarfile
import urllib
import urlparse
import sh
import socket

class location( object ):
    def __init__( self, url ):
        if isinstance( url, basestring ):
            url = urlparse.urlparse( url )
        self.url = url

    def source( self ):
        pass

    def destination( self, files ):
        pass

class local( location ):
    def source( self, recursive = False ):
        for fpath in os.listdir( self.url.path ):
            fpath = os.path.relpath( os.path.join( self.url.path, fpath ) )
            if os.path.isfile( fpath ):
                yield lazy_file( fpath )
            elif os.path.isdir( fpath ) and recursive:
                for f in local( fpath ).source( True ):
                    yield f

    def destination( self, files ):
        for f in files:
            dfn = os.path.join( self.url.path, f.name )

            # Ensure dest paths exist
            dpath = os.path.dirname( dfn )
            if not os.path.exists( dpath ):
                os.makedirs( dpath )

            if not f.delete:
                print 'local:%s' % dfn
                open( dfn, 'wb' ).write( f.read() )
            else:
                print 'local DELETE', dfn
                try:
                    os.remove( dfn )
                except OSError:
                    pass

class remote( location ):
    def __init__( self, url ):
        super( remote, self ).__init__( url )

        self.con = None

    def connect( self ):
        pass
<commit_msg>local loc recurse with base_path<commit_after>from deploy import lazy_file, pipe
import ftplib
import os
import paramiko
import tarfile
import urllib
import urlparse
import sh
import socket

class location( object ):
    def __init__( self, url ):
        if isinstance( url, basestring ):
            url = urlparse.urlparse( url )
        self.url = url

    def source( self ):
        pass

    def destination( self, files ):
        pass

class local( location ):
    def source( self, base_path = None, recursive = False ):
        if not base_path:
            base_path = self.url.path

        for fpath in os.listdir( base_path ):
            fpath = os.path.join( base_path, fpath )

            if os.path.isfile( fpath ):
                yield lazy_file( fpath )
            elif os.path.isdir( fpath ) and recursive:
                for f in local( fpath ).source( fpath, True ):
                    yield f

    def destination( self, files ):
        for f in files:
            dfn = os.path.join( self.url.path, f.name )

            # Ensure dest paths exist
            dpath = os.path.dirname( dfn )
            if not os.path.exists( dpath ):
                os.makedirs( dpath )

            if not f.delete:
                print 'local:%s' % dfn
                open( dfn, 'wb' ).write( f.read() )
            else:
                print 'local DELETE', dfn
                try:
                    os.remove( dfn )
                except OSError:
                    pass

class remote( location ):
    def __init__( self, url ):
        super( remote, self ).__init__( url )

        self.con = None

    def connect( self ):
        pass

{'fraction_non_alphanumeric': 0.08889572439249462, 'fraction_numerical': 0.0, 'mean_word_length': 1.5034642032332564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '447571', 'n_tokens_mistral': 948, 'n_tokens_neox': 905, 'n_words': 309}