<commit_before>import os.path
import StringIO
import sh
import sys

def debug_dest( files ):
    for f in files:
        print f.name
        if hasattr( f, 'mode' ):
            print 'Mode: ', f.mode
        print '-' * len( f.name )
        print f.read()
        print

class local_dest( object ):
    def __init__( self, path ):
        self.path = path

    def __call__( self, files ):
        for f in files:
            dfn = os.path.join( self.path, f.name )

            print 'Deploying %s as %s...' % ( f.name, dfn ),
            open( dfn, 'wb' ).write( f.read() )
            print 'Done'

class lazy_file( object ):
    def __init__( self, name, *a, **kw ):
        self.name = name
        self.file = None
        self.a = a
        self.kw = kw

    def open( self ):
        self.file = open( self.name, *self.a, **self.kw )

    def read( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.read( *a, **kw )

    def write( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.write( *a, **kw )

    def seek( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.seek( *a, **kw )

    def tell( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.tell( *a, **kw )

class queue_file( object ):
    def __init__( self, name, *a, **kw ):
        self.name = name
        self.q = []
        self.pos = 0

    def write( self, data ):
        self.q.append( data )
        self.pos += len( data )

    def read( self ):
        if len( self.q ):
            return self.q.pop( 0 )

    def tell( self ):
        return self.pos

class ftp_dest( object ):
    def __init__( self, url ):
        self.url = url
        self.con = None

    def __call__( files ):
        for f in files:
            if not self.con:
                self.connect()

            if f.delete:
                self.rm( f )
            else:
                self.put( f )

def test_splitter( files ):
    for f in files:
        if f.name in [ __file__, 'targets.py' ] or f.name.endswith( '.pyc' ):
            pass
        elif f.name.endswith( '.txt' ):
            yield uppercase( f )
        elif f.name.endswith( '.less' ):
            yield lessc( f, f.name.replace( '.less', '.css' ) )
        else:
            yield f

def dest_select( files, targets ):
    if len( sys.argv ) > 1:
        return getattr( targets, sys.argv[1] )( files )
    else:
        return targets.default( files )

def uppercase( f ):
    pipe = StringIO.StringIO()
    # todo: run in thread to allow concurrency
    pipe.write( f.read().upper() )
    pipe.seek( 0 )
    pipe.name = f.name
    return pipe

def lessc( f, d ):
    pipe = StringIO.StringIO()
    pipe.name = d
    sh.lessc( '-', _in = f, _out = pipe )
    pipe.seek( 0 )
    return pipe

def dirlist_source():
    for f in os.listdir( '.' ):
        if os.path.isfile( f ):
            yield open( f )
<commit_msg>recursive dirlist src<commit_after>import os.path
import StringIO
import sh
import sys

def debug_dest( files ):
    for f in files:
        print f.name
        if hasattr( f, 'mode' ):
            print 'Mode: ', f.mode
        print '-' * len( f.name )
        print f.read()
        print

class local_dest( object ):
    def __init__( self, path ):
        self.path = path

    def __call__( self, files ):
        for f in files:
            dfn = os.path.join( self.path, f.name )

            print 'Deploying %s as %s...' % ( f.name, dfn ),
            open( dfn, 'wb' ).write( f.read() )
            print 'Done'

class lazy_file( object ):
    def __init__( self, name, *a, **kw ):
        self.name = name
        self.file = None
        self.a = a
        self.kw = kw

    def open( self ):
        self.file = open( self.name, *self.a, **self.kw )

    def read( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.read( *a, **kw )

    def write( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.write( *a, **kw )

    def seek( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.seek( *a, **kw )

    def tell( self, *a, **kw ):
        if not self.file:
            self.open()

        return self.file.tell( *a, **kw )

class queue_file( object ):
    def __init__( self, name, *a, **kw ):
        self.name = name
        self.q = []
        self.pos = 0

    def write( self, data ):
        self.q.append( data )
        self.pos += len( data )

    def read( self ):
        if len( self.q ):
            return self.q.pop( 0 )

    def tell( self ):
        return self.pos

class ftp_dest( object ):
    def __init__( self, url ):
        self.url = url
        self.con = None

    def __call__( files ):
        for f in files:
            if not self.con:
                self.connect()

            if f.delete:
                self.rm( f )
            else:
                self.put( f )

def test_splitter( files ):
    for f in files:
        if f.name in [ __file__, 'targets.py' ] or f.name.endswith( '.pyc' ):
            pass
        elif f.name.endswith( '.txt' ):
            yield uppercase( f )
        elif f.name.endswith( '.less' ):
            yield lessc( f, f.name.replace( '.less', '.css' ) )
        else:
            yield f

def dest_select( files, targets ):
    if len( sys.argv ) > 1:
        return getattr( targets, sys.argv[1] )( files )
    else:
        return targets.default( files )

def uppercase( f ):
    pipe = StringIO.StringIO()
    # todo: run in thread to allow concurrency
    pipe.write( f.read().upper() )
    pipe.seek( 0 )
    pipe.name = f.name
    return pipe

def lessc( f, d ):
    pipe = StringIO.StringIO()
    pipe.name = d
    sh.lessc( '-', _in = f, _out = pipe )
    pipe.seek( 0 )
    return pipe

def dirlist_source():
    def _rec_dir_src( path ):
        for fpath in os.listdir( path ):
            fpath = os.path.relpath( os.path.join( path, fpath ) )
            if os.path.isfile( fpath ):
                yield open( fpath )
            elif os.path.isdir( fpath ):
                print 'dir', fpath
                for f in _rec_dir_src( fpath ):
                    yield f

    return _rec_dir_src( '.' )

{'fraction_non_alphanumeric': 0.13337527524378737, 'fraction_numerical': 0.0018873859704309531, 'mean_word_length': 1.5204122076892588, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '160046', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2066, 'n_words': 642}