<commit_before>import subprocess
import re
import collections


def sh(cmd, shell=True):
    return Sh(None, cmd, shell)

class Stream(object):
    _stream_classes = {}
    
    def __init__(self, stream):
        if not self._stream_classes:
            self.__reg_class(Stream)

        self._stream = stream
        print stream.__class__.__name__, 'stream'
        
        self.__is_end = False
        
    def __reg_class(self, cls):
        self._stream_classes[cls.__name__] = cls
        for subclass in cls.__subclasses__():
            self.__reg_class(subclass)

    def __getattr__(self, name):
        cap_name = name.capitalize()
        if cap_name not in self._stream_classes:
            # try to update the inheritance hierarchy 
            self.__reg_class(Stream)
        
        if cap_name in self._stream_classes:
            cls = self._stream_classes[cap_name]
            return lambda *args, **kwargs: cls(self, *args, **kwargs)
        else:
            return None

    
    def __iter__(self):
        for elem in self._do_iter():
            yield elem
            if self.__is_end:
                break
        
        self._end_iter()
    
    def _do_iter(self):
        raise NotImplemented
    
    def _end_iter(self):
        if not self.__is_end:
            self._do_end_iter()
            print self.__class__.__name__, 'end iter'
            if self._stream:
                print 'call instream _end_iter' 
                self._stream._end_iter()
            self.__is_end = True
    
    def _do_end_iter(self):
        pass

class Sh(Stream):
    ''' Run a shell program
    '''

    def __init__(self, stream, cmd, shell=True):
        Stream.__init__(self, stream)
        if shell:
            if not isinstance(cmd, str):
                cmd = ' '.join(cmd)
        else:
            if isinstance(cmd, str):
                cmd = cmd.split()

        self.__cmd = cmd
        stdin_pipe = subprocess.PIPE if stream else None
        self.__p = subprocess.Popen(cmd, shell=shell, 
                                    stdin=stdin_pipe, stdout=subprocess.PIPE)
        print 'hello'
        
    def get_process(self):
        return self.__p
    
    def _do_iter(self):
        if self._stream:
            for line in self._stream:
                print 'in sh:', line
                self.__p.stdin.write(line)
            self.__p.stdin.close()
        
        for line in self.__p.stdout.xreadlines():
            yield line
    
    def _do_end_iter(self):
        self.__p.stdout.close()
        self.__p.wait()
        print self.__cmd, 'process done'

class Grep(Stream):
    def __init__(self, stream, pattern):
        Stream.__init__(self, stream)
        self.__regex = re.compile(pattern)
    
    def _do_iter(self):
        for elem in self._stream:
            if self.__regex.search(elem):
                yield elem

class Col(Stream):
    def __init__(self, stream, column, sep=None, newline=True):
        Stream.__init__(self, stream)
        self.__column = column
        self.__sep = sep
        self.__line_end = '\n' if newline else ''
    <commit_msg>Fix the bug in Sh input handling.<commit_after>import subprocess
import re
import collections


def sh(cmd, shell=True):
    return Sh(None, cmd, shell)

class Stream(object):
    _stream_classes = {}
    
    def __init__(self, stream):
        if not self._stream_classes:
            self.__reg_class(Stream)

        self._stream = stream
        
        self.__is_end = False
        
    def __reg_class(self, cls):
        self._stream_classes[cls.__name__] = cls
        for subclass in cls.__subclasses__():
            self.__reg_class(subclass)

    def __getattr__(self, name):
        cap_name = name.capitalize()
        if cap_name not in self._stream_classes:
            # try to update the inheritance hierarchy 
            self.__reg_class(Stream)
        
        if cap_name in self._stream_classes:
            cls = self._stream_classes[cap_name]
            return lambda *args, **kwargs: cls(self, *args, **kwargs)
        else:
            return None

    
    def __iter__(self):
        for elem in self._do_iter():
            yield elem
            if self.__is_end:
                break
        
        self._end_iter()
    
    def _do_iter(self):
        raise NotImplemented
    
    def _end_iter(self):
        if not self.__is_end:
            self._do_end_iter()
            if self._stream:
                self._stream._end_iter()
            self.__is_end = True
    
    def _do_end_iter(self):
        pass

class Sh(Stream):
    ''' Run a shell program
    '''

    def __init__(self, stream, cmd, shell=True):
        Stream.__init__(self, stream)
        if shell:
            if not isinstance(cmd, str):
                cmd = ' '.join(cmd)
        else:
            if isinstance(cmd, str):
                cmd = cmd.split()

        self.__cmd = cmd
        if stream:
            if isinstance(stream, Sh):
                stdin_pipe = stream.get_process().stdout
            else:
                stdin_pipe = subprocess.PIPE
        else:
            stdin_pipe = None
        self.__p = subprocess.Popen(cmd, shell=shell, 
                                    stdin=stdin_pipe, stdout=subprocess.PIPE)
        
    def get_process(self):
        return self.__p
    
    def _do_iter(self):
        if self._stream and not isinstance(self._stream, Sh):
            for line in self._stream:
                self.__p.stdin.write(line)
            self.__p.stdin.close()
        
        for line in self.__p.stdout.xreadlines():
            yield line
    
    def _do_end_iter(self):
        self.__p.stdout.close()
        self.__p.wait()

class Grep(Stream):
    def __init__(self, stream, pattern):
        Stream.__init__(self, stream)
        self.__regex = re.compile(pattern)
    
    def _do_iter(self):
        for elem in self._stream:
            if self.__regex.search(elem):
                yield elem

class Col(Stream):
    def __init__(self, stream, column, sep=None, newline=True):
        Stream.__init__(self, stream)
        self.__column = column
        self.__sep = sep
        self.__line_end = '\n' if newline else ''
    
{'fraction_non_alphanumeric': 0.0907196666132393, 'fraction_numerical': 0.0, 'mean_word_length': 1.6723768736616702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1390729', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1782, 'n_words': 444}