<commit_before># -*- coding: utf-8 -
#
# This file is part of restkit released under the MIT license. 
# See the NOTICE for more information.


"""
TeeInput replace old FileInput. It use a file 
if size > MAX_BODY or memory. It's now possible to rewind
read or restart etc ... It's based on TeeInput from Gunicorn.

"""
import os
import StringIO
import tempfile

from restkit.sock import MAX_BODY, CHUNK_SIZE, recv

class TeeInput(object):
    
    def __init__(self, socket, parser, buf, maybe_close=None):
        self.buf = buf
        self.parser = parser
        self.socket = socket
        self.maybe_close = maybe_close
        self._is_socket = True
        self._len = parser.content_len
        
        if self._len and self._len < MAX_BODY:
            self.tmp = StringIO.StringIO()
        else:
            self.tmp = tempfile.TemporaryFile()
            
        if len(buf) > 0:
            chunk, self.buf = parser.filter_body(buf)
            if chunk:
                self.tmp.write(chunk)
                self.tmp.flush()
            self._finalize()
            self.tmp.seek(0)
        
    @property
    def len(self):
        if self._len: return self._len
        
        if self._is_socket:
            pos = self.tmp.tell()
            while True:
                self.tmp.seek(self._tmp_size())
                if not self._tee(CHUNK_SIZE):
                    break
            self.tmp.seek(pos)
        self._len = self._tmp_size()
        return self._len
        
    def seek(self, offset, whence=0):
        if self._is_socket:
            while True:
                self.tmp.seek(self._tmp_size())
                if not self._tee(CHUNK_SIZE):
                    break
        self.tmp.seek(offset, whence)

    def flush(self):
        self.tmp.flush()
        
    def read(self, length=-1):
        """ read """
        if not self._is_socket:
            return self.tmp.read(length)
        
        if length < 0:
            r = self.tmp.read() or ""
            while True:
                chunk = self._tee(CHUNK_SIZE)
                if not chunk: break
                r += chunk
            return r
        else:
            diff = self._tmp_size() - self.tmp.tell()
            if not diff:
                return self._ensure_length(self._tee(length), length)
            else:
                l = min(diff, length)
                return self._ensure_length(self.tmp.read(l), length)
                
    def readline(self, size=-1):
        if not self._is_socket:
            return self.tmp.readline(size)
        
        orig_size = self._tmp_size()
        if self.tmp.tell() == orig_size:
            if not self._tee(CHUNK_SIZE):
                return ''
            self.tmp.seek(orig_size)
        
        # now we can get line
        line = self.tmp.readline()
        i = line.find("\n")
        if i == -1:
            while True:
                orig_size = self.tmp.tell()
                if not self._tee(CHUNK_SIZE):
                    break
                self.tmp.seek(orig_size)
                line = self.tmp.readline()
                i = line.find("\n")
                if i != -1: 
                    break
                    <commit_msg>fix iter<commit_after># -*- coding: utf-8 -
#
# This file is part of restkit released under the MIT license. 
# See the NOTICE for more information.


"""
TeeInput replace old FileInput. It use a file 
if size > MAX_BODY or memory. It's now possible to rewind
read or restart etc ... It's based on TeeInput from Gunicorn.

"""
import os
import StringIO
import tempfile

from restkit.sock import MAX_BODY, CHUNK_SIZE, recv

class TeeInput(object):
    
    def __init__(self, socket, parser, buf, maybe_close=None):
        self.buf = buf
        self.parser = parser
        self.socket = socket
        self.maybe_close = maybe_close
        self._is_socket = True
        self._len = parser.content_len
        
        if self._len and self._len < MAX_BODY:
            self.tmp = StringIO.StringIO()
        else:
            self.tmp = tempfile.TemporaryFile()
            
        if len(buf) > 0:
            chunk, self.buf = parser.filter_body(buf)
            if chunk:
                self.tmp.write(chunk)
                self.tmp.flush()
            self._finalize()
            self.tmp.seek(0)
        
    @property
    def len(self):
        if self._len: return self._len
        
        if self._is_socket:
            pos = self.tmp.tell()
            while True:
                self.tmp.seek(self._tmp_size())
                if not self._tee(CHUNK_SIZE):
                    break
            self.tmp.seek(pos)
        self._len = self._tmp_size()
        return self._len
        
    def seek(self, offset, whence=0):
        if self._is_socket:
            while True:
                self.tmp.seek(self._tmp_size())
                if not self._tee(CHUNK_SIZE):
                    break
        self.tmp.seek(offset, whence)

    def flush(self):
        self.tmp.flush()
        
    def read(self, length=-1):
        """ read """
        if not self._is_socket:
            return self.tmp.read(length)
        
        if length < 0:
            r = self.tmp.read() or ""
            while True:
                chunk = self._tee(CHUNK_SIZE)
                if not chunk: break
                r += chunk
            return r
        else:
            diff = self._tmp_size() - self.tmp.tell()
            if not diff:
                return self._ensure_length(self._tee(length), length)
            else:
                l = min(diff, length)
                return self._ensure_length(self.tmp.read(l), length)
                
    def readline(self, size=-1):
        if not self._is_socket:
            return self.tmp.readline(size)
        
        orig_size = self._tmp_size()
        if self.tmp.tell() == orig_size:
            if not self._tee(CHUNK_SIZE):
                return ''
            self.tmp.seek(orig_size)
        
        # now we can get line
        line = self.tmp.readline()
        i = line.find("\n")
        if i == -1:
            while True:
                orig_size = self.tmp.tell()
                if not self._tee(CHUNK_SIZE):
                    break
                self.tmp.seek(orig_size)
                line += self.tmp.readline()
                i = line.find("\n")
                if i != -1: 
                    break
                    
{'fraction_non_alphanumeric': 0.09689922480620156, 'fraction_numerical': 0.0027906976744186047, 'mean_word_length': 1.3155061019382628, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2587936', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1819, 'n_words': 469}