<commit_before># $Id$


class FakeInput:
  name = "<fake input stream>"

  def __init__(self):
    self.closed = 0
    self.mode = "rb"
    self.softspace = 0
    self._buf = ""
    self.chunksize = 4096

  def _read(self, size):
    return ""

  def close(self):
    self.closed = 1

  def _check_open(self):
    if self.closed:
      raise IOError, "File is closed"

  def flush(self):
    self._check_open()

  def _read_more(self, size=-1):
    d = self._read(size)
    self._buf += d
    return len(d)

  def read(self, size=-1):
    self._check_open()
    if size < 0:
      while self._read_more():
        pass
      r = self._buf
      self._buf = ""
      return r
    while len(self._buf) < size and self._read_more(self.chunksize):
      pass
    if len(self._buf) == size:
      r = self._buf
      self._buf = ""
      return r
    r = self._buf[:size]
    self._buf = self._buf[size:]
    return r

  def readline(self, size=-1):
    self._check_open()
    start = 0
    while 1:
      if size < 0:
        pos = self._buf.find("\n", start)
      else:
        pos = self._buf.find("\n", start, size)
      start = len(self._buf)
      if pos >= 0:
        return self.read(pos + 1)
      if size >= 0 and len(self._buf) >= size:
        return self.read(size)
      if not self._read_more():
        return self.read(-1)

  def readlines(self, size=-1):
    self._check_open()
    lines = []
    while 1:
      line = self.readline()
      if line == "":
        return lines
      lines.append(line)
      if size >= 0:
        size -= len(line)
        if size <= 0:
          return lines

  def xreadlines(self):
    self._check_open()
    import xreadlines
    return xreadlines.xreadlines(self)

  def seek(self, offset, whence=0):
    raise IOError, "cannot seek()"

  def tell(self):<commit_msg>readlines should _read in chunksize blocks<commit_after># $Id$


class FakeInput:
  name = "<fake input stream>"

  def __init__(self):
    self.closed = 0
    self.mode = "rb"
    self.softspace = 0
    self._buf = ""
    self.chunksize = 4096

  def _read(self, size):
    return ""

  def close(self):
    self.closed = 1

  def _check_open(self):
    if self.closed:
      raise IOError, "File is closed"

  def flush(self):
    self._check_open()

  def _read_more(self, size=-1):
    d = self._read(size)
    self._buf += d
    return len(d)

  def read(self, size=-1):
    self._check_open()
    if size < 0:
      while self._read_more():
        pass
      r = self._buf
      self._buf = ""
      return r
    while len(self._buf) < size and self._read_more(self.chunksize):
      pass
    if len(self._buf) == size:
      r = self._buf
      self._buf = ""
      return r
    r = self._buf[:size]
    self._buf = self._buf[size:]
    return r

  def readline(self, size=-1):
    self._check_open()
    start = 0
    while 1:
      if size < 0:
        pos = self._buf.find("\n", start)
      else:
        pos = self._buf.find("\n", start, size)
      start = len(self._buf)
      if pos >= 0:
        return self.read(pos + 1)
      if size >= 0 and len(self._buf) >= size:
        return self.read(size)
      if not self._read_more(self.chunksize):
        return self.read()

  def readlines(self, size=-1):
    self._check_open()
    lines = []
    while 1:
      line = self.readline()
      if line == "":
        return lines
      lines.append(line)
      if size >= 0:
        size -= len(line)
        if size <= 0:
          return lines

  def xreadlines(self):
    self._check_open()
    import xreadlines
    return xreadlines.xreadlines(self)

  def seek(self, offset, whence=0):
    raise IOError, "cannot seek()"

  def tell(self):
{'fraction_non_alphanumeric': 0.1321011144332699, 'fraction_numerical': 0.012231584669747214, 'mean_word_length': 2.0463576158940397, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2681229', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1248, 'n_words': 334}