<commit_before>#!/usr/bin/env python -u

import readline
import sys
import os
import socket
import StringIO

global prompt
prompt = ' '
global waiting
waiting = True

histfile = os.path.join(os.path.expanduser("~"), ".wshist")
try:
  readline.read_history_file(histfile)
  # default history len is -1 (infinite), which may grow unruly
  readline.set_history_length(1000)
except IOError:
  pass

import atexit
atexit.register(readline.write_history_file, histfile)

global sio
sio = StringIO.StringIO()

def readUntilPrompt(s):
  global sio
  global prompt
  while True:
    buf = s.recv(8192)
    sio.write(buf)

    data = sio.getvalue()

    # Check if we have the beginning of the prompt in the buffer
    try:
      if data[0:2] == 'WS':
        idx = 0
      else:
        idx = data.index('\nWS')
    except ValueError:
      continue

    # Check if we have the end of the prompt in the buffer
    try:
      endidx = data.index('> ', idx)
    except ValueError:
      continue

    endidx = endidx + 2
    idx = idx + 1

    sys.stdout.write(data[0:idx])
    sys.stdout.flush()

    prompt = data[idx:endidx]
    data = data[endidx:]
<commit_msg>Modified to speed up display<commit_after>#!/usr/bin/env python -u

import readline
import sys
import os
import socket
import StringIO

global prompt
prompt = ' '
global waiting
waiting = True

histfile = os.path.join(os.path.expanduser("~"), ".wshist")
try:
  readline.read_history_file(histfile)
  # default history len is -1 (infinite), which may grow unruly
  readline.set_history_length(1000)
except IOError:
  pass

import atexit
atexit.register(readline.write_history_file, histfile)

global sio
sio = StringIO.StringIO()

def readUntilPrompt(s):
  global sio
  global prompt
  while True:
    buf = s.recv(8192)
    sio.write(buf)

    data = sio.getvalue()

    # Check if we have the beginning of the prompt in the buffer
    try:
      if data[0:2] == 'WS':
        idx = 0
      else:
        idx = data.index('\nWS')
    except ValueError:
      try:
        idx = data.rindex('\n')
        sys.stdout.write(data[0:idx])
        sys.stdout.flush()
        data = data[idx:]
        sio = StringIO.StringIO()
        sio.write(data)
      except ValueError:
        continue
      continue

    # Check if we have the end of the prompt in the buffer
    try:
      endidx = data.index('> ', idx)
    except ValueError:
      continue

    endidx = endidx + 2
    idx = idx + 1

    sys.stdout.write(data[0:idx])
    sys.stdout.flush()

    prompt = data[idx:endidx]
    data = data[endidx:]

{'fraction_non_alphanumeric': 0.10598357450136879, 'fraction_numerical': 0.012123582323034806, 'mean_word_length': 2.7343065693430657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '798042', 'n_tokens_mistral': 887, 'n_tokens_neox': 855, 'n_words': 247}