<commit_before>import subprocess
import sys



#-- message codes for the down protocol
START_MESSAGE=0
SET_JOB_CONF=1
SET_INPUT_TYPES=2
RUN_MAP=3
MAP_ITEM=4
RUN_REDUCE=5
REDUCE_KEY=6
REDUCE_VALUE=7
CLOSE=8
ABORT=9

#-- message codes for the up protocol
OUTPUT=50
PARTITIONED_OUTPUT=51
STATUS=52
PROGRESS=53
DONE=54
REGISTER_COUNTER=55
INCREMENT_COUNTER=56

def _true_false_int(v):
  return 1 if v else 0

import pydoop_pipes as pp

def serialize(t):
  tt = type(t)
  if tt == int:
    return pp.serialize_int(t)
  if tt == float:
    return pp.serialize_float(t)
  if tt == str:
    return pp.serialize_string(t)

class binary_down_protocol(object):
  def __init__(self, pipes_program, out_file=None):
    self.out_file = out_file
    self.pipes_program = pipes_program
    self.fd = open(self.out_file, "w")
    self.proc = subprocess.Popen([],
                                 executable=self.pipes_program,
                                 bufsize=0,
                                 stdin=subprocess.PIPE,
                                 stdout=self.fd)
  def __send(self, args):
    for v in args:
      self.proc.stdin.write(serialize(v))
      sys.stderr.write('ready to send: <%s>\n' % v)
  #--
  def start(self):
    self.__send([START_MESSAGE,  0])<commit_msg>lib::pydoop saved minor change in pydoop/binary_protocol.py<commit_after>import subprocess
import sys


#-- message codes for the down protocol
START_MESSAGE=0
SET_JOB_CONF=1
SET_INPUT_TYPES=2
RUN_MAP=3
MAP_ITEM=4
RUN_REDUCE=5
REDUCE_KEY=6
REDUCE_VALUE=7
CLOSE=8
ABORT=9

#-- message codes for the up protocol
OUTPUT=50
PARTITIONED_OUTPUT=51
STATUS=52
PROGRESS=53
DONE=54
REGISTER_COUNTER=55
INCREMENT_COUNTER=56

def _true_false_int(v):
  return 1 if v else 0

import pydoop_pipes as pp

def serialize(t):
  tt = type(t)
  if tt == int:
    return pp.serialize_int(t)
  if tt == float:
    return pp.serialize_float(t)
  if tt == str:
    return pp.serialize_string(t)

class binary_down_protocol(object):
  def __init__(self, pipes_program, out_file=None):
    self.out_file = out_file
    self.pipes_program = pipes_program
    self.fd = open(self.out_file, "w")
    self.proc = subprocess.Popen([],
                                 executable=self.pipes_program,
                                 bufsize=0,
                                 stdin=subprocess.PIPE,
                                 stdout=self.fd)
    #--
    open_server_socket()
    start_process_in_background()
    open_socket()
    #--
  def __send(self, args):
    for v in args:
      self.proc.stdin.write(serialize(v))
      sys.stderr.write('ready to send: <%s>\n' % v)
  #--
  def start(self):
    self.__send([START_MESSAGE,  0])
{'fraction_non_alphanumeric': 0.10149253731343283, 'fraction_numerical': 0.020895522388059702, 'mean_word_length': 2.7132963988919667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2101013', 'n_tokens_mistral': 981, 'n_tokens_neox': 955, 'n_words': 207}