<commit_before>from go import bufio, bytes, io
from go import net, sync, time
from go import crypto/rand

from . import dh, sym, keyring

SerialPrefix = mkbyt(12)  # Per Process nonce.
rand.Read(SerialPrefix)

SerialMutex = go_new(sync.Mutex)
SerialCounter = 100
def Serial():
  global SerialCounter
  SerialMutex.Lock()
  with defer SerialMutex.Unlock():
    SerialCounter += 1
    return (SerialPrefix, SerialCounter)

def AtMost(n, s):
  s = str(s)
  return s if len(s) < n else s[:n]

def ArgsSummary(args):
  return str([AtMost(80, repr(x)) for x in args])

class Request:
  def __init__(proc, args):
    .proc = proc
    .args = args
    .serial = None
    .replyQ = rye_chan(1)

CHUNK_MAGIC = 191
SHAKE_MAGIC = 222

def WriteChunk(w, data):
  n = len(data)
  #say w, n
  head = byt([CHUNK_MAGIC, n>>24, n>>16, n>>8, n])
  buf = bytes.NewBuffer(head)
  buf.Write(data)
  io.Copy(w, buf)

def ReadChunk(r):
  head = mkbyt(5)
  #say 'read head'
  io.ReadFull(r, head)
  must head[0] == CHUNK_MAGIC
  n = (head[1]<<24) | (head[2]<<16) | (head[3]<<8) | head[4]
  must n < (2 << 20)  # 2 Meg Max<commit_msg>raise limit 2Meg to 100Meg (temporarily).<commit_after>from go import bufio, bytes, io
from go import net, sync, time
from go import crypto/rand

from . import dh, sym, keyring

SerialPrefix = mkbyt(12)  # Per Process nonce.
rand.Read(SerialPrefix)

SerialMutex = go_new(sync.Mutex)
SerialCounter = 100
def Serial():
  global SerialCounter
  SerialMutex.Lock()
  with defer SerialMutex.Unlock():
    SerialCounter += 1
    return (SerialPrefix, SerialCounter)

def AtMost(n, s):
  s = str(s)
  return s if len(s) < n else s[:n] + '...'

def ArgsSummary(args):
  return str([AtMost(80, repr(x)) for x in args])

class Request:
  def __init__(proc, args):
    .proc = proc
    .args = args
    .serial = None
    .replyQ = rye_chan(1)

CHUNK_MAGIC = 191
SHAKE_MAGIC = 222

def WriteChunk(w, data):
  n = len(data)
  #say w, n
  head = byt([CHUNK_MAGIC, n>>24, n>>16, n>>8, n])
  buf = bytes.NewBuffer(head)
  buf.Write(data)
  io.Copy(w, buf)

def ReadChunk(r):
  head = mkbyt(5)
  #say 'read head'
  io.ReadFull(r, head)
  must head[0] == CHUNK_MAGIC
  n = (head[1]<<24) | (head[2]<<16) | (head[3]<<8) | head[4]
  must n < (100 << 20)  # 100 Meg Max
{'fraction_non_alphanumeric': 0.14849290780141844, 'fraction_numerical': 0.034574468085106384, 'mean_word_length': 3.625, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1227775', 'n_tokens_mistral': 937, 'n_tokens_neox': 900, 'n_words': 254}