<commit_before>/* v/term.c
**
** This file is in the public domain.
*/
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <setjmp.h>
#include <gmp.h>
#include <stdint.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <uv.h>
#include <errno.h>
#include <curses.h>
#include <termios.h>
#include <term.h>
#include <anachronism/common.h>
#include <anachronism/nvt.h>
#include "all.h"
#include "v/vere.h"

static void _term_read_tn_cb(uv_stream_t*, ssize_t, uv_buf_t);
static void _term_read_cb(uv_stream_t*, ssize_t, uv_buf_t);
static void _term_suck(u2_utty*, const c3_y*, ssize_t);
static void _tel_event(telnet_nvt*, telnet_event*);
static void _tel_opt(telnet_nvt*, telnet_byte, telnet_telopt_event*);

#define _T_ECHO 1    //  local echo
#define _T_CTIM 3    //  suppress GA/char-at-a-time
#define _T_NAWS 31   //  negotiate about window size

/* _term_alloc(): libuv buffer allocator.
*/
static uv_buf_t
_term_alloc(uv_handle_t* had_u, size_t len_i)
{
  return uv_buf_init(c3_malloc(len_i), len_i);
}

/* _term_close_cb(): free terminal.
*/
static void
_term_close_cb(uv_handle_t* han_t)
{
  u2_utty* tty_u = (void*) han_t;
  if ( u2_Host.uty_u == tty_u ) {
    u2_Host.uty_u = tty_u->nex_u;
  }
  else {
    for (u2_utty* uty_u = u2_Host.uty_u; uty_u; uty_u = uty_u->nex_u ) {
      if ( uty_u->nex_u == tty_u ) {
        uty_u->nex_u = tty_u->nex_u;
        break;
      }
    }
  }

  {
    u2_noun tid = u2_dc("scot", c3__ud, tty_u->tid_l);
    u2_noun pax = u2nq(c3__gold, c3__term, tid, u2_nul);
    u2_reck_plan(u2A, u2k(pax), u2nc(c3__hook, u2_nul));
    u2z(pax);
  }
  free(tty_u);
}

/* _tel_close_cb(): close telnet terminal
*/
static void
_tel_close_cb(uv_handle_t* han_t)
{
  u2_utel* pty_u = (u2_utel*)(void*)han_t;
  telnet_nvt_free(pty_u->tel_u);
  _term_close_cb(han_t);
}

/* u2_term_io_init(): initialize terminal.
*/
void
u2_term_io_init()
{
  u2_utty* uty_u = calloc(1, sizeof(u2_utty));

  if ( u2_yes == u2_Host.ops_u.dem ) {
    uty_u->fid_i = 1;

    uv_pipe_init(u2L, &(uty_u->pop_u), 0);
    uv_pipe_open(&(uty_u->pop_u), uty_u->fid_i);
  }
  else {
    //  Initialize event processing.  Rawdog it.
    //
    {
      uty_u->fid_i = 0;                       //  stdin, yes we write to it...

      uv_pipe_init(u2L, &(uty_u->pop_u), 0);
      uv_pipe_open(&(uty_u->pop_u), uty_u->fid_i);
      uv_read_start((uv_stream_t*)&(uty_u->pop_u), _term_alloc, _term_read_cb);
    }

    //  Configure horrible stateful terminfo api.
    //
    {
      if ( 0 != setupterm(0, 2, 0) ) {
        c3_assert(!"init-setupterm");
      }
    }

    //  Load terminfo strings.
    //
    {
      c3_w len_w;

#   define _utfo(way, nam) \
      { \
        uty_u->ufo_u.way.nam##_y = (const c3_y *) tigetstr(#nam); \
        c3_assert(uty_u->ufo_u.way.nam##_y); \
      }

      uty_u->ufo_u.inn.max_w = 0;

      _utfo(inn, kcuu1);
      _utfo(inn, kcud1);
      _utfo(inn, kcub1);
      _utfo(inn, kcuf1);

      _utfo(out, clear);
      _utfo(out, el);
      // _utfo(out, el1);
      _utfo(out, ed);
      _utfo(out, bel);
      _utfo(out, cub1);
      _utfo(out, cuf1);
      _utfo(out, cuu1);
      _utfo(out, cud1);
      // _utfo(out, cub);
      // _utfo(out, cuf);

      //  Terminfo chronically reports the wrong sequence for arrow
      //  keys on xterms.  Drastic fix for ridiculous unacceptable bug.
      //  Yes, we could fix this with smkx/rmkx, but this is retarded as well.
      {
        uty_u->ufo_u.inn.kcuu1_y = (const c3_y*)"\033[A";
        uty_u->ufo_u.inn.kcud1_y = (const c3_y*)"\033[B";
        uty_u->ufo_u.inn.kcuf1_y = (const c3_y*)"\033[C";
        uty_u->ufo_u.inn.kcub1_y = (const c3_y*)"\033[D";
      }

      uty_u->ufo_u.inn.max_w = 0;
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcuu1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcud1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcub1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcuf1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
    }

    //  Load old terminal state to restore.
    //
    {
      if ( 0 != tcgetattr(uty_u->fid_i, &uty_u->bak_u) ) {
        c3_assert(!"init-tcgetattr");
      }
      if ( -1 == fcntl(uty_u->fid_i, F_GETFL, &uty_u->cug_i) ) {
        c3_assert(!"init-fcntl");
      }
      uty_u->cug_i &= ~O_NONBLOCK;                // could fix?
      uty_u->nob_i = uty_u->cug_i | O_NONBLOCK;   // O_NDELAY on older unix
    }

    //  Construct raw termios configuration.
    //
    {
      uty_u->raw_u = uty_u->bak_u;

      uty_u->raw_u.c_lflag &= ~(ECHO | ECHONL | ICANON | IEXTEN);
      uty_u->raw_u.c_iflag &= ~(ICRNL | INPCK | ISTRIP);
      uty_u->raw_u.c_cflag &= ~(CSIZE | PARENB);
      uty_u->raw_u.c_cflag |= CS8;
      uty_u->raw_u.c_oflag &= ~(OPOST);
      uty_u->raw_u.c_cc[VMIN] = 0;
      uty_u->raw_u.c_cc[VTIME] = 0;
    }

    //  Initialize mirror and accumulator state.
    //
    {
      uty_u->tat_u.mir.lin_w = 0;
      uty_u->tat_u.mir.len_w = 0;
      uty_u->tat_u.mir.cus_w = 0;

      uty_u->tat_u.esc.ape = u2_no;
      uty_u->tat_u.esc.bra = u2_no;

      uty_u->tat_u.fut.len_w = 0;
      uty_u->tat_u.fut.wid_w = 0;
    }
  }

  //  This is terminal 1, linked in host.
  //
  {
    uty_u->tid_l = 1;
    uty_u->nex_u = 0;
    u2_Host.uty_u = uty_u;
  }

  if ( u2_no == u2_Host.ops_u.dem ) {
    //  Start raw input.
    //
    {
      if ( 0 != tcsetattr(uty_u->fid_i, TCSADRAIN, &uty_u->raw_u) ) {
        c3_assert(!"init-tcsetattr");
      }
      if ( -1 == fcntl(uty_u->fid_i, F_SETFL, uty_u->nob_i) ) {
        c3_assert(!"init-fcntl");
      }
    }
  }
}

void
_term_listen_cb(uv_stream_t *wax_u, int sas_i)
{
  u2_utel* pty_u = calloc(1, sizeof(*pty_u));
  u2_utty* tty_u = &pty_u->uty_t;
  uv_tcp_init(u2L, &tty_u->wax_u);
  if ( 0 != uv_accept(wax_u, (uv_stream_t*)&tty_u->wax_u) ) {
    uL(fprintf(uH, "term: accept: %s\n",
                    uv_strerror(uv_last_error(u2L))));

    uv_close((uv_handle_t*)&tty_u->wax_u, NULL);
    free(tty_u);
  }
  else {
    uv_read_start((uv_stream_t*)&tty_u->wax_u,
                  _term_alloc,
                  _term_read_tn_cb);

    tty_u->ufo_u.out.clear_y = (const c3_y*)"\033[H\033[J";
    tty_u->ufo_u.out.el_y    = (const c3_y*)"\033[K";
    tty_u->ufo_u.out.ed_y    = (const c3_y*)"\033[J";
    tty_u->ufo_u.out.bel_y   = (const c3_y*)"\007";
    tty_u->ufo_u.out.cub1_y  = (const c3_y*)"\010";
    tty_u->ufo_u.out.cud1_y  = (const c3_y*)"\033[B";
    tty_u->ufo_u.out.cuu1_y  = (const c3_y*)"\033[A";
    tty_u->ufo_u.out.cuf1_y  = (const c3_y*)"\033[C";

    tty_u->ufo_u.inn.kcuu1_y = (const c3_y*)"\033[A";
    tty_u->ufo_u.inn.kcud1_y = (const c3_y*)"\033[B";
    tty_u->ufo_u.inn.kcuf1_y = (const c3_y*)"\033[C";
    tty_u->ufo_u.inn.kcub1_y = (const c3_y*)"\033[D";
    tty_u->ufo_u.inn.max_w = strlen("\033[D");

    tty_u->fid_i = -1;

    tty_u->tat_u.mir.lin_w = 0;
    tty_u->tat_u.mir.len_w = 0;
    tty_u->tat_u.mir.cus_w = 0;

    tty_u->tat_u.esc.ape = u2_no;
    tty_u->tat_u.esc.bra = u2_no;

    tty_u->tat_u.fut.len_w = 0;
    tty_u->tat_u.fut.wid_w = 0;

    tty_u->tid_l = u2_Host.uty_u->tid_l + 1;
    tty_u->nex_u = u2_Host.uty_u;
    u2_Host.uty_u = tty_u;
    pty_u->tel_u = telnet_nvt_new(tty_u, _tel_event, _tel_opt, NULL);
    telnet_telopt_enable(pty_u->tel_u, _T_ECHO, TELNET_LOCAL);
    telnet_telopt_enable(pty_u->tel_u, _T_CTIM, TELNET_LOCAL);
    telnet_telopt_enable(pty_u->tel_u, _T_NAWS, TELNET_REMOTE);
    {
      u2_noun tid = u2_dc("scot", c3__ud, tty_u->tid_l);
      u2_noun pax = u2nq(c3__gold, c3__term, tid, u2_nul);
      //  u2_reck_plan(u2A, u2k(pax), u2nc(c3__blew, u2_term_get_blew(1)));
      u2_reck_plan(u2A, u2k(pax), u2nc(c3__hail, u2_nul));
      u2z(pax);
    }
  }
}

void
u2_term_io_talk(void)
{
  struct sockaddr_in add_u;
  u2_utel* tel_u = &u2_Host.tel_u;

  uv_tcp_init(u2L, &tel_u->uty_t.wax_u);
  tel_u->por_s = 10023;

  memset(&add_u, 0, sizeof(add_u));
  add_u.sin_family = AF_INET;
  add_u.sin_addr.s_addr = htonl(INADDR_LOOPBACK);

  /*  Try ascending ports.
  */
  while ( 1 ) {
    add_u.sin_port = htons(tel_u->por_s);

    if ( 0 != uv_tcp_bind(&tel_u->uty_t.wax_u, add_u)  ) {
      uv_err_t las_u = uv_last_error(u2L);

      if ( UV_EADDRINUSE == las_u.code ) {<commit_msg>default telnet windows to 80x25<commit_after>/* v/term.c
**
** This file is in the public domain.
*/
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <setjmp.h>
#include <gmp.h>
#include <stdint.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <uv.h>
#include <errno.h>
#include <curses.h>
#include <termios.h>
#include <term.h>
#include <anachronism/common.h>
#include <anachronism/nvt.h>
#include "all.h"
#include "v/vere.h"

static void _term_read_tn_cb(uv_stream_t*, ssize_t, uv_buf_t);
static void _term_read_cb(uv_stream_t*, ssize_t, uv_buf_t);
static void _term_suck(u2_utty*, const c3_y*, ssize_t);
static void _tel_event(telnet_nvt*, telnet_event*);
static void _tel_opt(telnet_nvt*, telnet_byte, telnet_telopt_event*);

#define _T_ECHO 1    //  local echo
#define _T_CTIM 3    //  suppress GA/char-at-a-time
#define _T_NAWS 31   //  negotiate about window size

/* _term_alloc(): libuv buffer allocator.
*/
static uv_buf_t
_term_alloc(uv_handle_t* had_u, size_t len_i)
{
  return uv_buf_init(c3_malloc(len_i), len_i);
}

/* _term_close_cb(): free terminal.
*/
static void
_term_close_cb(uv_handle_t* han_t)
{
  u2_utty* tty_u = (void*) han_t;
  if ( u2_Host.uty_u == tty_u ) {
    u2_Host.uty_u = tty_u->nex_u;
  }
  else {
    for (u2_utty* uty_u = u2_Host.uty_u; uty_u; uty_u = uty_u->nex_u ) {
      if ( uty_u->nex_u == tty_u ) {
        uty_u->nex_u = tty_u->nex_u;
        break;
      }
    }
  }

  {
    u2_noun tid = u2_dc("scot", c3__ud, tty_u->tid_l);
    u2_noun pax = u2nq(c3__gold, c3__term, tid, u2_nul);
    u2_reck_plan(u2A, u2k(pax), u2nc(c3__hook, u2_nul));
    u2z(pax);
  }
  free(tty_u);
}

/* _tel_close_cb(): close telnet terminal
*/
static void
_tel_close_cb(uv_handle_t* han_t)
{
  u2_utel* pty_u = (u2_utel*)(void*)han_t;
  telnet_nvt_free(pty_u->tel_u);
  _term_close_cb(han_t);
}

/* u2_term_io_init(): initialize terminal.
*/
void
u2_term_io_init()
{
  u2_utty* uty_u = calloc(1, sizeof(u2_utty));

  if ( u2_yes == u2_Host.ops_u.dem ) {
    uty_u->fid_i = 1;

    uv_pipe_init(u2L, &(uty_u->pop_u), 0);
    uv_pipe_open(&(uty_u->pop_u), uty_u->fid_i);
  }
  else {
    //  Initialize event processing.  Rawdog it.
    //
    {
      uty_u->fid_i = 0;                       //  stdin, yes we write to it...

      uv_pipe_init(u2L, &(uty_u->pop_u), 0);
      uv_pipe_open(&(uty_u->pop_u), uty_u->fid_i);
      uv_read_start((uv_stream_t*)&(uty_u->pop_u), _term_alloc, _term_read_cb);
    }

    //  Configure horrible stateful terminfo api.
    //
    {
      if ( 0 != setupterm(0, 2, 0) ) {
        c3_assert(!"init-setupterm");
      }
    }

    //  Load terminfo strings.
    //
    {
      c3_w len_w;

#   define _utfo(way, nam) \
      { \
        uty_u->ufo_u.way.nam##_y = (const c3_y *) tigetstr(#nam); \
        c3_assert(uty_u->ufo_u.way.nam##_y); \
      }

      uty_u->ufo_u.inn.max_w = 0;

      _utfo(inn, kcuu1);
      _utfo(inn, kcud1);
      _utfo(inn, kcub1);
      _utfo(inn, kcuf1);

      _utfo(out, clear);
      _utfo(out, el);
      // _utfo(out, el1);
      _utfo(out, ed);
      _utfo(out, bel);
      _utfo(out, cub1);
      _utfo(out, cuf1);
      _utfo(out, cuu1);
      _utfo(out, cud1);
      // _utfo(out, cub);
      // _utfo(out, cuf);

      //  Terminfo chronically reports the wrong sequence for arrow
      //  keys on xterms.  Drastic fix for ridiculous unacceptable bug.
      //  Yes, we could fix this with smkx/rmkx, but this is retarded as well.
      {
        uty_u->ufo_u.inn.kcuu1_y = (const c3_y*)"\033[A";
        uty_u->ufo_u.inn.kcud1_y = (const c3_y*)"\033[B";
        uty_u->ufo_u.inn.kcuf1_y = (const c3_y*)"\033[C";
        uty_u->ufo_u.inn.kcub1_y = (const c3_y*)"\033[D";
      }

      uty_u->ufo_u.inn.max_w = 0;
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcuu1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcud1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcub1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
      if ( (len_w = strlen((c3_c*)uty_u->ufo_u.inn.kcuf1_y)) >
            uty_u->ufo_u.inn.max_w )
      {
        uty_u->ufo_u.inn.max_w = len_w;
      }
    }

    //  Load old terminal state to restore.
    //
    {
      if ( 0 != tcgetattr(uty_u->fid_i, &uty_u->bak_u) ) {
        c3_assert(!"init-tcgetattr");
      }
      if ( -1 == fcntl(uty_u->fid_i, F_GETFL, &uty_u->cug_i) ) {
        c3_assert(!"init-fcntl");
      }
      uty_u->cug_i &= ~O_NONBLOCK;                // could fix?
      uty_u->nob_i = uty_u->cug_i | O_NONBLOCK;   // O_NDELAY on older unix
    }

    //  Construct raw termios configuration.
    //
    {
      uty_u->raw_u = uty_u->bak_u;

      uty_u->raw_u.c_lflag &= ~(ECHO | ECHONL | ICANON | IEXTEN);
      uty_u->raw_u.c_iflag &= ~(ICRNL | INPCK | ISTRIP);
      uty_u->raw_u.c_cflag &= ~(CSIZE | PARENB);
      uty_u->raw_u.c_cflag |= CS8;
      uty_u->raw_u.c_oflag &= ~(OPOST);
      uty_u->raw_u.c_cc[VMIN] = 0;
      uty_u->raw_u.c_cc[VTIME] = 0;
    }

    //  Initialize mirror and accumulator state.
    //
    {
      uty_u->tat_u.mir.lin_w = 0;
      uty_u->tat_u.mir.len_w = 0;
      uty_u->tat_u.mir.cus_w = 0;

      uty_u->tat_u.esc.ape = u2_no;
      uty_u->tat_u.esc.bra = u2_no;

      uty_u->tat_u.fut.len_w = 0;
      uty_u->tat_u.fut.wid_w = 0;
    }
  }

  //  This is terminal 1, linked in host.
  //
  {
    uty_u->tid_l = 1;
    uty_u->nex_u = 0;
    u2_Host.uty_u = uty_u;
  }

  if ( u2_no == u2_Host.ops_u.dem ) {
    //  Start raw input.
    //
    {
      if ( 0 != tcsetattr(uty_u->fid_i, TCSADRAIN, &uty_u->raw_u) ) {
        c3_assert(!"init-tcsetattr");
      }
      if ( -1 == fcntl(uty_u->fid_i, F_SETFL, uty_u->nob_i) ) {
        c3_assert(!"init-fcntl");
      }
    }
  }
}

void
_term_listen_cb(uv_stream_t *wax_u, int sas_i)
{
  u2_utel* pty_u = calloc(1, sizeof(*pty_u));
  u2_utty* tty_u = &pty_u->uty_t;
  uv_tcp_init(u2L, &tty_u->wax_u);
  if ( 0 != uv_accept(wax_u, (uv_stream_t*)&tty_u->wax_u) ) {
    uL(fprintf(uH, "term: accept: %s\n",
                    uv_strerror(uv_last_error(u2L))));

    uv_close((uv_handle_t*)&tty_u->wax_u, NULL);
    free(tty_u);
  }
  else {
    uv_read_start((uv_stream_t*)&tty_u->wax_u,
                  _term_alloc,
                  _term_read_tn_cb);

    tty_u->ufo_u.out.clear_y = (const c3_y*)"\033[H\033[J";
    tty_u->ufo_u.out.el_y    = (const c3_y*)"\033[K";
    tty_u->ufo_u.out.ed_y    = (const c3_y*)"\033[J";
    tty_u->ufo_u.out.bel_y   = (const c3_y*)"\007";
    tty_u->ufo_u.out.cub1_y  = (const c3_y*)"\010";
    tty_u->ufo_u.out.cud1_y  = (const c3_y*)"\033[B";
    tty_u->ufo_u.out.cuu1_y  = (const c3_y*)"\033[A";
    tty_u->ufo_u.out.cuf1_y  = (const c3_y*)"\033[C";

    tty_u->ufo_u.inn.kcuu1_y = (const c3_y*)"\033[A";
    tty_u->ufo_u.inn.kcud1_y = (const c3_y*)"\033[B";
    tty_u->ufo_u.inn.kcuf1_y = (const c3_y*)"\033[C";
    tty_u->ufo_u.inn.kcub1_y = (const c3_y*)"\033[D";
    tty_u->ufo_u.inn.max_w = strlen("\033[D");

    tty_u->fid_i = -1;

    tty_u->tat_u.mir.lin_w = 0;
    tty_u->tat_u.mir.len_w = 0;
    tty_u->tat_u.mir.cus_w = 0;

    tty_u->tat_u.esc.ape = u2_no;
    tty_u->tat_u.esc.bra = u2_no;

    tty_u->tat_u.fut.len_w = 0;
    tty_u->tat_u.fut.wid_w = 0;

    tty_u->tat_u.siz.col_l = 80;
    tty_u->tat_u.siz.row_l = 25;

    tty_u->tid_l = u2_Host.uty_u->tid_l + 1;
    tty_u->nex_u = u2_Host.uty_u;
    u2_Host.uty_u = tty_u;
    pty_u->tel_u = telnet_nvt_new(tty_u, _tel_event, _tel_opt, NULL);
    telnet_telopt_enable(pty_u->tel_u, _T_ECHO, TELNET_LOCAL);
    telnet_telopt_enable(pty_u->tel_u, _T_CTIM, TELNET_LOCAL);
    telnet_telopt_enable(pty_u->tel_u, _T_NAWS, TELNET_REMOTE);
    {
      u2_noun tid = u2_dc("scot", c3__ud, tty_u->tid_l);
      u2_noun pax = u2nq(c3__gold, c3__term, tid, u2_nul);
      u2_reck_plan(u2A, u2k(pax), u2nc(c3__blew, u2nc(80, 25)));
      u2_reck_plan(u2A, u2k(pax), u2nc(c3__hail, u2_nul));
      u2z(pax);
    }
  }
}

void
u2_term_io_talk(void)
{
  struct sockaddr_in add_u;
  u2_utel* tel_u = &u2_Host.tel_u;

  uv_tcp_init(u2L, &tel_u->uty_t.wax_u);
  tel_u->por_s = 10023;

  memset(&add_u, 0, sizeof(add_u));
  add_u.sin_family = AF_INET;
  add_u.sin_addr.s_addr = htonl(INADDR_LOOPBACK);

  /*  Try ascending ports.
  */
  while ( 1 ) {
    add_u.sin_port = htons(tel_u->por_s);

    if ( 0 != uv_tcp_bind(&tel_u->uty_t.wax_u, add_u)  ) {
      uv_err_t las_u = uv_last_error(u2L);

      if ( UV_EADDRINUSE == las_u.code ) {
{'fraction_non_alphanumeric': 0.167778672495111, 'fraction_numerical': 0.026860692511215922, 'mean_word_length': 2.5389782210462037, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 255, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6008052', 'n_tokens_mistral': 8903, 'n_tokens_neox': 8465, 'n_words': 1197}