<commit_before>#include "buffer.h"
#include "murmur2.h"
#include "cache.h"
#include "error.h"
#include "session.h"
#include "utils.h"
#include "log.h"

#include "config.h"

#include <cassert>
#include <cstdarg>
#include <cstring>

#include <boost/asio.hpp>

using namespace std;

enum session_state {
  write_prompt,
  read_command,
  execute_command,
  read_data,
  execute_write,
  write_data,
  write_result,
  stopping,
};

/*
static const char *
sss(session_state s)
{
  switch (s) {
  case write_prompt: return "write_prompt";
  case read_command: return "read_command";
  case execute_command: return "execute_command";
  case read_data: return "read_data";
  case execute_write: return "execute_write";
  case write_data: return "write_data";
  case write_result: return "write_result";
  case stopping: return "stopping";
  }
  return "unknown";
}
*/

class Session
{
  boost::asio::io_service &io_service;
  cache &money;
  stream &in;
  stream &out;
  std::ostream &log;
  const char *prompt_;
  bool noreply_;
  buffer ibuf;
  buffer obuf;
  session_state state_ = write_prompt;
  session_done done_;
  buffer args_;
  buffer cmd_;
  unsigned long flags_, exptime_;
  uint64_t unique_;
  buffer key_;
  mem *idata_ = NULL;
  const_rope odata_;
  std::function<void (boost::system::error_code, size_t)> callback_ =
    [this](boost::system::error_code ec, size_t bytes) -> void {
    callback(ec, bytes);
  };
  std::function<size_t (boost::system::error_code, size_t)> cmd_callback_ =
    [this](boost::system::error_code ec, size_t bytes) -> size_t {
    return cmd_callback(ec, bytes);
  };

  // Input
  bool recv_command();
  bool recv_data(size_t bytes);

  // Output
  bool send_prompt();
  void send_async(const char *msg, size_t bytes);
  void send_n(const char *msg, size_t bytes);
  void send(const char *msg);
  void sendln(const char *msg);
  void sendf(const char *fmt, ...);
  void vsendf(const char *fmt, va_list ap);
  void send_stat(const char *name, const char *val);
  void send_stat(const char *name, uint64_t val);
  void send_cache_result(cache_error_t res);
  void client_error(const char *fmt, ...);
  void server_error(const char *fmt, ...);
  bool send_data();
  bool flush();

  // Command handlers
  /*
  void flush_all(buffer &args);
  */
  bool cas();
  bool get(bool cas_unique);
  bool incr_decr(bool incr);
  bool del();
  bool stats();
  bool version();
  bool touch();

  // Command helpers
  bool parse_update(unsigned long *bytes);
  bool parse_key();
  void stored_if(bool stored);
  bool parse_noreply();

  // Session helpers
  void callback(boost::system::error_code ec, size_t bytes);
  size_t cmd_callback(boost::system::error_code ec, size_t bytes);
  void loop();

  bool dispatch();
  bool dispatch_write();

public:
  class closed { };

  Session(boost::asio::io_service &io_service,
          class cache &c, stream &in, stream &out,
          std::ostream &log, const char *prompt)
    : io_service(io_service), money(c), in(in), out(out), log(log),
      prompt_(prompt), noreply_(false), ibuf(2048), obuf(2048) { }
  ~Session() { }
  void interact(session_done done);
};

class stream_buffer : public std::streambuf
{
  buffer &buf_;
public:
  stream_buffer(buffer &buf) : buf_(buf) { }
  int overflow(int c = EOF)
  {
    if (buf_.available() > 0) {
      char a = c;
      buf_.write(&a, 1);
      return c;
    } else {
      return EOF;
    }
  }
};


class session_error_category : public std::error_category
{
  session_error_category() { }
  const char *name() const noexcept { return "session_error"; }
  string message (int val) const { return "some_error!?"; } // XXX

public:
  static session_error_category& instance() {
    static session_error_category instance_;
    return instance_;
  }
};

class cache_error_category : public std::error_category
{
  cache_error_category() { }
  const char *name() const noexcept { return "cache_error"; }
  string message (int val) const { return "some_error"; } // XXX

public:
  static cache_error_category& instance() {
    static cache_error_category instance_;
    return instance_;
  }
};

void
Session::send(const char *msg)
{
  send_n(msg, strlen(msg));
}

void
Session::sendln(const char *msg)
{
  send(msg);
  send(CRLF);
}

bool
Session::parse_noreply()
{
  const buffer nr = consume_token(args_);
  if (nr.empty()) {
    return false;
  } else if (!nr.is("noreplay")) {
    client_error("expected noreply or end of command");
    return true;
  } else {
    noreply_ = true;
    return false;
  }
}

static boost::system::error_code
success()
{
  return boost::system::error_code();
}

bool
Session::flush()
{
  if (obuf.empty()) {
    return false;
  } else {
    out.async_write(boost::asio::const_buffers_1(obuf.headp(), obuf.used()),
                    callback_);
    return true;
  }
}

void
Session::send_n(const char *msg, size_t bytes)
{
  if (!noreply_)
    obuf.write(msg, bytes);
}

void
Session::send_async(const char *msg, size_t bytes)
{
  assert(obuf.used() == 0);     // XXX - we could flush now
  // XXX - we could take a rope...
  out.async_write(boost::asio::const_buffers_1(msg, bytes), callback_);
}

void
Session::vsendf(const char *fmt, va_list ap)
{
  if (not noreply_) {
    int n = vsnprintf(obuf.tailp(), obuf.available(), fmt, ap);
    obuf.notify_write(n);
    send(CRLF);
  }
}

void
Session::sendf(const char *fmt, ...)
{
  va_list ap;
  va_start(ap, fmt);
  vsendf(fmt, ap);
  va_end(ap);
}

void
Session::send_stat(const char *name, const char *val)
{
  sendf("STAT %s %s", name, val);
}

void
Session::send_stat(const char *name, uint64_t val)
{
  char buf[64];
  snprintf(buf, sizeof(buf), "%lu", val);
  send_stat(name, buf);
}

bool
Session::send_data()
{
  const mem *m = odata_.pop();
  if (!m) {
    send(CRLF);
    state_ = execute_command;
    return false;
  }
  state_ = write_data;
  send_async(m->data, m->size);
  return true;
}

bool
Session::get(bool cas_unique)
{
  buffer key = consume_token(args_);
  if (key.empty()) {
    send("END" CRLF);
    state_ = write_result;
    return false;
  }

  cache::ref e = money.get(key);
  if (e == nullptr) {
    sendln("NOT_FOUND");
    state_ = write_result;
    return false;
  }

  odata_ = e->read();
  size_t size = odata_.size();
  if (cas_unique) {
    uint64_t version = odata_.hash(e->get_flags());
    sendf("VALUE %.*s %u %u %llu",
          key.used(), key.headp(), e->get_flags(), size, version);
  } else {
    sendf("VALUE %.*s %u %u",
          key.used(), key.headp(), e->get_flags(), size);
  }
  state_ = write_data;
  return flush();
}

void
Session::client_error(const char *fmt, ...)
{
  send("CLIENT ERROR ");
  va_list ap;
  va_start(ap, fmt);
  vsendf(fmt, ap);
  va_end(ap);
  send(CRLF);
  state_ = write_result;
}

void
Session::server_error(const char *fmt, ...)
{
  send("SERVER ERROR ");
  va_list ap;
  va_start(ap, fmt);
  vsendf(fmt, ap);
  va_end(ap);
  send(CRLF);
  state_ = write_result;
}

bool
Session::parse_key()
{
  key_ = consume_token(args_);
  if (key_.empty()) {
    client_error("missing key");
    return true;
  } else {
    return false;
  }
}

bool
Session::parse_update(unsigned long *bytes)
{
  if (parse_key())
    return true;
  if (!consume_int(args_, &flags_)) {
    client_error("missing flags");
    return true;
  } else if (!consume_int(args_, &exptime_)) {
    client_error("missing exptime");
    return true;
  } else if (!consume_int(args_, bytes)) {
    client_error("missing bytes");
    return true;
  }
  return false;
}

bool
Session::del()
{
  if (!parse_key() && !parse_noreply()) {
    cache_error_t res = money.del(key_);
    send_cache_result(res);
  }
  return false;
}

void
Session::send_cache_result(cache_error_t res)
{
  switch (res) {
  case cache_error_t::stored:
    sendln("STORED");
    break;
  case cache_error_t::deleted:
    sendln("DELETED");
    break;
  case cache_error_t::notfound:
    sendln("NOT_FOUND");
    break;
  case cache_error_t::set_error:
    sendln("NOT_STORED");
    break;
  case cache_error_t::cas_exists:
    sendln("EXISTS");
    break;
  }
  state_ = write_result;
}

bool
Session::incr_decr(bool incr)
{
  if (parse_key())
    return false;

  uint64_t v;
  if (!consume_u64(args_, &v)) {
    client_error("missing value");
    return false;
  }

  if (parse_noreply())
    return false;

  cache_error_t res = incr ? money.incr(key_, v, &v)
    : money.decr(key_, v, &v);
    
  if (res == cache_error_t::stored) {
    sendf("%llu", v);
  } else {
    send_cache_result(res);
  }
  return false;
}

bool
Session::cas()
{
  unsigned long bytes;
  if (parse_update(&bytes))
    return false;
  if (!consume_u64(args_, &unique_)) {
    client_error("missing cas unique");
    return false;
  }
  if (parse_noreply())
    return false;
  
  return recv_data(bytes);
}

bool
Session::touch()
{
  unsigned long exptime;
  if (parse_key())
    return false;
  if (!consume_int(args_, &exptime)) {
    client_error("missing exptime");
    return false;
  }
  if (parse_noreply())
    return false;
  money.touch(key_, exptime);
  sendln("TOUCHED");
  return false;
}

/*
void
Session::flush_all(buffer &args, session_result done)
{
  unsigned long delay;
  if (!consume_int(args, &delay))
    delay = 0;
  parse_noreply(args);
  money.flush_all(delay);
  result(done, success());
}
*/

bool
Session::version()
{
  sendln("VERSION " PACKAGE_VERSION);
  state_ = write_result;
  return false;
}

bool
Session::stats()
{
  stream_buffer buf(obuf);
  std::ostream os(&buf);
  send_stat("version", PACKAGE_VERSION);
  send_stat("pointer_size", sizeof(void*));
  send_stat("cmd_get", money.get_count());
  send_stat("cmd_set", money.set_count());
  send_stat("get_hits", money.get_hit_count());
  send_stat("get_misses", money.get_miss_count());
  send_stat("bytes", money.bytes());
  send_stat("buckets", money.buckets());
  send_stat("keys", money.keys());
  send("END" CRLF);
  state_ = write_result;
  return false;
}

bool                            // XXX - we never block
Session::dispatch_write()
{
  rope data = rope(idata_, idata_); // XXX
  cache_error_t res;
  if (cmd_.is("set")) {
    res = money.set(key_, flags_, exptime_, data);
  } else if (cmd_.is("add")) {
    res = money.add(key_, flags_, exptime_, data);
  } else if (cmd_.is("replace")) {
    res = money.replace(key_, flags_, exptime_, data);
  } else if (cmd_.is("append")) {
    res = money.append(key_, data);
  } else if (cmd_.is("prepend")) {
    res = money.prepend(key_, data);
  } else if (cmd_.is("cas")) {
    res = money.cas(key_, flags_, exptime_, unique_, data);
  } else {
    server_error("confused by command: %.*s", cmd_.used(), cmd_.headp());
    return false;
  }

  send_cache_result(res);
  return false;
}

bool
Session::dispatch()
{
  if (cmd_.empty()) {
    state_ = write_prompt;      // ignore empty commands
    return false;
  } else if (cmd_.is("get")) {
    return get(false);
  } else if (cmd_.is("gets")) {
    return get(true);
  } else if (cmd_.is("set") || cmd_.is("add") || cmd_.is("replace") ||
             cmd_.is("append") || cmd_.is("prepend")) {
    unsigned long bytes;
    if (parse_update(&bytes) || parse_noreply())
      return false;
    return recv_data(bytes);
  } else if (cmd_.is("incr")) {
    return incr_decr(true);
  } else if (cmd_.is("decr")) {
    return incr_decr(false);
  } else if (cmd_.is("delete")) {
    return del();
  } else if (cmd_.is("cas")) {
    return cas();
  } else if (cmd_.is("touch")) {
    return touch();
    /*
  } else if (cmd.is("flush_all")) {
    flush_all(cmdline, done);
    */
  } else if (cmd_.is("version")) {
    return version();
  } else if (cmd_.is("stats")) {<commit_msg>flush_all<commit_after>#include "buffer.h"
#include "murmur2.h"
#include "cache.h"
#include "error.h"
#include "session.h"
#include "utils.h"
#include "log.h"

#include "config.h"

#include <cassert>
#include <cstdarg>
#include <cstring>

#include <boost/asio.hpp>

using namespace std;

enum session_state {
  write_prompt,
  read_command,
  execute_command,
  read_data,
  execute_write,
  write_data,
  write_result,
  stopping,
};

/*
static const char *
sss(session_state s)
{
  switch (s) {
  case write_prompt: return "write_prompt";
  case read_command: return "read_command";
  case execute_command: return "execute_command";
  case read_data: return "read_data";
  case execute_write: return "execute_write";
  case write_data: return "write_data";
  case write_result: return "write_result";
  case stopping: return "stopping";
  }
  return "unknown";
}
*/

class Session
{
  boost::asio::io_service &io_service;
  cache &money;
  stream &in;
  stream &out;
  std::ostream &log;
  const char *prompt_;
  bool noreply_;
  buffer ibuf;
  buffer obuf;
  session_state state_ = write_prompt;
  session_done done_;
  buffer args_;
  buffer cmd_;
  unsigned long flags_, exptime_;
  uint64_t unique_;
  buffer key_;
  mem *idata_ = NULL;
  const_rope odata_;
  std::function<void (boost::system::error_code, size_t)> callback_ =
    [this](boost::system::error_code ec, size_t bytes) -> void {
    callback(ec, bytes);
  };
  std::function<size_t (boost::system::error_code, size_t)> cmd_callback_ =
    [this](boost::system::error_code ec, size_t bytes) -> size_t {
    return cmd_callback(ec, bytes);
  };

  // Input
  bool recv_command();
  bool recv_data(size_t bytes);

  // Output
  bool send_prompt();
  void send_async(const char *msg, size_t bytes);
  void send_n(const char *msg, size_t bytes);
  void send(const char *msg);
  void sendln(const char *msg);
  void sendf(const char *fmt, ...);
  void vsendf(const char *fmt, va_list ap);
  void send_stat(const char *name, const char *val);
  void send_stat(const char *name, uint64_t val);
  void send_cache_result(cache_error_t res);
  void client_error(const char *fmt, ...);
  void server_error(const char *fmt, ...);
  bool send_data();
  bool flush();

  // Command handlers
  bool flush_all();
  bool cas();
  bool get(bool cas_unique);
  bool incr_decr(bool incr);
  bool del();
  bool stats();
  bool version();
  bool touch();

  // Command helpers
  bool parse_update(unsigned long *bytes);
  bool parse_key();
  void stored_if(bool stored);
  bool parse_noreply();

  // Session helpers
  void callback(boost::system::error_code ec, size_t bytes);
  size_t cmd_callback(boost::system::error_code ec, size_t bytes);
  void loop();

  bool dispatch();
  bool dispatch_write();

public:
  class closed { };

  Session(boost::asio::io_service &io_service,
          class cache &c, stream &in, stream &out,
          std::ostream &log, const char *prompt)
    : io_service(io_service), money(c), in(in), out(out), log(log),
      prompt_(prompt), noreply_(false), ibuf(2048), obuf(2048) { }
  ~Session() { }
  void interact(session_done done);
};

class stream_buffer : public std::streambuf
{
  buffer &buf_;
public:
  stream_buffer(buffer &buf) : buf_(buf) { }
  int overflow(int c = EOF)
  {
    if (buf_.available() > 0) {
      char a = c;
      buf_.write(&a, 1);
      return c;
    } else {
      return EOF;
    }
  }
};


class session_error_category : public std::error_category
{
  session_error_category() { }
  const char *name() const noexcept { return "session_error"; }
  string message (int val) const { return "some_error!?"; } // XXX

public:
  static session_error_category& instance() {
    static session_error_category instance_;
    return instance_;
  }
};

class cache_error_category : public std::error_category
{
  cache_error_category() { }
  const char *name() const noexcept { return "cache_error"; }
  string message (int val) const { return "some_error"; } // XXX

public:
  static cache_error_category& instance() {
    static cache_error_category instance_;
    return instance_;
  }
};

void
Session::send(const char *msg)
{
  send_n(msg, strlen(msg));
}

void
Session::sendln(const char *msg)
{
  send(msg);
  send(CRLF);
}

bool
Session::parse_noreply()
{
  const buffer nr = consume_token(args_);
  if (nr.empty()) {
    return false;
  } else if (!nr.is("noreplay")) {
    client_error("expected noreply or end of command");
    return true;
  } else {
    noreply_ = true;
    return false;
  }
}

static boost::system::error_code
success()
{
  return boost::system::error_code();
}

bool
Session::flush()
{
  if (obuf.empty()) {
    return false;
  } else {
    out.async_write(boost::asio::const_buffers_1(obuf.headp(), obuf.used()),
                    callback_);
    return true;
  }
}

void
Session::send_n(const char *msg, size_t bytes)
{
  if (!noreply_)
    obuf.write(msg, bytes);
}

void
Session::send_async(const char *msg, size_t bytes)
{
  assert(obuf.used() == 0);     // XXX - we could flush now
  // XXX - we could take a rope...
  out.async_write(boost::asio::const_buffers_1(msg, bytes), callback_);
}

void
Session::vsendf(const char *fmt, va_list ap)
{
  if (not noreply_) {
    int n = vsnprintf(obuf.tailp(), obuf.available(), fmt, ap);
    obuf.notify_write(n);
    send(CRLF);
  }
}

void
Session::sendf(const char *fmt, ...)
{
  va_list ap;
  va_start(ap, fmt);
  vsendf(fmt, ap);
  va_end(ap);
}

void
Session::send_stat(const char *name, const char *val)
{
  sendf("STAT %s %s", name, val);
}

void
Session::send_stat(const char *name, uint64_t val)
{
  char buf[64];
  snprintf(buf, sizeof(buf), "%lu", val);
  send_stat(name, buf);
}

bool
Session::send_data()
{
  const mem *m = odata_.pop();
  if (!m) {
    send(CRLF);
    state_ = execute_command;
    return false;
  }
  state_ = write_data;
  send_async(m->data, m->size);
  return true;
}

bool
Session::get(bool cas_unique)
{
  buffer key = consume_token(args_);
  if (key.empty()) {
    send("END" CRLF);
    state_ = write_result;
    return false;
  }

  cache::ref e = money.get(key);
  if (e == nullptr) {
    sendln("NOT_FOUND");
    state_ = write_result;
    return false;
  }

  odata_ = e->read();
  size_t size = odata_.size();
  if (cas_unique) {
    uint64_t version = odata_.hash(e->get_flags());
    sendf("VALUE %.*s %u %u %llu",
          key.used(), key.headp(), e->get_flags(), size, version);
  } else {
    sendf("VALUE %.*s %u %u",
          key.used(), key.headp(), e->get_flags(), size);
  }
  state_ = write_data;
  return flush();
}

void
Session::client_error(const char *fmt, ...)
{
  send("CLIENT ERROR ");
  va_list ap;
  va_start(ap, fmt);
  vsendf(fmt, ap);
  va_end(ap);
  send(CRLF);
  state_ = write_result;
}

void
Session::server_error(const char *fmt, ...)
{
  send("SERVER ERROR ");
  va_list ap;
  va_start(ap, fmt);
  vsendf(fmt, ap);
  va_end(ap);
  send(CRLF);
  state_ = write_result;
}

bool
Session::parse_key()
{
  key_ = consume_token(args_);
  if (key_.empty()) {
    client_error("missing key");
    return true;
  } else {
    return false;
  }
}

bool
Session::parse_update(unsigned long *bytes)
{
  if (parse_key())
    return true;
  if (!consume_int(args_, &flags_)) {
    client_error("missing flags");
    return true;
  } else if (!consume_int(args_, &exptime_)) {
    client_error("missing exptime");
    return true;
  } else if (!consume_int(args_, bytes)) {
    client_error("missing bytes");
    return true;
  }
  return false;
}

bool
Session::del()
{
  if (!parse_key() && !parse_noreply()) {
    cache_error_t res = money.del(key_);
    send_cache_result(res);
  }
  return false;
}

void
Session::send_cache_result(cache_error_t res)
{
  switch (res) {
  case cache_error_t::stored:
    sendln("STORED");
    break;
  case cache_error_t::deleted:
    sendln("DELETED");
    break;
  case cache_error_t::notfound:
    sendln("NOT_FOUND");
    break;
  case cache_error_t::set_error:
    sendln("NOT_STORED");
    break;
  case cache_error_t::cas_exists:
    sendln("EXISTS");
    break;
  }
  state_ = write_result;
}

bool
Session::incr_decr(bool incr)
{
  if (parse_key())
    return false;

  uint64_t v;
  if (!consume_u64(args_, &v)) {
    client_error("missing value");
    return false;
  }

  if (parse_noreply())
    return false;

  cache_error_t res = incr ? money.incr(key_, v, &v)
    : money.decr(key_, v, &v);
    
  if (res == cache_error_t::stored) {
    sendf("%llu", v);
  } else {
    send_cache_result(res);
  }
  return false;
}

bool
Session::cas()
{
  unsigned long bytes;
  if (parse_update(&bytes))
    return false;
  if (!consume_u64(args_, &unique_)) {
    client_error("missing cas unique");
    return false;
  }
  if (parse_noreply())
    return false;
  
  return recv_data(bytes);
}

bool
Session::touch()
{
  unsigned long exptime;
  if (parse_key())
    return false;
  if (!consume_int(args_, &exptime)) {
    client_error("missing exptime");
    return false;
  }
  if (parse_noreply())
    return false;
  money.touch(key_, exptime);
  sendln("TOUCHED");
  return false;
}

bool
Session::flush_all()
{
  unsigned long delay;
  if (!consume_int(args_, &delay))
    delay = 0;
  if (parse_noreply())
    return false;
  money.flush_all(delay);
  return false;
}

bool
Session::version()
{
  sendln("VERSION " PACKAGE_VERSION);
  state_ = write_result;
  return false;
}

bool
Session::stats()
{
  stream_buffer buf(obuf);
  std::ostream os(&buf);
  send_stat("version", PACKAGE_VERSION);
  send_stat("pointer_size", sizeof(void*));
  send_stat("cmd_get", money.get_count());
  send_stat("cmd_set", money.set_count());
  send_stat("get_hits", money.get_hit_count());
  send_stat("get_misses", money.get_miss_count());
  send_stat("bytes", money.bytes());
  send_stat("buckets", money.buckets());
  send_stat("keys", money.keys());
  send("END" CRLF);
  state_ = write_result;
  return false;
}

bool                            // XXX - we never block
Session::dispatch_write()
{
  rope data = rope(idata_, idata_); // XXX
  cache_error_t res;
  if (cmd_.is("set")) {
    res = money.set(key_, flags_, exptime_, data);
  } else if (cmd_.is("add")) {
    res = money.add(key_, flags_, exptime_, data);
  } else if (cmd_.is("replace")) {
    res = money.replace(key_, flags_, exptime_, data);
  } else if (cmd_.is("append")) {
    res = money.append(key_, data);
  } else if (cmd_.is("prepend")) {
    res = money.prepend(key_, data);
  } else if (cmd_.is("cas")) {
    res = money.cas(key_, flags_, exptime_, unique_, data);
  } else {
    server_error("confused by command: %.*s", cmd_.used(), cmd_.headp());
    return false;
  }

  send_cache_result(res);
  return false;
}

bool
Session::dispatch()
{
  if (cmd_.empty()) {
    state_ = write_prompt;      // ignore empty commands
    return false;
  } else if (cmd_.is("get")) {
    return get(false);
  } else if (cmd_.is("gets")) {
    return get(true);
  } else if (cmd_.is("set") || cmd_.is("add") || cmd_.is("replace") ||
             cmd_.is("append") || cmd_.is("prepend")) {
    unsigned long bytes;
    if (parse_update(&bytes) || parse_noreply())
      return false;
    return recv_data(bytes);
  } else if (cmd_.is("incr")) {
    return incr_decr(true);
  } else if (cmd_.is("decr")) {
    return incr_decr(false);
  } else if (cmd_.is("delete")) {
    return del();
  } else if (cmd_.is("cas")) {
    return cas();
  } else if (cmd_.is("touch")) {
    return touch();
  } else if (cmd_.is("flush_all")) {
    return flush_all();
  } else if (cmd_.is("version")) {
    return version();
  } else if (cmd_.is("stats")) {
{'fraction_non_alphanumeric': 0.1558011524898942, 'fraction_numerical': 0.0026662079642212093, 'mean_word_length': 3.6472821742605914, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2173462', 'n_tokens_mistral': 8558, 'n_tokens_neox': 8395, 'n_words': 1995}