<commit_before>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <thrift/lib/cpp2/async/HeaderServerChannel.h>
#include <thrift/lib/cpp/async/TAsyncSocket.h>
#include <thrift/lib/cpp/TApplicationException.h>
#include <thrift/lib/cpp/transport/TTransportException.h>
#include <thrift/lib/cpp2/protocol/Serializer.h>
#include <folly/io/Cursor.h>
#include <folly/String.h>

#include <utility>
#include <exception>

using std::unique_ptr;
using std::pair;
using folly::IOBuf;
using folly::IOBufQueue;
using folly::make_unique;
using namespace apache::thrift::transport;
using namespace apache::thrift;
using folly::EventBase;
using apache::thrift::async::TAsyncSocket;
using apache::thrift::async::TAsyncTransport;
using apache::thrift::TApplicationException;
using apache::thrift::server::TServerObserver;
using apache::thrift::protocol::PROTOCOL_TYPES;

namespace apache { namespace thrift {

std::atomic<uint32_t> HeaderServerChannel::sample_(0);

HeaderServerChannel::HeaderServerChannel(
  const std::shared_ptr<TAsyncTransport>& transport)
    : HeaderServerChannel(
        std::shared_ptr<Cpp2Channel>(
            Cpp2Channel::newChannel(transport,
                make_unique<ServerFramingHandler>(*this))))
{}

HeaderServerChannel::HeaderServerChannel(
    const std::shared_ptr<Cpp2Channel>& cpp2Channel)
    : callback_(nullptr)
    , arrivalSeqId_(1)
    , lastWrittenSeqId_(0)
    , sampleRate_(0)
    , timeoutSASL_(5000)
    , saslServerCallback_(*this)
    , cpp2Channel_(cpp2Channel)
    , timer_(new folly::HHWheelTimer(getEventBase())) {}

void HeaderServerChannel::destroy() {
  DestructorGuard dg(this);

  saslServerCallback_.cancelTimeout();
  if (saslServer_) {
    saslServer_->detachEventBase();
  }

  if (callback_) {
    auto error = folly::make_exception_wrapper<TTransportException>(
        "Channel destroyed");
    callback_->channelClosed(std::move(error));
  }

  cpp2Channel_->closeNow();

  folly::DelayedDestruction::destroy();
}

// Header framing
unique_ptr<IOBuf>
HeaderServerChannel::ServerFramingHandler::addFrame(unique_ptr<IOBuf> buf,
                                                    THeader* header) {
  channel_.updateClientType(header->getClientType());

  // Note: This THeader function may throw.  However, we don't want to catch
  // it here, because this would send an empty message out on the wire.
  // Instead we have to catch it at sendMessage
  return header->addHeader(
    std::move(buf),
    channel_.getPersistentWriteHeaders(),
    false /* Data already transformed in AsyncProcessor.h */);
}

std::tuple<unique_ptr<IOBuf>, size_t, unique_ptr<THeader>>
HeaderServerChannel::ServerFramingHandler::removeFrame(IOBufQueue* q) {
  std::unique_ptr<THeader> header(new THeader(THeader::ALLOW_BIG_FRAMES));
  // removeHeader will set seqid in header.
  // For older clients with seqid in the protocol, header
  // will dig in to the protocol to get the seqid correctly.
  if (!q || !q->front() || q->front()->empty()) {
    return make_tuple(std::unique_ptr<IOBuf>(), 0, nullptr);
  }

  std::unique_ptr<folly::IOBuf> buf;
  size_t remaining = 0;
  try {
    buf = header->removeHeader(q, remaining,
                               channel_.getPersistentReadHeaders());
  } catch (const std::exception& e) {
    LOG(ERROR) << "Received invalid request from client: "
               << folly::exceptionStr(e) << " "
               << getTransportDebugString(channel_.getTransport());
    throw;
  }
  if (!buf) {
    return make_tuple(std::unique_ptr<IOBuf>(), remaining, nullptr);
  }

  CLIENT_TYPE ct = header->getClientType();
  if (!channel_.isSupportedClient(ct) &&
      ct != THRIFT_HEADER_SASL_CLIENT_TYPE) {
    LOG(ERROR) << "Server rejecting unsupported client type " << ct;
    channel_.checkSupportedClient(ct);
  }

  // Check if protocol used in the buffer is consistent with the protocol
  // id in the header.

  folly::io::Cursor c(buf.get());
  auto byte = c.read<uint8_t>();
  // Initialize it to a value never used on the wire
  PROTOCOL_TYPES protInBuf = PROTOCOL_TYPES::T_DEBUG_PROTOCOL;
  if (byte == 0x82) {
    protInBuf = PROTOCOL_TYPES::T_COMPACT_PROTOCOL;
  } else if (byte == 0x80) {
    protInBuf = PROTOCOL_TYPES::T_BINARY_PROTOCOL;
  } else if (ct != THRIFT_HTTP_SERVER_TYPE) {
    LOG(ERROR) << "Received corrupted request from client: "
               << getTransportDebugString(channel_.getTransport()) << ". "
               << "Corrupted payload in header message. In message header, "
               << "protoId: " << header->getProtocolId() << ", "
               << "clientType: " << folly::to<std::string>(ct) << ". "
               << "First few bytes of payload: "
               << getTHeaderPayloadString(buf.get());

  }

  if (protInBuf != PROTOCOL_TYPES::T_DEBUG_PROTOCOL &&
      header->getProtocolId() != protInBuf) {
    LOG(ERROR) << "Received corrupted request from client: "
               << getTransportDebugString(channel_.getTransport()) << ". "
               << "Protocol mismatch, in message header, protocolId: "
               << folly::to<std::string>(header->getProtocolId()) << ", "
               << "clientType: " << folly::to<std::string>(ct) << ", "
               << "in payload, protocolId: "
               << folly::to<std::string>(protInBuf)
               << ". First few bytes of payload: "
               << getTHeaderPayloadString(buf.get());
  }

  // In order to allow negotiation to happen when the client requests
  // sasl but it's not supported, we don't throw an exception in the
  // sasl case.  Instead, we let the message bubble up, and check if
  // the client is supported in handleSecurityMessage called from the
  // messageReceived callback.

  header->setMinCompressBytes(channel_.getMinCompressBytes());
  return make_tuple(std::move(buf), 0, std::move(header));
}

std::string
HeaderServerChannel::getTHeaderPayloadString(IOBuf* buf) {
  auto len = std::min<size_t>(buf->length(), 20);
  return folly::cEscape<std::string>(
      folly::StringPiece((const char*)buf->data(), len));
}

std::string
HeaderServerChannel::getTransportDebugString(TAsyncTransport* transport) {
  if (!transport) {
    return std::string();
  }

  auto ret = folly::to<std::string>("(transport ",
                                    folly::demangle(typeid(*transport)));

  try {
    folly::SocketAddress addr;
    transport->getPeerAddress(&addr);
    folly::toAppend(", address ", addr.getAddressStr(),
                    ", port ", addr.getPort(),
                    &ret);
  } catch (const std::exception& e) {
  }

  ret += ')';
  return ret;
}

// Client Interface

HeaderServerChannel::HeaderRequest::HeaderRequest(
      HeaderServerChannel* channel,
      unique_ptr<IOBuf>&& buf,
      unique_ptr<THeader>&& header,
      unique_ptr<sample> sample)
  : channel_(channel)
  , header_(std::move(header))
  , active_(true) {

  this->buf_ = std::move(buf);
  if (sample) {
    timestamps_.readBegin = sample->readBegin;
    timestamps_.readEnd = sample->readEnd;
  }
}

/**
 * send a reply to the client.
 *
 * Note that to be backwards compatible with thrift1, the generated
 * code calls sendReply(nullptr) for oneway calls where seqid !=
 * ONEWAY_SEQ_ID.  This is so that the sendCatchupRequests code runs
 * correctly for in-order responses to older clients.  sendCatchupRequests
 * does not actually send null buffers, it just ignores them.
 *
 */
void HeaderServerChannel::HeaderRequest::sendReply(
    unique_ptr<IOBuf>&& buf,
    MessageChannel::SendCallback* cb) {
  // This method is only called and active_ is only touched in evb, so
  // it is safe to use this flag from both timeout and normal responses.
  auto& header = active_ ? header_ : timeoutHeader_;
  if (!channel_->outOfOrder_.value()) {
    // In order processing, make sure the ordering is correct.
    if (InOrderRecvSeqId_ != channel_->lastWrittenSeqId_ + 1) {
      // Save it until we can send it in order.
      channel_->inOrderRequests_[InOrderRecvSeqId_] =
        std::make_tuple(cb, std::move(buf), std::move(header));
    } else {
      // Send it now, and send any subsequent requests in order.
      channel_->sendCatchupRequests(std::move(buf), cb, header.get());
    }
  } else {
    if (!buf) {
      // oneway calls are OK do this, but is a bug for twoway.
      DCHECK(isOneway());
      return;
    }
    try {
      // out of order, send as soon as it is done.
      channel_->sendMessage(cb, std::move(buf), header.get());
    } catch (const std::exception& e) {
      LOG(ERROR) << "Failed to send message: " << e.what();
    }
  }
}

/**
 * Send a serialized error back to the client.
 * For a header server, this means serializing the exception, and setting
 * an error flag in the header.
 */
void HeaderServerChannel::HeaderRequest::sendErrorWrapped(
    folly::exception_wrapper ew,
    std::string exCode,
    MessageChannel::SendCallback* cb) {

  // Other types are unimplemented.
  DCHECK(ew.is_compatible_with<TApplicationException>());

  header_->setHeader("ex", exCode);
  ew.with_exception([&](TApplicationException& tae) {
      std::unique_ptr<folly::IOBuf> exbuf;
      uint16_t proto = header_->getProtocolId();
      auto transforms = header_->getWriteTransforms();
      try {
        exbuf = serializeError(proto, tae, getBuf());
      } catch (const TProtocolException& pe) {
        LOG(ERROR) << "serializeError failed. type=" << pe.getType()
            << " what()=" << pe.what();
        channel_->closeNow();
        return;
      }
      exbuf = THeader::transform(std::move(exbuf),
                                 transforms,
                                 header_->getMinCompressBytes());
      sendReply(std::move(exbuf), cb);
    });
}

void HeaderServerChannel::HeaderRequest::sendTimeoutResponse(
    MessageChannel::SendCallback* cb,
    const std::map<std::string, std::string>& headers) {
  // Sending tiemout response always happens on eb thread, while normal
  // request handling might still be work-in-progress on tm thread and
  // touches the per-request THeader at any time. This builds a new THeader
  // and only reads certain fields from header_. To avoid race condition,
  // DO NOT read any header from the per-request THeader.
  timeoutHeader_ = folly::make_unique<THeader>();
  timeoutHeader_->setProtocolId(header_->getProtocolId());
  timeoutHeader_->setTransforms(header_->getWriteTransforms());
  timeoutHeader_->setMinCompressBytes(header_->getMinCompressBytes());
  timeoutHeader_->setSequenceNumber(header_->getSequenceNumber());
  timeoutHeader_->setHeader("ex", kTaskExpiredErrorCode);
  for (const auto& it : headers) {
    timeoutHeader_->setHeader(it.first, it.second);
  }

  std::unique_ptr<folly::IOBuf> exbuf;
  TApplicationException tae(
      TApplicationException::TApplicationExceptionType::TIMEOUT,
      "Task expired");
  try {
    exbuf = serializeError(timeoutHeader_->getProtocolId(), tae, getBuf());
  } catch (const TProtocolException& pe) {
    LOG(ERROR) << "serializeError failed. type=" << pe.getType()
      << " what()=" << pe.what();
    channel_->closeNow();
    return;
  }

  exbuf = THeader::transform(std::move(exbuf),
                             timeoutHeader_->getWriteTransforms(),
                             timeoutHeader_->getMinCompressBytes());
  sendReply(std::move(exbuf), cb);
}
<commit_msg>Fix exception in ProtectionHandler.cpp: Error unwrapping message<commit_after>/*
 * Copyright 2014 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <thrift/lib/cpp2/async/HeaderServerChannel.h>
#include <thrift/lib/cpp/async/TAsyncSocket.h>
#include <thrift/lib/cpp/TApplicationException.h>
#include <thrift/lib/cpp/transport/TTransportException.h>
#include <thrift/lib/cpp2/protocol/Serializer.h>
#include <folly/io/Cursor.h>
#include <folly/String.h>

#include <utility>
#include <exception>

using std::unique_ptr;
using std::pair;
using folly::IOBuf;
using folly::IOBufQueue;
using folly::make_unique;
using namespace apache::thrift::transport;
using namespace apache::thrift;
using folly::EventBase;
using apache::thrift::async::TAsyncSocket;
using apache::thrift::async::TAsyncTransport;
using apache::thrift::TApplicationException;
using apache::thrift::server::TServerObserver;
using apache::thrift::protocol::PROTOCOL_TYPES;

namespace apache { namespace thrift {

std::atomic<uint32_t> HeaderServerChannel::sample_(0);

HeaderServerChannel::HeaderServerChannel(
  const std::shared_ptr<TAsyncTransport>& transport)
    : HeaderServerChannel(
        std::shared_ptr<Cpp2Channel>(
            Cpp2Channel::newChannel(transport,
                make_unique<ServerFramingHandler>(*this))))
{}

HeaderServerChannel::HeaderServerChannel(
    const std::shared_ptr<Cpp2Channel>& cpp2Channel)
    : callback_(nullptr)
    , arrivalSeqId_(1)
    , lastWrittenSeqId_(0)
    , sampleRate_(0)
    , timeoutSASL_(5000)
    , saslServerCallback_(*this)
    , cpp2Channel_(cpp2Channel)
    , timer_(new folly::HHWheelTimer(getEventBase())) {}

void HeaderServerChannel::destroy() {
  DestructorGuard dg(this);

  saslServerCallback_.cancelTimeout();
  if (saslServer_) {
    saslServer_->detachEventBase();
  }

  if (callback_) {
    auto error = folly::make_exception_wrapper<TTransportException>(
        "Channel destroyed");
    callback_->channelClosed(std::move(error));
  }

  cpp2Channel_->closeNow();

  folly::DelayedDestruction::destroy();
}

// Header framing
unique_ptr<IOBuf>
HeaderServerChannel::ServerFramingHandler::addFrame(unique_ptr<IOBuf> buf,
                                                    THeader* header) {
  channel_.updateClientType(header->getClientType());

  // Note: This THeader function may throw.  However, we don't want to catch
  // it here, because this would send an empty message out on the wire.
  // Instead we have to catch it at sendMessage
  return header->addHeader(
    std::move(buf),
    channel_.getPersistentWriteHeaders(),
    false /* Data already transformed in AsyncProcessor.h */);
}

std::tuple<unique_ptr<IOBuf>, size_t, unique_ptr<THeader>>
HeaderServerChannel::ServerFramingHandler::removeFrame(IOBufQueue* q) {
  std::unique_ptr<THeader> header(new THeader(THeader::ALLOW_BIG_FRAMES));
  // removeHeader will set seqid in header.
  // For older clients with seqid in the protocol, header
  // will dig in to the protocol to get the seqid correctly.
  if (!q || !q->front() || q->front()->empty()) {
    return make_tuple(std::unique_ptr<IOBuf>(), 0, nullptr);
  }

  std::unique_ptr<folly::IOBuf> buf;
  size_t remaining = 0;
  try {
    buf = header->removeHeader(q, remaining,
                               channel_.getPersistentReadHeaders());
  } catch (const std::exception& e) {
    LOG(ERROR) << "Received invalid request from client: "
               << folly::exceptionStr(e) << " "
               << getTransportDebugString(channel_.getTransport());
    throw;
  }
  if (!buf) {
    return make_tuple(std::unique_ptr<IOBuf>(), remaining, nullptr);
  }

  CLIENT_TYPE ct = header->getClientType();
  if (!channel_.isSupportedClient(ct) &&
      ct != THRIFT_HEADER_SASL_CLIENT_TYPE) {
    LOG(ERROR) << "Server rejecting unsupported client type " << ct;
    channel_.checkSupportedClient(ct);
  }

  // Check if protocol used in the buffer is consistent with the protocol
  // id in the header.

  folly::io::Cursor c(buf.get());
  auto byte = c.read<uint8_t>();
  // Initialize it to a value never used on the wire
  PROTOCOL_TYPES protInBuf = PROTOCOL_TYPES::T_DEBUG_PROTOCOL;
  if (byte == 0x82) {
    protInBuf = PROTOCOL_TYPES::T_COMPACT_PROTOCOL;
  } else if (byte == 0x80) {
    protInBuf = PROTOCOL_TYPES::T_BINARY_PROTOCOL;
  } else if (ct != THRIFT_HTTP_SERVER_TYPE) {
    LOG(ERROR) << "Received corrupted request from client: "
               << getTransportDebugString(channel_.getTransport()) << ". "
               << "Corrupted payload in header message. In message header, "
               << "protoId: " << header->getProtocolId() << ", "
               << "clientType: " << folly::to<std::string>(ct) << ". "
               << "First few bytes of payload: "
               << getTHeaderPayloadString(buf.get());

  }

  if (protInBuf != PROTOCOL_TYPES::T_DEBUG_PROTOCOL &&
      header->getProtocolId() != protInBuf) {
    LOG(ERROR) << "Received corrupted request from client: "
               << getTransportDebugString(channel_.getTransport()) << ". "
               << "Protocol mismatch, in message header, protocolId: "
               << folly::to<std::string>(header->getProtocolId()) << ", "
               << "clientType: " << folly::to<std::string>(ct) << ", "
               << "in payload, protocolId: "
               << folly::to<std::string>(protInBuf)
               << ". First few bytes of payload: "
               << getTHeaderPayloadString(buf.get());
  }

  // In order to allow negotiation to happen when the client requests
  // sasl but it's not supported, we don't throw an exception in the
  // sasl case.  Instead, we let the message bubble up, and check if
  // the client is supported in handleSecurityMessage called from the
  // messageReceived callback.

  header->setMinCompressBytes(channel_.getMinCompressBytes());
  return make_tuple(std::move(buf), 0, std::move(header));
}

std::string
HeaderServerChannel::getTHeaderPayloadString(IOBuf* buf) {
  auto len = std::min<size_t>(buf->length(), 20);
  return folly::cEscape<std::string>(
      folly::StringPiece((const char*)buf->data(), len));
}

std::string
HeaderServerChannel::getTransportDebugString(TAsyncTransport* transport) {
  if (!transport) {
    return std::string();
  }

  auto ret = folly::to<std::string>("(transport ",
                                    folly::demangle(typeid(*transport)));

  try {
    folly::SocketAddress addr;
    transport->getPeerAddress(&addr);
    folly::toAppend(", address ", addr.getAddressStr(),
                    ", port ", addr.getPort(),
                    &ret);
  } catch (const std::exception& e) {
  }

  ret += ')';
  return ret;
}

// Client Interface

HeaderServerChannel::HeaderRequest::HeaderRequest(
      HeaderServerChannel* channel,
      unique_ptr<IOBuf>&& buf,
      unique_ptr<THeader>&& header,
      unique_ptr<sample> sample)
  : channel_(channel)
  , header_(std::move(header))
  , active_(true) {

  this->buf_ = std::move(buf);
  if (sample) {
    timestamps_.readBegin = sample->readBegin;
    timestamps_.readEnd = sample->readEnd;
  }
}

/**
 * send a reply to the client.
 *
 * Note that to be backwards compatible with thrift1, the generated
 * code calls sendReply(nullptr) for oneway calls where seqid !=
 * ONEWAY_SEQ_ID.  This is so that the sendCatchupRequests code runs
 * correctly for in-order responses to older clients.  sendCatchupRequests
 * does not actually send null buffers, it just ignores them.
 *
 */
void HeaderServerChannel::HeaderRequest::sendReply(
    unique_ptr<IOBuf>&& buf,
    MessageChannel::SendCallback* cb) {
  // This method is only called and active_ is only touched in evb, so
  // it is safe to use this flag from both timeout and normal responses.
  auto& header = active_ ? header_ : timeoutHeader_;
  if (!channel_->outOfOrder_.value()) {
    // In order processing, make sure the ordering is correct.
    if (InOrderRecvSeqId_ != channel_->lastWrittenSeqId_ + 1) {
      // Save it until we can send it in order.
      channel_->inOrderRequests_[InOrderRecvSeqId_] =
        std::make_tuple(cb, std::move(buf), std::move(header));
    } else {
      // Send it now, and send any subsequent requests in order.
      channel_->sendCatchupRequests(std::move(buf), cb, header.get());
    }
  } else {
    if (!buf) {
      // oneway calls are OK do this, but is a bug for twoway.
      DCHECK(isOneway());
      return;
    }
    try {
      // out of order, send as soon as it is done.
      channel_->sendMessage(cb, std::move(buf), header.get());
    } catch (const std::exception& e) {
      LOG(ERROR) << "Failed to send message: " << e.what();
    }
  }
}

/**
 * Send a serialized error back to the client.
 * For a header server, this means serializing the exception, and setting
 * an error flag in the header.
 */
void HeaderServerChannel::HeaderRequest::sendErrorWrapped(
    folly::exception_wrapper ew,
    std::string exCode,
    MessageChannel::SendCallback* cb) {

  // Other types are unimplemented.
  DCHECK(ew.is_compatible_with<TApplicationException>());

  header_->setHeader("ex", exCode);
  ew.with_exception([&](TApplicationException& tae) {
      std::unique_ptr<folly::IOBuf> exbuf;
      uint16_t proto = header_->getProtocolId();
      auto transforms = header_->getWriteTransforms();
      try {
        exbuf = serializeError(proto, tae, getBuf());
      } catch (const TProtocolException& pe) {
        LOG(ERROR) << "serializeError failed. type=" << pe.getType()
            << " what()=" << pe.what();
        channel_->closeNow();
        return;
      }
      exbuf = THeader::transform(std::move(exbuf),
                                 transforms,
                                 header_->getMinCompressBytes());
      sendReply(std::move(exbuf), cb);
    });
}

void HeaderServerChannel::HeaderRequest::sendTimeoutResponse(
    MessageChannel::SendCallback* cb,
    const std::map<std::string, std::string>& headers) {
  // Sending tiemout response always happens on eb thread, while normal
  // request handling might still be work-in-progress on tm thread and
  // touches the per-request THeader at any time. This builds a new THeader
  // and only reads certain fields from header_. To avoid race condition,
  // DO NOT read any header from the per-request THeader.
  timeoutHeader_ = folly::make_unique<THeader>();
  timeoutHeader_->setProtocolId(header_->getProtocolId());
  timeoutHeader_->setTransforms(header_->getWriteTransforms());
  timeoutHeader_->setMinCompressBytes(header_->getMinCompressBytes());
  timeoutHeader_->setSequenceNumber(header_->getSequenceNumber());
  timeoutHeader_->setClientType(header_->getClientType());
  timeoutHeader_->setHeader("ex", kTaskExpiredErrorCode);
  for (const auto& it : headers) {
    timeoutHeader_->setHeader(it.first, it.second);
  }

  std::unique_ptr<folly::IOBuf> exbuf;
  TApplicationException tae(
      TApplicationException::TApplicationExceptionType::TIMEOUT,
      "Task expired");
  try {
    exbuf = serializeError(timeoutHeader_->getProtocolId(), tae, getBuf());
  } catch (const TProtocolException& pe) {
    LOG(ERROR) << "serializeError failed. type=" << pe.getType()
      << " what()=" << pe.what();
    channel_->closeNow();
    return;
  }

  exbuf = THeader::transform(std::move(exbuf),
                             timeoutHeader_->getWriteTransforms(),
                             timeoutHeader_->getMinCompressBytes());
  sendReply(std::move(exbuf), cb);
}

{'fraction_non_alphanumeric': 0.12791865428069293, 'fraction_numerical': 0.0035986275002092226, 'mean_word_length': 3.5306161137440757, 'pattern_counts': {'":': 0, '<': 249, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1206728', 'n_tokens_mistral': 7096, 'n_tokens_neox': 7017, 'n_words': 1767}