<commit_before>/*
 *  Copyright (c) 2015, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include <proxygen/lib/http/codec/compress/GzipHeaderCodec.h>

#include <folly/Memory.h>
#include <folly/String.h>
#include <folly/ThreadLocal.h>
#include <folly/io/IOBuf.h>
#include <proxygen/lib/http/codec/SPDYCodec.h>
#include <proxygen/lib/http/codec/SPDYConstants.h>
#include <proxygen/lib/utils/UnionBasedStatic.h>
#include <string>

using folly::IOBuf;
using folly::ThreadLocalPtr;
using folly::io::Cursor;
using proxygen::compress::Header;
using proxygen::compress::HeaderPiece;
using proxygen::compress::HeaderPieceList;
using proxygen::spdy::kMaxFrameLength;
using std::string;
using std::unique_ptr;
using std::vector;

namespace {

// Maximum size of header names+values after expanding multi-value headers
const size_t kMaxExpandedHeaderLineBytes = 80 * 1024;

DEFINE_UNION_STATIC(ThreadLocalPtr<IOBuf>, IOBuf, s_buf);

folly::IOBuf& getStaticHeaderBufSpace(size_t size) {
  if (!s_buf.data) {
    s_buf.data.reset(new IOBuf(IOBuf::CREATE, size));
  } else {
    if (size > s_buf.data->capacity()) {
      s_buf.data.reset(new IOBuf(IOBuf::CREATE, size));
    } else {
      s_buf.data->clear();
    }
  }
  DCHECK(!s_buf.data->isShared());
  return *s_buf.data;
}

void appendString(uint8_t*& dst, const string& str) {
  size_t len = str.length();
  memcpy(dst, str.data(), len);
  dst += len;
}

} // anonymous namespace

namespace proxygen {

GzipHeaderCodec::GzipHeaderCodec(int compressionLevel,
                                 const SPDYVersionSettings& versionSettings)
    : versionSettings_(versionSettings) {
  // Create compression and decompression contexts by cloning thread-local
  // copies of the initial SPDY compression state
  auto context = getZlibContext(versionSettings, compressionLevel);
  deflateCopy(&deflater_, const_cast<z_stream*>(&(context->deflater)));
  inflateCopy(&inflater_, const_cast<z_stream*>(&(context->inflater)));
}

GzipHeaderCodec::GzipHeaderCodec(int compressionLevel,
                                 SPDYVersion version)
    : GzipHeaderCodec(
        compressionLevel,
        SPDYCodec::getVersionSettings(version)) {}

GzipHeaderCodec::~GzipHeaderCodec() {
  deflateEnd(&deflater_);
  inflateEnd(&inflater_);
}

folly::IOBuf& GzipHeaderCodec::getHeaderBuf() {
  return getStaticHeaderBufSpace(maxUncompressed_);
}

const GzipHeaderCodec::ZlibContext* GzipHeaderCodec::getZlibContext(
    SPDYVersionSettings versionSettings, int compressionLevel) {
  static folly::ThreadLocal<ZlibContextMap> zlibContexts_;
  ZlibConfig zlibConfig(versionSettings.version, compressionLevel);
  auto match = zlibContexts_->find(zlibConfig);
  if (match != zlibContexts_->end()) {
    return match->second.get();
  } else {
    // This is the first request for the specified SPDY version and compression
    // level in this thread, so we need to construct the initial compressor and
    // decompressor contexts.
    auto newContext = folly::make_unique<ZlibContext>();
    newContext->deflater.zalloc = Z_NULL;
    newContext->deflater.zfree = Z_NULL;
    newContext->deflater.opaque = Z_NULL;
    newContext->deflater.avail_in = 0;
    newContext->deflater.next_in = Z_NULL;
    int windowBits  = (compressionLevel == Z_NO_COMPRESSION) ? 8 : 11;
    int r = deflateInit2(
        &(newContext->deflater),
        compressionLevel,
        Z_DEFLATED, // compression method
        windowBits, // log2 of the compression window size, negative value
                    // means raw deflate output format w/o libz header
        1,          // memory size for internal compression state, 1-9
        Z_DEFAULT_STRATEGY);
    CHECK(r == Z_OK);
    if (compressionLevel != Z_NO_COMPRESSION) {
      r = deflateSetDictionary(&(newContext->deflater), versionSettings.dict,
                               versionSettings.dictSize);
      CHECK(r == Z_OK);
    }

    newContext->inflater.zalloc = Z_NULL;
    newContext->inflater.zfree = Z_NULL;
    newContext->inflater.opaque = Z_NULL;
    newContext->inflater.avail_in = 0;
    newContext->inflater.next_in = Z_NULL;
    r = inflateInit2(&(newContext->inflater), 0);
    CHECK(r == Z_OK);

    auto result = newContext.get();
    zlibContexts_->emplace(zlibConfig, std::move(newContext));
    return result;
  }
}

unique_ptr<IOBuf> GzipHeaderCodec::encode(vector<Header>& headers) noexcept {
  // Build a sequence of the header names and values, sorted by name.
  // The purpose of the sort is to make it easier to combine the
  // values of multiple headers with the same name.  The SPDY spec
  // prohibits any header name from appearing more than once in the
  // Name/Value list, so we must combine values when serializing.
  std::sort(headers.begin(), headers.end());

  auto& uncompressed = getHeaderBuf();
  // Compute the amount of space needed to hold the uncompressed
  // representation of the headers.  This is an upper bound on the
  // amount of space we'll actually need, because if we end up<commit_msg>use zlib with window optimization<commit_after>/*
 *  Copyright (c) 2015, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */
#include <proxygen/lib/http/codec/compress/GzipHeaderCodec.h>

#include <folly/Memory.h>
#include <folly/String.h>
#include <folly/ThreadLocal.h>
#include <folly/io/IOBuf.h>
#include <proxygen/lib/http/codec/SPDYCodec.h>
#include <proxygen/lib/http/codec/SPDYConstants.h>
#include <proxygen/lib/utils/UnionBasedStatic.h>
#include <string>

using folly::IOBuf;
using folly::ThreadLocalPtr;
using folly::io::Cursor;
using proxygen::compress::Header;
using proxygen::compress::HeaderPiece;
using proxygen::compress::HeaderPieceList;
using proxygen::spdy::kMaxFrameLength;
using std::string;
using std::unique_ptr;
using std::vector;

namespace {

// Maximum size of header names+values after expanding multi-value headers
const size_t kMaxExpandedHeaderLineBytes = 80 * 1024;

DEFINE_UNION_STATIC(ThreadLocalPtr<IOBuf>, IOBuf, s_buf);

folly::IOBuf& getStaticHeaderBufSpace(size_t size) {
  if (!s_buf.data) {
    s_buf.data.reset(new IOBuf(IOBuf::CREATE, size));
  } else {
    if (size > s_buf.data->capacity()) {
      s_buf.data.reset(new IOBuf(IOBuf::CREATE, size));
    } else {
      s_buf.data->clear();
    }
  }
  DCHECK(!s_buf.data->isShared());
  return *s_buf.data;
}

void appendString(uint8_t*& dst, const string& str) {
  size_t len = str.length();
  memcpy(dst, str.data(), len);
  dst += len;
}

} // anonymous namespace

namespace proxygen {

GzipHeaderCodec::GzipHeaderCodec(int compressionLevel,
                                 const SPDYVersionSettings& versionSettings)
    : versionSettings_(versionSettings) {
  // Create compression and decompression contexts by cloning thread-local
  // copies of the initial SPDY compression state
  auto context = getZlibContext(versionSettings, compressionLevel);
  deflateCopy(&deflater_, const_cast<z_stream*>(&(context->deflater)));
  inflateCopy(&inflater_, const_cast<z_stream*>(&(context->inflater)));
}

GzipHeaderCodec::GzipHeaderCodec(int compressionLevel,
                                 SPDYVersion version)
    : GzipHeaderCodec(
        compressionLevel,
        SPDYCodec::getVersionSettings(version)) {}

GzipHeaderCodec::~GzipHeaderCodec() {
  deflateEnd(&deflater_);
  inflateEnd(&inflater_);
}

folly::IOBuf& GzipHeaderCodec::getHeaderBuf() {
  return getStaticHeaderBufSpace(maxUncompressed_);
}

const GzipHeaderCodec::ZlibContext* GzipHeaderCodec::getZlibContext(
    SPDYVersionSettings versionSettings, int compressionLevel) {
  static folly::ThreadLocal<ZlibContextMap> zlibContexts_;
  ZlibConfig zlibConfig(versionSettings.version, compressionLevel);
  auto match = zlibContexts_->find(zlibConfig);
  if (match != zlibContexts_->end()) {
    return match->second.get();
  } else {
    // This is the first request for the specified SPDY version and compression
    // level in this thread, so we need to construct the initial compressor and
    // decompressor contexts.
    auto newContext = folly::make_unique<ZlibContext>();
    newContext->deflater.zalloc = Z_NULL;
    newContext->deflater.zfree = Z_NULL;
    newContext->deflater.opaque = Z_NULL;
    newContext->deflater.avail_in = 0;
    newContext->deflater.next_in = Z_NULL;
    int windowBits  = (compressionLevel == Z_NO_COMPRESSION) ? 8 : 11;
    int r = deflateInit2(
        &(newContext->deflater),
        compressionLevel,
        Z_DEFLATED, // compression method
        windowBits, // log2 of the compression window size, negative value
                    // means raw deflate output format w/o libz header
        1,          // memory size for internal compression state, 1-9
        Z_DEFAULT_STRATEGY);
    CHECK(r == Z_OK);
    if (compressionLevel != Z_NO_COMPRESSION) {
      r = deflateSetDictionary(&(newContext->deflater), versionSettings.dict,
                               versionSettings.dictSize);
      CHECK(r == Z_OK);
    }

    newContext->inflater.zalloc = Z_NULL;
    newContext->inflater.zfree = Z_NULL;
    newContext->inflater.opaque = Z_NULL;
    newContext->inflater.avail_in = 0;
    newContext->inflater.next_in = Z_NULL;
    // set zlib's reserved flag to allocate smaller initial sliding window, then
    // double it if necessary
    newContext->inflater.reserved = 0x01;
    r = inflateInit2(&(newContext->inflater), 0);
    CHECK(r == Z_OK);

    auto result = newContext.get();
    zlibContexts_->emplace(zlibConfig, std::move(newContext));
    return result;
  }
}

unique_ptr<IOBuf> GzipHeaderCodec::encode(vector<Header>& headers) noexcept {
  // Build a sequence of the header names and values, sorted by name.
  // The purpose of the sort is to make it easier to combine the
  // values of multiple headers with the same name.  The SPDY spec
  // prohibits any header name from appearing more than once in the
  // Name/Value list, so we must combine values when serializing.
  std::sort(headers.begin(), headers.end());

  auto& uncompressed = getHeaderBuf();
  // Compute the amount of space needed to hold the uncompressed
  // representation of the headers.  This is an upper bound on the
  // amount of space we'll actually need, because if we end up
{'fraction_non_alphanumeric': 0.10921213259460982, 'fraction_numerical': 0.004601371020753122, 'mean_word_length': 3.8255550521069326, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6738443', 'n_tokens_mistral': 3308, 'n_tokens_neox': 3267, 'n_words': 854}