<commit_before>/*
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef FCP_CLIENT_HTTP_HTTP_CLIENT_H_
#define FCP_CLIENT_HTTP_HTTP_CLIENT_H_

#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include "absl/base/attributes.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"

namespace fcp {
namespace client {
namespace http {

using Header = std::pair<std::string, std::string>;
// This is a vector of pairs and not a map since multiple request headers with
// the same name are allowed (see RFC2616 section 4.2).
using HeaderList = std::vector<Header>;

class HttpRequest;          // forward declaration
class HttpRequestCallback;  // forward declaration
class HttpRequestHandle;    // forward declaration
class HttpResponse;         // forward declaration

// An interface that allows the callers to make HTTP requests and receive their
// responses.
//
// Platforms will be required to pass an instance of this class to
// `RunFederatedComputation(...)`, and such instances must remain alive for at
// least the duration of those calls.
//
// Instances of this class must support being called from any thread. In such
// cases implementations should ideally handle multiple requests in parallel, at
// least up to an implementation-defined max number of parallel requests (i.e.
// ideally two `PerformRequests` calls on different threads can be handled in
// parallel, rather than having the 2nd call block until the 1st call is
// completed).
//
// Besides the requirements documented further below, the following high-level
// behavior is required of any `HttpClient` implementation:
// - Underlying protocols:
//   * Implementations must support at least HTTP/1.1 over TLS 1.2.
//   * Implementations are also allowed to serve requests using HTTP/2, QUIC or
//     other newer protocols.
//   * Implementations must support both IPv4 and IPv6 (but they are allowed to
//     fall back to IPv4).
// - Certificate validation:
//   * Implementations are responsible for TLS certificate validation and for
//     maintaining an up-to-date set of root certificates as well as an
//     up-to-date HTTP/TLS implementation.
//   * Implementations should not include user-added CAs, and should consider
//     restricting the set of CAs further to only those needed for connecting to
//     the expected endpoints.
// - Cookies:
//   * Implementations must not supply any cookies in requests (beyond those
//     that may be specified in the `HttpRequest::headers()` method).
//   * Implementations must not store any cookies returned by the server.
//   * Instead, they must return any server-specified "Set-Cookie" response
//     header via `HttpResponse::headers()`.
// - Redirects:
//   *  Implementations must follow HTTP redirects responses, up to an
//      implementation-defined maximum.
//   *  In such cases the response headers & body returned via the interfaces
//      below should be those of the final response.
//   *  See `HttpRequestCallback` docs below for more details.
// - Caching:
//   *  Implementations should not implement a cache as it is expected that
//      naive HTTP-level caching will not be effective (and since a cache may
//      ultimately be implemented over this interface, in the Federated Compute
//      library itself).
//   *  If implementations do implement one, however, they are expected to abide
//      by the standard HTTP caching rules (see the `HttpRequest::Method` docs
//      for more details).
// - Response body decompression & decoding:
//   *  If no "Accept-Encoding" request header is explicitly specified in
//      `HttpRequest::headers()`, then implementations must advertise an
//      "Accept-Encoding" request header themselves whose value includes at
//      least "gzip" (additional encodings are allowed to be specified in
//      addition to "gzip"), and must transparently decompress any compressed
//      server responses before returning the data via these interfaces.
//      Implementations are also allowed to advertise/support additional
//      encoding methods.
//   *  In such cases where no "Accept-Encoding" header is specified,
//      implementations must remove the "Content-Encoding" and
//      "Content-Length" headers from headers returned via
//      `HttpResponse::headers()` (since those wouldn't reflect the payload
//      delivered via this interface).
//   *  However, if an "Accept-Encoding" request header *is* explicitly
//      specified, then implementations must use that header verbatim and they
//      must not decompress the response (even if they natively support the
//      compression method), and they must leave the "Content-Encoding" and
//      "Content-Length" headers intact.
//   *  This ensures that the caller of this interface can take full control of
//      the decompression and/or choose to store decompressed payloads on disk
//      if it so chooses.
//   *  Implementations must transparently decode server responses served with
//      "Transfer-Encoding: gzip", "deflate", or "chunked". In such cases they
//      must remove the "Transfer-Encoding" response header.
// - Request body compression & encoding:
//   *  If implementations receive a "Content-Encoding" request header, this
//      means that the request body stream they receive has already been
//      compressed. The implementation must leave the header and request body
//      intact in such cases (i.e. not re-compress it).
//   *  If implementations receive a "Content-Length" request header, they must
//      use it verbatim and they should then assume that the request body will
//      be of exactly that size.
//   *  If they do not receive such a header then they must use the
//      "Transfer-encoding: chunked" mechanism to transmit the request body
//      (i.e. they shouldn't specify a "Content-Length" header and they should
//      transmit the body in chunks), or use an equivalent method of streaming
//      the data (such as HTTP/2's data streaming).
class HttpClient {
 public:
  virtual ~HttpClient() = default;<commit_msg>Remove `http_client.h` requirement for clients to support 'gzip' or 'deflate' 'Transfer-Encoding' headers.<commit_after>/*
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef FCP_CLIENT_HTTP_HTTP_CLIENT_H_
#define FCP_CLIENT_HTTP_HTTP_CLIENT_H_

#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include "absl/base/attributes.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"

namespace fcp {
namespace client {
namespace http {

using Header = std::pair<std::string, std::string>;
// This is a vector of pairs and not a map since multiple request headers with
// the same name are allowed (see RFC2616 section 4.2).
using HeaderList = std::vector<Header>;

class HttpRequest;          // forward declaration
class HttpRequestCallback;  // forward declaration
class HttpRequestHandle;    // forward declaration
class HttpResponse;         // forward declaration

// An interface that allows the callers to make HTTP requests and receive their
// responses.
//
// Platforms will be required to pass an instance of this class to
// `RunFederatedComputation(...)`, and such instances must remain alive for at
// least the duration of those calls.
//
// Instances of this class must support being called from any thread. In such
// cases implementations should ideally handle multiple requests in parallel, at
// least up to an implementation-defined max number of parallel requests (i.e.
// ideally two `PerformRequests` calls on different threads can be handled in
// parallel, rather than having the 2nd call block until the 1st call is
// completed).
//
// Besides the requirements documented further below, the following high-level
// behavior is required of any `HttpClient` implementation:
// - Underlying protocols:
//   * Implementations must support at least HTTP/1.1 over TLS 1.2.
//   * Implementations are also allowed to serve requests using HTTP/2, QUIC or
//     other newer protocols.
//   * Implementations must support both IPv4 and IPv6 (but they are allowed to
//     fall back to IPv4).
// - Certificate validation:
//   * Implementations are responsible for TLS certificate validation and for
//     maintaining an up-to-date set of root certificates as well as an
//     up-to-date HTTP/TLS implementation.
//   * Implementations should not include user-added CAs, and should consider
//     restricting the set of CAs further to only those needed for connecting to
//     the expected endpoints.
// - Cookies:
//   * Implementations must not supply any cookies in requests (beyond those
//     that may be specified in the `HttpRequest::headers()` method).
//   * Implementations must not store any cookies returned by the server.
//   * Instead, they must return any server-specified "Set-Cookie" response
//     header via `HttpResponse::headers()`.
// - Redirects:
//   *  Implementations must follow HTTP redirects responses, up to an
//      implementation-defined maximum.
//   *  In such cases the response headers & body returned via the interfaces
//      below should be those of the final response.
//   *  See `HttpRequestCallback` docs below for more details.
// - Caching:
//   *  Implementations should not implement a cache as it is expected that
//      naive HTTP-level caching will not be effective (and since a cache may
//      ultimately be implemented over this interface, in the Federated Compute
//      library itself).
//   *  If implementations do implement one, however, they are expected to abide
//      by the standard HTTP caching rules (see the `HttpRequest::Method` docs
//      for more details).
// - Response body decompression & decoding:
//   *  If no "Accept-Encoding" request header is explicitly specified in
//      `HttpRequest::headers()`, then implementations must advertise an
//      "Accept-Encoding" request header themselves whose value includes at
//      least "gzip" (additional encodings are allowed to be specified in
//      addition to "gzip"), and must transparently decompress any compressed
//      server responses before returning the data via these interfaces.
//      Implementations are also allowed to advertise/support additional
//      encoding methods.
//   *  In such cases where no "Accept-Encoding" header is specified,
//      implementations must remove the "Content-Encoding" and
//      "Content-Length" headers from headers returned via
//      `HttpResponse::headers()` (since those wouldn't reflect the payload
//      delivered via this interface).
//   *  However, if an "Accept-Encoding" request header *is* explicitly
//      specified, then implementations must use that header verbatim and they
//      must not decompress the response (even if they natively support the
//      compression method), and they must leave the "Content-Encoding" and
//      "Content-Length" headers intact.
//   *  This ensures that the caller of this interface can take full control of
//      the decompression and/or choose to store decompressed payloads on disk
//      if it so chooses.
//   *  Implementations must transparently decode server responses served with
//      "Transfer-Encoding: chunked". In such cases they must remove the
//      "Transfer-Encoding" response header.
// - Request body compression & encoding:
//   *  If implementations receive a "Content-Encoding" request header, this
//      means that the request body stream they receive has already been
//      compressed. The implementation must leave the header and request body
//      intact in such cases (i.e. not re-compress it).
//   *  If implementations receive a "Content-Length" request header, they must
//      use it verbatim and they should then assume that the request body will
//      be of exactly that size.
//   *  If they do not receive such a header then they must use the
//      "Transfer-encoding: chunked" mechanism to transmit the request body
//      (i.e. they shouldn't specify a "Content-Length" header and they should
//      transmit the body in chunks), or use an equivalent method of streaming
//      the data (such as HTTP/2's data streaming).
class HttpClient {
 public:
  virtual ~HttpClient() = default;
{'fraction_non_alphanumeric': 0.08130141554880309, 'fraction_numerical': 0.0037056251389609427, 'mean_word_length': 3.30019120458891, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7633128', 'n_tokens_mistral': 3379, 'n_tokens_neox': 3311, 'n_words': 1639}