<commit_before>/*
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef FCP_CLIENT_HTTP_IN_MEMORY_REQUEST_RESPONSE_H
#define FCP_CLIENT_HTTP_IN_MEMORY_REQUEST_RESPONSE_H

#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include "absl/base/thread_annotations.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/cord.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "fcp/client/http/http_client.h"
#include "fcp/client/interruptible_runner.h"

namespace fcp {
namespace client {
namespace http {

// Simple `HttpRequest` implementation with an in-memory request body.
class InMemoryHttpRequest : public HttpRequest {
 public:
  // Factory method for creating an instance.
  //
  // Callers are recommended to `std::move` the `body` parameter (or to rely on
  // copy elision), to avoid unnecessary copies of the data.
  //
  // Note that a "Content-Length" header will be constructed automatically, and
  // must not be provided by the caller.
  //
  // Returns an INVALID_ARGUMENT error if:
  // - the URI is a non-HTTPS URI,
  // - the request has a body but the request method doesn't allow it,
  // - the headers contain a "Content-Length" header.
  static absl::StatusOr<std::unique_ptr<HttpRequest>> Create(
      absl::string_view uri, Method method, HeaderList extra_headers,
      std::string body);

  absl::string_view uri() const override { return uri_; };
  Method method() const override { return method_; };
  const HeaderList& extra_headers() const override { return headers_; }
  bool HasBody() const override { return !body_.empty(); };

  absl::StatusOr<int64_t> ReadBody(char* buffer, int64_t requested) override;

 private:
  InMemoryHttpRequest(absl::string_view uri, Method method,
                      HeaderList extra_headers, std::string body)
      : uri_(uri),
        method_(method),
        body_(std::move(body)),
        headers_(std::move(extra_headers)) {}

  const std::string uri_;
  const Method method_;
  const std::string body_;
  const HeaderList headers_;
  int64_t cursor_ ABSL_GUARDED_BY(mutex_) = 0;
  mutable absl::Mutex mutex_;
};

// Simple container class for holding an HTTP response code, headers, and
// in-memory request body.
struct InMemoryHttpResponse {
  const int code;
  // This is empty if no "Content-Encoding" header was present in the response
  // headers.
  const std::string content_encoding;
  const absl::Cord body;
};

// Simple `HttpRequestCallback` implementation that stores the response and its
// body in an `InMemoryHttpResponse` object for later consumption.
class InMemoryHttpRequestCallback : public HttpRequestCallback {
 public:
  InMemoryHttpRequestCallback() = default;

  absl::Status OnResponseStarted(const HttpRequest& request,
                                 const HttpResponse& response) override;
  void OnResponseError(const HttpRequest& request,
                       const absl::Status& error) override;
  absl::Status OnResponseBody(const HttpRequest& request,
                              const HttpResponse& response,
                              absl::string_view data) override;
  void OnResponseBodyError(const HttpRequest& request,
                           const HttpResponse& response,
                           const absl::Status& error) override;
  void OnResponseCompleted(const HttpRequest& request,
                           const HttpResponse& response) override;
  absl::StatusOr<InMemoryHttpResponse> Response() const;

 private:
  absl::Status status_ ABSL_GUARDED_BY(mutex_) =
      absl::UnavailableError("No response received");
  std::optional<int> response_code_ ABSL_GUARDED_BY(mutex_);
  std::string content_encoding_ ABSL_GUARDED_BY(mutex_);
  std::optional<int64_t> expected_content_length_ ABSL_GUARDED_BY(mutex_);
  absl::Cord response_buffer_ ABSL_GUARDED_BY(mutex_);
  mutable absl::Mutex mutex_;
};

// Utility for performing a single HTTP request and returning the results (incl.
// the response body) via an in-memory object, in an interruptible way.
//
// If `bytes_received_acc` and `bytes_sent_acc` are non-null then those
// accumulators will also be incremented by the amount of data that was
// received/sent by the request.
//
// Returns an error if the request failed.
absl::StatusOr<InMemoryHttpResponse> PerformRequestInMemory(
    HttpClient& http_client, InterruptibleRunner& interruptible_runner,
    std::unique_ptr<http::HttpRequest> request, int64_t* bytes_received_acc,
    int64_t* bytes_sent_acc);

// Simple class representing a resource for which data is already available
// in-memory (`inline_data`) or for which data needs to be fetched by an HTTP
// GET request (via `uri`). Only one field can ever be set to a non-empty value,
// or both fields may be empty (indicating a zero-length resource for which
// nothing has to be fetched).
class UriOrInlineData {
 public:
  // Creates an instance representing a URI from which data has to be fetched.
  static UriOrInlineData CreateUri(std::string uri) {
    return UriOrInlineData(std::move(uri), absl::Cord());
  }
  // Creates an instance representing a resource's already-available (or empty)
  // data.
  static UriOrInlineData CreateInlineData(absl::Cord inline_data) {
    return UriOrInlineData("", std::move(inline_data));
  }

  std::string uri() const { return uri_; }
  absl::Cord inline_data() const { return inline_data_; }

 private:
  UriOrInlineData(std::string uri, absl::Cord inline_data)
      : uri_(std::move(uri)), inline_data_(std::move(inline_data)) {}

  const std::string uri_;
  const absl::Cord inline_data_;
};

// Utility for (potentially) fetching multiple resources at once, each of which
// either needs to be fetched from a URI using a HTTP GET request, or for which
// its data is already available, and returning the eventual results (incl. the
// response body) via in-memory objects, in an interruptible way. If resources
// do need to be fetched, then a single `HttpClient::PerformRequests` call will
// be made for all resources at once.
//
// This makes it a convenient way for callers to gather the data for a related
// set of resources (some of which might already have their data available) in
// one go, and to then access the data for the first resource at index 0, the
// second resource at index 1, etc., transparently handling the various
// permutations that are possible (e.g.  resource A having data inline but
// B having to be fetched, or both being inlined, or ...) via a unified access
// pattern and error handling mechanism.
//
// If `bytes_received_acc` and `bytes_sent_acc` are non-null then those
// accumulators will also be incremented by the aggregate amount of data that
// was received/sent by the HTTP requests that were issued.
//
// Returns an error if issuing the joint `HttpClient::PerformRequests` call
// failed.  Otherwise it returns a vector containing the result for each
// resource (in the same order the resources were provided in).
absl::StatusOr<std::vector<absl::StatusOr<InMemoryHttpResponse>>>
FetchResourcesInMemory(HttpClient& http_client,
                       InterruptibleRunner& interruptible_runner,
                       const std::vector<UriOrInlineData>& resources,
                       int64_t* bytes_received_acc, int64_t* bytes_sent_acc);

};  // namespace http
};  // namespace client
};  // namespace fcp

#endif  // FCP_CLIENT_HTTP_IN_MEMORY_REQUEST_RESPONSE_H<commit_msg>Fix missing trailing underscore for header guard, to align with style guide.<commit_after>/*
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#ifndef FCP_CLIENT_HTTP_IN_MEMORY_REQUEST_RESPONSE_H_
#define FCP_CLIENT_HTTP_IN_MEMORY_REQUEST_RESPONSE_H_

#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include "absl/base/thread_annotations.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/cord.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "fcp/client/http/http_client.h"
#include "fcp/client/interruptible_runner.h"

namespace fcp {
namespace client {
namespace http {

// Simple `HttpRequest` implementation with an in-memory request body.
class InMemoryHttpRequest : public HttpRequest {
 public:
  // Factory method for creating an instance.
  //
  // Callers are recommended to `std::move` the `body` parameter (or to rely on
  // copy elision), to avoid unnecessary copies of the data.
  //
  // Note that a "Content-Length" header will be constructed automatically, and
  // must not be provided by the caller.
  //
  // Returns an INVALID_ARGUMENT error if:
  // - the URI is a non-HTTPS URI,
  // - the request has a body but the request method doesn't allow it,
  // - the headers contain a "Content-Length" header.
  static absl::StatusOr<std::unique_ptr<HttpRequest>> Create(
      absl::string_view uri, Method method, HeaderList extra_headers,
      std::string body);

  absl::string_view uri() const override { return uri_; };
  Method method() const override { return method_; };
  const HeaderList& extra_headers() const override { return headers_; }
  bool HasBody() const override { return !body_.empty(); };

  absl::StatusOr<int64_t> ReadBody(char* buffer, int64_t requested) override;

 private:
  InMemoryHttpRequest(absl::string_view uri, Method method,
                      HeaderList extra_headers, std::string body)
      : uri_(uri),
        method_(method),
        body_(std::move(body)),
        headers_(std::move(extra_headers)) {}

  const std::string uri_;
  const Method method_;
  const std::string body_;
  const HeaderList headers_;
  int64_t cursor_ ABSL_GUARDED_BY(mutex_) = 0;
  mutable absl::Mutex mutex_;
};

// Simple container class for holding an HTTP response code, headers, and
// in-memory request body.
struct InMemoryHttpResponse {
  const int code;
  // This is empty if no "Content-Encoding" header was present in the response
  // headers.
  const std::string content_encoding;
  const absl::Cord body;
};

// Simple `HttpRequestCallback` implementation that stores the response and its
// body in an `InMemoryHttpResponse` object for later consumption.
class InMemoryHttpRequestCallback : public HttpRequestCallback {
 public:
  InMemoryHttpRequestCallback() = default;

  absl::Status OnResponseStarted(const HttpRequest& request,
                                 const HttpResponse& response) override;
  void OnResponseError(const HttpRequest& request,
                       const absl::Status& error) override;
  absl::Status OnResponseBody(const HttpRequest& request,
                              const HttpResponse& response,
                              absl::string_view data) override;
  void OnResponseBodyError(const HttpRequest& request,
                           const HttpResponse& response,
                           const absl::Status& error) override;
  void OnResponseCompleted(const HttpRequest& request,
                           const HttpResponse& response) override;
  absl::StatusOr<InMemoryHttpResponse> Response() const;

 private:
  absl::Status status_ ABSL_GUARDED_BY(mutex_) =
      absl::UnavailableError("No response received");
  std::optional<int> response_code_ ABSL_GUARDED_BY(mutex_);
  std::string content_encoding_ ABSL_GUARDED_BY(mutex_);
  std::optional<int64_t> expected_content_length_ ABSL_GUARDED_BY(mutex_);
  absl::Cord response_buffer_ ABSL_GUARDED_BY(mutex_);
  mutable absl::Mutex mutex_;
};

// Utility for performing a single HTTP request and returning the results (incl.
// the response body) via an in-memory object, in an interruptible way.
//
// If `bytes_received_acc` and `bytes_sent_acc` are non-null then those
// accumulators will also be incremented by the amount of data that was
// received/sent by the request.
//
// Returns an error if the request failed.
absl::StatusOr<InMemoryHttpResponse> PerformRequestInMemory(
    HttpClient& http_client, InterruptibleRunner& interruptible_runner,
    std::unique_ptr<http::HttpRequest> request, int64_t* bytes_received_acc,
    int64_t* bytes_sent_acc);

// Simple class representing a resource for which data is already available
// in-memory (`inline_data`) or for which data needs to be fetched by an HTTP
// GET request (via `uri`). Only one field can ever be set to a non-empty value,
// or both fields may be empty (indicating a zero-length resource for which
// nothing has to be fetched).
class UriOrInlineData {
 public:
  // Creates an instance representing a URI from which data has to be fetched.
  static UriOrInlineData CreateUri(std::string uri) {
    return UriOrInlineData(std::move(uri), absl::Cord());
  }
  // Creates an instance representing a resource's already-available (or empty)
  // data.
  static UriOrInlineData CreateInlineData(absl::Cord inline_data) {
    return UriOrInlineData("", std::move(inline_data));
  }

  std::string uri() const { return uri_; }
  absl::Cord inline_data() const { return inline_data_; }

 private:
  UriOrInlineData(std::string uri, absl::Cord inline_data)
      : uri_(std::move(uri)), inline_data_(std::move(inline_data)) {}

  const std::string uri_;
  const absl::Cord inline_data_;
};

// Utility for (potentially) fetching multiple resources at once, each of which
// either needs to be fetched from a URI using a HTTP GET request, or for which
// its data is already available, and returning the eventual results (incl. the
// response body) via in-memory objects, in an interruptible way. If resources
// do need to be fetched, then a single `HttpClient::PerformRequests` call will
// be made for all resources at once.
//
// This makes it a convenient way for callers to gather the data for a related
// set of resources (some of which might already have their data available) in
// one go, and to then access the data for the first resource at index 0, the
// second resource at index 1, etc., transparently handling the various
// permutations that are possible (e.g.  resource A having data inline but
// B having to be fetched, or both being inlined, or ...) via a unified access
// pattern and error handling mechanism.
//
// If `bytes_received_acc` and `bytes_sent_acc` are non-null then those
// accumulators will also be incremented by the aggregate amount of data that
// was received/sent by the HTTP requests that were issued.
//
// Returns an error if issuing the joint `HttpClient::PerformRequests` call
// failed.  Otherwise it returns a vector containing the result for each
// resource (in the same order the resources were provided in).
absl::StatusOr<std::vector<absl::StatusOr<InMemoryHttpResponse>>>
FetchResourcesInMemory(HttpClient& http_client,
                       InterruptibleRunner& interruptible_runner,
                       const std::vector<UriOrInlineData>& resources,
                       int64_t* bytes_received_acc, int64_t* bytes_sent_acc);

};  // namespace http
};  // namespace client
};  // namespace fcp

#endif  // FCP_CLIENT_HTTP_IN_MEMORY_REQUEST_RESPONSE_H_
{'fraction_non_alphanumeric': 0.08942888861477735, 'fraction_numerical': 0.0033304551622055016, 'mean_word_length': 3.8101453574606943, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4659604', 'n_tokens_mistral': 4567, 'n_tokens_neox': 4492, 'n_words': 1668}