<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "google/cloud/internal/getenv.h"
#include "google/cloud/log.h"
#include "google/cloud/storage/internal/curl_request_builder.h"
#include "google/cloud/storage/internal/nljson.h"
#include "google/cloud/storage/oauth2/anonymous_credentials.h"
#include "google/cloud/testing_util/assert_ok.h"
#include <gmock/gmock.h>
#include <cstdlib>
#include <vector>

namespace google {
namespace cloud {
namespace storage {
inline namespace STORAGE_CLIENT_NS {
namespace internal {
namespace {

using ::testing::HasSubstr;

std::string HttpBinEndpoint() {
  return google::cloud::internal::GetEnv("HTTPBIN_ENDPOINT")
      .value_or("https://nghttp2.org/httpbin");
}

TEST(CurlRequestTest, SimpleGET) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/get",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddQueryParameter("foo", "foo1&&&foo2");
  request.AddQueryParameter("bar", "bar1==bar2=");
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json args = parsed["args"];
  EXPECT_EQ("foo1&&&foo2", args["foo"].get<std::string>());
  EXPECT_EQ("bar1==bar2=", args["bar"].get<std::string>());
}

TEST(CurlRequestTest, FailedGET) {
  // This test fails if somebody manages to run a https server on port 0 (you
  // can't, but just documenting the assumptions in this test).
  storage::internal::CurlRequestBuilder request(
      "https://localhost:1/", storage::internal::GetDefaultCurlHandleFactory());

  auto response = request.BuildRequest().MakeRequest(std::string{});
  EXPECT_FALSE(response.ok());
}

TEST(CurlRequestTest, RepeatedGET) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/get",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddQueryParameter("foo", "foo1&&&foo2");
  request.AddQueryParameter("bar", "bar1==bar2=");
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto req = request.BuildRequest();
  auto response = req.MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);

  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json args = parsed["args"];
  EXPECT_EQ("foo1&&&foo2", args["foo"].get<std::string>());
  EXPECT_EQ("bar1==bar2=", args["bar"].get<std::string>());

  response = req.MakeRequest(std::string{});
  EXPECT_EQ(200, response->status_code);
  parsed = nl::json::parse(response->payload);
  args = parsed["args"];
  EXPECT_EQ("foo1&&&foo2", args["foo"].get<std::string>());
  EXPECT_EQ("bar1==bar2=", args["bar"].get<std::string>());
}

TEST(CurlRequestTest, SimplePOST) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/post",
      storage::internal::GetDefaultCurlHandleFactory());
  std::vector<std::pair<std::string, std::string>> form_parameters = {
      {"foo", "foo1&foo2 foo3"},
      {"bar", "bar1-bar2"},
      {"baz", "baz=baz2"},
  };
  std::string data;
  char const* sep = "";
  for (auto const& p : form_parameters) {
    data += sep;
    data += request.MakeEscapedString(p.first).get();
    data += '=';
    data += request.MakeEscapedString(p.second).get();
    sep = "&";
  }
  request.AddHeader("Accept: application/json");
  request.AddHeader("Content-Type: application/x-www-form-urlencoded");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(data);
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json form = parsed["form"];
  EXPECT_EQ("foo1&foo2 foo3", form["foo"].get<std::string>());
  EXPECT_EQ("bar1-bar2", form["bar"].get<std::string>());
  EXPECT_EQ("baz=baz2", form["baz"].get<std::string>());
}

TEST(CurlRequestTest, Handle404) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/status/404",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(404, response->status_code);
}

/// @test Verify the payload for error status is included in the return value.
TEST(CurlRequestTest, HandleTeapot) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/status/418",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(418, response->status_code);
  EXPECT_THAT(response->payload, HasSubstr("[ teapot ]"));
}

/// @test Verify the response includes the header values.
TEST(CurlRequestTest, CheckResponseHeaders) {
  // Test that headers are parsed correctly. We send capitalized headers
  // because some versions of httpbin capitalize and others do not, in real
  // code (as opposed to a test), we should search for headers in a
  // case-insensitive manner, but that is not the purpose of this test.
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() +
          "/response-headers"
          "?X-Test-Foo=bar"
          "&X-Test-Empty",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  EXPECT_EQ(1, response->headers.count("x-test-empty"));
  EXPECT_EQ("", response->headers.find("x-test-empty")->second);
  EXPECT_LE(1U, response->headers.count("x-test-foo"));
  EXPECT_EQ("bar", response->headers.find("x-test-foo")->second);
}

/// @test Verify the user agent prefix affects the request.
TEST(CurlRequestTest, UserAgentPrefix) {
  // Test that headers are parsed correctly. We send capitalized headers
  // because some versions of httpbin capitalize and others do not, in real
  // code (as opposed to a test), we should search for headers in a
  // case-insensitive manner, but that is not the purpose of this test.
  storage::internal::CurlRequestBuilder builder(
      HttpBinEndpoint() + "/headers",
      storage::internal::GetDefaultCurlHandleFactory());
  auto options =
      ClientOptions(std::make_shared<storage::oauth2::AnonymousCredentials>())
          .add_user_agent_prefx("test-user-agent-prefix");
  builder.ApplyClientOptions(options);
  builder.AddHeader("Accept: application/json");
  builder.AddHeader("charsets: utf-8");

  auto response = builder.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  auto payload = nl::json::parse(response->payload);
  ASSERT_EQ(1U, payload.count("headers"));
  auto headers = payload["headers"];
  EXPECT_THAT(headers.value("User-Agent", ""),
              HasSubstr("test-user-agent-prefix"));
}

/// @test Verify that the Projection parameter is included if set.
TEST(CurlRequestTest, WellKnownQueryParameters_Projection) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/get",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");
  request.AddOption(storage::Projection("full"));

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json args = parsed["args"];
  EXPECT_EQ("full", args.value("projection", ""));
  // The other well known parameters are not set.
  EXPECT_EQ(0, args.count("userProject"));
  EXPECT_EQ(0, args.count("ifGenerationMatch"));
  EXPECT_EQ(0, args.count("ifGenerationNotMatch"));
  EXPECT_EQ(0, args.count("ifMetagenerationMatch"));
  EXPECT_EQ(0, args.count("ifMetagenerationNotMatch"));
}

/// @test Verify that the UserProject parameter is included if set.
TEST(CurlRequestTest, WellKnownQueryParameters_UserProject) {<commit_msg>test: Add a test for telemetry header (#3048)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "google/cloud/internal/getenv.h"
#include "google/cloud/log.h"
#include "google/cloud/storage/internal/curl_request_builder.h"
#include "google/cloud/storage/internal/nljson.h"
#include "google/cloud/storage/oauth2/anonymous_credentials.h"
#include "google/cloud/testing_util/assert_ok.h"
#include <gmock/gmock.h>
#include <cstdlib>
#include <vector>

namespace google {
namespace cloud {
namespace storage {
inline namespace STORAGE_CLIENT_NS {
namespace internal {
namespace {

using ::testing::HasSubstr;

std::string HttpBinEndpoint() {
  return google::cloud::internal::GetEnv("HTTPBIN_ENDPOINT")
      .value_or("https://nghttp2.org/httpbin");
}

TEST(CurlRequestTest, SimpleGET) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/get",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddQueryParameter("foo", "foo1&&&foo2");
  request.AddQueryParameter("bar", "bar1==bar2=");
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json args = parsed["args"];
  EXPECT_EQ("foo1&&&foo2", args["foo"].get<std::string>());
  EXPECT_EQ("bar1==bar2=", args["bar"].get<std::string>());
}

TEST(CurlRequestTest, FailedGET) {
  // This test fails if somebody manages to run a https server on port 0 (you
  // can't, but just documenting the assumptions in this test).
  storage::internal::CurlRequestBuilder request(
      "https://localhost:1/", storage::internal::GetDefaultCurlHandleFactory());

  auto response = request.BuildRequest().MakeRequest(std::string{});
  EXPECT_FALSE(response.ok());
}

TEST(CurlRequestTest, RepeatedGET) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/get",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddQueryParameter("foo", "foo1&&&foo2");
  request.AddQueryParameter("bar", "bar1==bar2=");
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto req = request.BuildRequest();
  auto response = req.MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);

  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json args = parsed["args"];
  EXPECT_EQ("foo1&&&foo2", args["foo"].get<std::string>());
  EXPECT_EQ("bar1==bar2=", args["bar"].get<std::string>());

  response = req.MakeRequest(std::string{});
  EXPECT_EQ(200, response->status_code);
  parsed = nl::json::parse(response->payload);
  args = parsed["args"];
  EXPECT_EQ("foo1&&&foo2", args["foo"].get<std::string>());
  EXPECT_EQ("bar1==bar2=", args["bar"].get<std::string>());
}

TEST(CurlRequestTest, SimplePOST) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/post",
      storage::internal::GetDefaultCurlHandleFactory());
  std::vector<std::pair<std::string, std::string>> form_parameters = {
      {"foo", "foo1&foo2 foo3"},
      {"bar", "bar1-bar2"},
      {"baz", "baz=baz2"},
  };
  std::string data;
  char const* sep = "";
  for (auto const& p : form_parameters) {
    data += sep;
    data += request.MakeEscapedString(p.first).get();
    data += '=';
    data += request.MakeEscapedString(p.second).get();
    sep = "&";
  }
  request.AddHeader("Accept: application/json");
  request.AddHeader("Content-Type: application/x-www-form-urlencoded");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(data);
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json form = parsed["form"];
  EXPECT_EQ("foo1&foo2 foo3", form["foo"].get<std::string>());
  EXPECT_EQ("bar1-bar2", form["bar"].get<std::string>());
  EXPECT_EQ("baz=baz2", form["baz"].get<std::string>());
}

TEST(CurlRequestTest, Handle404) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/status/404",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(404, response->status_code);
}

/// @test Verify the payload for error status is included in the return value.
TEST(CurlRequestTest, HandleTeapot) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/status/418",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(418, response->status_code);
  EXPECT_THAT(response->payload, HasSubstr("[ teapot ]"));
}

/// @test Verify the response includes the header values.
TEST(CurlRequestTest, CheckResponseHeaders) {
  // Test that headers are parsed correctly. We send capitalized headers
  // because some versions of httpbin capitalize and others do not, in real
  // code (as opposed to a test), we should search for headers in a
  // case-insensitive manner, but that is not the purpose of this test.
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() +
          "/response-headers"
          "?X-Test-Foo=bar"
          "&X-Test-Empty",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  EXPECT_EQ(1, response->headers.count("x-test-empty"));
  EXPECT_EQ("", response->headers.find("x-test-empty")->second);
  EXPECT_LE(1U, response->headers.count("x-test-foo"));
  EXPECT_EQ("bar", response->headers.find("x-test-foo")->second);
}

/// @test Verify the user agent header.
TEST(CurlRequestTest, UserAgent) {
  // Test that headers are parsed correctly. We send capitalized headers
  // because some versions of httpbin capitalize and others do not, in real
  // code (as opposed to a test), we should search for headers in a
  // case-insensitive manner, but that is not the purpose of this test.
  // Also verifying the telemetry header is present.
  storage::internal::CurlRequestBuilder builder(
      HttpBinEndpoint() + "/headers",
      storage::internal::GetDefaultCurlHandleFactory());
  auto options =
      ClientOptions(std::make_shared<storage::oauth2::AnonymousCredentials>())
          .add_user_agent_prefx("test-user-agent-prefix");
  builder.ApplyClientOptions(options);
  builder.AddHeader("Accept: application/json");
  builder.AddHeader("charsets: utf-8");

  auto response = builder.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  auto payload = nl::json::parse(response->payload);
  ASSERT_EQ(1U, payload.count("headers"));
  auto headers = payload["headers"];
  EXPECT_THAT(headers.value("User-Agent", ""),
              HasSubstr("test-user-agent-prefix"));
  EXPECT_THAT(headers.value("User-Agent", ""), HasSubstr("gcloud-cpp/"));
}

/// @test Verify that the Projection parameter is included if set.
TEST(CurlRequestTest, WellKnownQueryParameters_Projection) {
  storage::internal::CurlRequestBuilder request(
      HttpBinEndpoint() + "/get",
      storage::internal::GetDefaultCurlHandleFactory());
  request.AddHeader("Accept: application/json");
  request.AddHeader("charsets: utf-8");
  request.AddOption(storage::Projection("full"));

  auto response = request.BuildRequest().MakeRequest(std::string{});
  ASSERT_STATUS_OK(response);
  EXPECT_EQ(200, response->status_code);
  nl::json parsed = nl::json::parse(response->payload);
  nl::json args = parsed["args"];
  EXPECT_EQ("full", args.value("projection", ""));
  // The other well known parameters are not set.
  EXPECT_EQ(0, args.count("userProject"));
  EXPECT_EQ(0, args.count("ifGenerationMatch"));
  EXPECT_EQ(0, args.count("ifGenerationNotMatch"));
  EXPECT_EQ(0, args.count("ifMetagenerationMatch"));
  EXPECT_EQ(0, args.count("ifMetagenerationNotMatch"));
}

/// @test Verify that the UserProject parameter is included if set.
TEST(CurlRequestTest, WellKnownQueryParameters_UserProject) {
{'fraction_non_alphanumeric': 0.15882515932391245, 'fraction_numerical': 0.010972568578553617, 'mean_word_length': 5.32970887407927, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 77, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5393767', 'n_tokens_mistral': 5530, 'n_tokens_neox': 5488, 'n_words': 1163}