<commit_before>/*
 * Copyright 2016 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/classification.hpp>

#include <memory>

#include <thrift/compiler/generate/common.h>
#include <thrift/compiler/generate/t_mstch_generator.h>

namespace {

class t_mstch_py3_generator : public t_mstch_generator {
  public:
    t_mstch_py3_generator(
        t_program* program,
        const std::map<std::string, std::string>& parsed_options,
        const std::string& /* option_string unused */)
        : t_mstch_generator(program, "py3", parsed_options) {

      this->out_dir_base_ = "gen-py3";
      auto include_prefix = this->get_option("include_prefix");
      if (include_prefix) {
        program->set_include_prefix(*include_prefix);
      }
    }

    void generate_program() override;
    mstch::map extend_program(const t_program&) const override;
    mstch::map extend_type(const t_type&) const override;
    mstch::map extend_service(const t_service&) const override;

  protected:
    void generate_init_files(const t_program&);
    void generate_structs(const t_program&);
    void generate_services(const t_program&);
    void generate_clients(const t_program&);
    boost::filesystem::path package_to_path(std::string package);
    mstch::array get_return_types(const t_program&) const;
    void add_container_types(const t_program&, mstch::map&) const;
    mstch::array get_cpp2_namespace(const t_program&) const;
    mstch::array get_py3_namespace(
      const t_program&,
      const string& tail = ""
    ) const;
    std::string flatten_type_name(const t_type&) const;

  private:
    void load_container_type(
      vector<t_type*>& container_types,
      std::set<string>& visited_names,
      t_type* type
    ) const;
};

mstch::map t_mstch_py3_generator::extend_program(
  const t_program& program
) const {
  const auto& cppNamespaces = this->get_cpp2_namespace(program);
  const auto& py3Namespaces = this->get_py3_namespace(program, "");
  mstch::array includeNamespaces;
  for (const auto included_program : program.get_includes()) {
    if (included_program->get_path() == program.get_path()) {
      continue;
    }
    const auto ns = this->get_py3_namespace(
      *included_program,
      included_program->get_name()
    );
    const mstch::map include_ns {{"includeNamespace", ns}};
    includeNamespaces.push_back(include_ns);
  }

  mstch::map result {
    {"returnTypes", this->get_return_types(program)},
    {"cppNamespaces", cppNamespaces},
    {"py3Namespaces", py3Namespaces},
    {"includeNamespaces", includeNamespaces},
  };
  this->add_container_types(program, result);
  return result;
}

mstch::map t_mstch_py3_generator::extend_type(const t_type& type) const {
  const auto type_program = type.get_program();
  const auto program = type_program? type_program: this->get_program();
  const auto modulePath = this->get_py3_namespace(
    *program,
    program->get_name() + ".types"
  );
  const auto& cppNamespaces = this->get_cpp2_namespace(*program);

  bool externalProgram = false;
  const auto& prog_path = program->get_path();
  if (prog_path != this->get_program()->get_path()) {
    externalProgram = true;
  }

  mstch::map result {
    {"modulePath", modulePath},
    {"externalProgram?", externalProgram},
    {"flat_name", this->flatten_type_name(type)},
    {"cppNamespaces", cppNamespaces},
  };
  return result;
}

mstch::map t_mstch_py3_generator::extend_service(
  const t_service& service
) const {
  const auto program = service.get_program();
  const auto& cppNamespaces = this->get_cpp2_namespace(*program);
  const auto& py3Namespaces = this->get_py3_namespace(*program);
  string include_prefix = program->get_include_prefix();
  // Used to avoid circular import in ServicesWrapper.h.mustache
  bool externalProgram = false;
  if (service.get_extends()) {
    const auto ext_program = service.get_extends()->get_program();
    const auto& ext_path = ext_program->get_path();
    const auto& prog_path = program->get_path();
    if (ext_path != prog_path) {
      externalProgram = true;
    }
  }
  mstch::map result {
    {"externalProgram?", externalProgram},
    {"cppNamespaces", cppNamespaces},
    {"py3Namespaces", py3Namespaces},
    {"programName", program->get_name()},
    {"includePrefix", include_prefix}
  };
  return result;
}

void t_mstch_py3_generator::generate_init_files(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));
  auto directory = boost::filesystem::path{};
  for (auto path_part : path) {
    directory /= path_part;
    this->write_output(directory / "__init__.py", "");
  }
}

void t_mstch_py3_generator::generate_structs(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));
  auto name = program.get_name();
  std::string module = "types";
  this->render_to_file(program, "Struct.pxd", path / name / (module + ".pxd"));
  this->render_to_file(program, "Struct.pyx", path / name / (module + ".pyx"));
}

void t_mstch_py3_generator::generate_services(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));

  auto name = program.get_name();
  this->render_to_file(
    program, "Services.pxd", path / name / "services.pxd");
  this->render_to_file(
    program, "CythonServices.pyx", path / name / "services.pyx");

  std::string basename = "services_wrapper";
  this->render_to_file(
    program, "ServicesWrapper.h", path / name / (basename + ".h"));
  this->render_to_file(
    program, "ServicesWrapper.cpp", path / name / (basename + ".cpp"));
  this->render_to_file(
    program, "ServicesWrapper.pxd", path / name / (basename + ".pxd"));
}

void t_mstch_py3_generator::generate_clients(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));

  auto name = program.get_name();
  this->render_to_file(
    program, "Clients.pxd", path / name / "clients.pxd");
  this->render_to_file(
    program, "CythonClients.pyx", path / name / "clients.pyx");

  std::string basename = "clients_wrapper";
  this->render_to_file(
    program, "ClientsWrapper.h", path / name / (basename + ".h"));
  this->render_to_file(
    program, "ClientsWrapper.cpp", path / name / (basename + ".cpp"));
  this->render_to_file(
    program, "ClientsWrapper.pxd", path / name / (basename + ".pxd"));
}

boost::filesystem::path t_mstch_py3_generator::package_to_path(
  std::string package
) {
  boost::algorithm::replace_all(package, ".", "/");
  return boost::filesystem::path{package};
}

mstch::array t_mstch_py3_generator::get_return_types(
  const t_program& program
) const {
  mstch::array distinct_return_types;
  std::set<string> visited_names;

  for (const auto service : program.get_services()) {
    for (const auto function : service->get_functions()) {
      const auto returntype = function->get_returntype();<commit_msg>Generate cpp where they are expected<commit_after>/*
 * Copyright 2016-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/classification.hpp>

#include <memory>

#include <thrift/compiler/generate/common.h>
#include <thrift/compiler/generate/t_mstch_generator.h>

namespace {

class t_mstch_py3_generator : public t_mstch_generator {
  public:
    t_mstch_py3_generator(
        t_program* program,
        const std::map<std::string, std::string>& parsed_options,
        const std::string& /* option_string unused */)
        : t_mstch_generator(program, "py3", parsed_options) {

      this->out_dir_base_ = "gen-py3";
      auto include_prefix = this->get_option("include_prefix");
      if (include_prefix) {
        program->set_include_prefix(*include_prefix);
      }
    }

    void generate_program() override;
    mstch::map extend_program(const t_program&) const override;
    mstch::map extend_type(const t_type&) const override;
    mstch::map extend_service(const t_service&) const override;

  protected:
    void generate_init_files(const t_program&);
    void generate_structs(const t_program&);
    void generate_services(const t_program&);
    void generate_clients(const t_program&);
    boost::filesystem::path package_to_path(std::string package);
    mstch::array get_return_types(const t_program&) const;
    void add_container_types(const t_program&, mstch::map&) const;
    mstch::array get_cpp2_namespace(const t_program&) const;
    mstch::array get_py3_namespace(
      const t_program&,
      const string& tail = ""
    ) const;
    std::string flatten_type_name(const t_type&) const;

  private:
    void load_container_type(
      vector<t_type*>& container_types,
      std::set<string>& visited_names,
      t_type* type
    ) const;
};

mstch::map t_mstch_py3_generator::extend_program(
  const t_program& program
) const {
  const auto& cppNamespaces = this->get_cpp2_namespace(program);
  const auto& py3Namespaces = this->get_py3_namespace(program, "");
  mstch::array includeNamespaces;
  for (const auto included_program : program.get_includes()) {
    if (included_program->get_path() == program.get_path()) {
      continue;
    }
    const auto ns = this->get_py3_namespace(
      *included_program,
      included_program->get_name()
    );
    const mstch::map include_ns {{"includeNamespace", ns}};
    includeNamespaces.push_back(include_ns);
  }

  mstch::map result {
    {"returnTypes", this->get_return_types(program)},
    {"cppNamespaces", cppNamespaces},
    {"py3Namespaces", py3Namespaces},
    {"includeNamespaces", includeNamespaces},
  };
  this->add_container_types(program, result);
  return result;
}

mstch::map t_mstch_py3_generator::extend_type(const t_type& type) const {
  const auto type_program = type.get_program();
  const auto program = type_program? type_program: this->get_program();
  const auto modulePath = this->get_py3_namespace(
    *program,
    program->get_name() + ".types"
  );
  const auto& cppNamespaces = this->get_cpp2_namespace(*program);

  bool externalProgram = false;
  const auto& prog_path = program->get_path();
  if (prog_path != this->get_program()->get_path()) {
    externalProgram = true;
  }

  mstch::map result {
    {"modulePath", modulePath},
    {"externalProgram?", externalProgram},
    {"flat_name", this->flatten_type_name(type)},
    {"cppNamespaces", cppNamespaces},
  };
  return result;
}

mstch::map t_mstch_py3_generator::extend_service(
  const t_service& service
) const {
  const auto program = service.get_program();
  const auto& cppNamespaces = this->get_cpp2_namespace(*program);
  const auto& py3Namespaces = this->get_py3_namespace(*program);
  string include_prefix = program->get_include_prefix();
  // Used to avoid circular import in ServicesWrapper.h.mustache
  bool externalProgram = false;
  if (service.get_extends()) {
    const auto ext_program = service.get_extends()->get_program();
    const auto& ext_path = ext_program->get_path();
    const auto& prog_path = program->get_path();
    if (ext_path != prog_path) {
      externalProgram = true;
    }
  }
  mstch::map result {
    {"externalProgram?", externalProgram},
    {"cppNamespaces", cppNamespaces},
    {"py3Namespaces", py3Namespaces},
    {"programName", program->get_name()},
    {"includePrefix", include_prefix}
  };
  return result;
}

void t_mstch_py3_generator::generate_init_files(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));
  auto directory = boost::filesystem::path{};
  for (auto path_part : path) {
    directory /= path_part;
    this->write_output(directory / "__init__.py", "");
  }
}

void t_mstch_py3_generator::generate_structs(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));
  auto name = program.get_name();
  std::string module = "types";
  this->render_to_file(program, "Struct.pxd", path / name / (module + ".pxd"));
  this->render_to_file(program, "Struct.pyx", path / name / (module + ".pyx"));
}

void t_mstch_py3_generator::generate_services(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));

  auto name = program.get_name();
  this->render_to_file(
    program, "Services.pxd", path / name / "services.pxd");
  this->render_to_file(
    program, "CythonServices.pyx", path / name / "services.pyx");

  std::string basename = "services_wrapper";
  auto cpp_path = boost::filesystem::path{name};
  this->render_to_file(
    program, "ServicesWrapper.h", cpp_path / (basename + ".h"));
  this->render_to_file(
    program, "ServicesWrapper.cpp", cpp_path / (basename + ".cpp"));
  this->render_to_file(
    program, "ServicesWrapper.pxd", path / name / (basename + ".pxd"));
}

void t_mstch_py3_generator::generate_clients(const t_program& program) {
  auto path = this->package_to_path(program.get_namespace("py3"));

  auto name = program.get_name();
  this->render_to_file(
    program, "Clients.pxd", path / name / "clients.pxd");
  this->render_to_file(
    program, "CythonClients.pyx", path / name / "clients.pyx");

  std::string basename = "clients_wrapper";
  auto cpp_path = boost::filesystem::path{name};
  this->render_to_file(
    program, "ClientsWrapper.h", cpp_path / (basename + ".h"));
  this->render_to_file(
    program, "ClientsWrapper.cpp", cpp_path / (basename + ".cpp"));
  this->render_to_file(
    program, "ClientsWrapper.pxd", path / name / (basename + ".pxd"));
}

boost::filesystem::path t_mstch_py3_generator::package_to_path(
  std::string package
) {
  boost::algorithm::replace_all(package, ".", "/");
  return boost::filesystem::path{package};
}

mstch::array t_mstch_py3_generator::get_return_types(
  const t_program& program
) const {
  mstch::array distinct_return_types;
  std::set<string> visited_names;

  for (const auto service : program.get_services()) {
    for (const auto function : service->get_functions()) {
      const auto returntype = function->get_returntype();
{'fraction_non_alphanumeric': 0.13211799801126947, 'fraction_numerical': 0.005303281405369572, 'mean_word_length': 4.519941456275156, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6809480', 'n_tokens_mistral': 4917, 'n_tokens_neox': 4871, 'n_words': 1048}