<commit_before>/* Copyright 2017 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.api.codegen.transformer.py;

import com.google.api.codegen.config.ApiModel;
import com.google.api.codegen.config.FieldModel;
import com.google.api.codegen.config.GapicMethodConfig;
import com.google.api.codegen.config.GapicProductConfig;
import com.google.api.codegen.config.InterfaceModel;
import com.google.api.codegen.config.MethodModel;
import com.google.api.codegen.config.PackageMetadataConfig;
import com.google.api.codegen.config.ProductConfig;
import com.google.api.codegen.config.ProductServiceConfig;
import com.google.api.codegen.config.ProtoApiModel;
import com.google.api.codegen.config.SampleSpec.SampleType;
import com.google.api.codegen.gapic.GapicCodePathMapper;
import com.google.api.codegen.gapic.GapicParser;
import com.google.api.codegen.gapic.ProtoFiles;
import com.google.api.codegen.transformer.DefaultFeatureConfig;
import com.google.api.codegen.transformer.DynamicLangApiMethodTransformer;
import com.google.api.codegen.transformer.FeatureConfig;
import com.google.api.codegen.transformer.FileHeaderTransformer;
import com.google.api.codegen.transformer.GapicInterfaceContext;
import com.google.api.codegen.transformer.GapicMethodContext;
import com.google.api.codegen.transformer.GrpcElementDocTransformer;
import com.google.api.codegen.transformer.GrpcStubTransformer;
import com.google.api.codegen.transformer.InitCodeTransformer;
import com.google.api.codegen.transformer.ModelToViewTransformer;
import com.google.api.codegen.transformer.ModelTypeTable;
import com.google.api.codegen.transformer.PageStreamingTransformer;
import com.google.api.codegen.transformer.PathTemplateTransformer;
import com.google.api.codegen.transformer.ServiceTransformer;
import com.google.api.codegen.transformer.SurfaceNamer;
import com.google.api.codegen.util.Name;
import com.google.api.codegen.util.NamePath;
import com.google.api.codegen.util.VersionMatcher;
import com.google.api.codegen.util.py.PythonTypeTable;
import com.google.api.codegen.viewmodel.BatchingDescriptorView;
import com.google.api.codegen.viewmodel.DynamicLangXApiView;
import com.google.api.codegen.viewmodel.GrpcDocView;
import com.google.api.codegen.viewmodel.GrpcElementDocView;
import com.google.api.codegen.viewmodel.GrpcMessageDocView;
import com.google.api.codegen.viewmodel.GrpcStreamingDetailView;
import com.google.api.codegen.viewmodel.ImportSectionView;
import com.google.api.codegen.viewmodel.LongRunningOperationDetailView;
import com.google.api.codegen.viewmodel.OptionalArrayMethodView;
import com.google.api.codegen.viewmodel.ParamDocView;
import com.google.api.codegen.viewmodel.PathTemplateGetterFunctionView;
import com.google.api.codegen.viewmodel.ViewModel;
import com.google.api.codegen.viewmodel.metadata.VersionIndexRequireView;
import com.google.api.codegen.viewmodel.metadata.VersionIndexView;
import com.google.api.tools.framework.model.MessageType;
import com.google.api.tools.framework.model.Model;
import com.google.api.tools.framework.model.ProtoContainerElement;
import com.google.api.tools.framework.model.ProtoFile;
import com.google.api.tools.framework.model.TypeRef;
import com.google.common.collect.ImmutableList;
import java.io.File;
import java.util.List;
import java.util.stream.Collectors;

/* Transforms a ProtoApiModel into the standard GAPIC surface for Python. */
public class PythonGapicSurfaceTransformer implements ModelToViewTransformer<ProtoApiModel> {
  private static final String XAPI_TEMPLATE_FILENAME = "py/main.snip";
  private static final String TRANSPORT_TEMPLATE_FILENAME = "py/transport.snip";
  private static final String ENUM_TEMPLATE_FILENAME = "py/enum.snip";
  private static final String TYPES_TEMPLATE_FILENAME = "py/types.snip";
  private static final String VERSIONED_INIT_TEMPLATE_FILENAME =
      "py/versioned_directory__init__.py.snip";
  private static final String TOP_LEVEL_ENTRY_POINT_FILENAME = "py/top_level_entry_point.snip";

  private static final SampleType sampleType = SampleType.IN_CODE;
  private final PythonImportSectionTransformer importSectionTransformer =
      new PythonImportSectionTransformer();
  private final FileHeaderTransformer fileHeaderTransformer =
      new FileHeaderTransformer(importSectionTransformer);
  private final DynamicLangApiMethodTransformer apiMethodTransformer =
      new DynamicLangApiMethodTransformer(
          new PythonApiMethodParamTransformer(),
          new InitCodeTransformer(importSectionTransformer),
          sampleType);
  private final PythonMethodViewGenerator methodGenerator =
      new PythonMethodViewGenerator(apiMethodTransformer);
  private final ServiceTransformer serviceTransformer = new ServiceTransformer();
  private final PageStreamingTransformer pageStreamingTransformer = new PageStreamingTransformer();
  private final PathTemplateTransformer pathTemplateTransformer = new PathTemplateTransformer();
  private final GrpcStubTransformer grpcStubTransformer = new GrpcStubTransformer();
  private final GrpcElementDocTransformer elementDocTransformer = new GrpcElementDocTransformer();
  private final GapicCodePathMapper pathMapper;
  private final PackageMetadataConfig packageConfig;
  private final ProductServiceConfig productServiceConfig = new ProductServiceConfig();

  public PythonGapicSurfaceTransformer(
      GapicCodePathMapper pathMapper, PackageMetadataConfig packageConfig) {
    this.pathMapper = pathMapper;
    this.packageConfig = packageConfig;
  }

  @Override
  public List<String> getTemplateFileNames() {
    return ImmutableList.of(
        XAPI_TEMPLATE_FILENAME,
        TRANSPORT_TEMPLATE_FILENAME,
        ENUM_TEMPLATE_FILENAME,
        TYPES_TEMPLATE_FILENAME,
        VERSIONED_INIT_TEMPLATE_FILENAME,
        TOP_LEVEL_ENTRY_POINT_FILENAME);
  }

  @Override
  public List<ViewModel> transform(ProtoApiModel apiModel, GapicProductConfig productConfig) {
    ImmutableList.Builder<ViewModel> views = ImmutableList.builder();
    views.addAll(generateServiceSurfaces(apiModel, productConfig));
    views.addAll(generateVersionedDirectoryViews(apiModel, productConfig));
    views.addAll(generateTopLevelViews(apiModel, productConfig));
    return views.build();
  }

  private Iterable<ViewModel> generateServiceSurfaces(
      ProtoApiModel apiModel, GapicProductConfig productConfig) {
    ModelTypeTable modelTypeTable =
        new ModelTypeTable(
            new PythonTypeTable(productConfig.getPackageName()),
            new PythonModelTypeNameConverter(productConfig.getPackageName()));
    SurfaceNamer namer = new PythonSurfaceNamer(productConfig.getPackageName());
    FeatureConfig featureConfig = new DefaultFeatureConfig();
    ImmutableList.Builder<ViewModel> serviceSurfaces = ImmutableList.builder();

    for (InterfaceModel apiInterface : apiModel.getInterfaces()) {
      GapicInterfaceContext context =
          GapicInterfaceContext.create(
              apiInterface, productConfig, modelTypeTable, namer, featureConfig);
      addApiImports(context);

      String subPath =
          pathMapper.getOutputPath(
              context.getInterfaceModel().getFullName(), context.getProductConfig());

      DynamicLangXApiView.Builder serviceClass = generateApiView(context);
      serviceClass.templateFileName(XAPI_TEMPLATE_FILENAME);
      serviceClass.outputPath(
          namer.getSourceFilePath(
              subPath, namer.getApiWrapperClassName(context.getInterfaceConfig())));
      serviceClass.fileHeader(fileHeaderTransformer.generateFileHeader(context));
      serviceSurfaces.add(serviceClass.build());

      DynamicLangXApiView.Builder grpcTransportClass = generateApiView(context);
      grpcTransportClass.templateFileName(TRANSPORT_TEMPLATE_FILENAME);
      grpcTransportClass.outputPath(
          namer.getSourceFilePath(
              subPath + "/transports",
              namer.getGrpcTransportClassName(context.getInterfaceConfig())));
      ImportSectionView grpcTransportImportSection =
          importSectionTransformer.generateGrpcTransportImportSection(context);
      grpcTransportClass.fileHeader(
          fileHeaderTransformer.generateFileHeader(
              productConfig, grpcTransportImportSection, namer));
      serviceSurfaces.add(grpcTransportClass.build());
    }

    GrpcDocView enumFile =
        generateEnumView(productConfig, modelTypeTable, namer, getProtoFiles(productConfig));
    if (!enumFile.elementDocs().isEmpty()) {
      serviceSurfaces.add(enumFile);
    }
    return serviceSurfaces.build();
  }

  private List<ProtoFile> getProtoFiles(GapicProductConfig productConfig) {
    return ImmutableList.copyOf(ProtoFiles.getProtoFiles(productConfig));
  }

  private void addApiImports(GapicInterfaceContext context) {
    for (TypeRef type : context.getInterface().getModel().getSymbolTable().getDeclaredTypes()) {
      if (type.isEnum() && type.getEnumType().isReachable()) {
        context.getImportTypeTable().getAndSaveNicknameFor(type);
        break;
      }
    }

    for (MethodModel method : context.getSupportedMethods()) {
      addMethodImports(context.asDynamicMethodContext(method));
    }
  }

  private void addMethodImports(GapicMethodContext context) {
    ModelTypeTable typeTable = context.getTypeTable();
    GapicMethodConfig methodConfig = context.getMethodConfig();
    if (methodConfig.isLongRunningOperation()) {
      typeTable.getAndSaveNicknameFor(methodConfig.getLongRunningConfig().getReturnType());
      typeTable.getAndSaveNicknameFor(methodConfig.getLongRunningConfig().getMetadataType());
    }

    typeTable.getAndSaveNicknameFor(context.getMethod().getInputType());
    addFieldsImports(typeTable, methodConfig.getRequiredFields());<commit_msg>Revert "Python: use 'ProtoFiles.getProtoFiles' to sort enums by filename." (#2442)<commit_after>/* Copyright 2017 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.api.codegen.transformer.py;

import com.google.api.codegen.config.ApiModel;
import com.google.api.codegen.config.FieldModel;
import com.google.api.codegen.config.GapicMethodConfig;
import com.google.api.codegen.config.GapicProductConfig;
import com.google.api.codegen.config.InterfaceModel;
import com.google.api.codegen.config.MethodModel;
import com.google.api.codegen.config.PackageMetadataConfig;
import com.google.api.codegen.config.ProductConfig;
import com.google.api.codegen.config.ProductServiceConfig;
import com.google.api.codegen.config.ProtoApiModel;
import com.google.api.codegen.config.SampleSpec.SampleType;
import com.google.api.codegen.gapic.GapicCodePathMapper;
import com.google.api.codegen.gapic.GapicParser;
import com.google.api.codegen.transformer.DefaultFeatureConfig;
import com.google.api.codegen.transformer.DynamicLangApiMethodTransformer;
import com.google.api.codegen.transformer.FeatureConfig;
import com.google.api.codegen.transformer.FileHeaderTransformer;
import com.google.api.codegen.transformer.GapicInterfaceContext;
import com.google.api.codegen.transformer.GapicMethodContext;
import com.google.api.codegen.transformer.GrpcElementDocTransformer;
import com.google.api.codegen.transformer.GrpcStubTransformer;
import com.google.api.codegen.transformer.InitCodeTransformer;
import com.google.api.codegen.transformer.ModelToViewTransformer;
import com.google.api.codegen.transformer.ModelTypeTable;
import com.google.api.codegen.transformer.PageStreamingTransformer;
import com.google.api.codegen.transformer.PathTemplateTransformer;
import com.google.api.codegen.transformer.ServiceTransformer;
import com.google.api.codegen.transformer.SurfaceNamer;
import com.google.api.codegen.util.Name;
import com.google.api.codegen.util.NamePath;
import com.google.api.codegen.util.VersionMatcher;
import com.google.api.codegen.util.py.PythonTypeTable;
import com.google.api.codegen.viewmodel.BatchingDescriptorView;
import com.google.api.codegen.viewmodel.DynamicLangXApiView;
import com.google.api.codegen.viewmodel.GrpcDocView;
import com.google.api.codegen.viewmodel.GrpcElementDocView;
import com.google.api.codegen.viewmodel.GrpcMessageDocView;
import com.google.api.codegen.viewmodel.GrpcStreamingDetailView;
import com.google.api.codegen.viewmodel.ImportSectionView;
import com.google.api.codegen.viewmodel.LongRunningOperationDetailView;
import com.google.api.codegen.viewmodel.OptionalArrayMethodView;
import com.google.api.codegen.viewmodel.ParamDocView;
import com.google.api.codegen.viewmodel.PathTemplateGetterFunctionView;
import com.google.api.codegen.viewmodel.ViewModel;
import com.google.api.codegen.viewmodel.metadata.VersionIndexRequireView;
import com.google.api.codegen.viewmodel.metadata.VersionIndexView;
import com.google.api.tools.framework.model.MessageType;
import com.google.api.tools.framework.model.Model;
import com.google.api.tools.framework.model.ProtoContainerElement;
import com.google.api.tools.framework.model.ProtoFile;
import com.google.api.tools.framework.model.TypeRef;
import com.google.common.collect.ImmutableList;
import java.io.File;
import java.util.List;
import java.util.stream.Collectors;

/* Transforms a ProtoApiModel into the standard GAPIC surface for Python. */
public class PythonGapicSurfaceTransformer implements ModelToViewTransformer<ProtoApiModel> {
  private static final String XAPI_TEMPLATE_FILENAME = "py/main.snip";
  private static final String TRANSPORT_TEMPLATE_FILENAME = "py/transport.snip";
  private static final String ENUM_TEMPLATE_FILENAME = "py/enum.snip";
  private static final String TYPES_TEMPLATE_FILENAME = "py/types.snip";
  private static final String VERSIONED_INIT_TEMPLATE_FILENAME =
      "py/versioned_directory__init__.py.snip";
  private static final String TOP_LEVEL_ENTRY_POINT_FILENAME = "py/top_level_entry_point.snip";

  private static final SampleType sampleType = SampleType.IN_CODE;
  private final PythonImportSectionTransformer importSectionTransformer =
      new PythonImportSectionTransformer();
  private final FileHeaderTransformer fileHeaderTransformer =
      new FileHeaderTransformer(importSectionTransformer);
  private final DynamicLangApiMethodTransformer apiMethodTransformer =
      new DynamicLangApiMethodTransformer(
          new PythonApiMethodParamTransformer(),
          new InitCodeTransformer(importSectionTransformer),
          sampleType);
  private final PythonMethodViewGenerator methodGenerator =
      new PythonMethodViewGenerator(apiMethodTransformer);
  private final ServiceTransformer serviceTransformer = new ServiceTransformer();
  private final PageStreamingTransformer pageStreamingTransformer = new PageStreamingTransformer();
  private final PathTemplateTransformer pathTemplateTransformer = new PathTemplateTransformer();
  private final GrpcStubTransformer grpcStubTransformer = new GrpcStubTransformer();
  private final GrpcElementDocTransformer elementDocTransformer = new GrpcElementDocTransformer();
  private final GapicCodePathMapper pathMapper;
  private final PackageMetadataConfig packageConfig;
  private final ProductServiceConfig productServiceConfig = new ProductServiceConfig();

  public PythonGapicSurfaceTransformer(
      GapicCodePathMapper pathMapper, PackageMetadataConfig packageConfig) {
    this.pathMapper = pathMapper;
    this.packageConfig = packageConfig;
  }

  @Override
  public List<String> getTemplateFileNames() {
    return ImmutableList.of(
        XAPI_TEMPLATE_FILENAME,
        TRANSPORT_TEMPLATE_FILENAME,
        ENUM_TEMPLATE_FILENAME,
        TYPES_TEMPLATE_FILENAME,
        VERSIONED_INIT_TEMPLATE_FILENAME,
        TOP_LEVEL_ENTRY_POINT_FILENAME);
  }

  @Override
  public List<ViewModel> transform(ProtoApiModel apiModel, GapicProductConfig productConfig) {
    ImmutableList.Builder<ViewModel> views = ImmutableList.builder();
    views.addAll(generateServiceSurfaces(apiModel, productConfig));
    views.addAll(generateVersionedDirectoryViews(apiModel, productConfig));
    views.addAll(generateTopLevelViews(apiModel, productConfig));
    return views.build();
  }

  private Iterable<ViewModel> generateServiceSurfaces(
      ProtoApiModel apiModel, GapicProductConfig productConfig) {
    ModelTypeTable modelTypeTable =
        new ModelTypeTable(
            new PythonTypeTable(productConfig.getPackageName()),
            new PythonModelTypeNameConverter(productConfig.getPackageName()));
    SurfaceNamer namer = new PythonSurfaceNamer(productConfig.getPackageName());
    FeatureConfig featureConfig = new DefaultFeatureConfig();
    ImmutableList.Builder<ViewModel> serviceSurfaces = ImmutableList.builder();

    for (InterfaceModel apiInterface : apiModel.getInterfaces()) {
      GapicInterfaceContext context =
          GapicInterfaceContext.create(
              apiInterface, productConfig, modelTypeTable, namer, featureConfig);
      addApiImports(context);

      String subPath =
          pathMapper.getOutputPath(
              context.getInterfaceModel().getFullName(), context.getProductConfig());

      DynamicLangXApiView.Builder serviceClass = generateApiView(context);
      serviceClass.templateFileName(XAPI_TEMPLATE_FILENAME);
      serviceClass.outputPath(
          namer.getSourceFilePath(
              subPath, namer.getApiWrapperClassName(context.getInterfaceConfig())));
      serviceClass.fileHeader(fileHeaderTransformer.generateFileHeader(context));
      serviceSurfaces.add(serviceClass.build());

      DynamicLangXApiView.Builder grpcTransportClass = generateApiView(context);
      grpcTransportClass.templateFileName(TRANSPORT_TEMPLATE_FILENAME);
      grpcTransportClass.outputPath(
          namer.getSourceFilePath(
              subPath + "/transports",
              namer.getGrpcTransportClassName(context.getInterfaceConfig())));
      ImportSectionView grpcTransportImportSection =
          importSectionTransformer.generateGrpcTransportImportSection(context);
      grpcTransportClass.fileHeader(
          fileHeaderTransformer.generateFileHeader(
              productConfig, grpcTransportImportSection, namer));
      serviceSurfaces.add(grpcTransportClass.build());
    }

    GrpcDocView enumFile =
        generateEnumView(productConfig, modelTypeTable, namer, apiModel.getProtoModel().getFiles());
    if (!enumFile.elementDocs().isEmpty()) {
      serviceSurfaces.add(enumFile);
    }
    return serviceSurfaces.build();
  }

  private void addApiImports(GapicInterfaceContext context) {
    for (TypeRef type : context.getInterface().getModel().getSymbolTable().getDeclaredTypes()) {
      if (type.isEnum() && type.getEnumType().isReachable()) {
        context.getImportTypeTable().getAndSaveNicknameFor(type);
        break;
      }
    }

    for (MethodModel method : context.getSupportedMethods()) {
      addMethodImports(context.asDynamicMethodContext(method));
    }
  }

  private void addMethodImports(GapicMethodContext context) {
    ModelTypeTable typeTable = context.getTypeTable();
    GapicMethodConfig methodConfig = context.getMethodConfig();
    if (methodConfig.isLongRunningOperation()) {
      typeTable.getAndSaveNicknameFor(methodConfig.getLongRunningConfig().getReturnType());
      typeTable.getAndSaveNicknameFor(methodConfig.getLongRunningConfig().getMetadataType());
    }

    typeTable.getAndSaveNicknameFor(context.getMethod().getInputType());
    addFieldsImports(typeTable, methodConfig.getRequiredFields());
{'fraction_non_alphanumeric': 0.0845532435740514, 'fraction_numerical': 0.0009791921664626683, 'mean_word_length': 5.347420758234929, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '834456', 'n_tokens_mistral': 5586, 'n_tokens_neox': 5596, 'n_words': 982}