<commit_before>/*
 * Copyright 2017-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.facebook.buck.android;

import com.facebook.buck.android.redex.ReDexStep;
import com.facebook.buck.android.redex.RedexOptions;
import com.facebook.buck.android.resources.ResourcesZipBuilder;
import com.facebook.buck.io.BuildCellRelativePath;
import com.facebook.buck.io.ProjectFilesystem;
import com.facebook.buck.jvm.java.AccumulateClassNamesStep;
import com.facebook.buck.jvm.java.JavaLibrary;
import com.facebook.buck.jvm.java.JavaRuntimeLauncher;
import com.facebook.buck.model.BuildTarget;
import com.facebook.buck.model.BuildTargets;
import com.facebook.buck.model.Pair;
import com.facebook.buck.rules.AddToRuleKey;
import com.facebook.buck.rules.AddsToRuleKey;
import com.facebook.buck.rules.BuildContext;
import com.facebook.buck.rules.BuildableContext;
import com.facebook.buck.rules.SourcePath;
import com.facebook.buck.rules.SourcePathResolver;
import com.facebook.buck.rules.args.Arg;
import com.facebook.buck.shell.AbstractGenruleStep;
import com.facebook.buck.step.AbstractExecutionStep;
import com.facebook.buck.step.ExecutionContext;
import com.facebook.buck.step.Step;
import com.facebook.buck.step.StepExecutionResult;
import com.facebook.buck.step.fs.CopyStep;
import com.facebook.buck.step.fs.MakeCleanDirectoryStep;
import com.facebook.buck.step.fs.MkdirStep;
import com.facebook.buck.step.fs.XzStep;
import com.facebook.buck.util.MoreCollectors;
import com.facebook.buck.util.RichStream;
import com.facebook.buck.zip.RepackZipEntriesStep;
import com.facebook.buck.zip.ZipScrubberStep;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import com.google.common.hash.HashCode;
import com.google.common.io.Files;
import com.google.common.util.concurrent.ListeningExecutorService;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.AbstractMap;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.annotation.Nullable;

class AndroidBinaryBuildable implements AddsToRuleKey {
  /**
   * The filename of the solidly compressed libraries if compressAssetLibraries is set to true. This
   * file can be found in assets/lib.
   */
  private static final String SOLID_COMPRESSED_ASSET_LIBRARY_FILENAME = "libs.xzs";

  /**
   * This is the path from the root of the APK that should contain the metadata.txt and
   * secondary-N.dex.jar files for secondary dexes.
   */
  static final String SMART_DEX_SECONDARY_DEX_SUBDIR =
      "assets/smart-dex-secondary-program-dex-jars";

  @AddToRuleKey private final SourcePath keystorePath;
  @AddToRuleKey private final SourcePath keystorePropertiesPath;
  @AddToRuleKey private final AndroidBinary.PackageType packageType;
  @AddToRuleKey private final DexSplitMode dexSplitMode;
  @AddToRuleKey private final ProGuardObfuscateStep.SdkProguardType sdkProguardConfig;
  @AddToRuleKey private final Optional<Integer> optimizationPasses;
  @AddToRuleKey private final Optional<SourcePath> proguardConfig;
  @AddToRuleKey private final Optional<SourcePath> proguardJarOverride;
  @AddToRuleKey private final Optional<RedexOptions> redexOptions;
  @AddToRuleKey private final String proguardMaxHeapSize;
  @AddToRuleKey private final Optional<List<String>> proguardJvmArgs;
  @AddToRuleKey private final Optional<String> proguardAgentPath;
  @AddToRuleKey private final ImmutableSet<NdkCxxPlatforms.TargetCpuType> cpuFilters;
  @AddToRuleKey private final EnumSet<AndroidBinary.ExopackageMode> exopackageModes;
  @AddToRuleKey private final Optional<Arg> preprocessJavaClassesBash;
  @AddToRuleKey private final boolean reorderClassesIntraDex;
  @AddToRuleKey private final Optional<SourcePath> dexReorderToolFile;
  @AddToRuleKey private final Optional<SourcePath> dexReorderDataDumpFile;
  @AddToRuleKey private final Optional<Integer> xzCompressionLevel;
  @AddToRuleKey private final boolean packageAssetLibraries;
  @AddToRuleKey private final boolean compressAssetLibraries;
  @AddToRuleKey private final boolean skipProguard;
  @AddToRuleKey private final JavaRuntimeLauncher javaRuntimeLauncher;
  @AddToRuleKey private final SourcePath androidManifestPath;
  @AddToRuleKey private final SourcePath resourcesApkPath;
  @AddToRuleKey private final ImmutableList<SourcePath> primaryApkAssetsZips;
  @AddToRuleKey private final SourcePath aaptGeneratedProguardConfigFile;
  @AddToRuleKey private final Optional<String> dxMaxHeapSize;
  @AddToRuleKey private final ImmutableList<SourcePath> proguardConfigs;
  @AddToRuleKey private final boolean isCompressResources;
  @AddToRuleKey private final ImmutableSortedSet<SourcePath> additionalJarsForProguard;
  @AddToRuleKey private final ImmutableSortedSet<SourcePath> pathsToThirdPartyJars;
  @AddToRuleKey private final boolean hasLinkableAssets;
  @AddToRuleKey private final ImmutableSortedSet<APKModule> apkModules;
  @AddToRuleKey private final boolean isPreDexed;
  @AddToRuleKey private final APKModule rootAPKModule;

  @AddToRuleKey private final Optional<ImmutableSet<SourcePath>> classpathEntriesToDexSourcePaths;

  @AddToRuleKey
  private final Optional<ImmutableSortedMap<APKModule, ImmutableList<SourcePath>>>
      moduleMappedClasspathEntriesToDex;

  @AddToRuleKey
  private final ImmutableSortedMap<APKModule, ImmutableList<SourcePath>> nativeLibAssetsDirectories;

  @AddToRuleKey
  private final ImmutableSortedMap<APKModule, ImmutableSortedSet<APKModule>> apkModuleMap;

  @AddToRuleKey
  @Nullable
  @SuppressWarnings("unused")
  private final SourcePath abiPath;

  // These should be the only things not added to the rulekey.
  private final ProjectFilesystem filesystem;
  private final BuildTarget buildTarget;
  private final ListeningExecutorService dxExecutorService;

  // Once these fields are properly reflected in the rulekey, we can remove the abiPath (and
  // delete ComputeExopackageDepsAbi).
  private final Optional<ImmutableMap<APKModule, CopyNativeLibraries>> copyNativeLibraries;
  private final Optional<Supplier<ImmutableSortedSet<SourcePath>>>
      predexedSecondaryDexDirectoriesSupplier;

  AndroidBinaryBuildable(
      BuildTarget buildTarget,
      ProjectFilesystem filesystem,
      SourcePath keystorePath,
      SourcePath keystorePropertiesPath,
      AndroidBinary.PackageType packageType,
      DexSplitMode dexSplitMode,
      ProGuardObfuscateStep.SdkProguardType sdkProguardConfig,
      Optional<Integer> optimizationPasses,
      Optional<SourcePath> proguardConfig,
      Optional<SourcePath> proguardJarOverride,
      Optional<RedexOptions> redexOptions,
      String proguardMaxHeapSize,
      Optional<List<String>> proguardJvmArgs,
      Optional<String> proguardAgentPath,
      ImmutableSet<NdkCxxPlatforms.TargetCpuType> cpuFilters,
      EnumSet<AndroidBinary.ExopackageMode> exopackageModes,
      Optional<Arg> preprocessJavaClassesBash,
      boolean reorderClassesIntraDex,
      Optional<SourcePath> dexReorderToolFile,
      Optional<SourcePath> dexReorderDataDumpFile,
      ImmutableSortedSet<JavaLibrary> rulesToExcludeFromDex,
      AndroidGraphEnhancementResult enhancementResult,
      ListeningExecutorService dxExecutorService,
      Optional<Integer> xzCompressionLevel,
      boolean packageAssetLibraries,
      boolean compressAssetLibraries,
      boolean skipProguard,
      JavaRuntimeLauncher javaRuntimeLauncher,
      SourcePath androidManifestPath,
      SourcePath resourcesApkPath,
      ImmutableList<SourcePath> primaryApkAssetsZips,
      SourcePath aaptGeneratedProguardConfigFile,
      Optional<String> dxMaxHeapSize,
      ImmutableList<SourcePath> proguardConfigs,
      boolean isCompressResources,
      SourcePath abiPath) {
    this.filesystem = filesystem;
    this.buildTarget = buildTarget;

    this.keystorePath = keystorePath;
    this.keystorePropertiesPath = keystorePropertiesPath;
    this.packageType = packageType;
    this.dexSplitMode = dexSplitMode;
    this.sdkProguardConfig = sdkProguardConfig;
    this.optimizationPasses = optimizationPasses;
    this.proguardConfig = proguardConfig;
    this.proguardJarOverride = proguardJarOverride;
    this.redexOptions = redexOptions;
    this.proguardMaxHeapSize = proguardMaxHeapSize;
    this.proguardJvmArgs = proguardJvmArgs;
    this.proguardAgentPath = proguardAgentPath;
    this.cpuFilters = cpuFilters;
    this.exopackageModes = exopackageModes;
    this.preprocessJavaClassesBash = preprocessJavaClassesBash;
    this.reorderClassesIntraDex = reorderClassesIntraDex;
    this.dexReorderToolFile = dexReorderToolFile;
    this.dexReorderDataDumpFile = dexReorderDataDumpFile;
    this.dxExecutorService = dxExecutorService;
    this.xzCompressionLevel = xzCompressionLevel;
    this.packageAssetLibraries = packageAssetLibraries;
    this.compressAssetLibraries = compressAssetLibraries;
    this.skipProguard = skipProguard;
    this.javaRuntimeLauncher = javaRuntimeLauncher;
    this.androidManifestPath = androidManifestPath;
    this.resourcesApkPath = resourcesApkPath;
    this.primaryApkAssetsZips = primaryApkAssetsZips;
    this.aaptGeneratedProguardConfigFile = aaptGeneratedProguardConfigFile;
    this.dxMaxHeapSize = dxMaxHeapSize;
    this.proguardConfigs = proguardConfigs;
    this.isCompressResources = isCompressResources;
    this.additionalJarsForProguard =
        rulesToExcludeFromDex
            .stream()
            .flatMap((javaLibrary) -> javaLibrary.getImmediateClasspaths().stream())
            .collect(MoreCollectors.toImmutableSortedSet());
    AndroidPackageableCollection packageableCollection =
        enhancementResult.getPackageableCollection();

    this.hasLinkableAssets = packageableCollection.getNativeLinkablesAssets().isEmpty();
    this.pathsToThirdPartyJars =
        ImmutableSortedSet.copyOf(packageableCollection.getPathsToThirdPartyJars());
    this.copyNativeLibraries = enhancementResult.getCopyNativeLibraries();
    this.predexedSecondaryDexDirectoriesSupplier =
        enhancementResult.getPreDexMerge().map(pdm -> pdm::getSecondaryDexDirectories);

    APKModuleGraph apkModuleGraph = enhancementResult.getAPKModuleGraph();
    this.nativeLibAssetsDirectories =
        convertToMapOfLists(packageableCollection.getNativeLibAssetsDirectories());
    this.apkModules =
        ImmutableSortedSet.copyOf(enhancementResult.getAPKModuleGraph().getAPKModules());
    this.apkModuleMap = apkModuleGraph.toOutgoingEdgesMap();
    this.rootAPKModule = apkModuleGraph.getRootAPKModule();

    this.isPreDexed = enhancementResult.getPreDexMerge().isPresent();

    if (isPreDexed) {
      Preconditions.checkState(!preprocessJavaClassesBash.isPresent());
      Preconditions.checkState(!packageType.isBuildWithObfuscation());
      this.classpathEntriesToDexSourcePaths = Optional.empty();
      this.moduleMappedClasspathEntriesToDex = Optional.empty();
    } else {
      this.classpathEntriesToDexSourcePaths =
          Optional.of(
              RichStream.from(enhancementResult.getClasspathEntriesToDex())
                  .concat(
                      RichStream.of(
                          enhancementResult.getCompiledUberRDotJava().getSourcePathToOutput()))
                  .collect(MoreCollectors.toImmutableSet()));
      this.moduleMappedClasspathEntriesToDex =
          Optional.of(
              convertToMapOfLists(packageableCollection.getModuleMappedClasspathEntriesToDex()));
    }

    this.abiPath = abiPath;
  }

  private <K extends Comparable<?>, V> ImmutableSortedMap<K, ImmutableList<V>> convertToMapOfLists(
      ImmutableMultimap<K, V> multimap) {
    return multimap
        .asMap()
        .entrySet()
        .stream()
        .collect(
            MoreCollectors.toImmutableSortedMap(
                e -> e.getKey(), e -> ImmutableList.copyOf(e.getValue())));
  }

  @SuppressWarnings("PMD.PrematureDeclaration")
  public ImmutableList<Step> getBuildSteps(
      BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();

    // The `HasInstallableApk` interface needs access to the manifest, so make sure we create our
    // own copy of this so that we don't have a runtime dep on the `AaptPackageResources` step.
    Path manifestPath = getManifestPath();
    steps.add(
        MkdirStep.of(
            BuildCellRelativePath.fromCellRelativePath(
                context.getBuildCellRootPath(), getProjectFilesystem(), manifestPath.getParent())));
    steps.add(
        CopyStep.forFile(
            getProjectFilesystem(),
            context.getSourcePathResolver().getRelativePath(androidManifestPath),
            manifestPath));
    buildableContext.recordArtifact(manifestPath);

    // Create the .dex files if we aren't doing pre-dexing.
    DexFilesInfo dexFilesInfo = addFinalDxSteps(buildableContext, context, steps);

    ////
    // BE VERY CAREFUL adding any code below here.
    // Any inputs to apkbuilder must be reflected in the hash returned by getAbiKeyForDeps.
    ////

    ImmutableSet.Builder<Path> nativeLibraryDirectoriesBuilder = ImmutableSet.builder();
    // Copy the transitive closure of native-libs-as-assets to a single directory, if any.
    ImmutableSet.Builder<Path> nativeLibraryAsAssetDirectories = ImmutableSet.builder();

    for (final APKModule module : apkModules) {
      boolean shouldPackageAssetLibraries = packageAssetLibraries || !module.isRootModule();

      if (!AndroidBinary.ExopackageMode.enabledForNativeLibraries(exopackageModes)
          && copyNativeLibraries.isPresent()
          && copyNativeLibraries.get().containsKey(module)) {
        CopyNativeLibraries copyNativeLibraries = this.copyNativeLibraries.get().get(module);
        if (shouldPackageAssetLibraries) {
          nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsDir());
        } else {
          nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsDir());
          nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsAssetsDir());
        }
      }

      if ((!nativeLibAssetsDirectories.isEmpty())
          || (!hasLinkableAssets && shouldPackageAssetLibraries)) {
        Preconditions.checkState(
            !AndroidBinary.ExopackageMode.enabledForResources(exopackageModes));
        Path pathForNativeLibsAsAssets = getPathForNativeLibsAsAssets();

        final Path libSubdirectory =
            pathForNativeLibsAsAssets
                .resolve("assets")
                .resolve(module.isRootModule() ? "lib" : module.getName());
        ImmutableCollection<SourcePath> nativeLibDirs = nativeLibAssetsDirectories.get(module);

        getStepsForNativeAssets(
            context,
            steps,
            nativeLibDirs == null ? Optional.empty() : Optional.of(nativeLibDirs),
            libSubdirectory,
            module.isRootModule() ? "metadata.txt" : "libs.txt",
            module);

        nativeLibraryAsAssetDirectories.add(pathForNativeLibsAsAssets);
      }
    }

    // If non-english strings are to be stored as assets, pass them to ApkBuilder.
    ImmutableSet.Builder<Path> zipFiles = ImmutableSet.builder();
    RichStream.from(primaryApkAssetsZips)
        .map(context.getSourcePathResolver()::getRelativePath)
        .forEach(zipFiles::add);

    if (AndroidBinary.ExopackageMode.enabledForNativeLibraries(exopackageModes)) {
      // We need to include a few dummy native libraries with our application so that Android knows
      // to run it as 32-bit.  Android defaults to 64-bit when no libraries are provided at all,
      // causing us to fail to load our 32-bit exopackage native libraries later.
      String fakeNativeLibraryBundle = System.getProperty("buck.native_exopackage_fake_path");

      if (fakeNativeLibraryBundle == null) {
        throw new RuntimeException("fake native bundle not specified in properties");
      }

      zipFiles.add(Paths.get(fakeNativeLibraryBundle));
    }

    ImmutableSet<Path> allAssetDirectories =
        ImmutableSet.<Path>builder()
            .addAll(nativeLibraryAsAssetDirectories.build())
            .addAll(dexFilesInfo.secondaryDexDirs)
            .build();

    SourcePathResolver resolver = context.getSourcePathResolver();
    Path signedApkPath = getSignedApkPath();
    final Path pathToKeystore = resolver.getAbsolutePath(keystorePath);
    Supplier<KeystoreProperties> keystoreProperties =
        Suppliers.memoize(
            () -> {
              try {
                return KeystoreProperties.createFromPropertiesFile(
                    pathToKeystore,
                    resolver.getAbsolutePath(keystorePropertiesPath),
                    getProjectFilesystem());
              } catch (IOException e) {
                throw new RuntimeException();
              }
            });

    ImmutableSet<Path> thirdPartyJars =
        pathsToThirdPartyJars
            .stream()
            .map(resolver::getAbsolutePath)
            .collect(MoreCollectors.toImmutableSet());

    if (AndroidBinary.ExopackageMode.enabledForResources(exopackageModes)) {
      steps.add(createMergedThirdPartyJarsStep(thirdPartyJars));
      buildableContext.recordArtifact(getMergedThirdPartyJarsPath());
    }

    ApkBuilderStep apkBuilderCommand =
        new ApkBuilderStep(
            getProjectFilesystem(),
            context.getSourcePathResolver().getAbsolutePath(resourcesApkPath),
            getSignedApkPath(),
            dexFilesInfo.primaryDexPath,
            allAssetDirectories,
            nativeLibraryDirectoriesBuilder.build(),
            zipFiles.build(),
            thirdPartyJars,
            pathToKeystore,
            keystoreProperties,
            /* debugMode */ false,
            javaRuntimeLauncher);
    steps.add(apkBuilderCommand);

    // The `ApkBuilderStep` delegates to android tools to build a ZIP with timestamps in it, making
    // the output non-deterministic.  So use an additional scrubbing step to zero these out.
    steps.add(ZipScrubberStep.of(getProjectFilesystem().resolve(signedApkPath)));

    Path apkToRedexAndAlign;
    // Optionally, compress the resources file in the .apk.
    if (isCompressResources) {
      Path compressedApkPath = getCompressedResourcesApkPath();
      apkToRedexAndAlign = compressedApkPath;
      RepackZipEntriesStep arscComp =
          new RepackZipEntriesStep(
              getProjectFilesystem(),
              signedApkPath,
              compressedApkPath,
              ImmutableSet.of("resources.arsc"));
      steps.add(arscComp);
    } else {
      apkToRedexAndAlign = signedApkPath;
    }

    boolean applyRedex = redexOptions.isPresent();
    Path apkPath = getFinalApkPath();
    Path apkToAlign = apkToRedexAndAlign;

    // redex
    if (applyRedex) {
      Path proguardConfigDir = getProguardTextFilesPath();
      Path redexedApk = getRedexedApkPath();
      apkToAlign = redexedApk;
      steps.add(
          MkdirStep.of(
              BuildCellRelativePath.fromCellRelativePath(
                  context.getBuildCellRootPath(), getProjectFilesystem(), redexedApk.getParent())));
      ImmutableList<Step> redexSteps =
          ReDexStep.createSteps(
              getProjectFilesystem(),
              resolver,
              redexOptions.get(),
              apkToRedexAndAlign,
              redexedApk,
              keystoreProperties,
              proguardConfigDir,
              buildableContext);
      steps.addAll(redexSteps);
    }

    steps.add(new ZipalignStep(getProjectFilesystem().getRootPath(), apkToAlign, apkPath));

    buildableContext.recordArtifact(apkPath);
    return steps.build();
  }

  private Step createMergedThirdPartyJarsStep(ImmutableSet<Path> thirdPartyJars) {
    return new AbstractExecutionStep("merging_third_party_jar_resources") {
      @Override
      public StepExecutionResult execute(ExecutionContext context)
          throws IOException, InterruptedException {
        try (ResourcesZipBuilder builder =
            new ResourcesZipBuilder(
                getProjectFilesystem().resolve(getMergedThirdPartyJarsPath()))) {
          for (Path jar : thirdPartyJars) {
            try (ZipFile base = new ZipFile(jar.toFile())) {
              for (ZipEntry inputEntry : Collections.list(base.entries())) {
                if (inputEntry.isDirectory()) {
                  continue;
                }
                String name = inputEntry.getName();
                String ext = Files.getFileExtension(name);
                String filename = Paths.get(name).getFileName().toString();
                // Android's ApkBuilder filters out a lot of files from Java resources. Try to
                // match its behavior.
                // See https://android.googlesource.com/platform/sdk/+/jb-release/sdkmanager/libs/sdklib/src/com/android/sdklib/build/ApkBuilder.java
                if (name.startsWith(".")
                    || name.endsWith("~")
                    || name.startsWith("META-INF")
                    || "aidl".equalsIgnoreCase(ext)
                    || "rs".equalsIgnoreCase(ext)
                    || "rsh".equalsIgnoreCase(ext)
                    || "d".equalsIgnoreCase(ext)
                    || "java".equalsIgnoreCase(ext)
                    || "scala".equalsIgnoreCase(ext)
                    || "class".equalsIgnoreCase(ext)
                    || "scc".equalsIgnoreCase(ext)
                    || "swp".equalsIgnoreCase(ext)
                    || "thumbs.db".equalsIgnoreCase(filename)
                    || "picasa.ini".equalsIgnoreCase(filename)
                    || "package.html".equalsIgnoreCase(filename)
                    || "overview.html".equalsIgnoreCase(filename)) {
                  continue;
                }
                try (InputStream inputStream = base.getInputStream(inputEntry)) {
                  builder.addEntry(
                      inputStream,
                      inputEntry.getSize(),
                      inputEntry.getCrc(),
                      name,
                      Deflater.NO_COMPRESSION,
                      false);
                }
              }
            }
          }
        }
        return StepExecutionResult.SUCCESS;
      }
    };
  }

  private void getStepsForNativeAssets(
      BuildContext context,
      ImmutableList.Builder<Step> steps,
      Optional<ImmutableCollection<SourcePath>> nativeLibDirs,
      final Path libSubdirectory,
      final String metadataFilename,
      final APKModule module) {

    steps.addAll(
        MakeCleanDirectoryStep.of(
            BuildCellRelativePath.fromCellRelativePath(
                context.getBuildCellRootPath(), getProjectFilesystem(), libSubdirectory)));

    // Filter, rename and copy the ndk libraries marked as assets.
    if (nativeLibDirs.isPresent()) {
      for (SourcePath nativeLibDir : nativeLibDirs.get()) {
        CopyNativeLibraries.copyNativeLibrary(
            context,
            getProjectFilesystem(),
            context.getSourcePathResolver().getAbsolutePath(nativeLibDir),
            libSubdirectory,
            cpuFilters,
            steps);
      }
    }

    // Input asset libraries are sorted in descending filesize order.
    final ImmutableSortedSet.Builder<Path> inputAssetLibrariesBuilder =
        ImmutableSortedSet.orderedBy(
            (libPath1, libPath2) -> {
              try {
                ProjectFilesystem filesystem = getProjectFilesystem();
                int filesizeResult =
                    -Long.compare(
                        filesystem.getFileSize(libPath1), filesystem.getFileSize(libPath2));
                int pathnameResult = libPath1.compareTo(libPath2);
                return filesizeResult != 0 ? filesizeResult : pathnameResult;
              } catch (IOException e) {
                return 0;
              }
            });

    if (packageAssetLibraries || !module.isRootModule()) {
      if (copyNativeLibraries.isPresent() && copyNativeLibraries.get().containsKey(module)) {
        // Copy in cxx libraries marked as assets. Filtering and renaming was already done
        // in CopyNativeLibraries.getBuildSteps().
        Path cxxNativeLibsSrc =
            copyNativeLibraries.get().get(module).getPathToNativeLibsAssetsDir();
        steps.add(
            CopyStep.forDirectory(
                getProjectFilesystem(),
                cxxNativeLibsSrc,
                libSubdirectory,
                CopyStep.DirectoryMode.CONTENTS_ONLY));
      }

      steps.add(
          // Step that populates a list of libraries and writes a metadata.txt to decompress.
          new AbstractExecutionStep("write_metadata_for_asset_libraries_" + module.getName()) {
            @Override
            public StepExecutionResult execute(ExecutionContext context)
                throws IOException, InterruptedException {
              ProjectFilesystem filesystem = getProjectFilesystem();
              // Walk file tree to find libraries
              filesystem.walkRelativeFileTree(
                  libSubdirectory,
                  new SimpleFileVisitor<Path>() {
                    @Override
                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                        throws IOException {
                      if (!file.toString().endsWith(".so")) {
                        throw new IOException("unexpected file in lib directory");
                      }<commit_msg>Remove CopyNativeLibraries field from AndroidBuildable<commit_after>/*
 * Copyright 2017-present Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License. You may obtain
 * a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */

package com.facebook.buck.android;

import com.facebook.buck.android.redex.ReDexStep;
import com.facebook.buck.android.redex.RedexOptions;
import com.facebook.buck.android.resources.ResourcesZipBuilder;
import com.facebook.buck.io.BuildCellRelativePath;
import com.facebook.buck.io.ProjectFilesystem;
import com.facebook.buck.jvm.java.AccumulateClassNamesStep;
import com.facebook.buck.jvm.java.JavaLibrary;
import com.facebook.buck.jvm.java.JavaRuntimeLauncher;
import com.facebook.buck.model.BuildTarget;
import com.facebook.buck.model.BuildTargets;
import com.facebook.buck.model.Pair;
import com.facebook.buck.rules.AddToRuleKey;
import com.facebook.buck.rules.AddsToRuleKey;
import com.facebook.buck.rules.BuildContext;
import com.facebook.buck.rules.BuildableContext;
import com.facebook.buck.rules.SourcePath;
import com.facebook.buck.rules.SourcePathResolver;
import com.facebook.buck.rules.args.Arg;
import com.facebook.buck.shell.AbstractGenruleStep;
import com.facebook.buck.step.AbstractExecutionStep;
import com.facebook.buck.step.ExecutionContext;
import com.facebook.buck.step.Step;
import com.facebook.buck.step.StepExecutionResult;
import com.facebook.buck.step.fs.CopyStep;
import com.facebook.buck.step.fs.MakeCleanDirectoryStep;
import com.facebook.buck.step.fs.MkdirStep;
import com.facebook.buck.step.fs.XzStep;
import com.facebook.buck.util.MoreCollectors;
import com.facebook.buck.util.RichStream;
import com.facebook.buck.zip.RepackZipEntriesStep;
import com.facebook.buck.zip.ZipScrubberStep;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import com.google.common.hash.HashCode;
import com.google.common.io.Files;
import com.google.common.util.concurrent.ListeningExecutorService;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.AbstractMap;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import javax.annotation.Nullable;

class AndroidBinaryBuildable implements AddsToRuleKey {
  /**
   * The filename of the solidly compressed libraries if compressAssetLibraries is set to true. This
   * file can be found in assets/lib.
   */
  private static final String SOLID_COMPRESSED_ASSET_LIBRARY_FILENAME = "libs.xzs";

  /**
   * This is the path from the root of the APK that should contain the metadata.txt and
   * secondary-N.dex.jar files for secondary dexes.
   */
  static final String SMART_DEX_SECONDARY_DEX_SUBDIR =
      "assets/smart-dex-secondary-program-dex-jars";

  @AddToRuleKey private final SourcePath keystorePath;
  @AddToRuleKey private final SourcePath keystorePropertiesPath;
  @AddToRuleKey private final AndroidBinary.PackageType packageType;
  @AddToRuleKey private final DexSplitMode dexSplitMode;
  @AddToRuleKey private final ProGuardObfuscateStep.SdkProguardType sdkProguardConfig;
  @AddToRuleKey private final Optional<Integer> optimizationPasses;
  @AddToRuleKey private final Optional<SourcePath> proguardConfig;
  @AddToRuleKey private final Optional<SourcePath> proguardJarOverride;
  @AddToRuleKey private final Optional<RedexOptions> redexOptions;
  @AddToRuleKey private final String proguardMaxHeapSize;
  @AddToRuleKey private final Optional<List<String>> proguardJvmArgs;
  @AddToRuleKey private final Optional<String> proguardAgentPath;
  @AddToRuleKey private final ImmutableSet<NdkCxxPlatforms.TargetCpuType> cpuFilters;
  @AddToRuleKey private final EnumSet<AndroidBinary.ExopackageMode> exopackageModes;
  @AddToRuleKey private final Optional<Arg> preprocessJavaClassesBash;
  @AddToRuleKey private final boolean reorderClassesIntraDex;
  @AddToRuleKey private final Optional<SourcePath> dexReorderToolFile;
  @AddToRuleKey private final Optional<SourcePath> dexReorderDataDumpFile;
  @AddToRuleKey private final Optional<Integer> xzCompressionLevel;
  @AddToRuleKey private final boolean packageAssetLibraries;
  @AddToRuleKey private final boolean compressAssetLibraries;
  @AddToRuleKey private final boolean skipProguard;
  @AddToRuleKey private final JavaRuntimeLauncher javaRuntimeLauncher;
  @AddToRuleKey private final SourcePath androidManifestPath;
  @AddToRuleKey private final SourcePath resourcesApkPath;
  @AddToRuleKey private final ImmutableList<SourcePath> primaryApkAssetsZips;
  @AddToRuleKey private final SourcePath aaptGeneratedProguardConfigFile;
  @AddToRuleKey private final Optional<String> dxMaxHeapSize;
  @AddToRuleKey private final ImmutableList<SourcePath> proguardConfigs;
  @AddToRuleKey private final boolean isCompressResources;
  @AddToRuleKey private final ImmutableSortedSet<SourcePath> additionalJarsForProguard;
  @AddToRuleKey private final ImmutableSortedSet<SourcePath> pathsToThirdPartyJars;
  @AddToRuleKey private final boolean hasLinkableAssets;
  @AddToRuleKey private final ImmutableSortedSet<APKModule> apkModules;
  @AddToRuleKey private final boolean isPreDexed;
  @AddToRuleKey private final APKModule rootAPKModule;

  @AddToRuleKey private final Optional<ImmutableSet<SourcePath>> classpathEntriesToDexSourcePaths;

  @AddToRuleKey
  private final Optional<ImmutableSortedMap<APKModule, ImmutableList<SourcePath>>>
      moduleMappedClasspathEntriesToDex;

  @AddToRuleKey
  private final ImmutableSortedMap<APKModule, ImmutableList<SourcePath>> nativeLibAssetsDirectories;

  @AddToRuleKey
  private final ImmutableSortedMap<APKModule, ImmutableSortedSet<APKModule>> apkModuleMap;

  @AddToRuleKey
  @Nullable
  @SuppressWarnings("unused")
  private final SourcePath abiPath;

  // These should be the only things not added to the rulekey.
  private final ProjectFilesystem filesystem;
  private final BuildTarget buildTarget;
  private final ListeningExecutorService dxExecutorService;

  // Once these fields are properly reflected in the rulekey, we can remove the abiPath (and
  // delete ComputeExopackageDepsAbi).
  private final Optional<Supplier<ImmutableSortedSet<SourcePath>>>
      predexedSecondaryDexDirectoriesSupplier;
  private Optional<ImmutableSortedMap<APKModule, Path>> nativeLibsDirs;
  // TODO(cjhopman): why is this derived differently than nativeLibAssetsDirectories?
  private Optional<ImmutableSortedMap<APKModule, Path>> nativeLibsAssetsDirs;

  AndroidBinaryBuildable(
      BuildTarget buildTarget,
      ProjectFilesystem filesystem,
      SourcePath keystorePath,
      SourcePath keystorePropertiesPath,
      AndroidBinary.PackageType packageType,
      DexSplitMode dexSplitMode,
      ProGuardObfuscateStep.SdkProguardType sdkProguardConfig,
      Optional<Integer> optimizationPasses,
      Optional<SourcePath> proguardConfig,
      Optional<SourcePath> proguardJarOverride,
      Optional<RedexOptions> redexOptions,
      String proguardMaxHeapSize,
      Optional<List<String>> proguardJvmArgs,
      Optional<String> proguardAgentPath,
      ImmutableSet<NdkCxxPlatforms.TargetCpuType> cpuFilters,
      EnumSet<AndroidBinary.ExopackageMode> exopackageModes,
      Optional<Arg> preprocessJavaClassesBash,
      boolean reorderClassesIntraDex,
      Optional<SourcePath> dexReorderToolFile,
      Optional<SourcePath> dexReorderDataDumpFile,
      ImmutableSortedSet<JavaLibrary> rulesToExcludeFromDex,
      AndroidGraphEnhancementResult enhancementResult,
      ListeningExecutorService dxExecutorService,
      Optional<Integer> xzCompressionLevel,
      boolean packageAssetLibraries,
      boolean compressAssetLibraries,
      boolean skipProguard,
      JavaRuntimeLauncher javaRuntimeLauncher,
      SourcePath androidManifestPath,
      SourcePath resourcesApkPath,
      ImmutableList<SourcePath> primaryApkAssetsZips,
      SourcePath aaptGeneratedProguardConfigFile,
      Optional<String> dxMaxHeapSize,
      ImmutableList<SourcePath> proguardConfigs,
      boolean isCompressResources,
      SourcePath abiPath) {
    this.filesystem = filesystem;
    this.buildTarget = buildTarget;

    this.keystorePath = keystorePath;
    this.keystorePropertiesPath = keystorePropertiesPath;
    this.packageType = packageType;
    this.dexSplitMode = dexSplitMode;
    this.sdkProguardConfig = sdkProguardConfig;
    this.optimizationPasses = optimizationPasses;
    this.proguardConfig = proguardConfig;
    this.proguardJarOverride = proguardJarOverride;
    this.redexOptions = redexOptions;
    this.proguardMaxHeapSize = proguardMaxHeapSize;
    this.proguardJvmArgs = proguardJvmArgs;
    this.proguardAgentPath = proguardAgentPath;
    this.cpuFilters = cpuFilters;
    this.exopackageModes = exopackageModes;
    this.preprocessJavaClassesBash = preprocessJavaClassesBash;
    this.reorderClassesIntraDex = reorderClassesIntraDex;
    this.dexReorderToolFile = dexReorderToolFile;
    this.dexReorderDataDumpFile = dexReorderDataDumpFile;
    this.dxExecutorService = dxExecutorService;
    this.xzCompressionLevel = xzCompressionLevel;
    this.packageAssetLibraries = packageAssetLibraries;
    this.compressAssetLibraries = compressAssetLibraries;
    this.skipProguard = skipProguard;
    this.javaRuntimeLauncher = javaRuntimeLauncher;
    this.androidManifestPath = androidManifestPath;
    this.resourcesApkPath = resourcesApkPath;
    this.primaryApkAssetsZips = primaryApkAssetsZips;
    this.aaptGeneratedProguardConfigFile = aaptGeneratedProguardConfigFile;
    this.dxMaxHeapSize = dxMaxHeapSize;
    this.proguardConfigs = proguardConfigs;
    this.isCompressResources = isCompressResources;
    this.additionalJarsForProguard =
        rulesToExcludeFromDex
            .stream()
            .flatMap((javaLibrary) -> javaLibrary.getImmediateClasspaths().stream())
            .collect(MoreCollectors.toImmutableSortedSet());
    AndroidPackageableCollection packageableCollection =
        enhancementResult.getPackageableCollection();

    this.hasLinkableAssets = packageableCollection.getNativeLinkablesAssets().isEmpty();
    this.pathsToThirdPartyJars =
        ImmutableSortedSet.copyOf(packageableCollection.getPathsToThirdPartyJars());
    this.predexedSecondaryDexDirectoriesSupplier =
        enhancementResult.getPreDexMerge().map(pdm -> pdm::getSecondaryDexDirectories);

    APKModuleGraph apkModuleGraph = enhancementResult.getAPKModuleGraph();
    this.nativeLibAssetsDirectories =
        convertToMapOfLists(packageableCollection.getNativeLibAssetsDirectories());
    this.apkModules =
        ImmutableSortedSet.copyOf(enhancementResult.getAPKModuleGraph().getAPKModules());
    this.apkModuleMap = apkModuleGraph.toOutgoingEdgesMap();
    this.rootAPKModule = apkModuleGraph.getRootAPKModule();

    this.isPreDexed = enhancementResult.getPreDexMerge().isPresent();

    Optional<ImmutableMap<APKModule, CopyNativeLibraries>> copyNativeLibraries =
        enhancementResult.getCopyNativeLibraries();

    boolean exopackageForNativeEnabled =
        AndroidBinary.ExopackageMode.enabledForNativeLibraries(exopackageModes);
    if (exopackageForNativeEnabled) {
      this.nativeLibsDirs = Optional.empty();
    } else {
      this.nativeLibsDirs =
          copyNativeLibraries.map(
              cnl ->
                  cnl.entrySet()
                      .stream()
                      .collect(
                          MoreCollectors.toImmutableSortedMap(
                              e -> e.getKey(), e -> e.getValue().getPathToNativeLibsDir())));
    }

    this.nativeLibsAssetsDirs =
        copyNativeLibraries.map(
            cnl ->
                cnl.entrySet()
                    .stream()
                    .filter(
                        entry ->
                            !exopackageForNativeEnabled
                                || packageAssetLibraries
                                || !entry.getKey().isRootModule())
                    .collect(
                        MoreCollectors.toImmutableSortedMap(
                            e -> e.getKey(), e -> e.getValue().getPathToNativeLibsAssetsDir())));

    if (isPreDexed) {
      Preconditions.checkState(!preprocessJavaClassesBash.isPresent());
      Preconditions.checkState(!packageType.isBuildWithObfuscation());
      this.classpathEntriesToDexSourcePaths = Optional.empty();
      this.moduleMappedClasspathEntriesToDex = Optional.empty();
    } else {
      this.classpathEntriesToDexSourcePaths =
          Optional.of(
              RichStream.from(enhancementResult.getClasspathEntriesToDex())
                  .concat(
                      RichStream.of(
                          enhancementResult.getCompiledUberRDotJava().getSourcePathToOutput()))
                  .collect(MoreCollectors.toImmutableSet()));
      this.moduleMappedClasspathEntriesToDex =
          Optional.of(
              convertToMapOfLists(packageableCollection.getModuleMappedClasspathEntriesToDex()));
    }

    this.abiPath = abiPath;
  }

  private <K extends Comparable<?>, V> ImmutableSortedMap<K, ImmutableList<V>> convertToMapOfLists(
      ImmutableMultimap<K, V> multimap) {
    return multimap
        .asMap()
        .entrySet()
        .stream()
        .collect(
            MoreCollectors.toImmutableSortedMap(
                e -> e.getKey(), e -> ImmutableList.copyOf(e.getValue())));
  }

  @SuppressWarnings("PMD.PrematureDeclaration")
  public ImmutableList<Step> getBuildSteps(
      BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();

    // The `HasInstallableApk` interface needs access to the manifest, so make sure we create our
    // own copy of this so that we don't have a runtime dep on the `AaptPackageResources` step.
    Path manifestPath = getManifestPath();
    steps.add(
        MkdirStep.of(
            BuildCellRelativePath.fromCellRelativePath(
                context.getBuildCellRootPath(), getProjectFilesystem(), manifestPath.getParent())));
    steps.add(
        CopyStep.forFile(
            getProjectFilesystem(),
            context.getSourcePathResolver().getRelativePath(androidManifestPath),
            manifestPath));
    buildableContext.recordArtifact(manifestPath);

    // Create the .dex files if we aren't doing pre-dexing.
    DexFilesInfo dexFilesInfo = addFinalDxSteps(buildableContext, context, steps);

    ////
    // BE VERY CAREFUL adding any code below here.
    // Any inputs to apkbuilder must be reflected in the hash returned by getAbiKeyForDeps.
    ////

    ImmutableSet.Builder<Path> nativeLibraryDirectoriesBuilder = ImmutableSet.builder();
    // Copy the transitive closure of native-libs-as-assets to a single directory, if any.
    ImmutableSet.Builder<Path> nativeLibraryAsAssetDirectories = ImmutableSet.builder();

    for (final APKModule module : apkModules) {
      boolean shouldPackageAssetLibraries = packageAssetLibraries || !module.isRootModule();
      if (!AndroidBinary.ExopackageMode.enabledForNativeLibraries(exopackageModes)
          && nativeLibsDirs.isPresent()
          && nativeLibsDirs.get().containsKey(module)) {
        if (shouldPackageAssetLibraries) {
          nativeLibraryDirectoriesBuilder.add(nativeLibsDirs.get().get(module));
        } else {
          nativeLibraryDirectoriesBuilder.add(nativeLibsDirs.get().get(module));
          nativeLibraryDirectoriesBuilder.add(nativeLibsAssetsDirs.get().get(module));
        }
      }

      if ((!nativeLibAssetsDirectories.isEmpty())
          || (!hasLinkableAssets && shouldPackageAssetLibraries)) {
        Preconditions.checkState(
            !AndroidBinary.ExopackageMode.enabledForResources(exopackageModes));
        Path pathForNativeLibsAsAssets = getPathForNativeLibsAsAssets();

        final Path libSubdirectory =
            pathForNativeLibsAsAssets
                .resolve("assets")
                .resolve(module.isRootModule() ? "lib" : module.getName());
        ImmutableCollection<SourcePath> nativeLibDirs = nativeLibAssetsDirectories.get(module);

        getStepsForNativeAssets(
            context,
            steps,
            nativeLibDirs == null ? Optional.empty() : Optional.of(nativeLibDirs),
            libSubdirectory,
            module.isRootModule() ? "metadata.txt" : "libs.txt",
            module);

        nativeLibraryAsAssetDirectories.add(pathForNativeLibsAsAssets);
      }
    }

    // If non-english strings are to be stored as assets, pass them to ApkBuilder.
    ImmutableSet.Builder<Path> zipFiles = ImmutableSet.builder();
    RichStream.from(primaryApkAssetsZips)
        .map(context.getSourcePathResolver()::getRelativePath)
        .forEach(zipFiles::add);

    if (AndroidBinary.ExopackageMode.enabledForNativeLibraries(exopackageModes)) {
      // We need to include a few dummy native libraries with our application so that Android knows
      // to run it as 32-bit.  Android defaults to 64-bit when no libraries are provided at all,
      // causing us to fail to load our 32-bit exopackage native libraries later.
      String fakeNativeLibraryBundle = System.getProperty("buck.native_exopackage_fake_path");

      if (fakeNativeLibraryBundle == null) {
        throw new RuntimeException("fake native bundle not specified in properties");
      }

      zipFiles.add(Paths.get(fakeNativeLibraryBundle));
    }

    ImmutableSet<Path> allAssetDirectories =
        ImmutableSet.<Path>builder()
            .addAll(nativeLibraryAsAssetDirectories.build())
            .addAll(dexFilesInfo.secondaryDexDirs)
            .build();

    SourcePathResolver resolver = context.getSourcePathResolver();
    Path signedApkPath = getSignedApkPath();
    final Path pathToKeystore = resolver.getAbsolutePath(keystorePath);
    Supplier<KeystoreProperties> keystoreProperties =
        Suppliers.memoize(
            () -> {
              try {
                return KeystoreProperties.createFromPropertiesFile(
                    pathToKeystore,
                    resolver.getAbsolutePath(keystorePropertiesPath),
                    getProjectFilesystem());
              } catch (IOException e) {
                throw new RuntimeException();
              }
            });

    ImmutableSet<Path> thirdPartyJars =
        pathsToThirdPartyJars
            .stream()
            .map(resolver::getAbsolutePath)
            .collect(MoreCollectors.toImmutableSet());

    if (AndroidBinary.ExopackageMode.enabledForResources(exopackageModes)) {
      steps.add(createMergedThirdPartyJarsStep(thirdPartyJars));
      buildableContext.recordArtifact(getMergedThirdPartyJarsPath());
    }

    ApkBuilderStep apkBuilderCommand =
        new ApkBuilderStep(
            getProjectFilesystem(),
            context.getSourcePathResolver().getAbsolutePath(resourcesApkPath),
            getSignedApkPath(),
            dexFilesInfo.primaryDexPath,
            allAssetDirectories,
            nativeLibraryDirectoriesBuilder.build(),
            zipFiles.build(),
            thirdPartyJars,
            pathToKeystore,
            keystoreProperties,
            /* debugMode */ false,
            javaRuntimeLauncher);
    steps.add(apkBuilderCommand);

    // The `ApkBuilderStep` delegates to android tools to build a ZIP with timestamps in it, making
    // the output non-deterministic.  So use an additional scrubbing step to zero these out.
    steps.add(ZipScrubberStep.of(getProjectFilesystem().resolve(signedApkPath)));

    Path apkToRedexAndAlign;
    // Optionally, compress the resources file in the .apk.
    if (isCompressResources) {
      Path compressedApkPath = getCompressedResourcesApkPath();
      apkToRedexAndAlign = compressedApkPath;
      RepackZipEntriesStep arscComp =
          new RepackZipEntriesStep(
              getProjectFilesystem(),
              signedApkPath,
              compressedApkPath,
              ImmutableSet.of("resources.arsc"));
      steps.add(arscComp);
    } else {
      apkToRedexAndAlign = signedApkPath;
    }

    boolean applyRedex = redexOptions.isPresent();
    Path apkPath = getFinalApkPath();
    Path apkToAlign = apkToRedexAndAlign;

    // redex
    if (applyRedex) {
      Path proguardConfigDir = getProguardTextFilesPath();
      Path redexedApk = getRedexedApkPath();
      apkToAlign = redexedApk;
      steps.add(
          MkdirStep.of(
              BuildCellRelativePath.fromCellRelativePath(
                  context.getBuildCellRootPath(), getProjectFilesystem(), redexedApk.getParent())));
      ImmutableList<Step> redexSteps =
          ReDexStep.createSteps(
              getProjectFilesystem(),
              resolver,
              redexOptions.get(),
              apkToRedexAndAlign,
              redexedApk,
              keystoreProperties,
              proguardConfigDir,
              buildableContext);
      steps.addAll(redexSteps);
    }

    steps.add(new ZipalignStep(getProjectFilesystem().getRootPath(), apkToAlign, apkPath));

    buildableContext.recordArtifact(apkPath);
    return steps.build();
  }

  private Step createMergedThirdPartyJarsStep(ImmutableSet<Path> thirdPartyJars) {
    return new AbstractExecutionStep("merging_third_party_jar_resources") {
      @Override
      public StepExecutionResult execute(ExecutionContext context)
          throws IOException, InterruptedException {
        try (ResourcesZipBuilder builder =
            new ResourcesZipBuilder(
                getProjectFilesystem().resolve(getMergedThirdPartyJarsPath()))) {
          for (Path jar : thirdPartyJars) {
            try (ZipFile base = new ZipFile(jar.toFile())) {
              for (ZipEntry inputEntry : Collections.list(base.entries())) {
                if (inputEntry.isDirectory()) {
                  continue;
                }
                String name = inputEntry.getName();
                String ext = Files.getFileExtension(name);
                String filename = Paths.get(name).getFileName().toString();
                // Android's ApkBuilder filters out a lot of files from Java resources. Try to
                // match its behavior.
                // See https://android.googlesource.com/platform/sdk/+/jb-release/sdkmanager/libs/sdklib/src/com/android/sdklib/build/ApkBuilder.java
                if (name.startsWith(".")
                    || name.endsWith("~")
                    || name.startsWith("META-INF")
                    || "aidl".equalsIgnoreCase(ext)
                    || "rs".equalsIgnoreCase(ext)
                    || "rsh".equalsIgnoreCase(ext)
                    || "d".equalsIgnoreCase(ext)
                    || "java".equalsIgnoreCase(ext)
                    || "scala".equalsIgnoreCase(ext)
                    || "class".equalsIgnoreCase(ext)
                    || "scc".equalsIgnoreCase(ext)
                    || "swp".equalsIgnoreCase(ext)
                    || "thumbs.db".equalsIgnoreCase(filename)
                    || "picasa.ini".equalsIgnoreCase(filename)
                    || "package.html".equalsIgnoreCase(filename)
                    || "overview.html".equalsIgnoreCase(filename)) {
                  continue;
                }
                try (InputStream inputStream = base.getInputStream(inputEntry)) {
                  builder.addEntry(
                      inputStream,
                      inputEntry.getSize(),
                      inputEntry.getCrc(),
                      name,
                      Deflater.NO_COMPRESSION,
                      false);
                }
              }
            }
          }
        }
        return StepExecutionResult.SUCCESS;
      }
    };
  }

  private void getStepsForNativeAssets(
      BuildContext context,
      ImmutableList.Builder<Step> steps,
      Optional<ImmutableCollection<SourcePath>> nativeLibDirs,
      final Path libSubdirectory,
      final String metadataFilename,
      final APKModule module) {

    steps.addAll(
        MakeCleanDirectoryStep.of(
            BuildCellRelativePath.fromCellRelativePath(
                context.getBuildCellRootPath(), getProjectFilesystem(), libSubdirectory)));

    // Filter, rename and copy the ndk libraries marked as assets.
    if (nativeLibDirs.isPresent()) {
      for (SourcePath nativeLibDir : nativeLibDirs.get()) {
        CopyNativeLibraries.copyNativeLibrary(
            context,
            getProjectFilesystem(),
            context.getSourcePathResolver().getAbsolutePath(nativeLibDir),
            libSubdirectory,
            cpuFilters,
            steps);
      }
    }

    // Input asset libraries are sorted in descending filesize order.
    final ImmutableSortedSet.Builder<Path> inputAssetLibrariesBuilder =
        ImmutableSortedSet.orderedBy(
            (libPath1, libPath2) -> {
              try {
                ProjectFilesystem filesystem = getProjectFilesystem();
                int filesizeResult =
                    -Long.compare(
                        filesystem.getFileSize(libPath1), filesystem.getFileSize(libPath2));
                int pathnameResult = libPath1.compareTo(libPath2);
                return filesizeResult != 0 ? filesizeResult : pathnameResult;
              } catch (IOException e) {
                return 0;
              }
            });

    if (packageAssetLibraries || !module.isRootModule()) {
      // TODO(cjhopman): Why is this packaging native libs as assets even when native exopackage is
      // enabled?
      if (nativeLibsAssetsDirs.isPresent() && nativeLibsAssetsDirs.get().containsKey(module)) {
        // Copy in cxx libraries marked as assets. Filtering and renaming was already done
        // in CopyNativeLibraries.getBuildSteps().
        Path cxxNativeLibsSrc = nativeLibsAssetsDirs.get().get(module);
        steps.add(
            CopyStep.forDirectory(
                getProjectFilesystem(),
                cxxNativeLibsSrc,
                libSubdirectory,
                CopyStep.DirectoryMode.CONTENTS_ONLY));
      }

      steps.add(
          // Step that populates a list of libraries and writes a metadata.txt to decompress.
          new AbstractExecutionStep("write_metadata_for_asset_libraries_" + module.getName()) {
            @Override
            public StepExecutionResult execute(ExecutionContext context)
                throws IOException, InterruptedException {
              ProjectFilesystem filesystem = getProjectFilesystem();
              // Walk file tree to find libraries
              filesystem.walkRelativeFileTree(
                  libSubdirectory,
                  new SimpleFileVisitor<Path>() {
                    @Override
                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                        throws IOException {
                      if (!file.toString().endsWith(".so")) {
                        throw new IOException("unexpected file in lib directory");
                      }
{'fraction_non_alphanumeric': 0.08052962882570003, 'fraction_numerical': 0.0007958903118442949, 'mean_word_length': 3.161147072105976, 'pattern_counts': {'":': 0, '<': 151, '<?xml version=': 0, '>': 170, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6644422', 'n_tokens_mistral': 15329, 'n_tokens_neox': 14786, 'n_words': 2902}