<commit_before>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using System.IO.Compression;

using Amazon.CloudFormation.Model;
using Amazon.Util;
using System.Text;

using ThirdParty.Json.LitJson;

namespace Amazon.Lambda.Tools
{
    /// <summary>
    /// This class will create the lambda zip package that can be upload to Lambda for deployment.
    /// </summary>
    public static class LambdaPackager
    {
        /// <summary>
        /// Execute the dotnet publish command and zip up the resulting publish folder.
        /// </summary>
        /// <param name="defaults"></param>
        /// <param name="logger"></param>
        /// <param name="workingDirectory"></param>
        /// <param name="projectLocation"></param>
        /// <param name="targetFramework"></param>
        /// <param name="configuration"></param>
        /// <param name="publishLocation"></param>
        /// <param name="zipArchivePath"></param>
        public static bool CreateApplicationBundle(LambdaToolsDefaults defaults, IToolLogger logger, string workingDirectory, string projectLocation, string configuration, string targetFramework,
            out string publishLocation, ref string zipArchivePath)
        {
            var cli = new DotNetCLIWrapper(logger, workingDirectory);

            publishLocation = Utilities.DeterminePublishLocation(workingDirectory, projectLocation, configuration, targetFramework);
            logger?.WriteLine("Executing publish command");
            if (cli.Publish(defaults, projectLocation, publishLocation, targetFramework, configuration) != 0)
                return false;

            var buildLocation = Utilities.DetermineBuildLocation(workingDirectory, projectLocation, configuration, targetFramework);

            // This is here for legacy reasons. Some older versions of the dotnet CLI were not 
            // copying the deps.json file into the publish folder.
            foreach(var file in Directory.GetFiles(buildLocation, "*.deps.json", SearchOption.TopDirectoryOnly))
            {
                var destinationPath = Path.Combine(publishLocation, Path.GetFileName(file));
                if(!File.Exists(destinationPath))
                    File.Copy(file, destinationPath);
            }

            bool flattenRuntime = FlattenRuntimeFolder(logger, publishLocation);

            if (zipArchivePath == null)
                zipArchivePath = Path.Combine(Directory.GetParent(publishLocation).FullName, new DirectoryInfo(workingDirectory).Name + ".zip");

            zipArchivePath = Path.GetFullPath(zipArchivePath);
            logger?.WriteLine($"Zipping publish folder {publishLocation} to {zipArchivePath}");
            if (File.Exists(zipArchivePath))
                File.Delete(zipArchivePath);

            var zipArchiveParentDirectory = Path.GetDirectoryName(zipArchivePath);
            if (!Directory.Exists(zipArchiveParentDirectory))
            {
                logger?.WriteLine($"Creating directory {zipArchiveParentDirectory}");
                new DirectoryInfo(zipArchiveParentDirectory).Create();
            }

            // Use the native zip utility if it exist which will maintain linux/osx file permissions
            var zipCLI = DotNetCLIWrapper.FindExecutableInPath("zip");
            if (!string.IsNullOrEmpty(zipCLI))
                BundleWithZipCLI(zipCLI, zipArchivePath, publishLocation, flattenRuntime, logger);
            else
                BundleWithDotNetCompression(zipArchivePath, publishLocation, flattenRuntime, logger);

            return true;
        }

        /// <summary>
        /// Process the runtime folder from the dotnet publish to flatten the platform specific dependencies to the
        /// root.
        /// </summary>
        /// <param name="logger"></param>
        /// <param name="publishLocation"></param>
        /// <returns>
        /// Returns true if flattening was successful. If the publishing folder changes in the future then flattening might fail. 
        /// In that case we want to publish the archive untouched so the tooling doesn't get in the way and let the user see if the  
        /// Lambda runtime has been updated to support the future changes. Warning messages will be written in case of failures.
        /// </returns>
        private static bool FlattenRuntimeFolder(IToolLogger logger, string publishLocation)
        {

            bool flattenAny = false;
            // Copy file function if the file hasn't already copied.
            var copyFileIfNotExist = new Action<string>(sourceRelativePath =>
            {
                var sourceFullPath = Path.Combine(publishLocation, sourceRelativePath);
                var targetFullPath = Path.Combine(publishLocation, Path.GetFileName(sourceFullPath));

                // Skip the copy if it has already been copied.
                if (File.Exists(targetFullPath))
                    return;

                // Only write the log message about flattening if we are actually going to flatten anything.
                if(!flattenAny)
                {
                    logger?.WriteLine("Flattening platform specific dependencies");
                    flattenAny = true;
                }

                logger?.WriteLine($"... flatten: {sourceRelativePath}");
                File.Copy(sourceFullPath, targetFullPath);
            });

            var depsJsonFilepath = Directory.GetFiles(publishLocation, "*.deps.json", SearchOption.TopDirectoryOnly).FirstOrDefault();
            if (!File.Exists(depsJsonFilepath))
            {
                logger?.WriteLine($"Missing deps.json file. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return false;
            }

            var depsRootData = JsonMapper.ToObject(File.ReadAllText(depsJsonFilepath));
            var runtimeTargetNode = depsRootData["runtimeTarget"];
            if (runtimeTargetNode == null)
            {
                logger?.WriteLine($"Missing runtimeTarget node. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return false;
            }

            string runtimeTarget;
            if (runtimeTargetNode.IsString)
            {
                runtimeTarget = runtimeTargetNode.ToString();
            }
            else
            {
                runtimeTarget = runtimeTargetNode["name"]?.ToString();
            }

            if (runtimeTarget == null)
            {
                logger?.WriteLine($"Missing runtimeTarget name. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return false;
            }

            var target = depsRootData["targets"]?[runtimeTarget];
            if (target == null)
            {
                logger?.WriteLine($"Missing targets node. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return false;
            }

            var runtimeHierarchy = CalculateRuntimeHierarchy();
            // Loop through all the valid runtimes in precedence order so we copy over the first match
            foreach (var runtime in runtimeHierarchy)
            {
                foreach (KeyValuePair<string, JsonData> dependencyNode in target)
                {
                    var depRuntimeTargets = dependencyNode.Value["runtimeTargets"];
                    if (depRuntimeTargets == null)
                        continue;

                    foreach (KeyValuePair<string, JsonData> depRuntimeTarget in depRuntimeTargets)
                    {
                        var rid = depRuntimeTarget.Value["rid"]?.ToString();

                        if(string.Equals(rid, runtime, StringComparison.Ordinal))
                        {
                            copyFileIfNotExist(depRuntimeTarget.Key);<commit_msg>Add validation to fail deployment if dependencies are bringing in later versions of .NET Core then the target framework.<commit_after>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using System.IO.Compression;

using Amazon.CloudFormation.Model;
using Amazon.Util;
using System.Text;

using ThirdParty.Json.LitJson;

namespace Amazon.Lambda.Tools
{
    /// <summary>
    /// This class will create the lambda zip package that can be upload to Lambda for deployment.
    /// </summary>
    public static class LambdaPackager
    {
        static IDictionary<string, Version> NETSTANDARD_LIBRARY_VERSIONS = new Dictionary<string, Version>
        {
            { "netcoreapp1.0", Version.Parse("1.6.0") },
            { "netcoreapp1.1", Version.Parse("1.6.1") }
        };

        /// <summary>
        /// Execute the dotnet publish command and zip up the resulting publish folder.
        /// </summary>
        /// <param name="defaults"></param>
        /// <param name="logger"></param>
        /// <param name="workingDirectory"></param>
        /// <param name="projectLocation"></param>
        /// <param name="targetFramework"></param>
        /// <param name="configuration"></param>
        /// <param name="publishLocation"></param>
        /// <param name="zipArchivePath"></param>
        public static bool CreateApplicationBundle(LambdaToolsDefaults defaults, IToolLogger logger, string workingDirectory, string projectLocation, string configuration, string targetFramework,
            out string publishLocation, ref string zipArchivePath)
        {
            var cli = new DotNetCLIWrapper(logger, workingDirectory);

            publishLocation = Utilities.DeterminePublishLocation(workingDirectory, projectLocation, configuration, targetFramework);
            logger?.WriteLine("Executing publish command");
            if (cli.Publish(defaults, projectLocation, publishLocation, targetFramework, configuration) != 0)
                return false;

            var buildLocation = Utilities.DetermineBuildLocation(workingDirectory, projectLocation, configuration, targetFramework);

            // This is here for legacy reasons. Some older versions of the dotnet CLI were not 
            // copying the deps.json file into the publish folder.
            foreach(var file in Directory.GetFiles(buildLocation, "*.deps.json", SearchOption.TopDirectoryOnly))
            {
                var destinationPath = Path.Combine(publishLocation, Path.GetFileName(file));
                if(!File.Exists(destinationPath))
                    File.Copy(file, destinationPath);
            }

            bool flattenRuntime = false;
            var depsJsonTargetNode = GetDepsJsonTargetNode(logger, publishLocation);
            // If there is no target node then this means the tool is being used on a future version of .NET Core
            // then was available when the this tool was written. Go ahead and continue the deployment with warnings so the
            // user can see if the future version will work.
            if (depsJsonTargetNode != null)
            {
                // Make sure the project is not pulling in dependencies requiring a later version of .NET Core then the declared target framework
                if (!ValidateDependencies(logger, targetFramework, depsJsonTargetNode))
                    return false;

                // Flatten the runtime folder which reduces the package size by not including native dependencies
                // for other platforms.
                flattenRuntime = FlattenRuntimeFolder(logger, publishLocation, depsJsonTargetNode);
            }

            if (zipArchivePath == null)
                zipArchivePath = Path.Combine(Directory.GetParent(publishLocation).FullName, new DirectoryInfo(workingDirectory).Name + ".zip");

            zipArchivePath = Path.GetFullPath(zipArchivePath);
            logger?.WriteLine($"Zipping publish folder {publishLocation} to {zipArchivePath}");
            if (File.Exists(zipArchivePath))
                File.Delete(zipArchivePath);

            var zipArchiveParentDirectory = Path.GetDirectoryName(zipArchivePath);
            if (!Directory.Exists(zipArchiveParentDirectory))
            {
                logger?.WriteLine($"Creating directory {zipArchiveParentDirectory}");
                new DirectoryInfo(zipArchiveParentDirectory).Create();
            }

            // Use the native zip utility if it exist which will maintain linux/osx file permissions
            var zipCLI = DotNetCLIWrapper.FindExecutableInPath("zip");
            if (!string.IsNullOrEmpty(zipCLI))
                BundleWithZipCLI(zipCLI, zipArchivePath, publishLocation, flattenRuntime, logger);
            else
                BundleWithDotNetCompression(zipArchivePath, publishLocation, flattenRuntime, logger);

            return true;
        }

        /// <summary>
        /// Return the targets node which declares all the dependencies for the project along with the dependency's dependencies.
        /// </summary>
        /// <param name="logger"></param>
        /// <param name="publishLocation"></param>
        /// <returns></returns>
        private static JsonData GetDepsJsonTargetNode(IToolLogger logger, string publishLocation)
        {
            var depsJsonFilepath = Directory.GetFiles(publishLocation, "*.deps.json", SearchOption.TopDirectoryOnly).FirstOrDefault();
            if (!File.Exists(depsJsonFilepath))
            {
                logger?.WriteLine($"Missing deps.json file. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return null;
            }

            var depsRootData = JsonMapper.ToObject(File.ReadAllText(depsJsonFilepath));
            var runtimeTargetNode = depsRootData["runtimeTarget"];
            if (runtimeTargetNode == null)
            {
                logger?.WriteLine($"Missing runtimeTarget node. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return null;
            }

            string runtimeTarget;
            if (runtimeTargetNode.IsString)
            {
                runtimeTarget = runtimeTargetNode.ToString();
            }
            else
            {
                runtimeTarget = runtimeTargetNode["name"]?.ToString();
            }

            if (runtimeTarget == null)
            {
                logger?.WriteLine($"Missing runtimeTarget name. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return null;
            }

            var target = depsRootData["targets"]?[runtimeTarget];
            if (target == null)
            {
                logger?.WriteLine($"Missing targets node. Skipping flattening runtime folder because {depsJsonFilepath} is an unrecognized format");
                return null;
            }

            return target;
        }

        /// <summary>
        /// Check to see if any of the dependencies listed in the deps.json file are pulling in later version of NETStandard.Library
        /// then the target framework supports.
        /// </summary>
        /// <param name="logger"></param>
        /// <param name="targetFramework"></param>
        /// <param name="depsJsonTargetNode"></param>
        /// <returns></returns>
        private static bool ValidateDependencies(IToolLogger logger, string targetFramework, JsonData depsJsonTargetNode)
        {
            Version maxNETStandardLibraryVersion;
            // If we don't know the NETStandard.Library NuGet package version then skip validation. This is to handle
            // the case we are packaging up for a future target framework verion then this version of the tooling knows about.
            // Skip validation so the tooling doesn't get in the way.
            if (!NETSTANDARD_LIBRARY_VERSIONS.TryGetValue(targetFramework, out maxNETStandardLibraryVersion))
                return true;

            var dependenciesUsingNETStandard = new List<string>();
            Version referencedNETStandardLibrary = null;

            foreach (KeyValuePair<string, JsonData> dependencyNode in depsJsonTargetNode)
            {
                var nameAndVersion = dependencyNode.Key.Split('/');
                if (nameAndVersion.Length != 2)
                    continue;

                if (string.Equals(nameAndVersion[0], "netstandard.library", StringComparison.OrdinalIgnoreCase))
                {
                    if(!Version.TryParse(nameAndVersion[1], out referencedNETStandardLibrary))
                    {
                        logger.WriteLine($"Error parsing version number for declared NETStandard.Library: {nameAndVersion[1]}");
                        return true;
                    }
                }
                // Collect the dependencies that are pulling in the NETStandard.Library metapackage
                else
                {
                    var subDependencies = dependencyNode.Value["dependencies"] as JsonData;
                    if (subDependencies != null)
                    {
                        foreach (KeyValuePair<string, JsonData> subDependency in subDependencies)
                        {
                            if (string.Equals(subDependency.Key, "netstandard.library", StringComparison.OrdinalIgnoreCase))
                            {
                                dependenciesUsingNETStandard.Add(nameAndVersion[0] + " : " + nameAndVersion[1]);
                                break;
                            }
                        }
                    }
                }
            }

            // If true the project is pulling in a new version of NETStandard.Library then the target framework supports.
            if(referencedNETStandardLibrary != null && maxNETStandardLibraryVersion < referencedNETStandardLibrary)
            {
                logger?.WriteLine($"Error: Project is referencing NETStandard.Library version {referencedNETStandardLibrary.ToString()}. Max version supported by {targetFramework} is {maxNETStandardLibraryVersion.ToString()}.");

                // See if we can find the target framework that does support the version the project is pulling in.
                // This can help the user know what framework their dependencies are targeting instead of understanding NuGet version numbers.
                var matchingTargetFramework = NETSTANDARD_LIBRARY_VERSIONS.FirstOrDefault(x =>
                {
                    return x.Value.Equals(referencedNETStandardLibrary);
                });

                if(!string.IsNullOrEmpty(matchingTargetFramework.Key))
                {
                    logger?.WriteLine($"Error: NETStandard.Library {referencedNETStandardLibrary.ToString()} is used for target framework {matchingTargetFramework.Key}.");
                }

                if (dependenciesUsingNETStandard.Count != 0)
                {
                    logger?.WriteLine($"Error: Check the following dependencies for versions compatible with {targetFramework}:");
                    foreach(var dependency in dependenciesUsingNETStandard)
                    {
                        logger?.WriteLine($"Error: \t{dependency}");
                    }
                }

                return false;
            }


            return true;
        }

        /// <summary>
        /// Process the runtime folder from the dotnet publish to flatten the platform specific dependencies to the
        /// root.
        /// </summary>
        /// <param name="logger"></param>
        /// <param name="publishLocation"></param>
        /// <param name="depsJsonTargetNode"></param>
        /// <returns>
        /// Returns true if flattening was successful. If the publishing folder changes in the future then flattening might fail. 
        /// In that case we want to publish the archive untouched so the tooling doesn't get in the way and let the user see if the  
        /// Lambda runtime has been updated to support the future changes. Warning messages will be written in case of failures.
        /// </returns>
        private static bool FlattenRuntimeFolder(IToolLogger logger, string publishLocation, JsonData depsJsonTargetNode)
        {

            bool flattenAny = false;
            // Copy file function if the file hasn't already copied.
            var copyFileIfNotExist = new Action<string>(sourceRelativePath =>
            {
                var sourceFullPath = Path.Combine(publishLocation, sourceRelativePath);
                var targetFullPath = Path.Combine(publishLocation, Path.GetFileName(sourceFullPath));

                // Skip the copy if it has already been copied.
                if (File.Exists(targetFullPath))
                    return;

                // Only write the log message about flattening if we are actually going to flatten anything.
                if(!flattenAny)
                {
                    logger?.WriteLine("Flattening platform specific dependencies");
                    flattenAny = true;
                }

                logger?.WriteLine($"... flatten: {sourceRelativePath}");
                File.Copy(sourceFullPath, targetFullPath);
            });

            var runtimeHierarchy = CalculateRuntimeHierarchy();
            // Loop through all the valid runtimes in precedence order so we copy over the first match
            foreach (var runtime in runtimeHierarchy)
            {
                foreach (KeyValuePair<string, JsonData> dependencyNode in depsJsonTargetNode)
                {
                    var depRuntimeTargets = dependencyNode.Value["runtimeTargets"];
                    if (depRuntimeTargets == null)
                        continue;

                    foreach (KeyValuePair<string, JsonData> depRuntimeTarget in depRuntimeTargets)
                    {
                        var rid = depRuntimeTarget.Value["rid"]?.ToString();

                        if(string.Equals(rid, runtime, StringComparison.Ordinal))
                        {
                            copyFileIfNotExist(depRuntimeTarget.Key);
{'fraction_non_alphanumeric': 0.08379738940615672, 'fraction_numerical': 0.001024635808794048, 'mean_word_length': 2.060812653395146, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4734312', 'n_tokens_mistral': 5222, 'n_tokens_neox': 4893, 'n_words': 1511}