<commit_before>// <copyright file="ProductModuleUtilities.cs" company="<NAME>">
//  Opus package
// </copyright>
// <summary>Publisher package</summary>
// <author><NAME></author>
namespace Publisher
{
    public static class ProductModuleUtilities
    {
        // TODO: out of all the dependents, how do we determine the metadata that they have associated with them
        // from the Publisher.ProductModule module, that is beyond just the need for graph building?
        private static Opus.Core.TypeArray
        GetModulesTypesWithAttribute(
            Publisher.ProductModule moduleToBuild,
            System.Type attributeType)
        {
            var moduleTypes = new Opus.Core.TypeArray();

            var flags = System.Reflection.BindingFlags.Instance |
                        System.Reflection.BindingFlags.NonPublic;
            var fields = moduleToBuild.GetType().GetFields(flags);
            foreach (var field in fields)
            {
                var attributes = field.GetCustomAttributes(true);
                if (attributes.Length != 1)
                {
                    throw new Opus.Core.Exception("Found {0} attributes on field {1} of module {2}. Should be just one",
                                                  attributes.Length, field.Name, moduleToBuild.OwningNode.ModuleName);
                }

                var currentAttr = attributes[0];
                if (currentAttr.GetType() == attributeType)
                {
                    var primaryTargetType = field.GetValue(moduleToBuild) as System.Type;
                    moduleTypes.AddUnique(primaryTargetType);
                }
            }

            return moduleTypes;
        }

        public static Opus.Core.DependencyNode GetPrimaryNode(Publisher.ProductModule moduleToBuild)
        {
            Opus.Core.DependencyNode primaryNode = null;

            var dependents = moduleToBuild.OwningNode.ExternalDependents;
            if (dependents.Count == 0)
            {
                return primaryNode;
            }

            var primaryTargetTypes = GetModulesTypesWithAttribute(moduleToBuild, typeof(Publisher.PrimaryTargetAttribute));
            if (primaryTargetTypes.Count == 0)
            {
                return primaryNode;
            }

            primaryNode = Opus.Core.ModuleUtilities.GetNode(primaryTargetTypes[0], (Opus.Core.BaseTarget)moduleToBuild.OwningNode.Target);
            return primaryNode;
        }

        public static string
        GetPublishedKeyName(
            Opus.Core.BaseModule primaryModule,
            Opus.Core.BaseModule module,
            Opus.Core.LocationKey key)
        {
            var builder = new System.Text.StringBuilder();
            builder.Append(module.OwningNode.ModuleName);
            builder.Append(".");
            builder.Append(key.ToString());
            builder.Append(".PublishedFile");
            if (primaryModule != module)
            {
                builder.Append(".For.");
                builder.Append(primaryModule.OwningNode.ModuleName);
            }
            return builder.ToString();
        }

        public static string
        GenerateDestinationPath(
            string sourcePath,
            string destinationDirectory,
            Opus.Core.BaseModule module,
            Opus.Core.LocationKey key)
        {
            var filename = System.IO.Path.GetFileName(sourcePath);
            var destPath = System.IO.Path.Combine(destinationDirectory, filename);
            module.Locations[key] = Opus.Core.FileLocation.Get(destPath, Opus.Core.Location.EExists.WillExist);
            return destPath;
        }

        public static void
        CopyFileToLocation(
            Opus.Core.Location sourceFile,
            string destinationDirectory,
            Opus.Core.BaseModule module,
            Opus.Core.LocationKey key)
        {
            var sourcePath = sourceFile.GetSingleRawPath();
            var destPath = GenerateDestinationPath(sourcePath, destinationDirectory, module, key);
            Opus.Core.Log.Info("Copying {0} to {1}", sourcePath, destPath);
            // TODO: this currently copies targets of symlinks
            System.IO.File.Copy(sourcePath, destPath, true);
        }
    }
}<commit_msg>[trunk] Running in Mono, checks file copies for symlinks, and only copies the symlink rather than the target<commit_after>// <copyright file="ProductModuleUtilities.cs" company="<NAME>">
//  Opus package
// </copyright>
// <summary>Publisher package</summary>
// <author><NAME></author>
namespace Publisher
{
    public static class ProductModuleUtilities
    {
        // TODO: out of all the dependents, how do we determine the metadata that they have associated with them
        // from the Publisher.ProductModule module, that is beyond just the need for graph building?
        private static Opus.Core.TypeArray
        GetModulesTypesWithAttribute(
            Publisher.ProductModule moduleToBuild,
            System.Type attributeType)
        {
            var moduleTypes = new Opus.Core.TypeArray();

            var flags = System.Reflection.BindingFlags.Instance |
                        System.Reflection.BindingFlags.NonPublic;
            var fields = moduleToBuild.GetType().GetFields(flags);
            foreach (var field in fields)
            {
                var attributes = field.GetCustomAttributes(true);
                if (attributes.Length != 1)
                {
                    throw new Opus.Core.Exception("Found {0} attributes on field {1} of module {2}. Should be just one",
                                                  attributes.Length, field.Name, moduleToBuild.OwningNode.ModuleName);
                }

                var currentAttr = attributes[0];
                if (currentAttr.GetType() == attributeType)
                {
                    var primaryTargetType = field.GetValue(moduleToBuild) as System.Type;
                    moduleTypes.AddUnique(primaryTargetType);
                }
            }

            return moduleTypes;
        }

        public static Opus.Core.DependencyNode GetPrimaryNode(Publisher.ProductModule moduleToBuild)
        {
            Opus.Core.DependencyNode primaryNode = null;

            var dependents = moduleToBuild.OwningNode.ExternalDependents;
            if (dependents.Count == 0)
            {
                return primaryNode;
            }

            var primaryTargetTypes = GetModulesTypesWithAttribute(moduleToBuild, typeof(Publisher.PrimaryTargetAttribute));
            if (primaryTargetTypes.Count == 0)
            {
                return primaryNode;
            }

            primaryNode = Opus.Core.ModuleUtilities.GetNode(primaryTargetTypes[0], (Opus.Core.BaseTarget)moduleToBuild.OwningNode.Target);
            return primaryNode;
        }

        public static string
        GetPublishedKeyName(
            Opus.Core.BaseModule primaryModule,
            Opus.Core.BaseModule module,
            Opus.Core.LocationKey key)
        {
            var builder = new System.Text.StringBuilder();
            builder.Append(module.OwningNode.ModuleName);
            builder.Append(".");
            builder.Append(key.ToString());
            builder.Append(".PublishedFile");
            if (primaryModule != module)
            {
                builder.Append(".For.");
                builder.Append(primaryModule.OwningNode.ModuleName);
            }
            return builder.ToString();
        }

        public static string
        GenerateDestinationPath(
            string sourcePath,
            string destinationDirectory,
            Opus.Core.BaseModule module,
            Opus.Core.LocationKey key)
        {
            var filename = System.IO.Path.GetFileName(sourcePath);
            var destPath = System.IO.Path.Combine(destinationDirectory, filename);
            module.Locations[key] = Opus.Core.FileLocation.Get(destPath, Opus.Core.Location.EExists.WillExist);
            return destPath;
        }

        public static void
        CopyFileToLocation(
            Opus.Core.Location sourceFile,
            string destinationDirectory,
            Opus.Core.BaseModule module,
            Opus.Core.LocationKey key)
        {
            var sourcePath = sourceFile.GetSingleRawPath();
            var destPath = GenerateDestinationPath(sourcePath, destinationDirectory, module, key);
            Opus.Core.Log.Info("Copying {0} to {1}", sourcePath, destPath);
#if __MonoCS__
            var buf = new Mono.Unix.Native.Stat();
            var statResult = Mono.Unix.Native.Syscall.lstat(sourcePath, out buf);
            if (0 != statResult)
            {
                throw new Opus.Core.Exception("Exception while stat'ing '{0}'", sourcePath);
            }
            var symLink = (buf.st_mode & Mono.Unix.Native.FilePermissions.S_IFLNK) == Mono.Unix.Native.FilePermissions.S_IFLNK;
            if (symLink)
            {
                Mono.Unix.Native.Syscall.symlink(sourcePath, destPath);
            }
            else
            {
                System.IO.File.Copy(sourcePath, destPath, true);
            }
#else
            System.IO.File.Copy(sourcePath, destPath, true);
#endif
        }
    }
}
{'fraction_non_alphanumeric': 0.08644506799050292, 'fraction_numerical': 0.0023742715303259228, 'mean_word_length': 1.834811869073111, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4229691', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2224, 'n_words': 496}