<commit_before>﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Build.ManagedReference
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Immutable;
    using System.Composition;
    using System.Linq;
    using System.IO;
    using System.Runtime.Serialization.Formatters.Binary;
    using System.Text;

    using Microsoft.DocAsCode.Build.Common;
    using Microsoft.DocAsCode.Build.ManagedReference.BuildOutputs;
    using Microsoft.DocAsCode.Common;
    using Microsoft.DocAsCode.DataContracts.Common;
    using Microsoft.DocAsCode.DataContracts.ManagedReference;
    using Microsoft.DocAsCode.Plugins;

    using Newtonsoft.Json;

    [Export(typeof(IDocumentProcessor))]
    public class ManagedReferenceDocumentProcessor
        : ReferenceDocumentProcessorBase, ISupportIncrementalDocumentProcessor
    {
        #region Fields
        private readonly ResourcePoolManager<JsonSerializer> _serializerPool;
        private static readonly string[] SystemKeys = {
            "uid",
            "isEii",
            "isExtensionMethod",
            "parent",
            "children",
            "href",
            "langs",
            "name",
            "nameWithType",
            "fullName",
            "type",
            "source",
            "documentation",
            "assemblies",
            "namespace",
            "summary",
            "remarks",
            "example",
            "syntax",
            "overridden",
            "overload",
            "exceptions",
            "seealso",
            "see",
            "inheritance",
            "derivedClasses",
            "level",
            "implements",
            "inheritedMembers",
            "extensionMethods",
            "conceptual",
            "platform",
            "attributes"
        };
        #endregion

        #region Constructors

        public ManagedReferenceDocumentProcessor()
        {
            _serializerPool = new ResourcePoolManager<JsonSerializer>(GetSerializer, 0x10);
        }

        #endregion

        #region ReferenceDocumentProcessorBase Members

        protected override string ProcessedDocumentType { get; } = "ManagedReference";

        protected override FileModel LoadArticle(FileAndType file, ImmutableDictionary<string, object> metadata)
        {
            var page = YamlUtility.Deserialize<PageViewModel>(file.File);
            if (page.Items == null || page.Items.Count == 0)
            {
                return null;
            }
            if (page.Metadata == null)
            {
                page.Metadata = metadata.ToDictionary(p => p.Key, p => p.Value);
            }
            else
            {
                foreach (var item in metadata)
                {
                    if (!page.Metadata.ContainsKey(item.Key))
                    {
                        page.Metadata[item.Key] = item.Value;
                    }
                }
            }
            page.Metadata[Constants.PropertyName.SystemKeys] = SystemKeys;

            var localPathFromRoot = PathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, EnvironmentContext.FileAbstractLayer.GetPhysicalPath(file.File));

            return new FileModel(file, page, serializer: new BinaryFormatter())
            {
                Uids = (from item in page.Items select new UidDefinition(item.Uid, localPathFromRoot)).ToImmutableArray(),
                LocalPathFromRoot = localPathFromRoot
            };
        }

        #endregion

        #region IDocumentProcessor Members

        [ImportMany(nameof(ManagedReferenceDocumentProcessor))]
        public override IEnumerable<IDocumentBuildStep> BuildSteps { get; set; }
<commit_msg>Fix #1807 that overload uids get lost (#1814)<commit_after>﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Build.ManagedReference
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Immutable;
    using System.Composition;
    using System.Linq;
    using System.IO;
    using System.Runtime.Serialization.Formatters.Binary;
    using System.Text;

    using Microsoft.DocAsCode.Build.Common;
    using Microsoft.DocAsCode.Build.ManagedReference.BuildOutputs;
    using Microsoft.DocAsCode.Common;
    using Microsoft.DocAsCode.DataContracts.Common;
    using Microsoft.DocAsCode.DataContracts.ManagedReference;
    using Microsoft.DocAsCode.Plugins;

    using Newtonsoft.Json;

    [Export(typeof(IDocumentProcessor))]
    public class ManagedReferenceDocumentProcessor
        : ReferenceDocumentProcessorBase, ISupportIncrementalDocumentProcessor
    {
        #region Fields
        private readonly ResourcePoolManager<JsonSerializer> _serializerPool;
        private static readonly string[] SystemKeys = {
            "uid",
            "isEii",
            "isExtensionMethod",
            "parent",
            "children",
            "href",
            "langs",
            "name",
            "nameWithType",
            "fullName",
            "type",
            "source",
            "documentation",
            "assemblies",
            "namespace",
            "summary",
            "remarks",
            "example",
            "syntax",
            "overridden",
            "overload",
            "exceptions",
            "seealso",
            "see",
            "inheritance",
            "derivedClasses",
            "level",
            "implements",
            "inheritedMembers",
            "extensionMethods",
            "conceptual",
            "platform",
            "attributes"
        };
        #endregion

        #region Constructors

        public ManagedReferenceDocumentProcessor()
        {
            _serializerPool = new ResourcePoolManager<JsonSerializer>(GetSerializer, 0x10);
        }

        #endregion

        #region ReferenceDocumentProcessorBase Members

        protected override string ProcessedDocumentType { get; } = "ManagedReference";

        protected override FileModel LoadArticle(FileAndType file, ImmutableDictionary<string, object> metadata)
        {
            var page = YamlUtility.Deserialize<PageViewModel>(file.File);
            if (page.Items == null || page.Items.Count == 0)
            {
                return null;
            }
            if (page.Metadata == null)
            {
                page.Metadata = metadata.ToDictionary(p => p.Key, p => p.Value);
            }
            else
            {
                foreach (var item in metadata)
                {
                    if (!page.Metadata.ContainsKey(item.Key))
                    {
                        page.Metadata[item.Key] = item.Value;
                    }
                }
            }
            page.Metadata[Constants.PropertyName.SystemKeys] = SystemKeys;

            var localPathFromRoot = PathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, EnvironmentContext.FileAbstractLayer.GetPhysicalPath(file.File));

            return new FileModel(file, page, serializer: new BinaryFormatter())
            {
                Uids = (from item in page.Items select item.Uid)
                .Concat(from item in page.Items where item.Overload != null select item.Overload)
                .Distinct().Select(s => new UidDefinition(s, localPathFromRoot)).ToImmutableArray(),
                LocalPathFromRoot = localPathFromRoot
            };
        }

        #endregion

        #region IDocumentProcessor Members

        [ImportMany(nameof(ManagedReferenceDocumentProcessor))]
        public override IEnumerable<IDocumentBuildStep> BuildSteps { get; set; }

{'fraction_non_alphanumeric': 0.08653724146681893, 'fraction_numerical': 0.0020301992132978047, 'mean_word_length': 1.886122299523984, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6964758', 'n_tokens_mistral': 1937, 'n_tokens_neox': 1869, 'n_words': 408}