<commit_before>﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Build.Engine
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Composition.Hosting;
    using System.Collections.Immutable;
    using System.Text;

    using Microsoft.DocAsCode.Build.Engine.Incrementals;
    using Microsoft.DocAsCode.Common;
    using Microsoft.DocAsCode.Exceptions;
    using Microsoft.DocAsCode.Plugins;
    using Microsoft.DocAsCode.Utility;

    using Newtonsoft.Json;

    public class SingleDocumentBuilder : IDisposable
    {
        private const string PhaseName = "Build Document";
        private const string XRefMapFileName = "xrefmap.yml";

        public CompositionHost Container { get; set; }
        public IEnumerable<IDocumentProcessor> Processors { get; set; }
        public IEnumerable<IInputMetadataValidator> MetadataValidators { get; set; }

        internal BuildInfo CurrentBuildInfo { get; set; }
        internal BuildInfo LastBuildInfo { get; set; }
        internal string IntermediateFolder { get; set; }

        private bool ShouldTraceIncrementalInfo => IntermediateFolder != null;
        private BuildVersionInfo _cbv;
        private BuildVersionInfo _lbv;
        private bool _canIncremental;

        public Manifest Build(DocumentBuildParameters parameters)
        {
            if (parameters == null)
            {
                throw new ArgumentNullException(nameof(parameters));
            }
            if (parameters.OutputBaseDir == null)
            {
                throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
            }
            if (parameters.Files == null)
            {
                throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
            }
            if (parameters.MaxParallelism <= 0)
            {
                parameters.MaxParallelism = Environment.ProcessorCount;
            }
            if (parameters.Metadata == null)
            {
                parameters.Metadata = ImmutableDictionary<string, object>.Empty;
            }

            return BuildCore(parameters);
        }

        private Manifest BuildCore(DocumentBuildParameters parameters)
        {
            using (new LoggerPhaseScope(PhaseName, true))
            {
                _canIncremental = false;
                _cbv = null;
                _lbv = null;
                Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}.");
                Directory.CreateDirectory(parameters.OutputBaseDir);
                var context = new DocumentBuildContext(
                    Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir),
                    parameters.Files.EnumerateFiles(),
                    parameters.ExternalReferencePackages,
                    parameters.XRefMaps,
                    parameters.MaxParallelism,
                    parameters.Files.DefaultBaseDir);
                if (ShouldTraceIncrementalInfo)
                {
                    string configHash = ComputeConfigHash(parameters);
                    var baseDir = Path.Combine(IntermediateFolder, CurrentBuildInfo.DirectoryName);
                    _lbv = LastBuildInfo?.Versions?.SingleOrDefault(v => v.VersionName == parameters.VersionName);
                    var fileAttributes = ComputeFileAttributes(parameters, _lbv?.Dependency);
                    _cbv = new BuildVersionInfo
                    {
                        VersionName = parameters.VersionName,
                        ConfigHash = configHash,
                        AttributesFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        DependencyFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        ManifestFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        XRefSpecMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        BuildMessageFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        Attributes = fileAttributes,
                        Dependency = context.DependencyGraph,
                    };
                    CurrentBuildInfo.Versions.Add(_cbv);
                    Logger.RegisterListener(_cbv.BuildMessage.GetListener());
                    _canIncremental = GetCanIncremental(configHash, parameters.VersionName);
                    if (_canIncremental && parameters.ForceRebuild)
                    {
                        _canIncremental = false;
                        Logger.LogVerbose($"Disable incremental build by force rebuild option.");
                    }
                    if (_canIncremental)
                    {
                        LoadChanges(parameters, context, fileAttributes);
                        Logger.LogVerbose($"Before expanding dependency before build, changes: {JsonUtility.Serialize(context.ChangeDict)}");
                        var dependencyGraph = _lbv.Dependency;
                        ExpandDependency(dependencyGraph, context, d => dependencyGraph.DependencyTypes[d.Type].TriggerBuild);
                        Logger.LogVerbose($"After expanding dependency before build, changes: {JsonUtility.Serialize(context.ChangeDict)}");
                    }
                }

                Logger.LogVerbose("Start building document...");

                // Start building document...
                List<HostService> hostServices = null;
                try
                {
                    using (var processor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor)
                    {
                        IMarkdownService markdownService;
                        using (new LoggerPhaseScope("CreateMarkdownService", true))
                        {
                            markdownService = CreateMarkdownService(parameters, processor.Tokens.ToImmutableDictionary());
                        }

                        using (new LoggerPhaseScope("Load", true))
                        {
                            hostServices = GetInnerContexts(parameters, Processors, processor, markdownService, context).ToList();
                        }<commit_msg>indent json for ops cannot handle too long json in a line (#788)<commit_after>﻿// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.DocAsCode.Build.Engine
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Linq;
    using System.Composition.Hosting;
    using System.Collections.Immutable;
    using System.Text;

    using Microsoft.DocAsCode.Build.Engine.Incrementals;
    using Microsoft.DocAsCode.Common;
    using Microsoft.DocAsCode.Exceptions;
    using Microsoft.DocAsCode.Plugins;
    using Microsoft.DocAsCode.Utility;

    using Newtonsoft.Json;

    public class SingleDocumentBuilder : IDisposable
    {
        private const string PhaseName = "Build Document";
        private const string XRefMapFileName = "xrefmap.yml";

        public CompositionHost Container { get; set; }
        public IEnumerable<IDocumentProcessor> Processors { get; set; }
        public IEnumerable<IInputMetadataValidator> MetadataValidators { get; set; }

        internal BuildInfo CurrentBuildInfo { get; set; }
        internal BuildInfo LastBuildInfo { get; set; }
        internal string IntermediateFolder { get; set; }

        private bool ShouldTraceIncrementalInfo => IntermediateFolder != null;
        private BuildVersionInfo _cbv;
        private BuildVersionInfo _lbv;
        private bool _canIncremental;

        public Manifest Build(DocumentBuildParameters parameters)
        {
            if (parameters == null)
            {
                throw new ArgumentNullException(nameof(parameters));
            }
            if (parameters.OutputBaseDir == null)
            {
                throw new ArgumentException("Output folder cannot be null.", nameof(parameters) + "." + nameof(parameters.OutputBaseDir));
            }
            if (parameters.Files == null)
            {
                throw new ArgumentException("Source files cannot be null.", nameof(parameters) + "." + nameof(parameters.Files));
            }
            if (parameters.MaxParallelism <= 0)
            {
                parameters.MaxParallelism = Environment.ProcessorCount;
            }
            if (parameters.Metadata == null)
            {
                parameters.Metadata = ImmutableDictionary<string, object>.Empty;
            }

            return BuildCore(parameters);
        }

        private Manifest BuildCore(DocumentBuildParameters parameters)
        {
            using (new LoggerPhaseScope(PhaseName, true))
            {
                _canIncremental = false;
                _cbv = null;
                _lbv = null;
                Logger.LogInfo($"Max parallelism is {parameters.MaxParallelism}.");
                Directory.CreateDirectory(parameters.OutputBaseDir);
                var context = new DocumentBuildContext(
                    Path.Combine(Directory.GetCurrentDirectory(), parameters.OutputBaseDir),
                    parameters.Files.EnumerateFiles(),
                    parameters.ExternalReferencePackages,
                    parameters.XRefMaps,
                    parameters.MaxParallelism,
                    parameters.Files.DefaultBaseDir);
                if (ShouldTraceIncrementalInfo)
                {
                    string configHash = ComputeConfigHash(parameters);
                    var baseDir = Path.Combine(IntermediateFolder, CurrentBuildInfo.DirectoryName);
                    _lbv = LastBuildInfo?.Versions?.SingleOrDefault(v => v.VersionName == parameters.VersionName);
                    var fileAttributes = ComputeFileAttributes(parameters, _lbv?.Dependency);
                    _cbv = new BuildVersionInfo
                    {
                        VersionName = parameters.VersionName,
                        ConfigHash = configHash,
                        AttributesFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        DependencyFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        ManifestFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        XRefSpecMapFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        BuildMessageFile = IncrementalUtility.CreateRandomFileName(baseDir),
                        Attributes = fileAttributes,
                        Dependency = context.DependencyGraph,
                    };
                    CurrentBuildInfo.Versions.Add(_cbv);
                    Logger.RegisterListener(_cbv.BuildMessage.GetListener());
                    _canIncremental = GetCanIncremental(configHash, parameters.VersionName);
                    if (_canIncremental && parameters.ForceRebuild)
                    {
                        _canIncremental = false;
                        Logger.LogVerbose($"Disable incremental build by force rebuild option.");
                    }
                    if (_canIncremental)
                    {
                        LoadChanges(parameters, context, fileAttributes);
                        Logger.LogVerbose($"Before expanding dependency before build, changes: {JsonUtility.Serialize(context.ChangeDict, Formatting.Indented)}");
                        var dependencyGraph = _lbv.Dependency;
                        ExpandDependency(dependencyGraph, context, d => dependencyGraph.DependencyTypes[d.Type].TriggerBuild);
                        Logger.LogVerbose($"After expanding dependency before build, changes: {JsonUtility.Serialize(context.ChangeDict, Formatting.Indented)}");
                    }
                }

                Logger.LogVerbose("Start building document...");

                // Start building document...
                List<HostService> hostServices = null;
                try
                {
                    using (var processor = parameters.TemplateManager?.GetTemplateProcessor(context, parameters.MaxParallelism) ?? TemplateProcessor.DefaultProcessor)
                    {
                        IMarkdownService markdownService;
                        using (new LoggerPhaseScope("CreateMarkdownService", true))
                        {
                            markdownService = CreateMarkdownService(parameters, processor.Tokens.ToImmutableDictionary());
                        }

                        using (new LoggerPhaseScope("Load", true))
                        {
                            hostServices = GetInnerContexts(parameters, Processors, processor, markdownService, context).ToList();
                        }
{'fraction_non_alphanumeric': 0.07413309052280143, 'fraction_numerical': 0.00037939145610440855, 'mean_word_length': 1.8265065408535277, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6986054', 'n_tokens_mistral': 2974, 'n_tokens_neox': 2781, 'n_words': 625}