<commit_before>﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.SourceBrowser.Common;
using Newtonsoft.Json;

namespace Microsoft.SourceBrowser.HtmlGenerator
{
    public class TypeScriptSupport
    {
        private static readonly HashSet<string> alreadyProcessed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        private Dictionary<string, List<Reference>> references;
        private List<string> declarations;
        public Dictionary<string, List<Tuple<string, long>>> SymbolIDToListOfLocationsMap { get; private set; }
        public static readonly string ProjectDestinationFolder = Path.Combine(Paths.SolutionDestinationFolder, Constants.TypeScriptFiles);

        public void Generate(IEnumerable<string> typeScriptFiles)
        {
            if (typeScriptFiles == null || !typeScriptFiles.Any())
            {
                return;
            }

            declarations = new List<string>();
            references = new Dictionary<string, List<Reference>>(StringComparer.OrdinalIgnoreCase);
            SymbolIDToListOfLocationsMap = new Dictionary<string, List<Tuple<string, long>>>();

            var list = new List<string>();
            string libFile = null;

            foreach(var file in typeScriptFiles)
            {
                if (!alreadyProcessed.Contains(file))
                {
                    if (libFile == null && string.Equals(Path.GetFileName(file), "lib.d.ts", StringComparison.OrdinalIgnoreCase))
                    {
                        libFile = file;
                    }
                    else
                    {
                        list.Add(file);
                    }
                }
            }

            if (libFile == null)
            {
                libFile = Path.Combine(Common.Paths.BaseAppFolder, "TypeScriptSupport", "lib.d.ts");
            }

            try
            {
                GenerateCore(list, libFile);
            }
            catch (Exception ex)
            {
                Log.Exception(ex, "Error when generating TypeScript files");
                return;
            }

            ProjectGenerator.GenerateReferencesDataFilesToAssembly(
                Paths.SolutionDestinationFolder,
                Constants.TypeScriptFiles,
                references);

            declarations.Sort();

            Serialization.WriteDeclaredSymbols(
                ProjectDestinationFolder,
                declarations);

            ProjectGenerator.GenerateSymbolIDToListOfDeclarationLocationsMap(
                ProjectDestinationFolder,
                SymbolIDToListOfLocationsMap);
        }

        private void GenerateCore(IEnumerable<string> fileNames, string libFile)
        {
            var output = Path.Combine(Common.Paths.BaseAppFolder, "output");
            if (Directory.Exists(output))
            {
                Directory.Delete(output, recursive: true);
            }

            var json = JsonConvert.SerializeObject(new { fileNames, libFile });
            var argumentsJson = Path.Combine(Common.Paths.BaseAppFolder, "TypeScriptAnalyzerArguments.json");
            File.WriteAllText(argumentsJson, json);

            var analyzerJs = Path.Combine(Common.Paths.BaseAppFolder, @"TypeScriptSupport\analyzer.js");
            var arguments = string.Format("\"{0}\" {1}", analyzerJs, argumentsJson);

            ProcessLaunchService.ProcessRunResult result;
            try
            {
                result = new ProcessLaunchService().RunAndRedirectOutput("node", arguments);
            }
            catch (Win32Exception)
            {
                Log.Write("Warning: Node.js is required to generate TypeScript files. Skipping generation. Download Node.js from https://nodejs.org.", ConsoleColor.Yellow);
                Log.Exception("Node.js is not installed.");
                return;
            }

            foreach (var file in Directory.GetFiles(output))
            {
                if (Path.GetFileNameWithoutExtension(file) == "ok")
                {
                    continue;
                }

                if (Path.GetFileNameWithoutExtension(file) == "error")
                {
                    var errorContent = File.ReadAllText(file);
                    Log.Exception(DateTime.Now.ToString() + " " + errorContent);
                    return;
                }

                var text = File.ReadAllText(file);
                AnalyzedFile analysis = JsonConvert.DeserializeObject<AnalyzedFile>(text);

                EnsureFileGeneratedAndGetUrl(analysis);
            }
        }

        public void EnsureFileGeneratedAndGetUrl(AnalyzedFile analysis)
        {
            string localFileSystemPath = analysis.fileName;
            localFileSystemPath = Path.GetFullPath(localFileSystemPath);

            string destinationFilePath = GetDestinationFilePath(localFileSystemPath);

            if (!File.Exists(destinationFilePath))
            {
                Generate(localFileSystemPath, destinationFilePath, analysis.syntacticClassifications, analysis.semanticClassifications);
            }
        }

        public static string GetDestinationFilePath(string sourceFilePath)
        {
            var url = sourceFilePath + ".html";
            url = url.Replace(":", "");
            url = url.Replace(" ", "");
            url = url.Replace(@"\bin\", @"\bin_\");
            if (url.StartsWith(@"\\"))
            {
                url = url.Substring(2);
            }

            url = Constants.TypeScriptFiles + @"\" + url;

            url = Path.Combine(Paths.SolutionDestinationFolder, url);<commit_msg>Measure how long it takes to call Node.js for TypeScript generation.<commit_after>﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Text;
using Microsoft.SourceBrowser.Common;
using Newtonsoft.Json;

namespace Microsoft.SourceBrowser.HtmlGenerator
{
    public class TypeScriptSupport
    {
        private static readonly HashSet<string> alreadyProcessed = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
        private Dictionary<string, List<Reference>> references;
        private List<string> declarations;
        public Dictionary<string, List<Tuple<string, long>>> SymbolIDToListOfLocationsMap { get; private set; }
        public static readonly string ProjectDestinationFolder = Path.Combine(Paths.SolutionDestinationFolder, Constants.TypeScriptFiles);

        public void Generate(IEnumerable<string> typeScriptFiles)
        {
            if (typeScriptFiles == null || !typeScriptFiles.Any())
            {
                return;
            }

            declarations = new List<string>();
            references = new Dictionary<string, List<Reference>>(StringComparer.OrdinalIgnoreCase);
            SymbolIDToListOfLocationsMap = new Dictionary<string, List<Tuple<string, long>>>();

            var list = new List<string>();
            string libFile = null;

            foreach (var file in typeScriptFiles)
            {
                if (!alreadyProcessed.Contains(file))
                {
                    if (libFile == null && string.Equals(Path.GetFileName(file), "lib.d.ts", StringComparison.OrdinalIgnoreCase))
                    {
                        libFile = file;
                    }
                    else
                    {
                        list.Add(file);
                    }
                }
            }

            if (libFile == null)
            {
                libFile = Path.Combine(Common.Paths.BaseAppFolder, "TypeScriptSupport", "lib.d.ts");
            }

            try
            {
                GenerateCore(list, libFile);
            }
            catch (Exception ex)
            {
                Log.Exception(ex, "Error when generating TypeScript files");
                return;
            }

            ProjectGenerator.GenerateReferencesDataFilesToAssembly(
                Paths.SolutionDestinationFolder,
                Constants.TypeScriptFiles,
                references);

            declarations.Sort();

            Serialization.WriteDeclaredSymbols(
                ProjectDestinationFolder,
                declarations);

            ProjectGenerator.GenerateSymbolIDToListOfDeclarationLocationsMap(
                ProjectDestinationFolder,
                SymbolIDToListOfLocationsMap);
        }

        private void GenerateCore(IEnumerable<string> fileNames, string libFile)
        {
            var output = Path.Combine(Common.Paths.BaseAppFolder, "output");
            if (Directory.Exists(output))
            {
                Directory.Delete(output, recursive: true);
            }

            var json = JsonConvert.SerializeObject(new { fileNames, libFile });
            var argumentsJson = Path.Combine(Common.Paths.BaseAppFolder, "TypeScriptAnalyzerArguments.json");
            File.WriteAllText(argumentsJson, json);

            var analyzerJs = Path.Combine(Common.Paths.BaseAppFolder, @"TypeScriptSupport\analyzer.js");
            var arguments = string.Format("\"{0}\" {1}", analyzerJs, argumentsJson);

            ProcessLaunchService.ProcessRunResult result;
            try
            {
                using (Disposable.Timing("Calling Node.js to process TypeScript"))
                {
                    result = new ProcessLaunchService().RunAndRedirectOutput("node", arguments);
                }
            }
            catch (Win32Exception)
            {
                Log.Write("Warning: Node.js is required to generate TypeScript files. Skipping generation. Download Node.js from https://nodejs.org.", ConsoleColor.Yellow);
                Log.Exception("Node.js is not installed.");
                return;
            }

            using (Disposable.Timing("Generating TypeScript files"))
            {
                foreach (var file in Directory.GetFiles(output))
                {
                    if (Path.GetFileNameWithoutExtension(file) == "ok")
                    {
                        continue;
                    }

                    if (Path.GetFileNameWithoutExtension(file) == "error")
                    {
                        var errorContent = File.ReadAllText(file);
                        Log.Exception(DateTime.Now.ToString() + " " + errorContent);
                        return;
                    }

                    var text = File.ReadAllText(file);
                    AnalyzedFile analysis = JsonConvert.DeserializeObject<AnalyzedFile>(text);

                    EnsureFileGeneratedAndGetUrl(analysis);
                }
            }
        }

        public void EnsureFileGeneratedAndGetUrl(AnalyzedFile analysis)
        {
            string localFileSystemPath = analysis.fileName;
            localFileSystemPath = Path.GetFullPath(localFileSystemPath);

            string destinationFilePath = GetDestinationFilePath(localFileSystemPath);

            if (!File.Exists(destinationFilePath))
            {
                Generate(localFileSystemPath, destinationFilePath, analysis.syntacticClassifications, analysis.semanticClassifications);
            }
        }

        public static string GetDestinationFilePath(string sourceFilePath)
        {
            var url = sourceFilePath + ".html";
            url = url.Replace(":", "");
            url = url.Replace(" ", "");
            url = url.Replace(@"\bin\", @"\bin_\");
            if (url.StartsWith(@"\\"))
            {
                url = url.Substring(2);
            }

            url = Constants.TypeScriptFiles + @"\" + url;

            url = Path.Combine(Paths.SolutionDestinationFolder, url);
{'fraction_non_alphanumeric': 0.09033186589908568, 'fraction_numerical': 0.0008465966813410091, 'mean_word_length': 1.758113471865515, 'pattern_counts': {'":': 2, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1213702', 'n_tokens_mistral': 2816, 'n_tokens_neox': 2685, 'n_words': 538}