<commit_before>﻿using Serenity.CodeGeneration;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;

namespace Serenity.CodeGenerator
{
    public class TSTypeLister
    {
        private string projectDir;

        public TSTypeLister(string projectDir)
        {
            this.projectDir = projectDir;
        }

        private string GetEmbeddedScript(string name)
        {
            using (var sr = new StreamReader(this.GetType()
                .GetTypeInfo().Assembly.GetManifestResourceStream(name)))
            {
                return sr.ReadToEnd();
            }
        }

        public List<ExternalType> List()
        {
            var files = Directory.GetFiles(Path.Combine(projectDir, @"Modules"), "*.ts", SearchOption.AllDirectories)
                .Concat(Directory.GetFiles(Path.Combine(projectDir, @"Imports"), "*.ts", SearchOption.AllDirectories))
                .Concat(Directory.GetFiles(Path.Combine(projectDir, @"typings"), "*.ts", SearchOption.AllDirectories))
                .Where(x => !x.EndsWith(".d.ts") || x.IndexOf("Serenity") >= 0);

            var tempDirectory = Path.ChangeExtension(Path.GetTempFileName(), null) + "__";
            Directory.CreateDirectory(tempDirectory);
            try
            {
                File.WriteAllText(Path.Combine(tempDirectory, "typeScriptServices.js"), GetEmbeddedScript("Serenity.CodeGenerator.typeScriptServices.js") + "\n\nif (exports)\nexports.ts = ts;\n");
                File.WriteAllText(Path.Combine(tempDirectory, "Serenity.CodeGeneration.js"), GetEmbeddedScript("Serenity.CodeGenerator.Serenity.CodeGeneration.js") + "\n\nif (exports)\nexports.Serenity = Serenity;\n");

                var sb = new StringBuilder();
                sb.AppendLine("var fs = require('fs');");
                sb.AppendLine("global.ts = require('./typescriptServices.js');");
                sb.AppendLine("global.Serenity = require('./Serenity.CodeGeneration.js').Serenity;");
                sb.AppendLine("var inputFiles = " + JSON.StringifyIndented(files.Select(x => x.Replace('\\', '/'))) + ";");
                sb.AppendLine("for (var i = 0; i < inputFiles.length; i++) {");
                sb.AppendLine("  var sourceText = fs.readFileSync(inputFiles[i]);");
                sb.AppendLine("  Serenity.CodeGeneration.addSourceFile(inputFiles[i], sourceText.toString());");
                sb.AppendLine("}");
                sb.AppendLine("var types = JSON.stringify(Serenity.CodeGeneration.parseTypes(sourceText));");
                sb.AppendLine("fs.writeFileSync('./typeList.json', types);");
                File.WriteAllText(Path.Combine(tempDirectory, "index.js"), sb.ToString());

                var process = Process.Start(new ProcessStartInfo()
                {
                    FileName = "node",
                    Arguments = "index.js",
                    WorkingDirectory = tempDirectory,
                    CreateNoWindow = true
                });
                process.WaitForExit(60000);

                var json = File.ReadAllText(Path.Combine(tempDirectory, "typeList.json"));
                return JSON.Parse<List<ExternalType>>(json);
            }
            finally
            {
                Directory.Delete(tempDirectory, true);
            }
        }
    }
}<commit_msg>improve build / transform time for sergen significantly by employing cache by input files<commit_after>﻿using Serenity.CodeGeneration;
using Serenity.IO;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;

namespace Serenity.CodeGenerator
{
    public class TSTypeLister
    {
        private string projectDir;

        public TSTypeLister(string projectDir)
        {
            this.projectDir = projectDir;
        }

        private string GetEmbeddedScript(string name)
        {
            using (var sr = new StreamReader(this.GetType()
                .GetTypeInfo().Assembly.GetManifestResourceStream(name)))
            {
                return sr.ReadToEnd();
            }
        }

        public List<ExternalType> List()
        {
            var files = Directory.GetFiles(Path.Combine(projectDir, @"Modules"), "*.ts", SearchOption.AllDirectories)
                .Concat(Directory.GetFiles(Path.Combine(projectDir, @"Imports"), "*.ts", SearchOption.AllDirectories))
                .Concat(Directory.GetFiles(Path.Combine(projectDir, @"typings"), "*.ts", SearchOption.AllDirectories))
                .Where(x => !x.EndsWith(".d.ts") || x.IndexOf("Serenity") >= 0).OrderBy(x => x);

            var tsServices = GetEmbeddedScript("Serenity.CodeGenerator.typeScriptServices.js");
            var codeGeneration = GetEmbeddedScript("Serenity.CodeGenerator.Serenity.CodeGeneration.js");

            StringBuilder sb = new StringBuilder();
            sb.AppendLine("var fs = require('fs');");
            sb.AppendLine(tsServices);
            sb.AppendLine(codeGeneration);
            foreach (var file in files)
                sb.AppendLine("Serenity.CodeGeneration.addSourceFile(" +
                    file.Replace('\\', '/').ToJson() + ", " +
                    File.ReadAllText(file).ToJson() + ");");
            sb.AppendLine("var types = JSON.stringify(Serenity.CodeGeneration.parseTypes());");
            sb.AppendLine("fs.writeFileSync('./typeList.json', types);");

            var cacheDir = Path.Combine(Path.GetTempPath(), ".tstypecache");

            var md5 = MD5.Create();
            var hash = BitConverter.ToString(md5.ComputeHash(Encoding.Unicode.GetBytes(sb.ToString())));
            var cacheFile = Path.Combine(cacheDir, hash + ".json");

            if (File.Exists(cacheFile))
            {
                try
                {
                    return JSON.Parse<List<ExternalType>>(File.ReadAllText(cacheFile));
                }
                catch
                {
                }
            }

            Action<string> writeCache = (json) =>
            {
                try
                {
                    Directory.CreateDirectory(cacheDir);
                    TemporaryFileHelper.PurgeDirectory(cacheDir, TimeSpan.Zero, 9, null);
                    File.WriteAllText(cacheFile, json);
                }
                catch
                {
                }
            };

            var tempDirectory = Path.ChangeExtension(Path.GetTempFileName(), null) + "__";
            Directory.CreateDirectory(tempDirectory);
            try
            {
                sb.Insert(0, "var fs = require('fs');\n");
                sb.AppendLine("var types = JSON.stringify(Serenity.CodeGeneration.parseTypes());");
                sb.AppendLine("fs.writeFileSync('./typeList.json', types);");
                File.WriteAllText(Path.Combine(tempDirectory, "index.js"), sb.ToString());

                var process = Process.Start(new ProcessStartInfo()
                {
                    FileName = "node",
                    Arguments = "index.js",
                    WorkingDirectory = tempDirectory,
                    CreateNoWindow = true
                });
                process.WaitForExit(60000);
                var json = File.ReadAllText(Path.Combine(tempDirectory, "typeList.json"));
                writeCache(json);
                return JSON.Parse<List<ExternalType>>(json);
            }
            finally
            {
                Directory.Delete(tempDirectory, true);
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.12534345152427057, 'fraction_numerical': 0.0023550961664267956, 'mean_word_length': 1.9570599613152804, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3622653', 'n_tokens_mistral': 2076, 'n_tokens_neox': 1963, 'n_words': 320}