<commit_before>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Hnx8.ReadJEnc;
using ICSharpCode.Decompiler;
using ICSharpCode.Decompiler.Ast;
using ICSharpCode.ILSpy.XmlDoc;
using ICSharpCode.NRefactory.CSharp;
using Microsoft.Owin;
using Mono.Cecil;
using NuGetCalcWeb.RazorSupport;
using NuGetCalcWeb.ViewModels.FilePreview;
using RazorEngine.Templating;

namespace NuGetCalcWeb
{
    public sealed class FilePreviewGenerator
    {
        private const string HighlightCss = @"<link rel=""stylesheet"" href=""https://cdnjs.cloudflare.com/ajax/libs/highlight.js/8.4/styles/vs.min.css"">";

        public static string GetHtmlFilePath(string input)
        {
            var splitedFileName = input.Split(Path.DirectorySeparatorChar);
            return Path.Combine("App_Data", "html", Path.Combine(
                splitedFileName.Skip(Array.LastIndexOf(splitedFileName, "packages") + 1).ToArray()));
        }

        private static ConcurrentDictionary<string, Task> tasks = new ConcurrentDictionary<string, Task>(StringComparer.OrdinalIgnoreCase);

        public FilePreviewGenerator(FileInfo input)
        {
            this.input = input;
            this.htmlFile = new FileInfo(GetHtmlFilePath(input.FullName));
        }

        private readonly FileInfo input;
        private FileInfo htmlFile;
        private IOwinContext owinContext;
        private HeaderModel header;

        public bool NeedsGenerate
        {
            get
            {
                return tasks.ContainsKey(this.input.FullName) || !this.htmlFile.Exists;
            }
        }

        public async Task GenerateHtml(IOwinContext owinContext, HeaderModel header)
        {
            this.owinContext = owinContext;
            this.header = header;
            var fullName = this.input.FullName;

            try
            {
                await tasks.GetOrAdd(fullName, _ => Task.Run(async () =>
                {
                    if (this.htmlFile.Exists) return;

                    Debug.WriteLine("Generating " + this.input.Name);
                    Directory.CreateDirectory(this.htmlFile.DirectoryName);
                    try
                    {
                        var ext = this.input.Extension;
                        await (ext == ".dll" || ext == ".exe"
                            ? this.GenerateFromAssemblyFile()
                            : this.GenerateFromFile()
                        ).ConfigureAwait(false);
                    }
                    catch
                    {
                        try
                        {
                            this.htmlFile.Delete();
                        }
                        catch { }
                        throw;
                    }
                })).ConfigureAwait(false);
            }
            finally
            {
                Task tmp;
                tasks.TryRemove(fullName, out tmp);
            }
        }

        private async Task GenerateFromAssemblyFile()
        {
            ModuleDefinition module = null;
            try
            {
                module = ModuleDefinition.ReadModule(this.input.FullName, new ReaderParameters()
                {
                    ReadingMode = ReadingMode.Immediate,
                    AssemblyResolver = new MyAssemblyResolver()
                });
            }
            catch (BadImageFormatException) { }

            if (module == null)
            {
                await this.GenerateFromFile().ConfigureAwait(false);
                return;
            }

            var model = new AssemblyModel()
            {
                AssemblyName = module.Assembly.Name.Name,
                AssemblyDescription = await HighlightCs(GenerateAssemblyDescription(module)).ConfigureAwait(false)
            };

            var types = new List<TypeDefinition>();
            model.Namespaces = module.Types
                .Where(t => t.IsPublic)
                .GroupBy(t => t.Namespace)
                .OrderBy(g => g.Key)
                .Select(g => new NamespaceModel()
                {
                    Name = g.Key,
                    Types = g.OrderBy(t => t.Name).Select(t => CreateTypeModel(t, types)).ToArray()
                })
                .ToArray();

            var tasks = new Task<TypeDescription>[types.Count];
            using (var genSemaphore = new SemaphoreSlim(4))
            {
                for (var i = 0; i < types.Count; i++)
                {
                    var type = types[i];
                    tasks[i] = Task.Run(async () =>
                    {
                        await genSemaphore.WaitAsync().ConfigureAwait(false);
                        try
                        {
                            Debug.WriteLine(type.FullName);
                            return new TypeDescription()
                            {
                                FullName = type.FullName,
                                Code = await HighlightCs(GenerateTypeDescription(module, type)).ConfigureAwait(false)
                            };
                        }
                        finally
                        {
                            genSemaphore.Release();
                        }
                    });
                }

                model.TypeDescriptions = await Task.WhenAll(tasks).ConfigureAwait(false);
            }

            this.RunTemplate("Assembly", model);
        }

        private static TypeModel CreateTypeModel(TypeDefinition type, List<TypeDefinition> types)
        {
            types.Add(type);
            return new TypeModel()
            {
                Name = type.Name,
                FullName = type.FullName,
                HtmlClass = type.IsEnum ? "enum"
                    : type.IsValueType ? "struct"
                    : type.IsInterface ? "interface"
                    : type.IsDelegate() ? "delegate"
                    : "class",
                NestedTypes = type.NestedTypes
                    .Where(t => t.IsPublic || t.IsNestedPublic)
                    .OrderBy(t => t.Name)
                    .Select(t => CreateTypeModel(t, types))
                    .ToArray()
            };
        }

        private static string GenerateAssemblyDescription(ModuleDefinition module)
        {
            var context = new DecompilerContext(module);
            var astBuilder = new AstBuilder(context) { DecompileMethodBodies = false };
            astBuilder.AddAssembly(module, true);
            var output = new PlainTextOutput();
            astBuilder.GenerateCode(output);
            return output.ToString();
        }

        private static string GenerateTypeDescription(ModuleDefinition module, TypeDefinition type)
        {
            var context = new DecompilerContext(module);
            var astBuilder = new AstBuilder(context) { DecompileMethodBodies = false };
            astBuilder.AddType(type);

            if (!(type.IsEnum || type.IsDelegate()))
            {
                var typeNode = astBuilder.SyntaxTree.GetTypes(false).First();

                // Remove nested types
                foreach (var node in typeNode.Children)
                    if (node is TypeDeclaration)
                        node.Remove();
<commit_msg>Fix #1: members of interfaces<commit_after>﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Hnx8.ReadJEnc;
using ICSharpCode.Decompiler;
using ICSharpCode.Decompiler.Ast;
using ICSharpCode.ILSpy.XmlDoc;
using ICSharpCode.NRefactory.CSharp;
using Microsoft.Owin;
using Mono.Cecil;
using NuGetCalcWeb.RazorSupport;
using NuGetCalcWeb.ViewModels.FilePreview;
using RazorEngine.Templating;

namespace NuGetCalcWeb
{
    public sealed class FilePreviewGenerator
    {
        private const string HighlightCss = @"<link rel=""stylesheet"" href=""https://cdnjs.cloudflare.com/ajax/libs/highlight.js/8.4/styles/vs.min.css"">";

        public static string GetHtmlFilePath(string input)
        {
            var splitedFileName = input.Split(Path.DirectorySeparatorChar);
            return Path.Combine("App_Data", "html", Path.Combine(
                splitedFileName.Skip(Array.LastIndexOf(splitedFileName, "packages") + 1).ToArray()));
        }

        private static ConcurrentDictionary<string, Task> tasks = new ConcurrentDictionary<string, Task>(StringComparer.OrdinalIgnoreCase);

        public FilePreviewGenerator(FileInfo input)
        {
            this.input = input;
            this.htmlFile = new FileInfo(GetHtmlFilePath(input.FullName));
        }

        private readonly FileInfo input;
        private FileInfo htmlFile;
        private IOwinContext owinContext;
        private HeaderModel header;

        public bool NeedsGenerate
        {
            get
            {
                return tasks.ContainsKey(this.input.FullName) || !this.htmlFile.Exists;
            }
        }

        public async Task GenerateHtml(IOwinContext owinContext, HeaderModel header)
        {
            this.owinContext = owinContext;
            this.header = header;
            var fullName = this.input.FullName;

            try
            {
                await tasks.GetOrAdd(fullName, _ => Task.Run(async () =>
                {
                    if (this.htmlFile.Exists) return;

                    Debug.WriteLine("Generating " + this.input.Name);
                    Directory.CreateDirectory(this.htmlFile.DirectoryName);
                    try
                    {
                        var ext = this.input.Extension;
                        await (ext == ".dll" || ext == ".exe"
                            ? this.GenerateFromAssemblyFile()
                            : this.GenerateFromFile()
                        ).ConfigureAwait(false);
                    }
                    catch
                    {
                        try
                        {
                            this.htmlFile.Delete();
                        }
                        catch { }
                        throw;
                    }
                })).ConfigureAwait(false);
            }
            finally
            {
                Task tmp;
                tasks.TryRemove(fullName, out tmp);
            }
        }

        private async Task GenerateFromAssemblyFile()
        {
            ModuleDefinition module = null;
            try
            {
                module = ModuleDefinition.ReadModule(this.input.FullName, new ReaderParameters()
                {
                    ReadingMode = ReadingMode.Immediate,
                    AssemblyResolver = new MyAssemblyResolver()
                });
            }
            catch (BadImageFormatException) { }

            if (module == null)
            {
                await this.GenerateFromFile().ConfigureAwait(false);
                return;
            }

            var model = new AssemblyModel()
            {
                AssemblyName = module.Assembly.Name.Name,
                AssemblyDescription = await HighlightCs(GenerateAssemblyDescription(module)).ConfigureAwait(false)
            };

            var types = new List<TypeDefinition>();
            model.Namespaces = module.Types
                .Where(t => t.IsPublic)
                .GroupBy(t => t.Namespace)
                .OrderBy(g => g.Key)
                .Select(g => new NamespaceModel()
                {
                    Name = g.Key,
                    Types = g.OrderBy(t => t.Name).Select(t => CreateTypeModel(t, types)).ToArray()
                })
                .ToArray();

            var tasks = new Task<TypeDescription>[types.Count];
            using (var genSemaphore = new SemaphoreSlim(4))
            {
                for (var i = 0; i < types.Count; i++)
                {
                    var type = types[i];
                    tasks[i] = Task.Run(async () =>
                    {
                        await genSemaphore.WaitAsync().ConfigureAwait(false);
                        try
                        {
                            Debug.WriteLine(type.FullName);
                            return new TypeDescription()
                            {
                                FullName = type.FullName,
                                Code = await HighlightCs(GenerateTypeDescription(module, type)).ConfigureAwait(false)
                            };
                        }
                        finally
                        {
                            genSemaphore.Release();
                        }
                    });
                }

                model.TypeDescriptions = await Task.WhenAll(tasks).ConfigureAwait(false);
            }

            this.RunTemplate("Assembly", model);
        }

        private static TypeModel CreateTypeModel(TypeDefinition type, List<TypeDefinition> types)
        {
            types.Add(type);
            return new TypeModel()
            {
                Name = type.Name,
                FullName = type.FullName,
                HtmlClass = type.IsEnum ? "enum"
                    : type.IsValueType ? "struct"
                    : type.IsInterface ? "interface"
                    : type.IsDelegate() ? "delegate"
                    : "class",
                NestedTypes = type.NestedTypes
                    .Where(t => t.IsPublic || t.IsNestedPublic)
                    .OrderBy(t => t.Name)
                    .Select(t => CreateTypeModel(t, types))
                    .ToArray()
            };
        }

        private static string GenerateAssemblyDescription(ModuleDefinition module)
        {
            var context = new DecompilerContext(module);
            var astBuilder = new AstBuilder(context) { DecompileMethodBodies = false };
            astBuilder.AddAssembly(module, true);
            var output = new PlainTextOutput();
            astBuilder.GenerateCode(output);
            return output.ToString();
        }

        private static string GenerateTypeDescription(ModuleDefinition module, TypeDefinition type)
        {
            var context = new DecompilerContext(module);
            var astBuilder = new AstBuilder(context) { DecompileMethodBodies = false };
            astBuilder.AddType(type);

            if (!(type.IsEnum || type.IsInterface || type.IsDelegate()))
            {
                var typeNode = astBuilder.SyntaxTree.GetTypes(false).First();

                // Remove nested types
                foreach (var node in typeNode.Children)
                    if (node is TypeDeclaration)
                        node.Remove();

{'fraction_non_alphanumeric': 0.08722720578390682, 'fraction_numerical': 0.000870263756861695, 'mean_word_length': 1.3422703041705863, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 39, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7373663', 'n_tokens_mistral': 3649, 'n_tokens_neox': 3433, 'n_words': 695}