<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Cci;
using Microsoft.Cci.Extensions;

namespace NotImplementedScanner
{
    internal static class Program
    {
        private static int Main(string[] args)
        {
            if (args.Length != 2)
            {
                var toolName = Path.GetFileNameWithoutExtension(Environment.GetCommandLineArgs()[0]);
                Console.Error.WriteLine($"Usage: {toolName} <directory-or-binary> <out-path>");
                return 1;
            }

            var inputPath = args[0];
            var outputPath = args[1];
            var isFile = File.Exists(inputPath);
            var isDirectory = Directory.Exists(inputPath);
            if (!isFile && !isDirectory)
            {
                Console.Error.WriteLine($"ERROR: '{inputPath}' must be a file or directory.");
                return 1;
            }

            try
            {
                Run(inputPath, outputPath);
                return 0;
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine($"ERROR: {ex.Message}");
                return 1;
            }
        }

        private static void Run(string inputPath, string outputPath)
        {
            using (var textWriter = new StreamWriter(outputPath))
            {
                textWriter.Write("DocId");
                textWriter.Write(",");
                textWriter.Write("Namespace");
                textWriter.Write(",");
                textWriter.Write("Type");
                textWriter.Write(",");
                textWriter.Write("Member");
                textWriter.Write(",");
                textWriter.Write("Nesting");
                textWriter.WriteLine();

                foreach (var assembly in LoadPaths(inputPath))
                    AnalyzeAssembly(textWriter, assembly);
            }
        }

        private static void AnalyzeAssembly(StreamWriter textWriter, IAssembly assembly)
        {
            foreach (var type in assembly.GetAllTypes())
                AnalyzeType(textWriter, type);
        }

        private static void AnalyzeType(StreamWriter textWriter, INamedTypeDefinition type)
        {
            if (!type.IsVisibleOutsideAssembly())
                return;

            foreach (var item in type.Members)
                AnalyzeMember(textWriter, item);
        }

        private static void AnalyzeMember(StreamWriter textWriter, ITypeDefinitionMember item)
        {
            if (!item.IsVisibleOutsideAssembly())
                return;

            var result = AnalyzePlatformNotSupported(item);

            if (result.Throws)
            {
                textWriter.WriteEscaped(item.DocId());
                textWriter.Write(",");
                textWriter.WriteEscaped(item.ContainingTypeDefinition.GetNamespaceName());
                textWriter.Write(",");
                textWriter.WriteEscaped(item.ContainingTypeDefinition.GetTypeName(false));
                textWriter.Write(",");
                textWriter.WriteEscaped(GetMemberSignature(item));
                textWriter.Write(",");
                textWriter.Write(result);
                textWriter.WriteLine();
            }
        }

        private static IEnumerable<IAssembly> LoadPaths(string input)
        {
            var inputPaths = HostEnvironment.SplitPaths(input);
            var filePaths = HostEnvironment.GetFilePaths(inputPaths, SearchOption.AllDirectories).ToArray();
            return HostEnvironment.LoadAssemblySet(filePaths);
        }

        private static ExceptionResult AnalyzePlatformNotSupported(ITypeDefinitionMember item)
        {
            if (item is IMethodDefinition m)
            {
                if (m.IsPropertyOrEventAccessor())
                    return ExceptionResult.DoesNotThrow;

                return AnalyzePlatformNotSupported(m);
            }
            else if (item is IPropertyDefinition p)
            {
                return AnalyzePlatformNotSupported(p.Accessors);
            }
            else if (item is IEventDefinition e)
            {
                return AnalyzePlatformNotSupported(e.Accessors);
            }
            else if (item is IFieldDefinition || item is ITypeDefinition)<commit_msg>Make sure we make the distinct<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Cci;
using Microsoft.Cci.Extensions;

namespace NotImplementedScanner
{
    internal static class Program
    {
        private static int Main(string[] args)
        {
            if (args.Length != 2)
            {
                var toolName = Path.GetFileNameWithoutExtension(Environment.GetCommandLineArgs()[0]);
                Console.Error.WriteLine($"Usage: {toolName} <directory-or-binary> <out-path>");
                return 1;
            }

            var inputPath = args[0];
            var outputPath = args[1];
            var isFile = File.Exists(inputPath);
            var isDirectory = Directory.Exists(inputPath);
            if (!isFile && !isDirectory)
            {
                Console.Error.WriteLine($"ERROR: '{inputPath}' must be a file or directory.");
                return 1;
            }

            try
            {
                Run(inputPath, outputPath);
                return 0;
            }
            catch (Exception ex)
            {
                Console.Error.WriteLine($"ERROR: {ex.Message}");
                return 1;
            }
        }

        private static void Run(string inputPath, string outputPath)
        {
            using (var textWriter = new StreamWriter(outputPath))
            {
                textWriter.Write("DocId");
                textWriter.Write(",");
                textWriter.Write("Namespace");
                textWriter.Write(",");
                textWriter.Write("Type");
                textWriter.Write(",");
                textWriter.Write("Member");
                textWriter.Write(",");
                textWriter.Write("Nesting");
                textWriter.WriteLine();

                foreach (var assembly in LoadPaths(inputPath))
                    AnalyzeAssembly(textWriter, assembly);
            }
        }

        private static void AnalyzeAssembly(StreamWriter textWriter, IAssembly assembly)
        {
            foreach (var type in assembly.GetAllTypes())
                AnalyzeType(textWriter, type);
        }

        private static void AnalyzeType(StreamWriter textWriter, INamedTypeDefinition type)
        {
            if (!type.IsVisibleOutsideAssembly())
                return;

            foreach (var item in type.Members)
                AnalyzeMember(textWriter, item);
        }

        private static void AnalyzeMember(StreamWriter textWriter, ITypeDefinitionMember item)
        {
            if (!item.IsVisibleOutsideAssembly())
                return;

            var result = AnalyzePlatformNotSupported(item);

            if (result.Throws)
            {
                textWriter.WriteEscaped(item.DocId());
                textWriter.Write(",");
                textWriter.WriteEscaped(item.ContainingTypeDefinition.GetNamespaceName());
                textWriter.Write(",");
                textWriter.WriteEscaped(item.ContainingTypeDefinition.GetTypeName(false));
                textWriter.Write(",");
                textWriter.WriteEscaped(GetMemberSignature(item));
                textWriter.Write(",");
                textWriter.Write(result);
                textWriter.WriteLine();
            }
        }

        private static IEnumerable<IAssembly> LoadPaths(string input)
        {
            var inputPaths = HostEnvironment.SplitPaths(input);
            var filePaths = HostEnvironment.GetFilePaths(inputPaths, SearchOption.AllDirectories).ToArray();
            return HostEnvironment.LoadAssemblySet(filePaths).Distinct();
        }

        private static ExceptionResult AnalyzePlatformNotSupported(ITypeDefinitionMember item)
        {
            if (item is IMethodDefinition m)
            {
                if (m.IsPropertyOrEventAccessor())
                    return ExceptionResult.DoesNotThrow;

                return AnalyzePlatformNotSupported(m);
            }
            else if (item is IPropertyDefinition p)
            {
                return AnalyzePlatformNotSupported(p.Accessors);
            }
            else if (item is IEventDefinition e)
            {
                return AnalyzePlatformNotSupported(e.Accessors);
            }
            else if (item is IFieldDefinition || item is ITypeDefinition)
{'fraction_non_alphanumeric': 0.08970824718584884, 'fraction_numerical': 0.001837813002526993, 'mean_word_length': 1.6700398650720638, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4184262', 'n_tokens_mistral': 2127, 'n_tokens_neox': 1989, 'n_words': 426}