<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Build.Utilities;
using Microsoft.Build.Framework;
using System.Runtime.Serialization;
using System.IO;
using System.Text.RegularExpressions;
using System.Reflection;
using System.Diagnostics;
using Microsoft.Win32;
using System.Threading;
using System.Globalization;
using System.Threading.Tasks;

namespace ClangCompile
{
    public static class StringSplitter
    {
        public static IEnumerable<string> Split(this string str, Func<char, bool> controller)
        {
            int nextPiece = 0;
            for (int c = 0; c < str.Length; c++)
            {
                if (controller(str[c]))
                {
                    yield return str.Substring(nextPiece, c - nextPiece);
                    nextPiece = c + 1;
                }
            }

            yield return str.Substring(nextPiece);
        }

        public static IEnumerable<string> SplitCommandLine(string commandLine)
        {
            bool inQuotes = false;
            return commandLine.Split(c =>
            {
                if (c == '\"')
                    inQuotes = !inQuotes;

                return !inQuotes && c == ' ';
            })
                                .Select(arg => arg.Trim())
                                .Where(arg => !string.IsNullOrEmpty(arg));
        }
    }
    
    [Rule(Name="Clang", PageTemplate="tool", DisplayName="Clang", Order="200")]
    [DataSource(Persistence="ProjectFile", ItemType="ClangCompile")]
    [PropertyCategory(Name="General", DisplayName="General", Order = 0)]
    [PropertyCategory(Name = "Paths", DisplayName = "Paths", Order = 1)]
    [PropertyCategory(Name = "Language", DisplayName = "Language", Order = 2)]
    [PropertyCategory(Name = "Preprocessor", DisplayName = "Preprocessor", Order = 3)]
    [PropertyCategory(Name = "All Options", DisplayName = "All Options", Subtype = "Search", Order = 4)]
    [PropertyCategory(Name = "Command Line", DisplayName = "Command Line", Subtype = "CommandLine", Order = 5)]
    [FileExtension(Name="*.c", ContentType="ClangCpp")]
    [FileExtension(Name="*.cpp", ContentType="ClangCpp")]
    [FileExtension(Name="*.cc", ContentType="ClangCpp")]
    [FileExtension(Name="*.cxx", ContentType="ClangCpp")]
    [FileExtension(Name="*.m", ContentType="ClangObjCpp")]
    [FileExtension(Name="*.mm", ContentType="ClangObjCpp")]
    [ItemType(Name="ClangCompile", DisplayName="Clang source")]
    [ContentType(Name="ClangCpp", DisplayName="Clang C/C++", ItemType="ClangCompile")]
    [ContentType(Name = "ClangObjCpp", DisplayName = "Clang Objective C/C++", ItemType = "ClangCompile")]
    public class Clang : ToolTask
    {
        /// <summary>
        /// This class holds an item to be compiled by clang.
        /// </summary>
        class ClangItem
        {
            private Clang clangTask = null;
            private List<string> stdOut;
            private List<string> stdErr;

            public ITaskItem Item { get; private set; }
            public string ObjFileName { get; private set; }
            public string DepFileName { get; private set; }
            public string CommandLine { get; private set; }
            public int ExitCode { get; private set; }

            public ClangItem(Clang clangTask, ITaskItem item, string objFileName, string depFileName, string commandLine)
            {
                this.clangTask = clangTask;
                this.Item = item;
                this.ObjFileName = objFileName;
                this.DepFileName = depFileName;
                this.CommandLine = commandLine;
                this.stdOut = new List<string>();
                this.stdErr = new List<string>();
            }

            public void RunClang(string executable)
            {
                ProcessStartInfo startInfo = new ProcessStartInfo();
                startInfo.FileName = executable;
                startInfo.Arguments = this.CommandLine;
                startInfo.UseShellExecute = false;
                startInfo.RedirectStandardOutput = true;
                startInfo.RedirectStandardError = true;

                string stdOutString = null;
                string stdErrString = null;

                Process process = Process.Start(startInfo);

                // Wait for the program to complete as we record all of its StdOut and StdErr
                Parallel.Invoke(
                    () => { stdOutString = process.StandardOutput.ReadToEnd(); },
                    () => { stdErrString = process.StandardError.ReadToEnd(); }
                    );

                process.WaitForExit();<commit_msg>Fix all the clang windows that sometimes pop up all over the place (#2258)<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.Build.Utilities;
using Microsoft.Build.Framework;
using System.Runtime.Serialization;
using System.IO;
using System.Text.RegularExpressions;
using System.Reflection;
using System.Diagnostics;
using Microsoft.Win32;
using System.Threading;
using System.Globalization;
using System.Threading.Tasks;

namespace ClangCompile
{
    public static class StringSplitter
    {
        public static IEnumerable<string> Split(this string str, Func<char, bool> controller)
        {
            int nextPiece = 0;
            for (int c = 0; c < str.Length; c++)
            {
                if (controller(str[c]))
                {
                    yield return str.Substring(nextPiece, c - nextPiece);
                    nextPiece = c + 1;
                }
            }

            yield return str.Substring(nextPiece);
        }

        public static IEnumerable<string> SplitCommandLine(string commandLine)
        {
            bool inQuotes = false;
            return commandLine.Split(c =>
            {
                if (c == '\"')
                    inQuotes = !inQuotes;

                return !inQuotes && c == ' ';
            })
                                .Select(arg => arg.Trim())
                                .Where(arg => !string.IsNullOrEmpty(arg));
        }
    }
    
    [Rule(Name="Clang", PageTemplate="tool", DisplayName="Clang", Order="200")]
    [DataSource(Persistence="ProjectFile", ItemType="ClangCompile")]
    [PropertyCategory(Name="General", DisplayName="General", Order = 0)]
    [PropertyCategory(Name = "Paths", DisplayName = "Paths", Order = 1)]
    [PropertyCategory(Name = "Language", DisplayName = "Language", Order = 2)]
    [PropertyCategory(Name = "Preprocessor", DisplayName = "Preprocessor", Order = 3)]
    [PropertyCategory(Name = "All Options", DisplayName = "All Options", Subtype = "Search", Order = 4)]
    [PropertyCategory(Name = "Command Line", DisplayName = "Command Line", Subtype = "CommandLine", Order = 5)]
    [FileExtension(Name="*.c", ContentType="ClangCpp")]
    [FileExtension(Name="*.cpp", ContentType="ClangCpp")]
    [FileExtension(Name="*.cc", ContentType="ClangCpp")]
    [FileExtension(Name="*.cxx", ContentType="ClangCpp")]
    [FileExtension(Name="*.m", ContentType="ClangObjCpp")]
    [FileExtension(Name="*.mm", ContentType="ClangObjCpp")]
    [ItemType(Name="ClangCompile", DisplayName="Clang source")]
    [ContentType(Name="ClangCpp", DisplayName="Clang C/C++", ItemType="ClangCompile")]
    [ContentType(Name = "ClangObjCpp", DisplayName = "Clang Objective C/C++", ItemType = "ClangCompile")]
    public class Clang : ToolTask
    {
        /// <summary>
        /// This class holds an item to be compiled by clang.
        /// </summary>
        class ClangItem
        {
            private Clang clangTask = null;
            private List<string> stdOut;
            private List<string> stdErr;

            public ITaskItem Item { get; private set; }
            public string ObjFileName { get; private set; }
            public string DepFileName { get; private set; }
            public string CommandLine { get; private set; }
            public int ExitCode { get; private set; }

            public ClangItem(Clang clangTask, ITaskItem item, string objFileName, string depFileName, string commandLine)
            {
                this.clangTask = clangTask;
                this.Item = item;
                this.ObjFileName = objFileName;
                this.DepFileName = depFileName;
                this.CommandLine = commandLine;
                this.stdOut = new List<string>();
                this.stdErr = new List<string>();
            }

            public void RunClang(string executable)
            {
                ProcessStartInfo startInfo = new ProcessStartInfo();
                startInfo.CreateNoWindow = true;
                startInfo.FileName = executable;
                startInfo.Arguments = this.CommandLine;
                startInfo.UseShellExecute = false;
                startInfo.RedirectStandardOutput = true;
                startInfo.RedirectStandardError = true;

                string stdOutString = null;
                string stdErrString = null;

                Process process = Process.Start(startInfo);

                // Wait for the program to complete as we record all of its StdOut and StdErr
                Parallel.Invoke(
                    () => { stdOutString = process.StandardOutput.ReadToEnd(); },
                    () => { stdErrString = process.StandardError.ReadToEnd(); }
                    );

                process.WaitForExit();
{'fraction_non_alphanumeric': 0.11186012218243101, 'fraction_numerical': 0.0033705498209395408, 'mean_word_length': 2.150298606502986, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6026948', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2435, 'n_words': 608}