<commit_before>// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Diagnostics;
using System.IO;
using System.Collections.Specialized;
using System.Collections;
using System.Threading.Tasks;
using System.Globalization;
using System.Runtime.InteropServices;

namespace MICore
{
    public class PipeTransport : StreamTransport
    {
        private Process _process;
        private StreamReader _stdErrReader;
        private int _remainingReaders;
        private ManualResetEvent _allReadersDone = new ManualResetEvent(false);
        private bool _killOnClose;
        private bool _filterStderr;

        public PipeTransport(bool killOnClose = false, bool filterStderr = false, bool filterStdout = false) : base(filterStdout)
        {
            _killOnClose = killOnClose;
            _filterStderr = filterStderr;
        }

        protected override string GetThreadName()
        {
            return "MI.PipeTransport";
        }

        protected virtual void InitProcess(Process proc, out StreamReader stdout, out StreamWriter stdin)
        {
            _process = proc;

            _process.EnableRaisingEvents = true;
            _process.Exited += OnProcessExit;

            _process.StartInfo.RedirectStandardInput = true;
            _process.StartInfo.RedirectStandardOutput = true;
            _process.StartInfo.RedirectStandardError = true;
            _process.StartInfo.UseShellExecute = false;
            _process.StartInfo.CreateNoWindow = true;

            lock (_process)
            {
                this.Callback.AppendToInitializationLog(string.Format(CultureInfo.InvariantCulture, "Starting: \"{0}\" {1}", _process.StartInfo.FileName, _process.StartInfo.Arguments));
                _process.Start();

                stdout = _process.StandardOutput;
                stdin = _process.StandardInput;
                _stdErrReader = _process.StandardError;
                _remainingReaders = 2;

                AsyncReadFromStdError();
            }
        }


        public override void InitStreams(LaunchOptions options, out StreamReader reader, out StreamWriter writer)
        {
            PipeLaunchOptions pipeOptions = (PipeLaunchOptions)options;

            Process proc = new Process();
            proc.StartInfo.FileName = pipeOptions.PipePath;
            proc.StartInfo.Arguments = pipeOptions.PipeArguments;
            proc.StartInfo.WorkingDirectory = System.IO.Path.GetDirectoryName(pipeOptions.PipePath);

            InitProcess(proc, out reader, out writer);
        }

        private void KillChildren(List<Tuple<int,int>> processes, int pid)
        {
            processes.ForEach((p) =>
            {
                if (p.Item2 == pid)
                {
                    KillChildren(processes, p.Item1);
                    var k = Process.Start("/bin/kill", String.Format(CultureInfo.InvariantCulture, "-9 {0}", p.Item1));
                    k.WaitForExit();
                }
            });
        }

        private void KillProcess(Process p)
        {
            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
            {
                // Run 'ps h -o "%p %P"', which generates a list of the process ids (%p) and parent process ids (%P).
                // Using this list, issue a 'kill' command for each child process. Kill the children (recursively) to eliminate
                // the entire process tree rooted at p. 
                Process ps = new Process();
                ps.StartInfo.FileName ="/bin/ps";
                ps.StartInfo.Arguments = "h -o \"%p %P\"";
                ps.StartInfo.RedirectStandardOutput = true;
                ps.StartInfo.UseShellExecute = false;
                ps.Start();
                string line;
                List<Tuple<int, int>> processAndParent = new List<Tuple<int, int>>();
                char[] whitespace = new char[] { ' ', '\t' };
                while ((line = ps.StandardOutput.ReadLine()) != null)
                {
                    line = line.Trim();
                    int id = Int32.Parse(line.Substring(0, line.IndexOfAny(whitespace)), CultureInfo.InvariantCulture);
                    int pid = Int32.Parse(line.Substring(line.IndexOfAny(whitespace)).Trim(), CultureInfo.InvariantCulture);
                    processAndParent.Add(new Tuple<int, int>(id, pid));
                }
                KillChildren(processAndParent, p.Id);
            }
            if (!p.HasExited)
            {
                p.Kill();
            }
        }

        public override void Close()
        {
            if (_writer != null)
            {
                try
                {
                    Echo("logout");
                }
                catch (Exception)
                {
                    // Ignore errors if logout couldn't be written
                }
            }

            base.Close();

            if (_stdErrReader != null)
            {
                _stdErrReader.Dispose();
            }
<commit_msg>Mac's ps is different from ubuntu's: doesn't support "h" (no headers) option, different output formatting specification.<commit_after>// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Diagnostics;
using System.IO;
using System.Collections.Specialized;
using System.Collections;
using System.Threading.Tasks;
using System.Globalization;
using System.Runtime.InteropServices;

namespace MICore
{
    public class PipeTransport : StreamTransport
    {
        private Process _process;
        private StreamReader _stdErrReader;
        private int _remainingReaders;
        private ManualResetEvent _allReadersDone = new ManualResetEvent(false);
        private bool _killOnClose;
        private bool _filterStderr;

        public PipeTransport(bool killOnClose = false, bool filterStderr = false, bool filterStdout = false) : base(filterStdout)
        {
            _killOnClose = killOnClose;
            _filterStderr = filterStderr;
        }

        protected override string GetThreadName()
        {
            return "MI.PipeTransport";
        }

        protected virtual void InitProcess(Process proc, out StreamReader stdout, out StreamWriter stdin)
        {
            _process = proc;

            _process.EnableRaisingEvents = true;
            _process.Exited += OnProcessExit;

            _process.StartInfo.RedirectStandardInput = true;
            _process.StartInfo.RedirectStandardOutput = true;
            _process.StartInfo.RedirectStandardError = true;
            _process.StartInfo.UseShellExecute = false;
            _process.StartInfo.CreateNoWindow = true;

            lock (_process)
            {
                this.Callback.AppendToInitializationLog(string.Format(CultureInfo.InvariantCulture, "Starting: \"{0}\" {1}", _process.StartInfo.FileName, _process.StartInfo.Arguments));
                _process.Start();

                stdout = _process.StandardOutput;
                stdin = _process.StandardInput;
                _stdErrReader = _process.StandardError;
                _remainingReaders = 2;

                AsyncReadFromStdError();
            }
        }


        public override void InitStreams(LaunchOptions options, out StreamReader reader, out StreamWriter writer)
        {
            PipeLaunchOptions pipeOptions = (PipeLaunchOptions)options;

            Process proc = new Process();
            proc.StartInfo.FileName = pipeOptions.PipePath;
            proc.StartInfo.Arguments = pipeOptions.PipeArguments;
            proc.StartInfo.WorkingDirectory = System.IO.Path.GetDirectoryName(pipeOptions.PipePath);

            InitProcess(proc, out reader, out writer);
        }

        private void KillChildren(List<Tuple<int,int>> processes, int pid)
        {
            processes.ForEach((p) =>
            {
                if (p.Item2 == pid)
                {
                    KillChildren(processes, p.Item1);
                    var k = Process.Start("/bin/kill", String.Format(CultureInfo.InvariantCulture, "-9 {0}", p.Item1));
                    k.WaitForExit();
                }
            });
        }

        private void KillProcess(Process p)
        {
            bool isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
            bool isOSX = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
            if (isLinux || isOSX)
            {
                // On linux run 'ps -x -o "%p %P"' (similarly on Mac), which generates a list of the process ids (%p) and parent process ids (%P).
                // Using this list, issue a 'kill' command for each child process. Kill the children (recursively) to eliminate
                // the entire process tree rooted at p. 
                Process ps = new Process();
                ps.StartInfo.FileName ="/bin/ps";
                ps.StartInfo.Arguments = isLinux ? "-x -o \"%p %P\"" : "-x -o \"pid ppid\"";
                ps.StartInfo.RedirectStandardOutput = true;
                ps.StartInfo.UseShellExecute = false;
                ps.Start();
                string line;
                List<Tuple<int, int>> processAndParent = new List<Tuple<int, int>>();
                char[] whitespace = new char[] { ' ', '\t' };
                while ((line = ps.StandardOutput.ReadLine()) != null)
                {
                    line = line.Trim();
                    int id, pid;
                    if (Int32.TryParse(line.Substring(0, line.IndexOfAny(whitespace)), NumberStyles.Integer, CultureInfo.InvariantCulture, out id)
                        && Int32.TryParse(line.Substring(line.IndexOfAny(whitespace)).Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out pid))
                    {
                        processAndParent.Add(new Tuple<int, int>(id, pid));
                    }
                }
                KillChildren(processAndParent, p.Id);
            }
            if (!p.HasExited)
            {
                p.Kill();
            }
        }

        public override void Close()
        {
            if (_writer != null)
            {
                try
                {
                    Echo("logout");
                }
                catch (Exception)
                {
                    // Ignore errors if logout couldn't be written
                }
            }

            base.Close();

            if (_stdErrReader != null)
            {
                _stdErrReader.Dispose();
            }

{'fraction_non_alphanumeric': 0.09090909090909091, 'fraction_numerical': 0.002397197123363452, 'mean_word_length': 1.736377396569122, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '115949', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2639, 'n_words': 662}