<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace LINQToTTreeLib.ExecutionCommon
{
    /// <summary>
    /// Execute a query on a remote proof cluster.
    /// </summary>
    public class ProofExecutor : IQueryExectuor
    {
        [Serializable]
        public class ProofException : Exception
        {
            /// <summary>
            /// Use the log info and a custom text message to generate an exception.
            /// </summary>
            /// <param name="message"></param>
            /// <param name="log"></param>
            /// <returns></returns>
            private static string MakeMessage(string message, ROOTNET.Interface.NTMacro log)
            {
                var bld = new StringBuilder();

                bld.AppendLine(message);
                if (log != null)
                {
                    bld.AppendLine("  PROOF log:");
                    foreach (var line in log.AsStrings())
                    {
                        bld.AppendLine(string.Format("  -> {0}", line));
                    }
                }

                return bld.ToString();
            }

            private static string MakeMessage(string message, IEnumerable<ROOTNET.Interface.NTMacro> logs)
            {
                var bld = new StringBuilder();

                bld.AppendLine(message);
                if (logs != null)
                {
                    bld.AppendLine("  PROOF log:");
                    foreach (var log in logs)
                    {
                        foreach (var line in log.AsStrings())
                        {
                            bld.AppendLine(string.Format("  -> {0}", line));
                        }
                    }
                }

                return bld.ToString();
            }

            public ProofException() { }
            public ProofException(string message, ROOTNET.Interface.NTMacro log) : base(MakeMessage(message, log)) { }
            public ProofException(string message, Exception inner, ROOTNET.Interface.NTMacro log) : base(MakeMessage(message, log), inner) { }

            public ProofException(string message, IEnumerable<ROOTNET.Interface.NTMacro> log) : base(MakeMessage(message, log)) { }
            public ProofException(string message, Exception inner, IEnumerable<ROOTNET.Interface.NTMacro> log) : base(MakeMessage(message, log), inner) { }
            protected ProofException(
              System.Runtime.Serialization.SerializationInfo info,
              System.Runtime.Serialization.StreamingContext context)
                : base(info, context) { }
        }

        /// <summary>
        /// Keep track of all modules that we've loaded
        /// </summary>
        private List<string> _loadedModuleNames = new List<string>();

        /// <summary>
        /// Run the execution on the PROOF cluster specified in the list of URI's we need to execute.
        /// </summary>
        /// <param name="templateFile"></param>
        /// <param name="queryDirectory"></param>
        /// <param name="varsToTransfer"></param>
        /// <returns></returns>
        public IDictionary<string, ROOTNET.Interface.NTObject> Execute(FileInfo templateFile, DirectoryInfo queryDirectory, IEnumerable<KeyValuePair<string, object>> varsToTransfer)
        {
            //
            // x-check parameters
            //

            if (templateFile == null)
                throw new ArgumentNullException("A template file must exist for this to work!");
            if (!templateFile.Exists)
                throw new FileNotFoundException("Unable to find C++ code to use for PROOF query", templateFile.FullName);

            if (Environment.ExtraComponentFiles != null && Environment.ExtraComponentFiles.Length > 0)
                throw new NotSupportedException("PROOF executor does not support extra component files!");

            //
            // Make sure everythign is inialized and ready for this query.
            //

            ExecutionUtilities.Init();
            OrganizeUris();

            //
            // Put us in the query directory
            //

            var oldDir = System.Environment.CurrentDirectory;
            System.Environment.CurrentDirectory = queryDirectory.FullName;
            ROOTNET.NTSystem.gSystem.ChangeDirectory(queryDirectory.FullName);
            var cdr = ROOTNET.NTSystem.gSystem.WorkingDirectory();

            //
            // A key "funny" thing about this is we need to track all the files used by this query and make sure they make it up to the
            // PROOF server. So we need to scan for all files and figure out where they are, and add them to a list of files that needs
            // to be sent.
            //

            var pc = GetProofConnection();

            var allIncludeFiles = ExecutionUtilities.FindGoodIncludeFilesRecursive(templateFile);
            var fList = new StringBuilder();
            fList.Append(string.Format("{0}+", templateFile.Name));
            foreach (var f in allIncludeFiles)
            {
                fList.Append(string.Format(",{0}", f.Name));
                PushFileToMaster(pc, f.Name);
            }

            //
            // Next, we need to build the proof job and the objects, and make sure something silly didn't go wrong!
            //

            var r = pc.Load(fList.ToString());
            if (r != 0)
                throw new InvalidOperationException(string.Format("Unable to compile and load selector in PROOF. Error: {0}", r));
            _loadedModuleNames.Add(templateFile.Name.Replace(".", "_"));

            var objName = Path.GetFileNameWithoutExtension(templateFile.Name);
            var cls = ROOTNET.NTClass.GetClass(objName);
            if (cls == null)
                throw new InvalidOperationException(string.Format("Unable to locate compiled object named '{0}'.", objName));

            //
            // Load up teh variables that need to go over to the cluster.
            //

            TraceHelpers.TraceInfo(20, "RunNtupleQuery: Saving the objects we are going to ship over");
            pc.InputList.Clear();

            foreach (var item in varsToTransfer)
            {
                var obj = item.Value as ROOTNET.Interface.NTNamed;
                if (obj == null)
                    throw new InvalidOperationException("Can only deal with named objects");
                var cloned = obj.Clone(item.Key);
                pc.InputList.Add(cloned);
            }

            //
            // Now run the PROOF query
            //

            var rResult = pc.Process(PROOFDSNames(), objName, "", 10000);

            Console.WriteLine("result is {0}", rResult);

            var logMessages = pc.GetLastLog();

            if (rResult == 0 || logMessages.ContainsString("kPROOF_FATAL"))
            {
                var mgr = pc.GetManager();
                var logs = mgr.GetSessionLogs();
                var macroLogs = logs.ListOfLogs.Cast<ROOTNET.Interface.NTProofLogElem>().Select(mc => mc.Macro);
                throw new ProofException(string.Format("Faild to run PROOF query (error from Process method was {0})", rResult), new ROOTNET.Interface.NTMacro[] { logMessages }.Concat(macroLogs));
            }

            //
            // Next, grab all the PROOF objects
            //
<commit_msg>Proper testing for an error coming back from the proof server.<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;

namespace LINQToTTreeLib.ExecutionCommon
{
    /// <summary>
    /// Execute a query on a remote proof cluster.
    /// </summary>
    public class ProofExecutor : IQueryExectuor
    {
        [Serializable]
        public class ProofException : Exception
        {
            /// <summary>
            /// Use the log info and a custom text message to generate an exception.
            /// </summary>
            /// <param name="message"></param>
            /// <param name="log"></param>
            /// <returns></returns>
            private static string MakeMessage(string message, ROOTNET.Interface.NTMacro log)
            {
                var bld = new StringBuilder();

                bld.AppendLine(message);
                if (log != null)
                {
                    bld.AppendLine("  PROOF log:");
                    foreach (var line in log.AsStrings())
                    {
                        bld.AppendLine(string.Format("  -> {0}", line));
                    }
                }

                return bld.ToString();
            }

            private static string MakeMessage(string message, IEnumerable<ROOTNET.Interface.NTMacro> logs)
            {
                var bld = new StringBuilder();

                bld.AppendLine(message);
                if (logs != null)
                {
                    bld.AppendLine("  PROOF log:");
                    foreach (var log in logs)
                    {
                        foreach (var line in log.AsStrings())
                        {
                            bld.AppendLine(string.Format("  -> {0}", line));
                        }
                    }
                }

                return bld.ToString();
            }

            public ProofException() { }
            public ProofException(string message, ROOTNET.Interface.NTMacro log) : base(MakeMessage(message, log)) { }
            public ProofException(string message, Exception inner, ROOTNET.Interface.NTMacro log) : base(MakeMessage(message, log), inner) { }

            public ProofException(string message, IEnumerable<ROOTNET.Interface.NTMacro> log) : base(MakeMessage(message, log)) { }
            public ProofException(string message, Exception inner, IEnumerable<ROOTNET.Interface.NTMacro> log) : base(MakeMessage(message, log), inner) { }
            protected ProofException(
              System.Runtime.Serialization.SerializationInfo info,
              System.Runtime.Serialization.StreamingContext context)
                : base(info, context) { }
        }

        /// <summary>
        /// Keep track of all modules that we've loaded
        /// </summary>
        private List<string> _loadedModuleNames = new List<string>();

        /// <summary>
        /// Run the execution on the PROOF cluster specified in the list of URI's we need to execute.
        /// </summary>
        /// <param name="templateFile"></param>
        /// <param name="queryDirectory"></param>
        /// <param name="varsToTransfer"></param>
        /// <returns></returns>
        public IDictionary<string, ROOTNET.Interface.NTObject> Execute(FileInfo templateFile, DirectoryInfo queryDirectory, IEnumerable<KeyValuePair<string, object>> varsToTransfer)
        {
            //
            // x-check parameters
            //

            if (templateFile == null)
                throw new ArgumentNullException("A template file must exist for this to work!");
            if (!templateFile.Exists)
                throw new FileNotFoundException("Unable to find C++ code to use for PROOF query", templateFile.FullName);

            if (Environment.ExtraComponentFiles != null && Environment.ExtraComponentFiles.Length > 0)
                throw new NotSupportedException("PROOF executor does not support extra component files!");

            //
            // Make sure everythign is inialized and ready for this query.
            //

            ExecutionUtilities.Init();
            OrganizeUris();

            //
            // Put us in the query directory
            //

            var oldDir = System.Environment.CurrentDirectory;
            System.Environment.CurrentDirectory = queryDirectory.FullName;
            ROOTNET.NTSystem.gSystem.ChangeDirectory(queryDirectory.FullName);
            var cdr = ROOTNET.NTSystem.gSystem.WorkingDirectory();

            //
            // A key "funny" thing about this is we need to track all the files used by this query and make sure they make it up to the
            // PROOF server. So we need to scan for all files and figure out where they are, and add them to a list of files that needs
            // to be sent.
            //

            var pc = GetProofConnection();

            var allIncludeFiles = ExecutionUtilities.FindGoodIncludeFilesRecursive(templateFile);
            var fList = new StringBuilder();
            fList.Append(string.Format("{0}+", templateFile.Name));
            foreach (var f in allIncludeFiles)
            {
                fList.Append(string.Format(",{0}", f.Name));
                PushFileToMaster(pc, f.Name);
            }

            //
            // Next, we need to build the proof job and the objects, and make sure something silly didn't go wrong!
            //

            var r = pc.Load(fList.ToString());
            if (r != 0)
                throw new InvalidOperationException(string.Format("Unable to compile and load selector in PROOF. Error: {0}", r));
            _loadedModuleNames.Add(templateFile.Name.Replace(".", "_"));

            var objName = Path.GetFileNameWithoutExtension(templateFile.Name);
            var cls = ROOTNET.NTClass.GetClass(objName);
            if (cls == null)
                throw new InvalidOperationException(string.Format("Unable to locate compiled object named '{0}'.", objName));

            //
            // Load up teh variables that need to go over to the cluster.
            //

            TraceHelpers.TraceInfo(20, "RunNtupleQuery: Saving the objects we are going to ship over");
            pc.InputList.Clear();

            foreach (var item in varsToTransfer)
            {
                var obj = item.Value as ROOTNET.Interface.NTNamed;
                if (obj == null)
                    throw new InvalidOperationException("Can only deal with named objects");
                var cloned = obj.Clone(item.Key);
                pc.InputList.Add(cloned);
            }

            //
            // Now run the PROOF query
            //

            var rResult = pc.Process(PROOFDSNames(), objName, "", 10000);

            //
            // There is an error if the result is non zero or there is an error in the short log.
            // If there is an error, build a full log message and throw it!
            //

            var logMessages = pc.GetLastLog();
            if (rResult != 0 || logMessages.ContainsString("kPROOF_FATAL"))
            {
                var mgr = pc.GetManager();
                var logs = mgr.GetSessionLogs();
                var macroLogs = logs.ListOfLogs.Cast<ROOTNET.Interface.NTProofLogElem>().Select(mc => mc.Macro);
                throw new ProofException(string.Format("Faild to run PROOF query (error from Process method was {0})", rResult), new ROOTNET.Interface.NTMacro[] { logMessages }.Concat(macroLogs));
            }

            //
            // Next, grab all the PROOF objects
            //

{'fraction_non_alphanumeric': 0.10025946377486528, 'fraction_numerical': 0.0023285210564832678, 'mean_word_length': 1.668560269838452, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '934687', 'n_tokens_mistral': 3752, 'n_tokens_neox': 3596, 'n_words': 1076}