<commit_before>﻿using CredentialManagement;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace AtlasSSH
{
    /// <summary>
    /// Thrown when there is a config error on the Linux side of things; and that commands that
    /// should always work on the Linux host, are not.
    /// </summary>
    [Serializable]
    public class LinuxConfigException : Exception
    {
        public LinuxConfigException() { }
        public LinuxConfigException(string message) : base(message) { }
        public LinuxConfigException(string message, Exception inner) : base(message, inner) { }
        protected LinuxConfigException(
          System.Runtime.Serialization.SerializationInfo info,
          System.Runtime.Serialization.StreamingContext context) : base(info, context)
        { }
    }

    /// <summary>
    /// Thrown when something on Linux was tried, but the config prevented it (e.g. missing release).
    /// </summary>
    [Serializable]
    public class LinuxMissingConfigurationException : Exception
    {
        public LinuxMissingConfigurationException() { }
        public LinuxMissingConfigurationException(string message) : base(message) { }
        public LinuxMissingConfigurationException(string message, Exception inner) : base(message, inner) { }
        protected LinuxMissingConfigurationException(
          System.Runtime.Serialization.SerializationInfo info,
          System.Runtime.Serialization.StreamingContext context) : base(info, context)
        { }
    }


    [Serializable]
    public class LinuxCommandErrorException : Exception
    {
        public LinuxCommandErrorException() { }
        public LinuxCommandErrorException(string message) : base(message) { }
        public LinuxCommandErrorException(string message, Exception inner) : base(message, inner) { }
        protected LinuxCommandErrorException(
          System.Runtime.Serialization.SerializationInfo info,
          System.Runtime.Serialization.StreamingContext context) : base(info, context)
        { }
    }

    /// <summary>
    /// Some commands
    /// </summary>
    public static class AtlasCommands
    {
        /// <summary>
        /// Run the setup ATLAS command
        /// </summary>
        /// <param name="connection">The connection that will understand the setupATLAS command</param>
        /// <returns>A reconfigured SSH shell connection (same as what went in)</returns>
        public static ISSHConnection setupATLAS(this ISSHConnection connection)
        {
            bool foundalias = false;
            connection
                .ExecuteCommand("setupATLAS")
                .ExecuteCommand("alias", l => foundalias = foundalias || l.Contains("rcSetup"));

            if (!foundalias)
            {
                throw new LinuxConfigException("The setupATLAS command did not have the expected effect - rcSetup was not defined as an alias");
            }

            return connection;
        }

        /// <summary>
        /// Setup Rucio. ATLAS must have been previously configured, or this will "crash".
        /// </summary>
        /// <param name="connection">Connection on-which we will set everything up</param>
        /// <param name="rucioUsername">The user alias used on the grid</param>
        /// <returns>A shell on-which rucio has been setup (the same connection that went in)</returns>
        public static ISSHConnection setupRucio(this ISSHConnection connection, string rucioUsername)
        {
            int hashCount = 0;
            connection
                .ExecuteCommand(string.Format("export RUCIO_ACCOUNT={0}", rucioUsername))
                .ExecuteCommand("localSetupRucioClients")
                .ExecuteCommand("hash rucio", l => hashCount++);

            if (hashCount != 0)
            {
                throw new LinuxConfigException("Unable to setup Rucio... did you forget to setup ATLAS first?");
            }
            return connection;
        }

        /// <summary>
        /// Initializes the VOMS proxy for use on the GRID. The connection must have already been configured so that
        /// the command voms-proxy-init works.
        /// </summary>
        /// <param name="connection">The configured SSH shell</param>
        /// <param name="GRIDUsername">The username to use to fetch the password for the voms proxy file</param>
        /// <param name="voms">The name of the voms to connect to</param>
        /// <returns>Connection on which the grid is setup and ready to go</returns>
        public static ISSHConnection VomsProxyInit(this ISSHConnection connection, string voms, Func<bool> failNow = null)
        {
            // Get the GRID VOMS password
            var sclist = new CredentialSet("GRID");
            var passwordInfo = sclist.Load().FirstOrDefault();
            if (passwordInfo == null)
            {
                throw new ArgumentException("There is no generic windows credential targeting the network address 'GRID' for username. This password should be your cert <PASSWORD> phrase and your on-the-grid username. Please create one on this machine.");
            }

            // Run the command
            bool goodProxy = false;
            var whatHappened = new List<string>();

            connection
                .ExecuteCommand(string.Format("echo {0} | voms-proxy-init -voms {1}", passwordInfo.Password, voms),
                    l =>
                    {
                        goodProxy = goodProxy || l.Contains("Your proxy is valid");
                        whatHappened.Add(l);
                    },
                    secondsTimeout: 20, failNow: failNow
                    );

            // If we failed to get the proxy, then build an error message that can be understood. Since this
            // could be for a large range of reasons, we are going to pass back a lot of info to the user
            // so they can figure it out (not likely a program will be able to sort this out).

            if (goodProxy == false)
            {
                var error = new StringBuilder();
                error.AppendLine("Failed to get the proxy: ");
                foreach (var l in whatHappened)
                {
                    error.AppendLine(string.Format("  -> {0}", l));
                }
                throw new ArgumentException(error.ToString());
            }

            return connection;
        }

        /// <summary>
        /// Fetch a dataset from the grid using Rucio to a local directory.
        /// </summary>
        /// <param name="connection">A previously configured connection with everything ready to go for GRID access.</param>
        /// <param name="datasetName">The rucio dataset name</param>
        /// <param name="localDirectory">The local directory (on Linux) where the file should be downloaded</param>
        /// <param name="fileStatus">Gets updates as new files are downloaded. This will contain just the filename.</param>
        /// <param name="fileNameFilter">Filter function to alter the files that are to be downloaded</param>
        /// <returns></returns>
        public static ISSHConnection DownloadFromGRID(this ISSHConnection connection, string datasetName, string localDirectory,
            Action<string> fileStatus = null,
            Func<string[], string[]> fileNameFilter = null,
            Func<bool> failNow = null,
            int timeout = 3600)
        {
            // Does the dataset exist?
            if (fileStatus != null)
            {
                fileStatus("Checking the dataset exists");
            }
            var response = new List<string>();
            connection.ExecuteCommand(string.Format("rucio ls {0}", datasetName), l => response.Add(l), secondsTimeout: 60, failNow: failNow);

            var dsnames = response
                .Where(l => l.Contains("DATASET") | l.Contains("CONTAINER"))
                .Select(l => l.Split(' '))
                .Where(sl => sl.Length > 1)
                .Select(sl => sl[1])
                .ToArray();

            if (!dsnames.Where(n => n.SantizeDSName() == datasetName.SantizeDSName()).Any())
            {
                throw new ArgumentException(string.Format("Unable to find any datasets with the name '{0}'.", datasetName));
            }

            // Get the complete list of files in the dataset.
            if (fileStatus != null)
            {
                fileStatus("Getting the complete list of files from the dataset");
            }
            var fileNameList = connection.FilelistFromGRID(datasetName, failNow: failNow);

            // Filter them if need be.
            var goodFiles = fileNameFilter != null
                ? fileNameFilter(fileNameList)
                : fileNameList;

            // If we have no files to download, then we are totally done!
            if (goodFiles.Length == 0)
            {
                return connection;
            }<commit_msg>Improve error message<commit_after>﻿using CredentialManagement;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace AtlasSSH
{
    /// <summary>
    /// Thrown when there is a config error on the Linux side of things; and that commands that
    /// should always work on the Linux host, are not.
    /// </summary>
    [Serializable]
    public class LinuxConfigException : Exception
    {
        public LinuxConfigException() { }
        public LinuxConfigException(string message) : base(message) { }
        public LinuxConfigException(string message, Exception inner) : base(message, inner) { }
        protected LinuxConfigException(
          System.Runtime.Serialization.SerializationInfo info,
          System.Runtime.Serialization.StreamingContext context) : base(info, context)
        { }
    }

    /// <summary>
    /// Thrown when something on Linux was tried, but the config prevented it (e.g. missing release).
    /// </summary>
    [Serializable]
    public class LinuxMissingConfigurationException : Exception
    {
        public LinuxMissingConfigurationException() { }
        public LinuxMissingConfigurationException(string message) : base(message) { }
        public LinuxMissingConfigurationException(string message, Exception inner) : base(message, inner) { }
        protected LinuxMissingConfigurationException(
          System.Runtime.Serialization.SerializationInfo info,
          System.Runtime.Serialization.StreamingContext context) : base(info, context)
        { }
    }


    [Serializable]
    public class LinuxCommandErrorException : Exception
    {
        public LinuxCommandErrorException() { }
        public LinuxCommandErrorException(string message) : base(message) { }
        public LinuxCommandErrorException(string message, Exception inner) : base(message, inner) { }
        protected LinuxCommandErrorException(
          System.Runtime.Serialization.SerializationInfo info,
          System.Runtime.Serialization.StreamingContext context) : base(info, context)
        { }
    }

    /// <summary>
    /// Some commands
    /// </summary>
    public static class AtlasCommands
    {
        /// <summary>
        /// Run the setup ATLAS command
        /// </summary>
        /// <param name="connection">The connection that will understand the setupATLAS command</param>
        /// <returns>A reconfigured SSH shell connection (same as what went in)</returns>
        public static ISSHConnection setupATLAS(this ISSHConnection connection)
        {
            bool foundalias = false;
            connection
                .ExecuteCommand("setupATLAS")
                .ExecuteCommand("alias", l => foundalias = foundalias || l.Contains("rcSetup"));

            if (!foundalias)
            {
                throw new LinuxConfigException("The setupATLAS command did not have the expected effect - rcSetup was not defined as an alias");
            }

            return connection;
        }

        /// <summary>
        /// Setup Rucio. ATLAS must have been previously configured, or this will "crash".
        /// </summary>
        /// <param name="connection">Connection on-which we will set everything up</param>
        /// <param name="rucioUsername">The user alias used on the grid</param>
        /// <returns>A shell on-which rucio has been setup (the same connection that went in)</returns>
        public static ISSHConnection setupRucio(this ISSHConnection connection, string rucioUsername)
        {
            int hashCount = 0;
            connection
                .ExecuteCommand(string.Format("export RUCIO_ACCOUNT={0}", rucioUsername))
                .ExecuteCommand("localSetupRucioClients")
                .ExecuteCommand("hash rucio", l => hashCount++);

            if (hashCount != 0)
            {
                throw new LinuxConfigException("Unable to setup Rucio... did you forget to setup ATLAS first?");
            }
            return connection;
        }

        /// <summary>
        /// Initializes the VOMS proxy for use on the GRID. The connection must have already been configured so that
        /// the command voms-proxy-init works.
        /// </summary>
        /// <param name="connection">The configured SSH shell</param>
        /// <param name="GRIDUsername">The username to use to fetch the password for the voms proxy file</param>
        /// <param name="voms">The name of the voms to connect to</param>
        /// <returns>Connection on which the grid is setup and ready to go</returns>
        public static ISSHConnection VomsProxyInit(this ISSHConnection connection, string voms, Func<bool> failNow = null)
        {
            // Get the GRID VOMS password
            var sclist = new CredentialSet("GRID");
            var passwordInfo = sclist.Load().FirstOrDefault();
            if (passwordInfo == null)
            {
                throw new ArgumentException("There is no generic windows credential targeting the network address 'GRID' for username. This password should be your cert pass phrase and your on-the-grid username. Please create one on this machine.");
            }

            // Run the command
            bool goodProxy = false;
            var whatHappened = new List<string>();

            connection
                .ExecuteCommand(string.Format("echo {0} | voms-proxy-init -voms {1}", passwordInfo.Password, voms),
                    l =>
                    {
                        goodProxy = goodProxy || l.Contains("Your proxy is valid");
                        whatHappened.Add(l);
                    },
                    secondsTimeout: 20, failNow: failNow
                    );

            // If we failed to get the proxy, then build an error message that can be understood. Since this
            // could be for a large range of reasons, we are going to pass back a lot of info to the user
            // so they can figure it out (not likely a program will be able to sort this out).

            if (goodProxy == false)
            {
                var error = new StringBuilder();
                error.AppendLine("Failed to get the proxy: ");
                foreach (var l in whatHappened)
                {
                    error.AppendLine(string.Format("  -> {0}", l));
                }
                throw new ArgumentException(error.ToString());
            }

            return connection;
        }

        /// <summary>
        /// Fetch a dataset from the grid using Rucio to a local directory.
        /// </summary>
        /// <param name="connection">A previously configured connection with everything ready to go for GRID access.</param>
        /// <param name="datasetName">The rucio dataset name</param>
        /// <param name="localDirectory">The local directory (on Linux) where the file should be downloaded</param>
        /// <param name="fileStatus">Gets updates as new files are downloaded. This will contain just the filename.</param>
        /// <param name="fileNameFilter">Filter function to alter the files that are to be downloaded</param>
        /// <returns></returns>
        public static ISSHConnection DownloadFromGRID(this ISSHConnection connection, string datasetName, string localDirectory,
            Action<string> fileStatus = null,
            Func<string[], string[]> fileNameFilter = null,
            Func<bool> failNow = null,
            int timeout = 3600)
        {
            // Does the dataset exist?
            if (fileStatus != null)
            {
                fileStatus("Checking the dataset exists");
            }
            var response = new List<string>();
            connection.ExecuteCommand(string.Format("rucio ls {0}", datasetName), l => response.Add(l), secondsTimeout: 60, failNow: failNow);

            var dsnames = response
                .Where(l => l.Contains("DATASET") | l.Contains("CONTAINER"))
                .Select(l => l.Split(' '))
                .Where(sl => sl.Length > 1)
                .Select(sl => sl[1])
                .ToArray();

            if (!dsnames.Where(n => n.SantizeDSName() == datasetName.SantizeDSName()).Any())
            {
                throw new ArgumentException(string.Format("Unable to find any datasets on the GRID (in rucuio) with the name '{0}'.", datasetName));
            }

            // Get the complete list of files in the dataset.
            if (fileStatus != null)
            {
                fileStatus("Getting the complete list of files from the dataset");
            }
            var fileNameList = connection.FilelistFromGRID(datasetName, failNow: failNow);

            // Filter them if need be.
            var goodFiles = fileNameFilter != null
                ? fileNameFilter(fileNameList)
                : fileNameList;

            // If we have no files to download, then we are totally done!
            if (goodFiles.Length == 0)
            {
                return connection;
            }
{'fraction_non_alphanumeric': 0.0949794968414053, 'fraction_numerical': 0.002105729801618087, 'mean_word_length': 2.0229480737018424, 'pattern_counts': {'":': 0, '<': 104, '<?xml version=': 0, '>': 126, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '86367', 'n_tokens_mistral': 4391, 'n_tokens_neox': 4238, 'n_words': 1459}