<commit_before>using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using log4net;
using LogSearchShipper.Core.ConfigurationSections;

namespace LogSearchShipper.Core
{
	public class EDBEnvironment
	{
		public string Name { get; set; }
		public List<EDBServerGroup> ServerGroups { get; set; }
	}

	public class EDBServerGroup
	{
		public string Name;
		public List<EdbServer> Servers;
	}

	public class EdbServer
	{
		public string Name;
		public string Description;
		public string Tags;
		public string Domain;
		public string Environment;
		public string NetworkArea;
		public List<XElement> Services;
	}

	public class EdbService
	{
		public EdbService(XElement source)
		{
			Name = source.Element("Name").Value;
			State = source.Element("State").Value;

			for (int i = 0; ; i++)
			{
				var sourceName = "LogPath" + (i == 0 ? "" : i.ToString());
				var sourceElem = source.Element(sourceName);
				if (sourceElem == null)
					break;

				EventSources.Add(sourceElem.Value);
			}
		}

		public string Name;
		public string State;
		public List<string> EventSources = new List<string>();
	}

	public class EDBEnvironmentComparer : IEqualityComparer<EDBEnvironment>
	{
		public bool Equals(EDBEnvironment e1, EDBEnvironment e2)
		{
			return e1.Name.ToUpper() == e2.Name.ToUpper();
		}

		public int GetHashCode(EDBEnvironment obj)
		{
			return obj.Name.GetHashCode();
		}
	}

	public static class SearchExtensionMethods
	{
		public static bool RegExMatches(this string str, string regEx)
		{
			if (string.IsNullOrEmpty(regEx))
				return false;
			return Regex.Match(str, regEx, RegexOptions.IgnoreCase).Success;
		}
	}

	public class EDBFileWatchParser
	{
		private static readonly ILog _log = LogManager.GetLogger(typeof(EDBFileWatchParser));

		private readonly EnvironmentWatchElement _environmentWatchElement;

		public EDBFileWatchParser(EnvironmentWatchElement environmentWatchElement)
		{
			_environmentWatchElement = environmentWatchElement;
		}

		public List<FileWatchElement> ToFileWatchCollection()
		{
			XDocument environmentDataXml = LoadEDBXml();

			var serversFiltered = environmentDataXml.Descendants("Servers").Descendants("Server").Where(
				server => server.Element("Name").Value.RegExMatches(_environmentWatchElement.ServerNames) &&
					!server.Element("Name").Value.RegExMatches(_environmentWatchElement.ServerNamesNotMatch) &&
					server.Element("NetworkArea").Value.RegExMatches(_environmentWatchElement.NetworkAreas) &&
					!server.Element("NetworkArea").Value.RegExMatches(_environmentWatchElement.NetworkAreasNotMatch)
				).ToArray();

			var servers = (from server in serversFiltered
							select new
							{
								Name = server.Element("Name").Value,
								NetworkArea = server.Element("NetworkArea").Value,
								Services = from service in server.Descendants("Services").Descendants("Entity")
										where service.Element("Name").Value.RegExMatches(_environmentWatchElement.ServiceNames) &&
											!service.Element("Name").Value.RegExMatches(_environmentWatchElement.ServiceNamesNotMatch)
										select new
										{
											Name = service.Element("Name").Value,
											LogFile = (string)service.Elements("LogPath").FirstOrDefault(),
											LogType = (string)service.Elements("LogPathType").FirstOrDefault(),
											LogFile1 = (string)service.Elements("LogPath1").FirstOrDefault(),
											LogType1 = (string)service.Elements("LogPath1Type").FirstOrDefault(),
											LogFile2 = (string)service.Elements("LogPath2").FirstOrDefault(),
											LogType2 = (string)service.Elements("LogPath2Type").FirstOrDefault()
										}
							}).ToArray();

			var watches = new List<FileWatchElement>();
			foreach (var server in servers)
			{
				foreach (var service in server.Services)
				{
					var fields = new FieldCollection
					{
						new FieldElement { Key = "host", Value = server.Name },
						new FieldElement { Key = "service", Value = service.Name },
					};
					foreach (FieldElement field in _environmentWatchElement.Fields)
					{
						fields.Add(field);
					}

					AddFileWatchElementForLogFile(service.LogFile, service.LogType, watches, fields, server.NetworkArea, server.Name,
						service.Name);
					AddFileWatchElementForLogFile(service.LogFile1, service.LogType1, watches, fields, server.NetworkArea, server.Name,
						service.Name);
					AddFileWatchElementForLogFile(service.LogFile2, service.LogType2, watches, fields, server.NetworkArea, server.Name,
						service.Name);
				}
			}
			return watches;
		}

		private void AddFileWatchElementForLogFile(string logFile, string logType, ICollection<FileWatchElement> watches,
			FieldCollection fields, string serverNetworkArea, string serverName, string serviceName)
		{
			//Don't ship logs without a type or with an empty type
			if (!string.IsNullOrEmpty(logType) && !string.IsNullOrWhiteSpace(logType))
			{
				var newWatch = new FileWatchElement
				{
					Files = logFile,
					Type = logType,
					Fields = fields
				};

				var overrideConfig = FindOverride(serviceName);
				if (overrideConfig != null)
				{
					newWatch.CloseWhenIdle = overrideConfig.CloseWhenIdle;
					if (overrideConfig.CustomNxlogConfig != null)
					{
						newWatch.CustomNxlogConfig = new CustomNxlogConfig
							{
								Value = overrideConfig.CustomNxlogConfig.Value
							};<commit_msg>refactoring<commit_after>using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml.Linq;

using log4net;
using LogSearchShipper.Core.ConfigurationSections;

namespace LogSearchShipper.Core
{
	public class EDBEnvironment
	{
		public string Name { get; set; }
		public List<EDBServerGroup> ServerGroups { get; set; }
	}

	public class EDBServerGroup
	{
		public string Name;
		public List<EdbServer> Servers;
	}

	public class EdbServer
	{
		public string Name;
		public string Description;
		public string Tags;
		public string Domain;
		public string Environment;
		public string NetworkArea;
		public List<XElement> Services;
	}

	public class EdbService
	{
		public EdbService(XElement source)
		{
			Name = source.Element("Name").Value;
			State = source.Element("State").Value;

			for (int i = 0; ; i++)
			{
				var sourceName = "LogPath" + (i == 0 ? "" : i.ToString());
				var sourceElem = source.Element(sourceName);
				if (sourceElem == null)
					break;

				EventSources.Add(sourceElem.Value);
			}
		}

		public string Name;
		public string State;
		public List<string> EventSources = new List<string>();
	}

	public class EDBEnvironmentComparer : IEqualityComparer<EDBEnvironment>
	{
		public bool Equals(EDBEnvironment e1, EDBEnvironment e2)
		{
			return e1.Name.ToUpper() == e2.Name.ToUpper();
		}

		public int GetHashCode(EDBEnvironment obj)
		{
			return obj.Name.GetHashCode();
		}
	}

	public static class SearchExtensionMethods
	{
		public static bool RegExMatches(this string str, string regEx)
		{
			if (string.IsNullOrEmpty(regEx))
				return false;
			return Regex.Match(str, regEx, RegexOptions.IgnoreCase).Success;
		}
	}

	public class EDBFileWatchParser
	{
		private static readonly ILog _log = LogManager.GetLogger(typeof(EDBFileWatchParser));

		private readonly EnvironmentWatchElement _environmentWatchElement;

		public EDBFileWatchParser(EnvironmentWatchElement environmentWatchElement)
		{
			_environmentWatchElement = environmentWatchElement;
		}

		public List<FileWatchElement> ToFileWatchCollection()
		{
			var environmentDataXml = LoadEDBXml();

			var serversFiltered = environmentDataXml.Descendants("Servers").Descendants("Server").Where(
				server =>
				{
					var serverName = server.Element("Name").Value;
					var networkArea = server.Element("NetworkArea").Value;

					return serverName.RegExMatches(_environmentWatchElement.ServerNames) &&
						!serverName.RegExMatches(_environmentWatchElement.ServerNamesNotMatch) &&
						networkArea.RegExMatches(_environmentWatchElement.NetworkAreas) &&
						!networkArea.RegExMatches(_environmentWatchElement.NetworkAreasNotMatch);
				}).ToArray();

			var watches = new List<FileWatchElement>();
			foreach (var serverNode in serversFiltered)
			{
				var serverName = serverNode.Element("Name").Value;
				var serverNetworkArea = serverNode.Element("NetworkArea").Value;

				foreach (var serviceNode in serverNode.Descendants("Services").Descendants("Entity"))
				{
					var serviceName = serviceNode.Element("Name").Value;

					if (!serviceName.RegExMatches(_environmentWatchElement.ServiceNames) ||
							serviceName.RegExMatches(_environmentWatchElement.ServiceNamesNotMatch))
						continue;

					var fields = new FieldCollection
					{
						new FieldElement { Key = "host", Value = serverName },
						new FieldElement { Key = "service", Value = serviceName },
					};
					foreach (FieldElement field in _environmentWatchElement.Fields)
					{
						fields.Add(field);
					}

					var serviceLogFile = (string)serviceNode.Elements("LogPath").FirstOrDefault();
					var serviceLogType = (string)serviceNode.Elements("LogPathType").FirstOrDefault();
					var serviceLogFile1 = (string)serviceNode.Elements("LogPath1").FirstOrDefault();
					var serviceLogType1 = (string)serviceNode.Elements("LogPath1Type").FirstOrDefault();
					var serviceLogFile2 = (string)serviceNode.Elements("LogPath2").FirstOrDefault();
					var serviceLogType2 = (string)serviceNode.Elements("LogPath2Type").FirstOrDefault();

					AddFileWatchElementForLogFile(serviceLogFile, serviceLogType, watches, fields, serverNetworkArea, serverName,
						serviceName);
					AddFileWatchElementForLogFile(serviceLogFile1, serviceLogType1, watches, fields, serverNetworkArea, serverName,
						serviceName);
					AddFileWatchElementForLogFile(serviceLogFile2, serviceLogType2, watches, fields, serverNetworkArea, serverName,
						serviceName);
				}
			}

			return watches;
		}

		private void AddFileWatchElementForLogFile(string logFile, string logType, ICollection<FileWatchElement> watches,
			FieldCollection fields, string serverNetworkArea, string serverName, string serviceName)
		{
			//Don't ship logs without a type or with an empty type
			if (!string.IsNullOrEmpty(logType) && !string.IsNullOrWhiteSpace(logType))
			{
				var newWatch = new FileWatchElement
				{
					Files = logFile,
					Type = logType,
					Fields = fields
				};

				var overrideConfig = FindOverride(serviceName);
				if (overrideConfig != null)
				{
					newWatch.CloseWhenIdle = overrideConfig.CloseWhenIdle;
					if (overrideConfig.CustomNxlogConfig != null)
					{
						newWatch.CustomNxlogConfig = new CustomNxlogConfig
							{
								Value = overrideConfig.CustomNxlogConfig.Value
							};
{'fraction_non_alphanumeric': 0.0998792158320171, 'fraction_numerical': 0.0035306141410387437, 'mean_word_length': 4.086956521739131, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5758170', 'n_tokens_mistral': 3844, 'n_tokens_neox': 3064, 'n_words': 653}