<commit_before>﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace HackMaineIrcBot.Irc.Hooks
{
    [IRCHook("URL Helper","Fetches information on URL's")]
    [TypePriority(MemberPriority.Lowest)]
    class UrlHook : BaseIrcHook
    {
        const string urlMatcher = @"(https?)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&amp;%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&amp;%\$#\=~_\-\(\)]+))*";
        Regex urlEx = new Regex(@"(?nix).*(?<URL>\b(" + urlMatcher + "))+.*");

        protected override async Task<bool> Handle(BaseIrcResponder responder, ChannelMessageEventArgs e)
        {
            IEnumerable<string> found;
            if (FindUrl(e.Data.Message, out found))
            {
                bool handledAny = false;
                foreach (var url in found)
                {
                    string response = await GetPageTitle(url);
                    if (!string.IsNullOrWhiteSpace(response))
                    {
                        responder.SendResponseLine(" [{0}]", response);
                        handledAny = true;
                    }
                }
                return handledAny;
            }
            return false;
        }

        private async Task<string> GetPageTitle(string url)
        {
            string html;
            try
            {
                html = await GetHtml(url);
            }
            catch (System.Net.WebException ex)
            {
                return ex.Message;
            }
            catch (ArgumentException ex)
            {
                return ex.Message;
            }
            catch (FormatException ex)
            {
                return ex.Message;
            }
            var doc = ParseHtml(html);
            return GetDocumentTitle(doc);
        }

        private async Task<string> GetHtml(string url)
        {
            System.Net.HttpWebRequest request=null;
            System.Net.WebResponse response = null;
            request = System.Net.HttpWebRequest.CreateHttp(url);
            response = await request.GetResponseAsync();

            var header = response.Headers;
            if (header == null || !((string)header["Content-Type"]).StartsWith("text/html"))
                return null;

            string html;
            using (var sr = new System.IO.StreamReader(response.GetResponseStream()))
                html = await sr.ReadToEndAsync();

            return html;
        }

        internal static string GetDocumentTitle(HtmlDocument doc)
        {
            var title = doc.DocumentNode.SelectNodes("/html/head/title");
            if (title == null)
                return null;
            var titleLine= title.First().InnerText.Trim().Split(new char[] { '\n', '\r' }, 1, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
            return titleLine.Trim();
        }

        internal static HtmlDocument ParseHtml(string html)
        {
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            return doc;
        }

        internal bool CanHandle(string message)
        {
            IEnumerable<string> found;
            return FindUrl(message, out found);
        }

        internal bool FindUrl(string message, out IEnumerable<string> found)
        {
            found=null;
            var match = urlEx.Match(message);
            if(!match.Success) return false;
            var group = match.Groups["URL"];
            found = new List<string>(group.Captures.Cast<Capture>().Select(m=>m.Value));
            return true;
        }
    }
}<commit_msg>Url RegEx fix. Thanks Dan!<commit_after>﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace HackMaineIrcBot.Irc.Hooks
{
    [IRCHook("URL Helper", "Fetches information on URL's")]
    [TypePriority(MemberPriority.Lowest)]
    class UrlHook : BaseIrcHook
    {
        const string urlMatcher = @"(https?)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&amp;%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)?(/($|[a-zA-Z0-9\.\,\?\'\\\+&amp;%\$#\=~_\-\(\)]+))*";
        Regex urlEx = new Regex(@"(?nix)(?<URL>\b(" + urlMatcher + "))+");

        protected override async Task<bool> Handle(BaseIrcResponder responder, ChannelMessageEventArgs e)
        {
            IEnumerable<string> found;
            if (FindUrl(e.Data.Message, out found))
            {
                bool handledAny = false;
                foreach (var url in found)
                {
                    string response = await GetPageTitle(url);
                    if (!string.IsNullOrWhiteSpace(response))
                    {
                        responder.SendResponseLine(" [{0}]", response);
                        handledAny = true;
                    }
                }
                return handledAny;
            }
            return false;
        }

        private async Task<string> GetPageTitle(string url)
        {
            string html;
            try
            {
                html = await GetHtml(url);
            }
            catch (System.Net.WebException ex)
            {
                return ex.Message;
            }
            catch (ArgumentException ex)
            {
                return ex.Message;
            }
            catch (FormatException ex)
            {
                return ex.Message;
            }
            var doc = ParseHtml(html);
            return GetDocumentTitle(doc);
        }

        private async Task<string> GetHtml(string url)
        {
            System.Net.HttpWebRequest request = null;
            System.Net.WebResponse response = null;
            request = System.Net.HttpWebRequest.CreateHttp(url);
            response = await request.GetResponseAsync();

            var header = response.Headers;
            if (header == null || !((string)header["Content-Type"]).StartsWith("text/html"))
                return null;

            string html;
            using (var sr = new System.IO.StreamReader(response.GetResponseStream()))
                html = await sr.ReadToEndAsync();

            return html;
        }

        internal static string GetDocumentTitle(HtmlDocument doc)
        {
            var title = doc.DocumentNode.SelectNodes("/html/head/title");
            if (title == null)
                return null;
            var titleLine = title.First().InnerText.Trim().Split(new char[] { '\n', '\r' }, 1, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
            return titleLine.Trim();
        }

        internal static HtmlDocument ParseHtml(string html)
        {
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
            return doc;
        }

        internal bool CanHandle(string message)
        {
            IEnumerable<string> found;
            return FindUrl(message, out found);
        }

        internal bool FindUrl(string message, out IEnumerable<string> found)
        {
            found = null;
            var matches = urlEx.Matches(message);
            if (matches.Count < 1) return false;

            found = new List<string>(matches.Cast<Match>().Select(m => m.Value));
            return true;
        }
    }
}
{'fraction_non_alphanumeric': 0.15972562469377757, 'fraction_numerical': 0.026825085742283195, 'mean_word_length': 1.7603110209601083, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5511580', 'n_tokens_mistral': 2588, 'n_tokens_neox': 2472, 'n_words': 417}