<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Runtime.Serialization.Formatters.Binary;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
using System.Web.Script.Serialization;

namespace PegBot
{
    class PluginUtils
    {
        public static class IrcConstants
        {
            public const char CtcpChar = '\x1';
            public const char IrcBold = '\x2';
            public const char IrcColor = '\x3';
            public const char IrcReverse = '\x16';
            public const char IrcNormal = '\xf';
            public const char IrcUnderline = '\x1f';
            public const char CtcpQuoteChar = '\x20';

        }

        public enum IrcColors
        {
            White = 0,
            Black = 1,
            Blue = 2,
            Green = 3,
            LightRed = 4,
            Brown = 5,
            Purple = 6,
            Orange = 7,
            Yellow = 8,
            LightGreen = 9,
            Cyan = 10,
            LightCyan = 11,
            LightBlue = 12,
            Pink = 13,
            Grey = 14,
            LightGrey = 15,
            Transparent = 99
        }

        public static void SaveObject(object o, string filename)
        {
            try
            {
                using (Stream stream = File.Open(filename, FileMode.Create))
                {
                    new BinaryFormatter().Serialize(stream, o);
                }
            }
            catch (IOException ex)
            {
                Log(ex.Message);
            }
        }

        public static object LoadObject(string filename)
        {
            using (Stream stream = File.Open(filename, FileMode.Open))
            {
                return new BinaryFormatter().Deserialize(stream);
            }
        }

        public static string DownloadWebPage(string url)
        {
            return DownloadWebPage(url, false);
        }

        public static string DownloadWebPage(string url, bool identifyAsBrowser)
        {
            string response = String.Empty;
            var oldCallback = ServicePointManager.ServerCertificateValidationCallback;
            try
            {
                using (WebClient w = new WebClient())
                {
                    ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate;
                    if(identifyAsBrowser)
                        w.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2";
                    response = w.DownloadString(url);
                }
            }
            catch (Exception e)<commit_msg>Added support for noscript redirect<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Runtime.Serialization.Formatters.Binary;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
using System.Web.Script.Serialization;

namespace PegBot
{
    class PluginUtils
    {
        public static class IrcConstants
        {
            public const char CtcpChar = '\x1';
            public const char IrcBold = '\x2';
            public const char IrcColor = '\x3';
            public const char IrcReverse = '\x16';
            public const char IrcNormal = '\xf';
            public const char IrcUnderline = '\x1f';
            public const char CtcpQuoteChar = '\x20';

        }

        public enum IrcColors
        {
            White = 0,
            Black = 1,
            Blue = 2,
            Green = 3,
            LightRed = 4,
            Brown = 5,
            Purple = 6,
            Orange = 7,
            Yellow = 8,
            LightGreen = 9,
            Cyan = 10,
            LightCyan = 11,
            LightBlue = 12,
            Pink = 13,
            Grey = 14,
            LightGrey = 15,
            Transparent = 99
        }

        public static void SaveObject(object o, string filename)
        {
            try
            {
                using (Stream stream = File.Open(filename, FileMode.Create))
                {
                    new BinaryFormatter().Serialize(stream, o);
                }
            }
            catch (IOException ex)
            {
                Log(ex.Message);
            }
        }

        public static object LoadObject(string filename)
        {
            using (Stream stream = File.Open(filename, FileMode.Open))
            {
                return new BinaryFormatter().Deserialize(stream);
            }
        }

        public static string DownloadWebPage(string url)
        {
            return DownloadWebPage(url, false);
        }

        public static string DownloadWebPage(string url, bool identifyAsBrowser)
        {
            string response = String.Empty;
            var metaRedirectRegex = new Regex(@"content=\""\d+\s*;\s*?url=(?<URL>(.*?))\""", RegexOptions.IgnoreCase);
            var oldCallback = ServicePointManager.ServerCertificateValidationCallback;
            try
            {
                using (WebClient w = new WebClient())
                {
                    ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate;
                    if (identifyAsBrowser)
                        w.Headers[HttpRequestHeader.UserAgent] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2";

                    for (int i = 0; i < 5; i++)
                    {
                        response = w.DownloadString(url);
                        var m = metaRedirectRegex.Match(response);
                        if (m.Success)
                        {
                            var redirect = m.Groups["URL"].Value;
                            if (url != redirect)
                            {
                                url = redirect;
                                continue;
                            }
                        }
                        break;
                    }
                }
            }
            catch (Exception e)
{'fraction_non_alphanumeric': 0.08255722178374113, 'fraction_numerical': 0.018626677190213103, 'mean_word_length': 1.3174835405998537, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '619367', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1540, 'n_words': 395}