<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Xml;

namespace Claymore.SharpMediaWiki
{
    public class Wiki
    {
        private CookieCollection _cookies;
        private readonly Uri _uri;
        private string _userAgent;
        private string _token;
        private DateTime _lastQueryTime;
        private bool _highLimits;
        private string _username;
        private bool _isBot;
        private int _sleepBetweenEdits;
        private int _sleepBetweenQueries;
        private Dictionary<string, int> _namespaces;
        private static readonly bool _isRunningOnMono = (Type.GetType("Mono.Runtime") != null);

        /// <summary>
        /// Initializes a new instance of the Wiki class with the specified URI.
        /// </summary>
        /// <param name="url">A URI string.</param>
        /// <example>Wiki wiki = new Wiki("http://en.wikipedia.org/w/");</example>
        /// <exception cref="System.ArgumentNullException" />
        /// <exception cref="System.UriFormatException" />
        public Wiki(string uri)
        {
            UriBuilder ub = new UriBuilder(uri);
            _uri = ub.Uri;
            _highLimits = false;
            _isBot = false;
            SleepBetweenEdits = 10;
            SleepBetweenQueries = 10;
            Version version = Assembly.GetExecutingAssembly().GetName().Version;
            _userAgent = string.Format("SharpMediaWiki/{0}.{1}",
                version.Major, version.Minor);
            _namespaces = new Dictionary<string, int>();
        }

        public int SleepBetweenEdits
        {
            set { _sleepBetweenEdits = Math.Max(2, value) * 1000; }
        }

        public int SleepBetweenQueries
        {
            set { _sleepBetweenQueries = Math.Max(2, value) * 1000; }
        }

        public Uri Uri
        {
            get { return _uri; }
        }

        public bool HighLimits
        {
            get { return _highLimits; }
        }

        public bool IsBot
        {
            get { return _isBot; }
        }

        public string Token
        {
            get { return _token; }
        }

        public string User
        {
            get { return _username; }
        }

        /// <summary>
        /// Logs into the MediaWiki as 'username' using 'password'.
        /// </summary>
        /// <param name="username">A username.</param>
        /// <param name="password">A password.</param>
        /// <exception cref="Claymore.SharpMediaWiki.WikiException">Thrown when an error occurs.</exception>
        /// <remarks><see cref="http://www.mediawiki.org/wiki/API:Login"/></remarks>
        public void Login(string username, string password)
        {
            if (string.IsNullOrEmpty(username))
            {
                throw new ArgumentException("Username shoudln't be empty.", "username");
            }
            if (string.IsNullOrEmpty(password))
            {
                throw new ArgumentException("Password shoudln't be empty.", "password");
            }
            if (_cookies != null && _cookies.Count > 0 && username == _username)
            {
                return;
            }
            ParameterCollection parameters = new ParameterCollection
            {
                { "lgname", username },
                { "lgpassword", password }
            };

            try
            {
                XmlDocument xml = MakeRequest(Action.Login, parameters);
                string result = xml.SelectSingleNode("//login").Attributes["result"].Value;
                if (result != "Success")
                {
                    throw new WikiException("Login failed, server returned '" + result + "'.");
                }
                _username = username;

                parameters = new ParameterCollection
                {
                    { "prop", "info" },
                    { "meta", "userinfo" },
                    { "uiprop", "rights" },
                    { "intoken", "edit" }
                };

                XmlDocument doc = Query(QueryBy.Titles, parameters, "Main Page");
                _highLimits = doc.SelectSingleNode("//rights[r=\"apihighlimits\"]/r") != null;
                _isBot = doc.SelectSingleNode("//rights[r=\"bot\"]/r") != null;
                XmlNode pageNode = doc.SelectSingleNode("//page");
                _token = pageNode != null ? pageNode.Attributes["edittoken"].Value : "";
            }
            catch (WebException e)
            {
                throw new WikiException("Login failed", e);
            }
        }

        public void Login()
        {
            try
            {
                ParameterCollection parameters = new ParameterCollection
                {
                    { "prop", "info" },
                    { "meta", "userinfo" },
                    { "uiprop", "rights" },
                    { "intoken", "edit" }
                };

                XmlDocument doc = Query(QueryBy.Titles, parameters, "Main Page");
                _highLimits = doc.SelectSingleNode("//rights[r=\"apihighlimits\"]/r") != null;
                _isBot = doc.SelectSingleNode("//rights[r=\"bot\"]/r") != null;
                XmlNode pageNode = doc.SelectSingleNode("//page");
                XmlNode userNode = doc.SelectSingleNode("//userinfo");
                _username = userNode != null ? userNode.Attributes["name"].Value : "";
                _token = pageNode != null ? pageNode.Attributes["edittoken"].Value : "";
            }
            catch (WebException e)
            {
                throw new WikiException("Login failed", e);
            }
        }

        /// <summary>
        /// Logs out from the Wiki.
        /// </summary>
        /// <exception cref="System.Net.ProtocolViolationException" />
        /// <exception cref="System.Net.WebException" />
        /// <remarks><see cref="http://www.mediawiki.org/wiki/API:Logout"/></remarks>
        public void Logout()
        {
            try
            {
                HttpWebRequest request = PrepareRequest();
                using (StreamWriter sw = new StreamWriter(request.GetRequestStream()))
                {
                    sw.Write("action=logout");
                }
                WebResponse response = request.GetResponse();
                response.Close();
                _username = "";
                _cookies = null;
            }
            catch (WebException e)
            {
                throw new WikiException("Logout failed", e);
            }
        }

        public XmlDocument Enumerate(ParameterCollection parameters, bool getAll)
        {
            XmlDocument result = new XmlDocument();
            string query = PrepareQuery(Action.Query, parameters);
            Parameter parameter = null;

            while (true)
            {
                try
                {
                    parameter = FillDocumentWithQueryResults(query, result);
                }
                catch (WebException e)
                {
                    throw new WikiException("Enumerating failed", e);
                }
                if (parameter == null || !getAll)
                {
                    break;
                }
                string limitName = parameter.Name.Substring(0, 2) + "limit";
                ParameterCollection localParameters = new ParameterCollection(parameters);
                localParameters.Set(parameter.Name, parameter.Value);
                localParameters.Set(limitName, "max");
                query = PrepareQuery(Action.Query, localParameters);
            }
            return result;
        }

        private void Enumerate(ParameterCollection parameters, XmlDocument result, bool all)
        {
            string query = PrepareQuery(Action.Query, parameters);
            Parameter parameter = null;

            while (true)
            {
                try
                {
                    parameter = FillDocumentWithQueryResults(query, result);
                }
                catch (WebException e)
                {
                    throw new WikiException("Query failed.", e);
                }

                if (parameter == null || !all)
                {
                    break;
                }
                string limitName = parameter.Name.Substring(0, 2) + "limit";
                ParameterCollection localParameters = new ParameterCollection(parameters);
                localParameters.Set(parameter.Name, parameter.Value);
                localParameters.Set(limitName, "max");
                query = PrepareQuery(Action.Query, localParameters);
            }
        }

        public XmlDocument Query(QueryBy queryBy,
            ParameterCollection parameters,
            string id)
        {
            return Query(queryBy, parameters, new string[] { id }, _highLimits ? 500 : 50, true);
        }

        public XmlDocument Query(QueryBy queryBy,
            ParameterCollection parameters,
            IEnumerable<string> ids)
        {
            return Query(queryBy, parameters, ids, _highLimits ? 500 : 50, true);
        }

        public XmlDocument Query(QueryBy queryBy,
            ParameterCollection parameters,
            IEnumerable<string> ids,
            int limit,
            bool all)
        {
            string keyword = "";
            switch (queryBy)
            {
                case QueryBy.IDs:
                    keyword = "pageids";
                    break;
                case QueryBy.Revisions:
                    keyword = "revids";
                    break;
                case QueryBy.Titles:
                    keyword = "titles";
                    break;
            }
            XmlDocument document = new XmlDocument();
            StringBuilder idsString = new StringBuilder();
            int index = 0;
            foreach (string id in ids)
            {
                if (index < limit)
                {
                    idsString.Append("|" + id);
                    ++index;
                }
                else
                {
                    idsString.Remove(0, 1);
                    ParameterCollection localParameters = new ParameterCollection(parameters);
                    localParameters.Add(keyword, idsString.ToString());
                    string query = PrepareQuery(Action.Query, localParameters);
                    Enumerate(localParameters, document, all);

                    index = 1;
                    idsString = new StringBuilder("|" + id);
                }
            }
            if (index > 0)
            {
                idsString.Remove(0, 1);
                ParameterCollection localParameters = new ParameterCollection(parameters);
                localParameters.Add(keyword, idsString.ToString());
                string query = PrepareQuery(Action.Query, localParameters);
                Enumerate(localParameters, document, all);
            }
            return document;
        }

        public string LoadText(string title)
        {
            if (string.IsNullOrEmpty(title))
            {
                throw new ArgumentException("Title shouldn't be empty.", "title");
            }
            UriBuilder ub = new UriBuilder(Uri);
            ub.Path = "/w/index.php";
            ub.Query = string.Format("title={0}&redirect=no&action=raw&ctype=text/plain&dontcountme=1",
                    Uri.EscapeDataString(title));
            try
            {
                return MakeRequest(ub.Uri, RequestMethod.Get);
            }
            catch (WebException e)
            {
                if (e.Status == WebExceptionStatus.ProtocolError &&
                    e.Message.Contains("(404)"))
                {
                    throw new WikiPageNotFound(title + " not found", e);
                }
                else
                {
                    throw new WikiException("Failed to load page " + title, e);
                }
            }
        }

        public string Append(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.NoCreate,
                        WatchFlags.None,
                        SaveFlags.Append,
                        true,
                        "",
                        "",
                        Token);
        }

        public string Prepend(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.NoCreate,
                        WatchFlags.None,
                        SaveFlags.Prepend,
                        true,
                        "",
                        "",
                        Token);
        }
        
        public string Create(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.CreateOnly,
                        WatchFlags.None,
                        SaveFlags.Replace,
                        true,
                        "",
                        "",
                        Token);
        }

        public string SaveSection(string title, string section, string text, string summary)
        {
            if (string.IsNullOrEmpty(section))
            {
                throw new ArgumentException("Section shouldn't be empty.", "section");
            }
            return Save(title,
                        section,
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.None,
                        WatchFlags.None,
                        SaveFlags.Replace,
                        true,
                        "",
                        "",
                        Token);
        }

        public string Save(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.None,
                        WatchFlags.None,
                        SaveFlags.Replace,<commit_msg>added a few convient functions<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Xml;

namespace Claymore.SharpMediaWiki
{
    public class Wiki
    {
        private CookieCollection _cookies;
        private readonly Uri _uri;
        private string _userAgent;
        private string _token;
        private DateTime _lastQueryTime;
        private bool _highLimits;
        private string _username;
        private bool _isBot;
        private int _sleepBetweenEdits;
        private int _sleepBetweenQueries;
        private Dictionary<string, int> _namespaces;
        private static readonly bool _isRunningOnMono = (Type.GetType("Mono.Runtime") != null);

        /// <summary>
        /// Initializes a new instance of the Wiki class with the specified URI.
        /// </summary>
        /// <param name="url">A URI string.</param>
        /// <example>Wiki wiki = new Wiki("http://en.wikipedia.org/w/");</example>
        /// <exception cref="System.ArgumentNullException" />
        /// <exception cref="System.UriFormatException" />
        public Wiki(string uri)
        {
            UriBuilder ub = new UriBuilder(uri);
            _uri = ub.Uri;
            _highLimits = false;
            _isBot = false;
            SleepBetweenEdits = 10;
            SleepBetweenQueries = 10;
            Version version = Assembly.GetExecutingAssembly().GetName().Version;
            _userAgent = string.Format("SharpMediaWiki/{0}.{1}",
                version.Major, version.Minor);
            _namespaces = new Dictionary<string, int>();
        }

        public int SleepBetweenEdits
        {
            set { _sleepBetweenEdits = Math.Max(2, value) * 1000; }
        }

        public int SleepBetweenQueries
        {
            set { _sleepBetweenQueries = Math.Max(2, value) * 1000; }
        }

        public Uri Uri
        {
            get { return _uri; }
        }

        public bool HighLimits
        {
            get { return _highLimits; }
        }

        public bool IsBot
        {
            get { return _isBot; }
        }

        public string Token
        {
            get { return _token; }
        }

        public string User
        {
            get { return _username; }
        }

        /// <summary>
        /// Logs into the MediaWiki as 'username' using 'password'.
        /// </summary>
        /// <param name="username">A username.</param>
        /// <param name="password">A password.</param>
        /// <exception cref="Claymore.SharpMediaWiki.WikiException">Thrown when an error occurs.</exception>
        /// <remarks><see cref="http://www.mediawiki.org/wiki/API:Login"/></remarks>
        public void Login(string username, string password)
        {
            if (string.IsNullOrEmpty(username))
            {
                throw new ArgumentException("Username shoudln't be empty.", "username");
            }
            if (string.IsNullOrEmpty(password))
            {
                throw new ArgumentException("Password shoudln't be empty.", "password");
            }
            if (_cookies != null && _cookies.Count > 0 && username == _username)
            {
                return;
            }
            ParameterCollection parameters = new ParameterCollection
            {
                { "lgname", username },
                { "lgpassword", password }
            };

            try
            {
                XmlDocument xml = MakeRequest(Action.Login, parameters);
                string result = xml.SelectSingleNode("//login").Attributes["result"].Value;
                if (result != "Success")
                {
                    throw new WikiException("Login failed, server returned '" + result + "'.");
                }
                _username = username;

                parameters = new ParameterCollection
                {
                    { "prop", "info" },
                    { "meta", "userinfo" },
                    { "uiprop", "rights" },
                    { "intoken", "edit" }
                };

                XmlDocument doc = Query(QueryBy.Titles, parameters, "Main Page");
                _highLimits = doc.SelectSingleNode("//rights[r=\"apihighlimits\"]/r") != null;
                _isBot = doc.SelectSingleNode("//rights[r=\"bot\"]/r") != null;
                XmlNode pageNode = doc.SelectSingleNode("//page");
                _token = pageNode != null ? pageNode.Attributes["edittoken"].Value : "";
            }
            catch (WebException e)
            {
                throw new WikiException("Login failed", e);
            }
        }

        public void Login()
        {
            try
            {
                ParameterCollection parameters = new ParameterCollection
                {
                    { "prop", "info" },
                    { "meta", "userinfo" },
                    { "uiprop", "rights" },
                    { "intoken", "edit" }
                };

                XmlDocument doc = Query(QueryBy.Titles, parameters, "Main Page");
                _highLimits = doc.SelectSingleNode("//rights[r=\"apihighlimits\"]/r") != null;
                _isBot = doc.SelectSingleNode("//rights[r=\"bot\"]/r") != null;
                XmlNode pageNode = doc.SelectSingleNode("//page");
                XmlNode userNode = doc.SelectSingleNode("//userinfo");
                _username = userNode != null ? userNode.Attributes["name"].Value : "";
                _token = pageNode != null ? pageNode.Attributes["edittoken"].Value : "";
            }
            catch (WebException e)
            {
                throw new WikiException("Login failed", e);
            }
        }

        /// <summary>
        /// Logs out from the Wiki.
        /// </summary>
        /// <exception cref="System.Net.ProtocolViolationException" />
        /// <exception cref="System.Net.WebException" />
        /// <remarks><see cref="http://www.mediawiki.org/wiki/API:Logout"/></remarks>
        public void Logout()
        {
            try
            {
                HttpWebRequest request = PrepareRequest();
                using (StreamWriter sw = new StreamWriter(request.GetRequestStream()))
                {
                    sw.Write("action=logout");
                }
                WebResponse response = request.GetResponse();
                response.Close();
                _username = "";
                _cookies = null;
            }
            catch (WebException e)
            {
                throw new WikiException("Logout failed", e);
            }
        }

        public XmlDocument Enumerate(ParameterCollection parameters, bool getAll)
        {
            XmlDocument result = new XmlDocument();
            string query = PrepareQuery(Action.Query, parameters);
            Parameter parameter = null;

            while (true)
            {
                try
                {
                    parameter = FillDocumentWithQueryResults(query, result);
                }
                catch (WebException e)
                {
                    throw new WikiException("Enumerating failed", e);
                }
                if (parameter == null || !getAll)
                {
                    break;
                }
                string limitName = parameter.Name.Substring(0, 2) + "limit";
                ParameterCollection localParameters = new ParameterCollection(parameters);
                localParameters.Set(parameter.Name, parameter.Value);
                localParameters.Set(limitName, "max");
                query = PrepareQuery(Action.Query, localParameters);
            }
            return result;
        }

        private void Enumerate(ParameterCollection parameters, XmlDocument result, bool all)
        {
            string query = PrepareQuery(Action.Query, parameters);
            Parameter parameter = null;

            while (true)
            {
                try
                {
                    parameter = FillDocumentWithQueryResults(query, result);
                }
                catch (WebException e)
                {
                    throw new WikiException("Query failed.", e);
                }

                if (parameter == null || !all)
                {
                    break;
                }
                string limitName = parameter.Name.Substring(0, 2) + "limit";
                ParameterCollection localParameters = new ParameterCollection(parameters);
                localParameters.Set(parameter.Name, parameter.Value);
                localParameters.Set(limitName, "max");
                query = PrepareQuery(Action.Query, localParameters);
            }
        }

        public XmlDocument Query(QueryBy queryBy,
            ParameterCollection parameters,
            string id)
        {
            return Query(queryBy, parameters, new string[] { id }, _highLimits ? 500 : 50, true);
        }

        public XmlDocument Query(QueryBy queryBy,
            ParameterCollection parameters,
            IEnumerable<string> ids)
        {
            return Query(queryBy, parameters, ids, _highLimits ? 500 : 50, true);
        }

        public XmlDocument Query(QueryBy queryBy,
            ParameterCollection parameters,
            IEnumerable<string> ids,
            int limit,
            bool all)
        {
            string keyword = "";
            switch (queryBy)
            {
                case QueryBy.IDs:
                    keyword = "pageids";
                    break;
                case QueryBy.Revisions:
                    keyword = "revids";
                    break;
                case QueryBy.Titles:
                    keyword = "titles";
                    break;
            }
            XmlDocument document = new XmlDocument();
            StringBuilder idsString = new StringBuilder();
            int index = 0;
            foreach (string id in ids)
            {
                if (index < limit)
                {
                    idsString.Append("|" + id);
                    ++index;
                }
                else
                {
                    idsString.Remove(0, 1);
                    ParameterCollection localParameters = new ParameterCollection(parameters);
                    localParameters.Add(keyword, idsString.ToString());
                    string query = PrepareQuery(Action.Query, localParameters);
                    Enumerate(localParameters, document, all);

                    index = 1;
                    idsString = new StringBuilder("|" + id);
                }
            }
            if (index > 0)
            {
                idsString.Remove(0, 1);
                ParameterCollection localParameters = new ParameterCollection(parameters);
                localParameters.Add(keyword, idsString.ToString());
                string query = PrepareQuery(Action.Query, localParameters);
                Enumerate(localParameters, document, all);
            }
            return document;
        }

        public string LoadText(string title)
        {
            if (string.IsNullOrEmpty(title))
            {
                throw new ArgumentException("Title shouldn't be empty.", "title");
            }
            UriBuilder ub = new UriBuilder(Uri);
            ub.Path = "/w/index.php";
            ub.Query = string.Format("title={0}&redirect=no&action=raw&ctype=text/plain&dontcountme=1",
                    Uri.EscapeDataString(title));
            try
            {
                return MakeRequest(ub.Uri, RequestMethod.Get);
            }
            catch (WebException e)
            {
                if (e.Status == WebExceptionStatus.ProtocolError &&
                    e.Message.Contains("(404)"))
                {
                    throw new WikiPageNotFound(title + " not found", e);
                }
                else
                {
                    throw new WikiException("Failed to load page " + title, e);
                }
            }
        }

        public string Append(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.NoCreate,
                        WatchFlags.None,
                        SaveFlags.Append,
                        true,
                        "",
                        "",
                        Token);
        }

        public string Append(string title, string text, string summary, MinorFlags minor, bool botEdit)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        minor,
                        CreateFlags.None,
                        WatchFlags.None,
                        SaveFlags.Append,
                        botEdit,
                        "",
                        "",
                        Token);
        }

        public string Prepend(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.NoCreate,
                        WatchFlags.None,
                        SaveFlags.Prepend,
                        true,
                        "",
                        "",
                        Token);
        }
        
        public string Create(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.CreateOnly,
                        WatchFlags.None,
                        SaveFlags.Replace,
                        true,
                        "",
                        "",
                        Token);
        }

        public string SaveSection(string title, string section, string text, string summary)
        {
            if (string.IsNullOrEmpty(section))
            {
                throw new ArgumentException("Section shouldn't be empty.", "section");
            }
            return Save(title,
                        section,
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.None,
                        WatchFlags.None,
                        SaveFlags.Replace,
                        true,
                        "",
                        "",
                        Token);
        }

        public string Save(string title, string text, string summary, MinorFlags minor, bool botEdit)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        minor,
                        CreateFlags.None,
                        WatchFlags.None,
                        SaveFlags.Replace,
                        botEdit,
                        "",
                        "",
                        Token);
        }

        public string Save(string title, string text, string summary)
        {
            return Save(title,
                        "",
                        text,
                        summary,
                        MinorFlags.None,
                        CreateFlags.None,
                        WatchFlags.None,
                        SaveFlags.Replace,
{'fraction_non_alphanumeric': 0.09258291982240793, 'fraction_numerical': 0.002807521546095586, 'mean_word_length': 1.15892592853619, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1702080', 'n_tokens_mistral': 7252, 'n_tokens_neox': 6696, 'n_words': 1569}