<commit_before><?php
/**
 Copyright 2013 Myers Enterprises II

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

namespace com_brucemyers\MediaWiki;

use ChrisG\wikipedia;
use com_brucemyers\Util\FileCache;
use com_brucemyers\Util\Config;
use com_brucemyers\Util\Logger;
use Exception;
use DateTime;

/**
 * Wrapper for ChrisG's bot classes.
 */
class MediaWiki extends wikipedia
{
    const WIKIURLKEY = 'wiki.url';
    const WIKIUSERNAMEKEY = 'wiki.username';
    const WIKIPASSWORDKEY = '<PASSWORD>';
    const WIKIPAGEINCREMENT = 'wiki.pagefetchincrement';
    const WIKICHANGESINCREMENT = 'wiki.recentchangesincrement';
    const MAX_PAGE_SIZE = 2000000;

    static public $namespaces = array(
                    0 => 'Article',
                    1 => 'Article talk',
                    2 => 'User',
                    3 => 'User talk',
                    4 => 'Wikipedia',
                    5 => 'Wikipedia talk',
                    6 => 'File',
                    7 => 'File talk',
                    8 => 'MediaWiki',
                    9 => 'MediaWiki talk',
                    10 => 'Template',
                    11 => 'Template talk',
                    12 => 'Help',
                    13 => 'Help talk',
                    14 => 'Category',
                    15 => 'Category talk',
                    100 => 'Portal',
                    101 => 'Portal talk',
                    108 => 'Book',
                    109 => 'Book talk',
                    118 => 'Draft',
                    119 => 'Draft talk',
    				120 => 'Property',
    				121 => 'Property talk',
    				200 => 'Grants',
    				201 => 'Grants talk',
    				202 => 'Research',
    				203 => 'Research talk',
    				204 => 'Participation',
    				205 => 'Participation talk',
    				206 => 'Iberocoop',
    				207 => 'Iberocoop talk',
    				208 => 'Programs',
    				209 => 'Programs talk',
    				446 => 'Education Program',
                    447 => 'Education Program talk',
    				470 => 'Schema',
                    471 => 'Schema talk',
    				484 => 'Graph',
                    485 => 'Graph talk',
    				486 => 'Data',
                    487 => 'Data talk',
    				710 => 'TimedText',
                    711 => 'TimedText talk',
                    828 => 'Module',
                    829 => 'Module talk',
                    866 => 'CNBanner',
                    867 => 'CNBanner talk',
                    1198 => 'Translations',
                    1199 => 'Translations talk'
    );

    static $flipped_namespaces = false;

    /**
     * Constructor
     *
     * @param $url String mediawiki url
     */
    public function __construct($url)
    {
        parent::__construct($url);
        $this->http->quiet = true;
        curl_setopt($this->http->ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
    }

    /**
     * Login to mediawiki
     *
     * @param $username String username
     * @param $password String password
     */
    public function login($username, $password)
    {
    	static $saveusername = null;
    	static $savepassword = null;

    	if ($username == null && $password == null) {
    		$username = $saveusername;
    		$password = $savepassword;
    		$this->http->reset(); // Clear stored cookies
    	} else {
    		$saveusername = $username;
    		$savepassword = $password;
    	}

    	$post = array('lgname' => $username, 'lgpassword' => $password);
        $ret = $this->query('?action=login&format=php', $post);

        /* This is now required - see https://bugzilla.wikimedia.org/show_bug.cgi?id=23076 */
        if ($ret['login']['result'] == 'NeedToken') {
        	$post['lgtoken'] = $ret['login']['token'];
        	$ret = $this->query('?action=login&format=php', $post);
        }

        if ($ret['login']['result'] != 'Success') {
            throw new Exception('Login Error ' . print_r($ret, true));
        }
    }

    /**
     * Query mediawiki
     *
     * @param $query string query string
     * @param $post array Post data using key=>value
     * @param $repeat int Retry start value, max = 5
     * @return array Response
     */
    public function query($query, $post = null, $repeat = 0)
    {
        if ($post == null) {
            $ret = $this->http->get($this->url . $query);
        } else {
            $ret = $this->http->post($this->url . $query, $post);
        }

        $http_code = $this->http->http_code();

        $ok = false;
        if ($http_code == '200') $ok = true;
        elseif (strpos($query, 'action=edit') !== false && ($http_code == '504' || $http_code == '503')){
            return array(); // Proxy timeout on large edit requests
        }

		if (! $ok) {
			if ($repeat < 5) {
			    Logger::log("*** query retry #$repeat $query http_code:" . $http_code . ' errortext:' . $this->http->http_errortext());
			    sleep($repeat * 10);
				return $this->query($query, $post, ++$repeat);
			} else {
				throw new Exception('HTTP Error ' . $this->http->http_code());
			}
		}

        return unserialize($ret);
    }

    /**
     * Edits a page.
     * @param $page Page name to edit.
     * @param $data Data to post to page.
     * @param $summary Edit summary to use.
     * @param $minor Whether or not to mark edit as minor.  (Default false)
     * @param $bot Whether or not to mark edit as a bot edit.  (Default true)
     * @param $repeat int Retry start value, max = 5
     * @return api result
     **/
    public function edit($page, &$data, $summary = '', $minor = false, $bot = true, $section = null, $detectEC=false, $maxlag='', $repeat = 0) {
    	if ($this->token==null) {
    		$this->token = $this->getedittoken();
    	}
    	$params = array(
    			'title' => $page,
    			'text' => $data,
    			'token' => $this->token,
    			'summary' => $summary,
    			($minor?'minor':'notminor') => '1',
    			($bot?'bot':'notbot') => '1'
    	);
    	if ($section != null) {
    		$params['section'] = $section;
    	}
    	if ($this->ecTimestamp != null && $detectEC == true) {
    		$params['basetimestamp'] = $this->ecTimestamp;
    		$this->ecTimestamp = null;<commit_msg>MediaWiki - catch unserialize error<commit_after><?php
/**
 Copyright 2013 Myers Enterprises II

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

namespace com_brucemyers\MediaWiki;

use ChrisG\wikipedia;
use com_brucemyers\Util\FileCache;
use com_brucemyers\Util\Config;
use com_brucemyers\Util\Logger;
use Exception;
use DateTime;

/**
 * Wrapper for ChrisG's bot classes.
 */
class MediaWiki extends wikipedia
{
    const WIKIURLKEY = 'wiki.url';
    const WIKIUSERNAMEKEY = 'wiki.username';
    const WIKIPASSWORDKEY = '<PASSWORD>';
    const WIKIPAGEINCREMENT = 'wiki.pagefetchincrement';
    const WIKICHANGESINCREMENT = 'wiki.recentchangesincrement';
    const MAX_PAGE_SIZE = 2000000;

    static public $namespaces = array(
                    0 => 'Article',
                    1 => 'Article talk',
                    2 => 'User',
                    3 => 'User talk',
                    4 => 'Wikipedia',
                    5 => 'Wikipedia talk',
                    6 => 'File',
                    7 => 'File talk',
                    8 => 'MediaWiki',
                    9 => 'MediaWiki talk',
                    10 => 'Template',
                    11 => 'Template talk',
                    12 => 'Help',
                    13 => 'Help talk',
                    14 => 'Category',
                    15 => 'Category talk',
                    100 => 'Portal',
                    101 => 'Portal talk',
                    108 => 'Book',
                    109 => 'Book talk',
                    118 => 'Draft',
                    119 => 'Draft talk',
    				120 => 'Property',
    				121 => 'Property talk',
    				200 => 'Grants',
    				201 => 'Grants talk',
    				202 => 'Research',
    				203 => 'Research talk',
    				204 => 'Participation',
    				205 => 'Participation talk',
    				206 => 'Iberocoop',
    				207 => 'Iberocoop talk',
    				208 => 'Programs',
    				209 => 'Programs talk',
    				446 => 'Education Program',
                    447 => 'Education Program talk',
    				470 => 'Schema',
                    471 => 'Schema talk',
    				484 => 'Graph',
                    485 => 'Graph talk',
    				486 => 'Data',
                    487 => 'Data talk',
    				710 => 'TimedText',
                    711 => 'TimedText talk',
                    828 => 'Module',
                    829 => 'Module talk',
                    866 => 'CNBanner',
                    867 => 'CNBanner talk',
                    1198 => 'Translations',
                    1199 => 'Translations talk'
    );

    static $flipped_namespaces = false;

    /**
     * Constructor
     *
     * @param $url String mediawiki url
     */
    public function __construct($url)
    {
        parent::__construct($url);
        $this->http->quiet = true;
        curl_setopt($this->http->ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
    }

    /**
     * Login to mediawiki
     *
     * @param $username String username
     * @param $password String password
     */
    public function login($username, $password)
    {
    	static $saveusername = null;
    	static $savepassword = null;

    	if ($username == null && $password == null) {
    		$username = $saveusername;
    		$password = $<PASSWORD>;
    		$this->http->reset(); // Clear stored cookies
    	} else {
    		$saveusername = $username;
    		$savepassword = $password;
    	}

    	$post = array('lgname' => $username, 'lgpassword' => $password);
        $ret = $this->query('?action=login&format=php', $post);

        /* This is now required - see https://bugzilla.wikimedia.org/show_bug.cgi?id=23076 */
        if ($ret['login']['result'] == 'NeedToken') {
        	$post['lgtoken'] = $ret['login']['token'];
        	$ret = $this->query('?action=login&format=php', $post);
        }

        if ($ret['login']['result'] != 'Success') {
            throw new Exception('Login Error ' . print_r($ret, true));
        }
    }

    /**
     * Query mediawiki
     *
     * @param $query string query string
     * @param $post array Post data using key=>value
     * @param $repeat int Retry start value, max = 5
     * @return array Response
     */
    public function query($query, $post = null, $repeat = 0)
    {
        if ($post == null) {
            $ret = $this->http->get($this->url . $query);
        } else {
            $ret = $this->http->post($this->url . $query, $post);
        }

        $http_code = $this->http->http_code();

        $ok = false;
        if ($http_code == '200') $ok = true;
        elseif (strpos($query, 'action=edit') !== false && ($http_code == '504' || $http_code == '503')){
            return array(); // Proxy timeout on large edit requests
        }

		if (! $ok) {
			if ($repeat < 5) {
			    Logger::log("*** query retry #$repeat $query http_code:" . $http_code . ' errortext:' . $this->http->http_errortext());
			    sleep($repeat * 10);
				return $this->query($query, $post, ++$repeat);
			} else {
				throw new Exception('HTTP Error ' . $this->http->http_code());
			}
		}

		$retval = unserialize($ret);
		if ($retval === false) {
			throw new Exception("unserialize failed = $ret");
		}
        return $retval;
    }

    /**
     * Edits a page.
     * @param $page Page name to edit.
     * @param $data Data to post to page.
     * @param $summary Edit summary to use.
     * @param $minor Whether or not to mark edit as minor.  (Default false)
     * @param $bot Whether or not to mark edit as a bot edit.  (Default true)
     * @param $repeat int Retry start value, max = 5
     * @return api result
     **/
    public function edit($page, &$data, $summary = '', $minor = false, $bot = true, $section = null, $detectEC=false, $maxlag='', $repeat = 0) {
    	if ($this->token==null) {
    		$this->token = $this->getedittoken();
    	}
    	$params = array(
    			'title' => $page,
    			'text' => $data,
    			'token' => $this->token,
    			'summary' => $summary,
    			($minor?'minor':'notminor') => '1',
    			($bot?'bot':'notbot') => '1'
    	);
    	if ($section != null) {
    		$params['section'] = $section;
    	}
    	if ($this->ecTimestamp != null && $detectEC == true) {
    		$params['basetimestamp'] = $this->ecTimestamp;
    		$this->ecTimestamp = null;
{'fraction_non_alphanumeric': 0.14645445714712696, 'fraction_numerical': 0.024658148397220355, 'mean_word_length': 1.878279569892473, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 180, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '245589', 'n_tokens_mistral': 4429, 'n_tokens_neox': 3945, 'n_words': 1104}