<commit_before><?php

/**
* 
* Parse for URLS in the source text.
* 
* @category Text
* 
* @package Text_Wiki
* 
* @author <NAME> <<EMAIL>>
* 
* @license LGPL
* 
* @version $Id$
* 
*/

/**
* 
* Parse for URLS in the source text.
* 
* Various URL markings are supported: inline (the URL by itself),
* numbered or footnote reference (where the URL is enclosed in square
* brackets), and named reference (where the URL is enclosed in square
* brackets and has a name included inside the brackets).  E.g.:
*
* inline    -- http://example.com or www.example.com
* numbered  -- [http://example.com]
* described -- [http://example.com Example Description]
*
* When rendering a URL token, this will convert URLs pointing to a .gif,
* .jpg, or .png image into an inline <img /> tag (for the 'xhtml'
* format).
*
* Token options are:
* 
* 'type' => ['inline'|'footnote'|'descr'] the type of URL
* 
* 'href' => the URL link href portion
* 
* 'text' => the displayed text of the URL link
* 
* @category Text
* 
* @package Text_Wiki
* 
* @author <NAME> <<EMAIL>>
* 
*/

class Text_Wiki_Parse_Url extends Text_Wiki_Parse {

    /**
    * 
    * Keeps a running count of numbered-reference URLs.
    * 
    * @access public
    * 
    * @var int
    * 
    */
    
    public $footnoteCount = 0;

    /**
    * 
    * URL schemes recognized by this rule.
    * 
    * @access public
    * 
    * @var array
    * 
    */
    
    public $conf = array(
        'schemes' => array(
            'http://',
            'https://',
            'ftp://',
            'gopher://',
            'news://',
            'mailto:',
            'mms://'
        )
    );

    /**
    * 
    * Constructor.
    * 
    * We override the constructor so we can comment the regex nicely.
    * 
    * @access public
    * 
    */
    
    function Text_Wiki_Parse_Url(&$obj)
    {
        parent::Text_Wiki_Parse($obj);
        
        // convert the list of recognized schemes to a regex-safe string,
        // where the pattern delim is a slash
        $tmp = array();
        $list = $this->getConf('schemes', array());
        foreach ($list as $val) {
            $tmp[] = preg_quote($val, '/');
        }
        $schemes = implode('|', $tmp);
        
        // build the regex
        $this->regex =
            "(?:(?:$schemes)" . // allowed schemes
            "(?:" . // start pattern
            "[^ \\/\"\'{$this->wiki->delim}]*\\/" . // no spaces, backslashes, slashes, double-quotes, single quotes, or delimiters;
            ")*" . // end pattern
            "[^ \\t\\n\\/\"\'{$this->wiki->delim}]*" .
            "[A-Za-z%0-9\\/?=&~_])";
        $this->regexLiberal =
            "(?:(?:$schemes)" . // allowed schemes
            "(?:" . // start pattern
            "[^ \\/\"\'{$this->wiki->delim}]*\\/" . // no spaces, backslashes, slashes, double-quotes, single quotes, or delimiters;
            ")*" . // end pattern
            "[^ \\t\\n\\/\"{$this->wiki->delim}]*)";
            
    }

    /**
    * 
    * Find three different kinds of URLs in the source text.
    *
    * @access public
    * 
    */
    
    function parse()
    {
        // -------------------------------------------------------------
        // 
        // Described-reference (named) URLs.
        // 
        
        // the regular expression for this kind of URL
        $tmp_regex = '/\[(\*)?((?:' . $this->regexLiberal . ')|(?:#[a-zA-Z0-9_\-%]*)|(?:\/)) ([^\]'.$this->wiki->delim.']+)\]/';
        
        // use a custom callback processing method to generate
        // the replacement text for matches.
        $this->wiki->source = preg_replace_callback(
            $tmp_regex,
            array(&$this, 'processDescr'),
            $this->wiki->source
        );
        
        $postVars = $this->getConf("post_vars");
        
        if($postVars){
        		// enable %%foo%% variables to act in described links
        		$tmp_regex = 	'/\[(\*)?(%%[^%]+%%[^ ]*) ([^\]]+)\]/';
        		$this->wiki->source = preg_replace_callback(
	            $tmp_regex,
	            array(&$this, 'processPV'),
	            $this->wiki->source
	        );
        }

        // -------------------------------------------------------------
        // 
        // Normal inline URLs.
        // <commit_msg>* Allowed [/pagename/more-path linkname] local links to be processed in the source.<commit_after><?php

/**
* 
* Parse for URLS in the source text.
* 
* @category Text
* 
* @package Text_Wiki
* 
* @author <NAME> <<EMAIL>>
* 
* @license LGPL
* 
* @version $Id$
* 
*/

/**
* 
* Parse for URLS in the source text.
* 
* Various URL markings are supported: inline (the URL by itself),
* numbered or footnote reference (where the URL is enclosed in square
* brackets), and named reference (where the URL is enclosed in square
* brackets and has a name included inside the brackets).  E.g.:
*
* inline    -- http://example.com or www.example.com
* numbered  -- [http://example.com]
* described -- [http://example.com Example Description]
*
* When rendering a URL token, this will convert URLs pointing to a .gif,
* .jpg, or .png image into an inline <img /> tag (for the 'xhtml'
* format).
*
* Token options are:
* 
* 'type' => ['inline'|'footnote'|'descr'] the type of URL
* 
* 'href' => the URL link href portion
* 
* 'text' => the displayed text of the URL link
* 
* @category Text
* 
* @package Text_Wiki
* 
* @author <NAME> <<EMAIL>.net>
* 
*/

class Text_Wiki_Parse_Url extends Text_Wiki_Parse {

    /**
    * 
    * Keeps a running count of numbered-reference URLs.
    * 
    * @access public
    * 
    * @var int
    * 
    */
    
    public $footnoteCount = 0;

    /**
    * 
    * URL schemes recognized by this rule.
    * 
    * @access public
    * 
    * @var array
    * 
    */
    
    public $conf = array(
        'schemes' => array(
            'http://',
            'https://',
            'ftp://',
            'gopher://',
            'news://',
            'mailto:',
            'mms://'
        )
    );

    /**
    * 
    * Constructor.
    * 
    * We override the constructor so we can comment the regex nicely.
    * 
    * @access public
    * 
    */
    
    function Text_Wiki_Parse_Url(&$obj)
    {
        parent::Text_Wiki_Parse($obj);
        
        // convert the list of recognized schemes to a regex-safe string,
        // where the pattern delim is a slash
        $tmp = array();
        $list = $this->getConf('schemes', array());
        foreach ($list as $val) {
            $tmp[] = preg_quote($val, '/');
        }
        $schemes = implode('|', $tmp);
        
        // build the regex
        $this->regex =
            "(?:(?:$schemes)" . // allowed schemes
            "(?:" . // start pattern
            "[^ \\/\"\'{$this->wiki->delim}]*\\/" . // no spaces, backslashes, slashes, double-quotes, single quotes, or delimiters;
            ")*" . // end pattern
            "[^ \\t\\n\\/\"\'{$this->wiki->delim}]*" .
            "[A-Za-z%0-9\\/?=&~_])";
        $this->regexLiberal =
            "(?:(?:$schemes)" . // allowed schemes
            "(?:" . // start pattern
            "[^ \\/\"\'{$this->wiki->delim}]*\\/" . // no spaces, backslashes, slashes, double-quotes, single quotes, or delimiters;
            ")*" . // end pattern
            "[^ \\t\\n\\/\"{$this->wiki->delim}]*)";
            
    }

    /**
    * 
    * Find three different kinds of URLs in the source text.
    *
    * @access public
    * 
    */
    
    function parse()
    {
        // -------------------------------------------------------------
        // 
        // Described-reference (named) URLs.
        // 
        
        // the regular expression for this kind of URL
        $tmp_regex = '/\[(\*)?((?:' . $this->regexLiberal . ')|(?:#[a-zA-Z0-9_\-%]*)|(?:\/[^ \\t\\n\"\''.$this->wiki->delim.']*)) ([^\]'.$this->wiki->delim.']+)\]/';
        
        // use a custom callback processing method to generate
        // the replacement text for matches.
        $this->wiki->source = preg_replace_callback(
            $tmp_regex,
            array(&$this, 'processDescr'),
            $this->wiki->source
        );
        
        $postVars = $this->getConf("post_vars");
        
        if($postVars){
        		// enable %%foo%% variables to act in described links
        		$tmp_regex = 	'/\[(\*)?(%%[^%]+%%[^ ]*) ([^\]]+)\]/';
        		$this->wiki->source = preg_replace_callback(
	            $tmp_regex,
	            array(&$this, 'processPV'),
	            $this->wiki->source
	        );
        }

        // -------------------------------------------------------------
        // 
        // Normal inline URLs.
        // 
{'fraction_non_alphanumeric': 0.20142972443214574, 'fraction_numerical': 0.0011530035743110803, 'mean_word_length': 1.5215116279069767, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 73, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6098079', 'n_tokens_mistral': 2757, 'n_tokens_neox': 2636, 'n_words': 678}