<commit_before><?php
/** 
* Arc90's Readability ported to PHP for FiveFilters.org
* Based on readability.js version 1.7.1 (without multi-page support)
* ------------------------------------------------------
* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
* Arc90's project URL: http://lab.arc90.com/experiments/readability/
* JS Source: http://code.google.com/p/arc90labs-readability
* Ported by: <NAME>, http://www.keyvan.net
* More information: http://fivefilters.org/content-only/
* License: Apache License, Version 2.0
* Requires: PHP5
* Date: 2011-07-22
* 
* Differences between the PHP port and the original
* ------------------------------------------------------
* Arc90's Readability is designed to run in the browser. It works on the DOM 
* tree (the parsed HTML) after the page's CSS styles have been applied and 
* Javascript code executed. This PHP port does not run inside a browser. 
* We use PHP's ability to parse HTML to build our DOM tree, but we cannot 
* rely on CSS or Javascript support. As such, the results will not always 
* match Arc90's Readability. (For example, if a web page contains CSS style 
* rules or Javascript code which hide certain HTML elements from display, 
* Arc90's Readability will dismiss those from consideration but our PHP port, 
* unable to understand CSS or Javascript, will not know any better.)
* 
* Another significant difference is that the aim of Arc90's Readability is 
* to re-present the main content block of a given web page so users can 
* read it more easily in their browsers. Correct identification, clean up, 
* and separation of the content block is only a part of this process. 
* This PHP port is only concerned with this part, it does not include code 
* that relates to presentation in the browser - Arc90 already do 
* that extremely well, and for PDF output there's FiveFilters.org's 
* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
* 
* Finally, this class contains methods that might be useful for developers 
* working on HTML document fragments. So without deviating too much from 
* the original code (which I don't want to do because it makes debugging 
* and updating more difficult), I've tried to make it a little more 
* developer friendly. You should be able to use the methods here on 
* existing DOMElement objects without passing an entire HTML document to 
* be parsed.
*/

// This class allows us to do JavaScript like assignements to innerHTML
require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');

// Alternative usage (for testing only!)
// uncomment the lines below and call Readability.php in your browser 
// passing it the URL of the page you'd like content from, e.g.:
// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php

/*
if (!isset($_GET['url']) || $_GET['url'] == '') {
	die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
}
$url = $_GET['url'];
if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
$html = file_get_contents($url);
$r = new Readability($html, $url);
$r->init();
echo $r->articleContent->innerHTML;
*/

	
class Readability
{
	/* constants */
	const FLAG_STRIP_UNLIKELYS = 1;
	const FLAG_WEIGHT_CLASSES = 2;
	const FLAG_CLEAN_CONDITIONALLY = 4;

	public $version = '1.7.1-without-multi-page';
	public $convertLinksToFootnotes = false;
	public $revertForcedParagraphElements = true;
	public $articleTitle;
	public $articleContent;
	public $dom;
	public $url = null; // optional - URL where HTML was retrieved
	public $debug = false;
	protected $body = null; // 
	protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
	protected $flags =  Self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4;   // Start with all flags set.
	protected $success = false; // indicates whether we were able to extract or not
	
	/**
	* All of the regular expressions in use within readability.
	* Defined up here so we don't instantiate them repeatedly in loops.
	**/
	public $regexps = array(
		'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i',
		'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
		'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i',
		'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
		'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i',
		'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
		'replaceFonts' => '/<(\/?)font[^>]*>/i',
		// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
		'normalize' => '/\s{2,}/',
		'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
		'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i',
		'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
	);	
	
	/**
	* Create instance of Readability
	* @param string UTF-8 encoded string
	* @param string (optional) URL associated with HTML (used for footnotes)
	*/
	function __construct($html, $url=null)
	{
		/* Turn all double br's into p's */<commit_msg>Correction erreur sur le Self<commit_after><?php
/** 
* Arc90's Readability ported to PHP for FiveFilters.org
* Based on readability.js version 1.7.1 (without multi-page support)
* ------------------------------------------------------
* Original URL: http://lab.arc90.com/experiments/readability/js/readability.js
* Arc90's project URL: http://lab.arc90.com/experiments/readability/
* JS Source: http://code.google.com/p/arc90labs-readability
* Ported by: <NAME>, http://www.keyvan.net
* More information: http://fivefilters.org/content-only/
* License: Apache License, Version 2.0
* Requires: PHP5
* Date: 2011-07-22
* 
* Differences between the PHP port and the original
* ------------------------------------------------------
* Arc90's Readability is designed to run in the browser. It works on the DOM 
* tree (the parsed HTML) after the page's CSS styles have been applied and 
* Javascript code executed. This PHP port does not run inside a browser. 
* We use PHP's ability to parse HTML to build our DOM tree, but we cannot 
* rely on CSS or Javascript support. As such, the results will not always 
* match Arc90's Readability. (For example, if a web page contains CSS style 
* rules or Javascript code which hide certain HTML elements from display, 
* Arc90's Readability will dismiss those from consideration but our PHP port, 
* unable to understand CSS or Javascript, will not know any better.)
* 
* Another significant difference is that the aim of Arc90's Readability is 
* to re-present the main content block of a given web page so users can 
* read it more easily in their browsers. Correct identification, clean up, 
* and separation of the content block is only a part of this process. 
* This PHP port is only concerned with this part, it does not include code 
* that relates to presentation in the browser - Arc90 already do 
* that extremely well, and for PDF output there's FiveFilters.org's 
* PDF Newspaper: http://fivefilters.org/pdf-newspaper/.
* 
* Finally, this class contains methods that might be useful for developers 
* working on HTML document fragments. So without deviating too much from 
* the original code (which I don't want to do because it makes debugging 
* and updating more difficult), I've tried to make it a little more 
* developer friendly. You should be able to use the methods here on 
* existing DOMElement objects without passing an entire HTML document to 
* be parsed.
*/

// This class allows us to do JavaScript like assignements to innerHTML
require_once(dirname(__FILE__).'/JSLikeHTMLElement.php');

// Alternative usage (for testing only!)
// uncomment the lines below and call Readability.php in your browser 
// passing it the URL of the page you'd like content from, e.g.:
// Readability.php?url=http://medialens.org/alerts/09/090615_the_guardian_climate.php

/*
if (!isset($_GET['url']) || $_GET['url'] == '') {
	die('Please pass a URL to the script. E.g. Readability.php?url=bla.com/story.html');
}
$url = $_GET['url'];
if (!preg_match('!^https?://!i', $url)) $url = 'http://'.$url;
$html = file_get_contents($url);
$r = new Readability($html, $url);
$r->init();
echo $r->articleContent->innerHTML;
*/

	
class Readability
{
	/* constants */
	const FLAG_STRIP_UNLIKELYS = 1;
	const FLAG_WEIGHT_CLASSES = 2;
	const FLAG_CLEAN_CONDITIONALLY = 4;

	public $version = '1.7.1-without-multi-page';
	public $convertLinksToFootnotes = false;
	public $revertForcedParagraphElements = true;
	public $articleTitle;
	public $articleContent;
	public $dom;
	public $url = null; // optional - URL where HTML was retrieved
	public $debug = false;
	protected $body = null; // 
	protected $bodyCache = null; // Cache the body HTML in case we need to re-use it later
	protected $flags =  self::FLAG_CLEAN_CONDITIONALLY; // 1 | 2 | 4;   // Start with all flags set.
	protected $success = false; // indicates whether we were able to extract or not
	
	/**
	* All of the regular expressions in use within readability.
	* Defined up here so we don't instantiate them repeatedly in loops.
	**/
	public $regexps = array(
		'unlikelyCandidates' => '/combx|comment|comments|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter/i',
		'okMaybeItsACandidate' => '/and|article|body|column|main|shadow/i',
		'positive' => '/article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|attachment/i',
		'negative' => '/combx|comment|comments|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget/i',
		'divToPElements' => '/<(a|blockquote|dl|div|ol|p|pre|table|ul)/i',
		'replaceBrs' => '/(<br[^>]*>[ \n\r\t]*){2,}/i',
		'replaceFonts' => '/<(\/?)font[^>]*>/i',
		// 'trimRe' => '/^\s+|\s+$/g', // PHP has trim()
		'normalize' => '/\s{2,}/',
		'killBreaks' => '/(<br\s*\/?>(\s|&nbsp;?)*){1,}/',
		'video' => '/http:\/\/(www\.)?(youtube|vimeo|dailymotion)\.com/i',
		'skipFootnoteLink' => '/^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i'
	);	
	
	/**
	* Create instance of Readability
	* @param string UTF-8 encoded string
	* @param string (optional) URL associated with HTML (used for footnotes)
	*/
	function __construct($html, $url=null)
	{
		/* Turn all double br's into p's */
{'fraction_non_alphanumeric': 0.1537734064686968, 'fraction_numerical': 0.011159447701910347, 'mean_word_length': 3.995276334435522, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2928110', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3277, 'n_words': 1098}