<commit_before><?php
	
	class Extension_HTMLFormatter extends Extension {
		public function about() {
			return array(
				'name'			=> 'HTML Formatter',
				'version'		=> '2.0.1',
				'release-date'	=> '2009-06-26',
				'author'		=> array(
					'name'			=> '<NAME>',
					'website'		=> 'http://pixelcarnage.com/',
					'email'			=> '<EMAIL>'
				),
				'description'	=> 'Allows HTML to be used as input, safely.'
			);
		}
		
		public function format($source, $pretty = false) {
			header('content-type: text/plain');
			
			// Switch tabs for space:
			$this->reindent($source);
			
			// Tidy up input:
			$this->prepare($source);
			
			// Wrap stray text with paragraphs:
			$this->wrap($source);
			
			// Tidy again:
			$this->cleanup($source);
			
			// Make it pretty?
			if ($pretty) $this->pretty($source);
			
			// Remove body:
			$source = preg_replace(
				'/^<body>|<\/body>$/i', '', $source
			);
			
			$source = trim($source);
			
			//var_dump($source); exit;
			
			return $source;
		}
		
		protected function pretty(&$source) {
			$document = new DOMDocument('1.0', 'UTF-8');
			$document->loadXML($source);
			$xpath = new DOMXPath($document);
			$nodes = array();
			$results = $xpath->query('//dd | //h1 | //h2 | //h3 | //h4 | //h5 | //h6 | //li | //p');
			
			// Find nodes that may contain prettyable bits:
			foreach ($results as $node) {
				array_unshift($nodes, $node);
			}
			
			// Loop through the nodes, now in reverse order:
			foreach ($nodes as $node) {
				$content = '';
				
				// Find content:
				while ($node->hasChildNodes()) {
					$content .= $document->saveXML($node->firstChild);
					$node->removeChild($node->firstChild);
				}
				
				// Replace characters with special meaning:
				$search = array(
					'/(\w)\'(\w)|(\s)\'(\d+\w?)\b(?!\')/',				// apostrophe's
					'/(\S)\'(?=\s|[[:punct:]]|<|$)/',					// single closing
					'/\'/',												// single opening
					'/(\S)\"(?=\s|[[:punct:]]|<|$)/',					// double closing
					'/"/',												// double opening
					'/\b([A-Z][A-Z0-9]{2,})\b(?:[(]([^)]*)[)])/',		// 3+ acronym
					'/\b( )?\.{3}/',									// ellipsis
					'/--/',												// em dash
					'/-/',												// en dash
					'/(\d+)( ?)x( ?)(?=\d+)/',							// dimension sign
					'/\b ?[([]TM[])]/i',								// trademark
					'/\b ?[([]R[])]/i',									// registered
					'/\b ?[([]C[])]/i'									// copyright
				);
				
				$replace = array(
					'\1&#8217;\2',										// apostrophe's
					'\1&#8217;',										// single closing
					'&#8216;',											// single opening
					'\1&#8221;',										// double closing
					'&#8220;',											// double opening
					'<acronym title="\2">\1</acronym>',					// 3+ uppercase acronym
					'\1&#8230;',										// ellipsis
					'&#8212;',											// em dash
					'&#8211;',											// en dash
					'\1\2&#215;\3',										// dimension sign<commit_msg>Small fix for elipsis match.<commit_after><?php
	
	class Extension_HTMLFormatter extends Extension {
		public function about() {
			return array(
				'name'			=> 'HTML Formatter',
				'version'		=> '2.0.1',
				'release-date'	=> '2009-06-26',
				'author'		=> array(
					'name'			=> '<NAME>',
					'website'		=> 'http://pixelcarnage.com/',
					'email'			=> '<EMAIL>'
				),
				'description'	=> 'Allows HTML to be used as input, safely.'
			);
		}
		
		public function format($source, $pretty = false) {
			header('content-type: text/plain');
			
			// Switch tabs for space:
			$this->reindent($source);
			
			// Tidy up input:
			$this->prepare($source);
			
			// Wrap stray text with paragraphs:
			$this->wrap($source);
			
			// Tidy again:
			$this->cleanup($source);
			
			// Make it pretty?
			if ($pretty) $this->pretty($source);
			
			$source = preg_replace(
				'/^<body>|<\/body>$/i', '', $source
			);
			
			$source = trim($source);
			
			//var_dump($source); exit;
			
			return $source;
		}
		
		protected function pretty(&$source) {
			$document = new DOMDocument('1.0', 'UTF-8');
			$document->loadXML($source);
			$xpath = new DOMXPath($document);
			$nodes = array();
			$results = $xpath->query('//dd | //h1 | //h2 | //h3 | //h4 | //h5 | //h6 | //li | //p');
			
			// Find nodes that may contain prettyable bits:
			foreach ($results as $node) {
				array_unshift($nodes, $node);
			}
			
			// Loop through the nodes, now in reverse order:
			foreach ($nodes as $node) {
				$content = '';
				
				// Find content:
				while ($node->hasChildNodes()) {
					$content .= $document->saveXML($node->firstChild);
					$node->removeChild($node->firstChild);
				}
				
				// Replace characters with special meaning:
				$search = array(
					'/(\w)\'(\w)|(\s)\'(\d+\w?)\b(?!\')/',				// apostrophe's
					'/(\S)\'(?=\s|[[:punct:]]|<|$)/',					// single closing
					'/\'/',												// single opening
					'/(\S)\"(?=\s|[[:punct:]]|<|$)/',					// double closing
					'/"/',												// double opening
					'/\b([A-Z][A-Z0-9]{2,})\b(?:[(]([^)]*)[)])/',		// 3+ acronym
					'/\.{3}/',											// ellipsis
					'/--/',												// em dash
					'/-/',												// en dash
					'/(\d+)( ?)x( ?)(?=\d+)/',							// dimension sign
					'/\b ?[([]TM[])]/i',								// trademark
					'/\b ?[([]R[])]/i',									// registered
					'/\b ?[([]C[])]/i'									// copyright
				);
				
				$replace = array(
					'\1&#8217;\2',										// apostrophe's
					'\1&#8217;',										// single closing
					'&#8216;',											// single opening
					'\1&#8221;',										// double closing
					'&#8220;',											// double opening
					'<acronym title="\2">\1</acronym>',					// 3+ uppercase acronym
					'\1&#8230;',										// ellipsis
					'&#8212;',											// em dash
					'&#8211;',											// en dash
					'\1\2&#215;\3',										// dimension sign
{'fraction_non_alphanumeric': 0.23649584487534625, 'fraction_numerical': 0.02458448753462604, 'mean_word_length': 1.7962245885769603, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471079', 'n_tokens_mistral': 3012, 'n_tokens_neox': 2047, 'n_words': 416}