<commit_before><?php
namespace HtmlParser;

use tidyNode;
use tidy;
/**
 * Copyright (c) 2013, 俊杰Jerry
 * All rights reserved.
 * @description: html解析器
 * @author: 俊杰Jerry<<EMAIL>>
 * @date: 2013-6-10
 * @version: 1.0
 */
class Parser {

	private $tidy_node = null;
	private $find_rs = array();

	/**
	 * @param tidyNode|string $tidy_node
	 */
	public function __construct($tidy_node = null){
		if(!function_exists('tidy_parse_string')){
			exit('tidy模块未加载');
		}
		if($tidy_node !== null){
			if($tidy_node instanceof tidyNode){
				$this->tidy_node = $tidy_node;
			}else{
				$this->parseStr($tidy_node);
			}
		}
	}

	public function __destruct(){
		$this->clearNode($this->tidy_node);
	}

	public function __get($name){
		if(isset($this->tidy_node->attribute [$name])){
			return $this->tidy_node->attribute [$name];
		}
		if(isset($this->tidy_node->$name)){
			return $this->tidy_node->$name;
		}
		return false;
	}

	/**
	 * 使用tidy解析html
	 * @param $str
	 * @param array $config_options
	 * @param string $encoding
	 */
	public function parseStr($str, $config_options = array(), $encoding = 'utf8'){
		$str = $this->remove_html ( $str, "'<!--(.*?)-->'is" );
// 		$str = $this->remove_html ( $str, "'<!\[CDATA\[(.*?)\]\]>'is" );
		$str = $this->remove_html ( $str, "'<\s*script[^>]*[^/]>(.*?)<\s*/\s*script\s*>'is" );
		$str = $this->remove_html ( $str, "'<\s*script\s*>(.*?)<\s*/\s*script\s*>'is" );
		$str = $this->remove_html ( $str, "'<\s*style[^>]*[^/]>(.*?)<\s*/\s*style\s*>'is" );
		$str = $this->remove_html ( $str, "'<\s*style\s*>(.*?)<\s*/\s*style\s*>'is" );
// 		$str = $this->remove_html ( $str, "'<\s*(?:code)[^>]*>(.*?)<\s*/\s*(?:code)\s*>'is" );
		/* 		$str = $this->remove_html ( $str, "'(<\?)(.*?)(\?>)'s" );
				 $str = $this->remove_html ( $str, "'(\{\w)(.*?)(\})'s" );*/
		$tidy = new tidy();
		$tidy->parseString($str, $config_options, $encoding);
		$this->tidy_node = $tidy->html();
	}

	/**
	 * 广度优先查询
	 * @param string $selector
	 * @param number $idx 找第几个,从0开始计算，null 表示都返回, 负数表示倒数第几个
	 * @return Parser|array
	 */
	public function find($selector, $idx = null){
		if(empty($this->tidy_node->child)){
			return false;
		}
		$selectors = $this->parse_selector ( $selector );
		if (($count = count ( $selectors )) === 0){
			return false;
		}
		$found = array();
		for($c = 0; $c < $count; $c ++) {
			if (($level = count ( $selectors [$c] )) === 0){
				return false;
			}
			$need_to_search = $this->tidy_node->child;
			$search_level = 1;
			while (!empty($need_to_search)){
				$temp = array();
				foreach ($need_to_search as $search){
					if($search_level >= $level){
						$rs = $this->seek ($search, $selectors [$c], $level - 1 );
						if($rs !== false && $idx !== null){
							if($idx == count($found)){
								return new Parser($rs);
							}else{
								$found[] = new Parser($rs);
							}
						}elseif($rs !== false){
							$found[] = new Parser($rs);
						}
					}
					$temp[] = $search;
					array_shift($need_to_search);
				}
				foreach ($temp as $temp_val){
					if(!empty($temp_val->child)){
						foreach ($temp_val->child as $key => $val){
							$temp_val->child[$key]->parent = $temp_val;
							$need_to_search[] = $temp_val->child[$key];
						}
					}
				}
				$search_level++;
			}
		}
		if($idx !== null){
			if($idx < 0){
				$idx = count($found) + $idx;
			}
			if(isset($found[$idx])){
				return $found[$idx];
			}else{
				return false;
			}
		}
		return $found;
	}


	/**
	 * 深度优先查询
	 * @param string $selector
	 * @param number $idx 找第几个,从0开始计算，null 表示都返回, 负数表示倒数第几个
	 * @return Parser|array
	 */
	public function find2($selector, $idx = null){
		if(empty($this->tidy_node->child)){
			return false;
		}
		$selectors = $this->parse_selector ( $selector );
		if (($count = count ( $selectors )) === 0){
			return false;
		}
		for($c = 0; $c < $count; $c ++) {
			if (($level = count ( $selectors [$c] )) === 0){
				return false;
			}
			$this->search($this->tidy_node, $idx, $selectors [$c], $level);
		}
		$found = $this->find_rs;
		$this->find_rs = array();
		if($idx !== null){
			if($idx < 0){
				$idx = count($found) + $idx;
			}
			if(isset($found[$idx])){
				return $found[$idx];
			}else{
				return false;
			}
		}
		return $found;
	}

	/**
	 * 返回文本信息
	 * @return string, mixed
	 */
	public function getPlainText(){
		return $this->text($this->tidy_node);
	}

	/**
	 * 深度查询
	 * @param $search
	 * @param $idx
	 * @param $selectors
	 * @param $level
	 * @param int $search_levle
	 * @return bool
	 */
	private function search(&$search, $idx, $selectors, $level, $search_levle = 0){
		if($search_levle >= $level){
			$rs = $this->seek ($search, $selectors , $level - 1 );
			if($rs !== false && $idx !== null){
				if($idx == count($this->find_rs)){
					$this->find_rs[] = new Parser($rs);
					return true;
				}else{
					$this->find_rs[] = new Parser($rs);
				}
			}elseif($rs !== false){
				$this->find_rs[] = new Parser($rs);
			}
		}
		if(!empty($search->child)){<commit_msg>add getAttr function<commit_after><?php
namespace HtmlParser;

use tidyNode;
use tidy;
/**
 * Copyright (c) 2013, 俊杰Jerry
 * All rights reserved.
 * @description: html解析器
 * @author: 俊杰Jerry<<EMAIL>>
 * @date: 2013-6-10
 * @version: 1.0
 */
class Parser {

	private $tidy_node = null;
	private $find_rs = array();

	/**
	 * @param tidyNode|string $tidy_node
	 */
	public function __construct($tidy_node = null){
		if(!function_exists('tidy_parse_string')){
			exit('tidy模块未加载');
		}
		if($tidy_node !== null){
			if($tidy_node instanceof tidyNode){
				$this->tidy_node = $tidy_node;
			}else{
				$this->parseStr($tidy_node);
			}
		}
	}

	public function __destruct(){
		$this->clearNode($this->tidy_node);
	}

	public function __get($name){
		if(isset($this->tidy_node->attribute [$name])){
			return $this->tidy_node->attribute [$name];
		}
		if(isset($this->tidy_node->$name)){
			return $this->tidy_node->$name;
		}
		return false;
	}

	/**
	 * 使用tidy解析html
	 * @param $str
	 * @param array $config_options
	 * @param string $encoding
	 */
	public function parseStr($str, $config_options = array(), $encoding = 'utf8'){
		$str = $this->remove_html ( $str, "'<!--(.*?)-->'is" );
// 		$str = $this->remove_html ( $str, "'<!\[CDATA\[(.*?)\]\]>'is" );
		$str = $this->remove_html ( $str, "'<\s*script[^>]*[^/]>(.*?)<\s*/\s*script\s*>'is" );
		$str = $this->remove_html ( $str, "'<\s*script\s*>(.*?)<\s*/\s*script\s*>'is" );
		$str = $this->remove_html ( $str, "'<\s*style[^>]*[^/]>(.*?)<\s*/\s*style\s*>'is" );
		$str = $this->remove_html ( $str, "'<\s*style\s*>(.*?)<\s*/\s*style\s*>'is" );
// 		$str = $this->remove_html ( $str, "'<\s*(?:code)[^>]*>(.*?)<\s*/\s*(?:code)\s*>'is" );
		/* 		$str = $this->remove_html ( $str, "'(<\?)(.*?)(\?>)'s" );
				 $str = $this->remove_html ( $str, "'(\{\w)(.*?)(\})'s" );*/
		$tidy = new tidy();
		$tidy->parseString($str, $config_options, $encoding);
		$this->tidy_node = $tidy->html();
	}

	/**
	 * 广度优先查询
	 * @param string $selector
	 * @param number $idx 找第几个,从0开始计算，null 表示都返回, 负数表示倒数第几个
	 * @return Parser|array
	 */
	public function find($selector, $idx = null){
		if(empty($this->tidy_node->child)){
			return false;
		}
		$selectors = $this->parse_selector ( $selector );
		if (($count = count ( $selectors )) === 0){
			return false;
		}
		$found = array();
		for($c = 0; $c < $count; $c ++) {
			if (($level = count ( $selectors [$c] )) === 0){
				return false;
			}
			$need_to_search = $this->tidy_node->child;
			$search_level = 1;
			while (!empty($need_to_search)){
				$temp = array();
				foreach ($need_to_search as $search){
					if($search_level >= $level){
						$rs = $this->seek ($search, $selectors [$c], $level - 1 );
						if($rs !== false && $idx !== null){
							if($idx == count($found)){
								return new Parser($rs);
							}else{
								$found[] = new Parser($rs);
							}
						}elseif($rs !== false){
							$found[] = new Parser($rs);
						}
					}
					$temp[] = $search;
					array_shift($need_to_search);
				}
				foreach ($temp as $temp_val){
					if(!empty($temp_val->child)){
						foreach ($temp_val->child as $key => $val){
							$temp_val->child[$key]->parent = $temp_val;
							$need_to_search[] = $temp_val->child[$key];
						}
					}
				}
				$search_level++;
			}
		}
		if($idx !== null){
			if($idx < 0){
				$idx = count($found) + $idx;
			}
			if(isset($found[$idx])){
				return $found[$idx];
			}else{
				return false;
			}
		}
		return $found;
	}


	/**
	 * 深度优先查询
	 * @param string $selector
	 * @param number $idx 找第几个,从0开始计算，null 表示都返回, 负数表示倒数第几个
	 * @return Parser|array
	 */
	public function find2($selector, $idx = null){
		if(empty($this->tidy_node->child)){
			return false;
		}
		$selectors = $this->parse_selector ( $selector );
		if (($count = count ( $selectors )) === 0){
			return false;
		}
		for($c = 0; $c < $count; $c ++) {
			if (($level = count ( $selectors [$c] )) === 0){
				return false;
			}
			$this->search($this->tidy_node, $idx, $selectors [$c], $level);
		}
		$found = $this->find_rs;
		$this->find_rs = array();
		if($idx !== null){
			if($idx < 0){
				$idx = count($found) + $idx;
			}
			if(isset($found[$idx])){
				return $found[$idx];
			}else{
				return false;
			}
		}
		return $found;
	}

	/**
	 * 返回文本信息
	 * @return string, mixed
	 */
	public function getPlainText(){
		return $this->text($this->tidy_node);
	}
	
	/**
	 * 获取html的元属值
	 * @return string|bool
	 */
	public function getAttr($name) {
		return isset($this->tidy_node->attribute [$name]) ? $this->tidy_node->attribute [$name] : false;
	}

	/**
	 * 深度查询
	 * @param $search
	 * @param $idx
	 * @param $selectors
	 * @param $level
	 * @param int $search_levle
	 * @return bool
	 */
	private function search(&$search, $idx, $selectors, $level, $search_levle = 0){
		if($search_levle >= $level){
			$rs = $this->seek ($search, $selectors , $level - 1 );
			if($rs !== false && $idx !== null){
				if($idx == count($this->find_rs)){
					$this->find_rs[] = new Parser($rs);
					return true;
				}else{
					$this->find_rs[] = new Parser($rs);
				}
			}elseif($rs !== false){
				$this->find_rs[] = new Parser($rs);
			}
		}
		if(!empty($search->child)){
{'fraction_non_alphanumeric': 0.23528830963665087, 'fraction_numerical': 0.005726698262243286, 'mean_word_length': 3.124185667752443, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 149, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6149918', 'n_tokens_mistral': 4734, 'n_tokens_neox': 4029, 'n_words': 751}