<commit_before><?php
	ini_set('max_execution_time', 0);
	ini_set('memory_limit', -1);

	require(__DIR__ . '/phpQuery/phpQuery.php');

	class WorkerThread extends Thread {
		private $source;

		public function __construct ($source) {
			$this->source = $source;
		}

		public function run () {
			$source = $this->source;
			file_put_contents(__DIR__ . '/temp/r.' . $source, json_encode($this->$source()));
		}

		private function chinatimes () {
			$url = 'http://www.chinatimes.com/syndication/rss';
			$tokens = explode('<li>即時新聞', file_get_contents($url));
			$token = trim($tokens[1]);
			$doc = phpQuery::newDocument(substr($token, 0, strpos($token, '</ul>') + 5));

			$map = array();

			foreach ($doc['.rssli'] as $li) {
				$li = pq($li);

				$map[] = array(
						'label' => $li['span']->eq(0)->text(),
						'url' => $li['a']->attr('href')
					);
			}

			return $map;
		}

		private function appledaily () {
			$url = 'http://www.appledaily.com.tw/rss';
			$doc = phpQuery::newDocument(file_get_contents($url));

			$map = array();

			foreach ($doc['.each_level'] as $section) {
				$section = pq($section);

				if (str_replace(' ', '', $section['header > h1 > a']->text()) !== '即時新聞總覽') {
					continue;
				}

				foreach ($section['ul li a'] as $anchor) {
					$anchor = pq($anchor);

					$map[] = array(
							'label' => $anchor->text(),
							'url' => 'http://www.appledaily.com.tw/' . $anchor->attr('href')
						);
				}
			}

			return $map;
		}

		private function libertytimes () {
			$doc = phpQuery::newDocument(file_get_contents('http://news.ltn.com.tw/service?p=8'));
			$map = array();

			foreach ($doc['.ltnrss tr']->slice(1) as $tr) {
				$tr = pq($tr);
				$td = $tr['td'];

				$map[] = array(
						'label' => $td->eq(0)->text(),
						'url' => $td->eq(2)->find('a')->attr('href')
					);<commit_msg>Fixed Apple Daily RSS<commit_after><?php
	ini_set('max_execution_time', 0);
	ini_set('memory_limit', -1);

	require(__DIR__ . '/phpQuery/phpQuery.php');

	class WorkerThread extends Thread {
		private $source;

		public function __construct ($source) {
			$this->source = $source;
		}

		public function run () {
			$source = $this->source;
			file_put_contents(__DIR__ . '/temp/r.' . $source, json_encode($this->$source()));
		}

		private function chinatimes () {
			$url = 'http://www.chinatimes.com/syndication/rss';
			$tokens = explode('<li>即時新聞', file_get_contents($url));
			$token = trim($tokens[1]);
			$doc = phpQuery::newDocument(substr($token, 0, strpos($token, '</ul>') + 5));

			$map = array();

			foreach ($doc['.rssli'] as $li) {
				$li = pq($li);

				$map[] = array(
						'label' => $li['span']->eq(0)->text(),
						'url' => $li['a']->attr('href')
					);
			}

			return $map;
		}

		private function appledaily () {
			$url = 'https://tw.appledaily.com/rss';
			$doc = phpQuery::newDocument(file_get_contents($url));

			$map = array();

			foreach ($doc['.each_level'] as $section) {
				$section = pq($section);

				if (str_replace(' ', '', $section['header > h1 > a']->text()) !== '即時新聞總覽') {
					continue;
				}

				foreach ($section['ul li a'] as $anchor) {
					$anchor = pq($anchor);

					$map[] = array(
							'label' => $anchor->text(),
							'url' => 'https://tw.appledaily.com/' . $anchor->attr('href')
						);
				}
			}

			return $map;
		}

		private function libertytimes () {
			$doc = phpQuery::newDocument(file_get_contents('http://news.ltn.com.tw/service?p=8'));
			$map = array();

			foreach ($doc['.ltnrss tr']->slice(1) as $tr) {
				$tr = pq($tr);
				$td = $tr['td'];

				$map[] = array(
						'label' => $td->eq(0)->text(),
						'url' => $td->eq(2)->find('a')->attr('href')
					);
{'fraction_non_alphanumeric': 0.23030961434003258, 'fraction_numerical': 0.005975013579576317, 'mean_word_length': 3.101336302895323, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1136677', 'n_tokens_mistral': 1681, 'n_tokens_neox': 1355, 'n_words': 245}