<commit_before><?php
/**
 * 奇书网www.qishu.com的爬虫
 * Created by PhpStorm.
 * User: zoowii
 * Date: 14-2-3
 * Time: 下午11:23
 */

namespace RP\SiteCrawl;

use Sunra\PhpSimple\HtmlDomParser;

class QishuCrawler
{
    public function search($name, $page = 1)
    {
        // p=1 表示页数
        $p = $page - 1;
        $url = "http://s.qisuu.com/cse/search?q=" . urlencode(trim($name)) . "&p=$p&s=2672242722776283010";
        $res = \RP\util\HttpClient::fetch_page('qishu', $url);
        $dom = HtmlDomParser::str_get_html($res);
        $elements = $dom->find('div.result');
        $result = array('data' => array(), 'paginator' => array());
        foreach ($elements as $ele) {
            $title = $ele->find('h3.c-title', 0)->find('a', 0)->innertext;
            $title = str_replace(' - txt全集下载,电子书 - 奇书网', '', $title);
            $title = str_replace('<em>', '', $title);
            $title = str_replace('</em>', '', $title);<commit_msg>cache crawler result<commit_after><?php
/**
 * 奇书网www.qishu.com的爬虫
 * Created by PhpStorm.
 * User: zoowii
 * Date: 14-2-3
 * Time: 下午11:23
 */

namespace RP\SiteCrawl;

use Sunra\PhpSimple\HtmlDomParser;

class QishuCrawler
{
    public function search($name, $page = 1)
    {
        // p=1 表示页数
        $p = $page - 1;
        $url = "http://s.qisuu.com/cse/search?q=" . urlencode(trim($name)) . "&p=$p&s=2672242722776283010";
        $cache = \RP\core\CCache::instance();
        $cache_key = "crawler_$url";
        $res = $cache->get($cache_key);
        if ($res === null) {
            $res = \RP\util\HttpClient::fetch_page('qishu', $url);
            $cache->set($cache_key, $res, 3600);
        }
        $dom = HtmlDomParser::str_get_html($res);
        $elements = $dom->find('div.result');
        $result = array('data' => array(), 'paginator' => array());
        foreach ($elements as $ele) {
            $title = $ele->find('h3.c-title', 0)->find('a', 0)->innertext;
            $title = str_replace(' - txt全集下载,电子书 - 奇书网', '', $title);
            $title = str_replace('<em>', '', $title);
            $title = str_replace('</em>', '', $title);
{'fraction_non_alphanumeric': 0.20579847908745247, 'fraction_numerical': 0.03326996197718631, 'mean_word_length': 2.5437710437710437, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7571436', 'n_tokens_mistral': 844, 'n_tokens_neox': 822, 'n_words': 137}