<commit_before><?php
/**
 * Created by PhpStorm.
 * User: 1
 * Date: 2017-07-07
 * Time: 13:47
 */

namespace app\src\crawler\action;


use app\src\base\helper\ResultHelper;
use app\src\base\helper\ValidateHelper;
use app\src\crawler\constants\CrawlerUrlType;
use app\src\crawler\logic\CrawlerUrlLogic;

class JavformeRssAction
{
    public function read($url){
        $xml = file_get_contents($url);
        $xml = simplexml_load_string($xml);
        $items = $xml->xpath("channel/item");
        $now = time();
        $allEntity = [];
        foreach ($items as $item){
            $url = ''.$item->link;
            $map = ['url'=>$url];
            $result = (new CrawlerUrlLogic())->getInfo($map);

            if(ValidateHelper::legalArrayResult($result) && $result['info']['url'] == $url) {
                //已经存在则不添加了
                continue;
            }
            array_push($allEntity,[
                'url'=>$url,
                'create_time'=>$now,
                'update_time'=>$now,
                'climb_status'=>0,
                'url_type'=>CrawlerUrlType::JAV_FOR_ME,
            ]);
        }
        if(count($allEntity) > 0){
            $result = (new CrawlerUrlLogic())->addAll($allEntity);
            return $result;
        }
        return ResultHelper::error('没有可插入的');
    }
}<commit_msg>【完善】增加视频演员关联表<commit_after><?php
/**
 * Created by PhpStorm.
 * User: 1
 * Date: 2017-07-07
 * Time: 13:47
 */

namespace app\src\crawler\action;


use app\src\base\helper\ResultHelper;
use app\src\base\helper\ValidateHelper;
use app\src\crawler\constants\CrawlerUrlType;
use app\src\crawler\logic\CrawlerUrlLogic;

class JavformeRssAction
{
    public function read($url){
        $xml = file_get_contents($url);
        $xml = simplexml_load_string($xml);
        $items = $xml->xpath("channel/item");
        $now = time();
        $allEntity = [];
        echo 'read count = '.count($items);
        foreach ($items as $item){
            $url = ''.$item->link;
            $map = ['url'=>$url];
            $result = (new CrawlerUrlLogic())->getInfo($map);

            if(ValidateHelper::legalArrayResult($result) && $result['info']['url'] == $url) {
                //已经存在则不添加了
                continue;
            }
            array_push($allEntity,[
                'url'=>$url,
                'create_time'=>$now,
                'update_time'=>$now,
                'climb_status'=>0,
                'url_type'=>CrawlerUrlType::JAV_FOR_ME,
            ]);
        }
        echo 'new url count = '.count($allEntity);
        if(count($allEntity) > 0){
            $result = (new CrawlerUrlLogic())->addAll($allEntity);
            return $result;
        }
        return ResultHelper::error('没有可插入的');
    }
}
{'fraction_non_alphanumeric': 0.1618874773139746, 'fraction_numerical': 0.010889292196007259, 'mean_word_length': 2.0219298245614037, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1211569', 'n_tokens_mistral': 933, 'n_tokens_neox': 912, 'n_words': 139}