<commit_before><?php

/* 
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

class GS_Module_Spider extends GS_Module_Base{
    public function run(){ 
        $aParam = $this->_aParam["query"];
        $oModule = new GS_Module($this->_aParam['business'], "Entity", "spider", "gets",$aParam);        
        $aSpider = $oModule->run();
        if(!empty($this->_aParam["query"]["data"])){
            $sSpiderUrl = $this->parseUrlByParam($aSpider[0]->spider_url);
        }else{
            $sSpiderUrl = $aSpider[0]->spider_url;
        }
        $aData = $this->getData($sSpiderUrl,$aSpider[0]->keyword_rule);
        if(!empty($aData[0])){
            $this->saveData($aData,$aSpider[0]->save_rule);
        }
    }   
    
    private function parseUrlByParam($sSpiderUrl){
        $aParams = $this->_aParam["query"];
        $sSpiderUrl = Util_DataType::replace($sSpiderUrl, $aParams["data"]);
        unset($aParams["data"]);
        $sSpiderUrl = Util_DataType::replace($sSpiderUrl, $aParams);  
        $sSpiderUrl = Util_DataType::replaceDate($sSpiderUrl); 
        return $sSpiderUrl;
    }
    
    private function getData($sSpiderUrl,$sRule){
        $aContent = Util_Curl::execute($sSpiderUrl);
        $aRules = explode("|", $sRule);
        $sPrev = "";
        foreach($aRules as $sRuleOption){
            $this->parseRule($sRuleOption,$aContent,$sPrev);
        }
        return $aContent;        
    }
    
    private function saveData($aData,$sSaveRule){        
        $aRules = explode(":", $sSaveRule);
        if($aRules[0] == "table"){
            $aFields = explode(",", $aRules[2]);
            foreach($aFields as $row){
                $aOption = explode("@",$row);    
                $sField = $aOption[0];
                $sIndex = $aOption[1];
                if(strstr($sIndex,"{")){
                    $this->_aParam["query"][$sField] = Util_DataType::replace($sIndex, $this->_aParam["query"]["data"]);
                }else{   
                    $this->_aParam["query"][$sField] = $aData[$sIndex];
                    if(!empty($aOption[2])){    
                        $aOption[2] = str_replace("-", "@", $aOption[2]);
                        $sPrev = "";
                        $this->parseRule($aOption[2], $this->_aParam["query"][$sField], $sPrev);
                    }
                }                   
            }      
            $oModule = new GS_Module($this->_aParam['business'], "Entity", $aRules[1], "updateOrInsert",$this->_aParam["query"]);        
            $oModule->run();
        }     
    }
    
    private function parseRule($sRuleOption,&$aContent,&$sPrev){
        $aOption = explode("@",$sRuleOption);
        if($aOption[0] == "strpos"){
            if(empty($aOption[2])){
                $sPrev = strpos($aContent,$aOption[1],0);
            }else{
                $sPrev = strpos($aContent,$aOption[1],0)+$aOption[2];
            }
        }elseif($aOption[0] == "reg"){
            preg_match_all(addslashes($aOption[1]), $aContent, $aMatchs);
            $aContent = substr($aMatchs[0][0],1,-1);
            return "";
        }elseif($aOption[0] == "substr"){
            if($aOption[1] == "start"){
                $aContent = substr($aContent,$sPrev);
                return "";
            }elseif($aOption[1] == "0" and $aOption[2] == "end"){
                $aContent = substr($aContent,0,$sPrev);
                return "";
            }else{
                $aContent = substr($aContent,$aOption[1],$aOption[2]);
                return "";
            }
        }elseif($aOption[0] == "str_replace"){
            $aContent = str_replace($aOption[1], $aOption[2], $aContent);
            return "";
        }elseif($aOption[0] == "strtolower"){
            $aContent = strtolower($aContent);
            return "";
        }elseif($aOption[0] == "explode"){
            $aContent = explode($aOption[1],$aContent);
            return "";
        }
    }
    
}<commit_msg>修改spider<commit_after><?php

/* 
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

class GS_Module_Spider extends GS_Module_Base{
    public function run(){ 
        $aParam = $this->_aParam["query"];
        $oModule = new GS_Module($this->_aParam['business'], "Entity", "spider", "gets",$aParam);        
        $aSpider = $oModule->run();
        if(!empty($this->_aParam["query"]["data"])){
            $sSpiderUrl = $this->parseUrlByParam($aSpider[0]->spider_url);
        }else{
            $sSpiderUrl = $aSpider[0]->spider_url;
        }
        $aData = $this->getData($sSpiderUrl,$aSpider[0]->keyword_rule);
        if(!empty($aData[0])){
            if(strstr($aSpider[0]->save_rule,":more:")){
                foreach($aData as $row){
                    $this->saveData(array($row),$aSpider[0]->save_rule);
                }
            }else{
                $this->saveData($aData,$aSpider[0]->save_rule);
            }
        }
    }   
    
    private function parseUrlByParam($sSpiderUrl){
        $aParams = $this->_aParam["query"];
        $sSpiderUrl = Util_DataType::replace($sSpiderUrl, $aParams["data"]);
        unset($aParams["data"]);
        $sSpiderUrl = Util_DataType::replace($sSpiderUrl, $aParams);  
        $sSpiderUrl = Util_DataType::replaceDate($sSpiderUrl); 
        return $sSpiderUrl;
    }
    
    private function getData($sSpiderUrl,$sRule){
        $aContent = Util_Curl::execute($sSpiderUrl);
        $aRules = explode("|", $sRule);
        $sPrev = "";
        foreach($aRules as $sRuleOption){
            $this->parseRule($sRuleOption,$aContent,$sPrev);
        }
        return $aContent;        
    }
    
    private function saveData($aData,$sSaveRule){        
        $aRules = explode(":", $sSaveRule);
        if($aRules[0] == "table"){
            $aFields = explode(",", $aRules[3]);
            foreach($aFields as $row){
                $aOption = explode("@",$row);    
                $sField = $aOption[0];
                $sIndex = $aOption[1];
                if(strstr($sIndex,"{")){
                    $this->_aParam["query"][$sField] = Util_DataType::replace($sIndex, $this->_aParam["query"]["data"]);
                }else{   
                    $this->_aParam["query"][$sField] = $aData[$sIndex];
                    if(!empty($aOption[2])){    
                        $aOption[2] = str_replace("-", "@", $aOption[2]);
                        $sPrev = "";
                        $this->parseRule($aOption[2], $this->_aParam["query"][$sField], $sPrev);
                    }
                }                   
            }      
            $oModule = new GS_Module($this->_aParam['business'], "Entity", $aRules[1], "updateOrInsert",$this->_aParam["query"]);        
            $oModule->run();
        }     
    }
    
    private function parseRule($sRuleOption,&$aContent,&$sPrev){
        $aOption = explode("@",$sRuleOption);
        if($aOption[0] == "strpos"){
            if(empty($aOption[2])){
                $sPrev = strpos($aContent,$aOption[1],0);
            }else{
                $sPrev = strpos($aContent,$aOption[1],0)+$aOption[2];
            }
        }elseif($aOption[0] == "reg"){
            preg_match_all(addslashes($aOption[1]), $aContent, $aMatchs);
            if(!empty($aOption[2])){                
                $aContent = $aMatchs[0][$aOption[2]];
            }else{
                $aContent = $aMatchs[0];
            }
            return "";
        }elseif($aOption[0] == "substr"){
            if($aOption[1] == "start"){
                $aContent = substr($aContent,$sPrev);
                return "";
            }elseif($aOption[1] == "0" and $aOption[2] == "end"){
                $aContent = substr($aContent,0,$sPrev);
                return "";
            }else{
                $aContent = substr($aContent,$aOption[1],$aOption[2]);
                return "";
            }
        }elseif($aOption[0] == "str_replace"){
            $aContent = str_replace($aOption[1], $aOption[2], $aContent);
            return "";
        }elseif($aOption[0] == "strtolower"){
            $aContent = strtolower($aContent);
            return "";
        }elseif($aOption[0] == "explode"){
            $aContent = explode($aOption[1],$aContent);
            return "";
        }
    }
    
}
{'fraction_non_alphanumeric': 0.17691136869219062, 'fraction_numerical': 0.009834913944502985, 'mean_word_length': 1.9263446385748544, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4654154', 'n_tokens_mistral': 2669, 'n_tokens_neox': 2634, 'n_words': 378}