<commit_before>package processor;

import model.ExtractField;
import model.ExpressionType;
import model.FieldSourceType;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.auth.UsernamePasswordCredentials;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.ProxyPool;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.selector.Selector;

import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Created by mian on 2017/1/12.
 * 通用页面抽取处理类
 * PageProcessor的定制分为三个部分，分别是爬虫的配置、页面元素的抽取和链接的发现
 */
public class CommonPageProcessor implements PageProcessor {
    private Site site;

    private String targetRequestCssSelector;
    private String targetRequestRegex;
    private List<ExtractField> extractFields;

    public CommonPageProcessor(){
        this.site = Site.me();
    }

    public void process(Page page) {

        // 部分二：定义如何抽取页面信息，并保存下来
        this.initExtractFields(page);

        // 部分三：从页面发现后续的url地址来抓取
        List<String> urls = this.initTargetRequests(page);
        page.addTargetRequests(urls);
    }

    public Site getSite() {
        return this.site;
    }

    private List<String> initTargetRequests(Page page){
        List<String> urls = null;
        if(StringUtils.isNotEmpty(this.targetRequestCssSelector) && StringUtils.isNotEmpty(this.targetRequestRegex)){
            urls = page.getHtml().css(this.targetRequestCssSelector).links().regex(this.targetRequestRegex).all();
        }

        if(StringUtils.isNotEmpty(this.targetRequestCssSelector)){
            urls = page.getHtml().css(this.targetRequestCssSelector).links().all();
        }

        if(StringUtils.isNotEmpty(this.targetRequestRegex)){
            urls = page.getHtml().links().regex(this.targetRequestRegex).all();
        }

        return urls;
    }

    private void initExtractFields(Page page){
        if(this.extractFields != null){
            for (ExtractField field : extractFields) {
                if(field != null){
                    String fieldName = field.getFieldName();
                    ExpressionType expressionType = field.getExpressionType();
                    FieldSourceType fieldSourceType = field.getFieldSourceType();
                    Selector selector = field.getSelector();

                    Selectable extractContent = null;
                    if(fieldSourceType == FieldSourceType.Html){
                        extractContent = page.getHtml();
                    }else if(fieldSourceType == FieldSourceType.Url){
                        extractContent = page.getUrl();
                    }else if(fieldSourceType == FieldSourceType.Json){
                        extractContent = page.getJson();
                    }

                    if(extractContent != null && selector != null) {
                        if (field.isMulti()) {
                            List<String> results = page.getHtml().selectDocumentForList(field.getSelector());
                            if (field.isNeed() && results.size() == 0) {
                                //如果是必须字段，字段内容为空的时候跳过这页面
                                page.setSkip(true);
                            } else {
                                page.getResultItems().put(field.getFieldName(), results);
                            }
                        }else{
                            String result = extractContent.select(selector).toString();
                            if (field.isNeed() && result == null) {
                                //如果是必须字段，字段内容为空的时候跳过这页面
                                page.setSkip(true);
                            } else {
                                if (expressionType != ExpressionType.JsonPath) {
                                    page.putField(fieldName, extractContent.select(selector).toString());
                                } else {

                                }
                            }
                        }
                    }
                }
            }
        }
    }

    public CommonPageProcessor setTargetRequests(String cssSelector, String urlRegex){
        this.targetRequestCssSelector = cssSelector;
        this.targetRequestRegex = urlRegex;<commit_msg>CommonPageProcessor<commit_after>package processor;

import model.ExtractField;
import model.ExpressionType;
import model.FieldSourceType;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpHost;
import org.apache.http.auth.UsernamePasswordCredentials;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.ProxyPool;
import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.webmagic.selector.Selector;

import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * Created by mian on 2017/1/12.
 * 通用页面抽取处理类
 * PageProcessor的定制分为三个部分，分别是爬虫的配置、页面元素的抽取和链接的发现
 */
public class CommonPageProcessor implements PageProcessor {
    private Site site;

    private String targetRequestCssSelector;
    private String targetRequestRegex;
    private List<ExtractField> extractFields;

    public CommonPageProcessor(){
        this.site = Site.me();
    }

    public void process(Page page) {

        // 部分二：定义如何抽取页面信息，并保存下来
        this.initExtractFields(page);

        // 部分三：从页面发现后续的url地址来抓取
        List<String> urls = this.initTargetRequests(page);
        page.addTargetRequests(urls);
    }

    public Site getSite() {
        return this.site;
    }

    private List<String> initTargetRequests(Page page){
        List<String> urls = null;
        if(StringUtils.isNotEmpty(this.targetRequestCssSelector) && StringUtils.isNotEmpty(this.targetRequestRegex)){
            urls = page.getHtml().css(this.targetRequestCssSelector).links().regex(this.targetRequestRegex).all();
        }

        if(StringUtils.isNotEmpty(this.targetRequestCssSelector)){
            urls = page.getHtml().css(this.targetRequestCssSelector).links().all();
        }

        if(StringUtils.isNotEmpty(this.targetRequestRegex)){
            urls = page.getHtml().links().regex(this.targetRequestRegex).all();
        }

        return urls;
    }

    private void initExtractFields(Page page){
        if(this.extractFields != null){
            for (ExtractField field : extractFields) {
                if(field != null){
                    String fieldName = field.getFieldName();
                    ExpressionType expressionType = field.getExpressionType();
                    FieldSourceType fieldSourceType = field.getFieldSourceType();
                    Selector selector = field.getSelector();

                    Selectable extractContent = null;
                    if(fieldSourceType == FieldSourceType.Html){
                        extractContent = page.getHtml();
                    }else if(fieldSourceType == FieldSourceType.Url){
                        extractContent = page.getUrl();
                    }else if(fieldSourceType == FieldSourceType.Json){
                        extractContent = page.getJson();
                    }

                    if(extractContent != null && selector != null) {
                        if (field.isMulti()) {
                            List<String> results = page.getHtml().selectDocumentForList(field.getSelector());
                            if (field.isNeed() && results.size() == 0) {
                                //如果是必须字段，字段内容为空的时候跳过这页面
                                page.setSkip(true);
                            } else {
                                page.getResultItems().put(field.getFieldName(), results);
                            }
                        }else{
                            String result = extractContent.select(selector).toString();
                            if (field.isNeed() && result == null) {
                                //如果是必须字段，字段内容为空的时候跳过这页面
                                page.setSkip(true);
                            } else {
                                if (expressionType != ExpressionType.JsonPath) {
                                    page.putField(fieldName, extractContent.select(selector).toString());
                                } else {
                                    String expressionValue = field.getExpressionValue();
                                    if(StringUtils.isNotEmpty(expressionValue)) {
                                        page.putField(fieldName, extractContent.jsonPath(expressionValue).get());
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    public CommonPageProcessor setTargetRequests(String cssSelector, String urlRegex){
        this.targetRequestCssSelector = cssSelector;
        this.targetRequestRegex = urlRegex;
{'fraction_non_alphanumeric': 0.09469613259668508, 'fraction_numerical': 0.0017679558011049724, 'mean_word_length': 1.7327898550724639, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '936112', 'n_tokens_mistral': 2412, 'n_tokens_neox': 2374, 'n_words': 367}