<commit_before>package com.zuoxiaolong.reptile;

/*
 * Copyright 2002-2015 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import com.zuoxiaolong.client.HttpClient;
import com.zuoxiaolong.client.HttpUriEnums;
import com.zuoxiaolong.config.Configuration;
import com.zuoxiaolong.dao.*;
import com.zuoxiaolong.model.Status;
import com.zuoxiaolong.model.Type;
import com.zuoxiaolong.orm.DaoFactory;
import com.zuoxiaolong.util.IOUtil;
import com.zuoxiaolong.util.ImageUtil;
import com.zuoxiaolong.util.JsoupUtil;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author 左潇龙
 * @since 2015年5月12日 下午9:24:21
 */
public abstract class Cnblogs {
	
	private static final Logger logger = Logger.getLogger(Cnblogs.class);
	
	private static final String article_url = "http://www.cnblogs.com/zuoxiaolong/default.html";
	private static final String digg_url = "http://www.cnblogs.com/zuoxiaolong/ajax/BlogPostInfo.aspx";
	private static final String category_url = "https://www.cnblogs.com/zuoxiaolong/ajax/CategoriesTags.aspx";
	private static final String comment_url = "https://www.cnblogs.com/zuoxiaolong/ajax/GetComments.aspx";

    private static void saveTagAndCategory(Integer id,String tagsHtml) {
        Document document = Jsoup.parse(tagsHtml);
        Element tagElement = document.getElementById("EntryTag");
        if (tagElement != null) {
            Elements elements = tagElement.getElementsByTag("a");
            if (elements != null) {
                for (Element element : elements) {
                    String tag = element.text().trim();
                    if (tag != null && tag.length() > 0 && !"NULL".equals(tag)) {
                        Integer tagId = HttpClient.get(Integer.class, HttpUriEnums.TAG_GET_ID, new String[]{"tag"}, tag);
                        if (tagId == null) {
                            HttpClient.get(HttpUriEnums.TAG_SAVE, new String[]{"tag"}, tag);
                        }
                        if (!HttpClient.get(Boolean.class, HttpUriEnums.ARTICLE_TAG_EXISTS, new String[]{"articleId", "tagId"}, id, tagId)) {
                            HttpClient.get(HttpUriEnums.ARTICLE_TAG_SAVE, new String[]{"articleId", "tagId"}, id, tagId);
                        }
                    }
                }
            }
        }
        Element categoryElement = document.getElementById("BlogPostCategory");
        if (categoryElement != null) {
            Elements elements = categoryElement.getElementsByTag("a");
            if (elements != null) {
                for (Element element : elements) {
                    String category = element.text().trim();
                    Integer categoryId = DaoFactory.getDao(CategoryDao.class).getId(category);
                    if (categoryId == null) {
                        categoryId = DaoFactory.getDao(CategoryDao.class).save(category);
                    }
                    if (!DaoFactory.getDao(ArticleCategoryDao.class).exsits(id, categoryId)) {
                        DaoFactory.getDao(ArticleCategoryDao.class).save(id, categoryId);
                    }
                }
            }
        }
    }
    
    private static void saveComment(Integer id, Integer postId) throws IOException {
    	String commentsUrl = comment_url + "?blogApp=zuoxiaolong&postId=" + postId + "&pageIndex=";
        for (int i = 1 ; ; i++ ) {
            String currentCommentUrl = commentsUrl + i;
            String commentsHtml = getArticleHtml(currentCommentUrl);
            Document commentsDocument = Jsoup.parse(commentsHtml);
            Elements commentElements = commentsDocument.getElementsByClass("feedbackItem");
            for (Element commentElement : commentElements) {
                String commentResourceId = commentElement.getElementsByClass("layer").first().attr("href");
                if (commentResourceId.startsWith("#")) {
                    commentResourceId = commentResourceId.substring(1);<commit_msg>polish<commit_after>package com.zuoxiaolong.reptile;

/*
 * Copyright 2002-2015 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import com.zuoxiaolong.client.HttpClient;
import com.zuoxiaolong.client.HttpUriEnums;
import com.zuoxiaolong.config.Configuration;
import com.zuoxiaolong.dao.*;
import com.zuoxiaolong.model.Status;
import com.zuoxiaolong.model.Type;
import com.zuoxiaolong.orm.DaoFactory;
import com.zuoxiaolong.util.IOUtil;
import com.zuoxiaolong.util.ImageUtil;
import com.zuoxiaolong.util.JsoupUtil;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author 左潇龙
 * @since 2015年5月12日 下午9:24:21
 */
public abstract class Cnblogs {
	
	private static final Logger logger = Logger.getLogger(Cnblogs.class);
	
	private static final String article_url = "http://www.cnblogs.com/zuoxiaolong/default.html";
	private static final String digg_url = "http://www.cnblogs.com/zuoxiaolong/ajax/BlogPostInfo.aspx";
	private static final String category_url = "https://www.cnblogs.com/zuoxiaolong/ajax/CategoriesTags.aspx";
	private static final String comment_url = "https://www.cnblogs.com/zuoxiaolong/ajax/GetComments.aspx";

    private static void saveTagAndCategory(Integer articleId,String tagsHtml) {
        Document document = Jsoup.parse(tagsHtml);
        Element tagElement = document.getElementById("EntryTag");
        if (tagElement != null) {
            Elements elements = tagElement.getElementsByTag("a");
            if (elements != null) {
                for (Element element : elements) {
                    String tag = element.text().trim();
                    if (tag != null && tag.length() > 0 && !"NULL".equals(tag)) {
                        Integer tagId = HttpClient.get(Integer.class, HttpUriEnums.TAG_GET_ID, new String[]{"tag"}, tag);
                        if (tagId == null) {
                            logger.info("tag save : " + tag + ", articleId : " + articleId + ",tagElement : " +  element);
                            HttpClient.get(HttpUriEnums.TAG_SAVE, new String[]{"tag"}, tag);
                        }
                        if (!HttpClient.get(Boolean.class, HttpUriEnums.ARTICLE_TAG_EXISTS, new String[]{"articleId", "tagId"}, articleId, tagId)) {
                            HttpClient.get(HttpUriEnums.ARTICLE_TAG_SAVE, new String[]{"articleId", "tagId"}, articleId, tagId);
                        }
                    }
                }
            }
        }
        Element categoryElement = document.getElementById("BlogPostCategory");
        if (categoryElement != null) {
            Elements elements = categoryElement.getElementsByTag("a");
            if (elements != null) {
                for (Element element : elements) {
                    String category = element.text().trim();
                    Integer categoryId = DaoFactory.getDao(CategoryDao.class).getId(category);
                    if (categoryId == null) {
                        categoryId = DaoFactory.getDao(CategoryDao.class).save(category);
                    }
                    if (!DaoFactory.getDao(ArticleCategoryDao.class).exsits(articleId, categoryId)) {
                        DaoFactory.getDao(ArticleCategoryDao.class).save(articleId, categoryId);
                    }
                }
            }
        }
    }
    
    private static void saveComment(Integer id, Integer postId) throws IOException {
    	String commentsUrl = comment_url + "?blogApp=zuoxiaolong&postId=" + postId + "&pageIndex=";
        for (int i = 1 ; ; i++ ) {
            String currentCommentUrl = commentsUrl + i;
            String commentsHtml = getArticleHtml(currentCommentUrl);
            Document commentsDocument = Jsoup.parse(commentsHtml);
            Elements commentElements = commentsDocument.getElementsByClass("feedbackItem");
            for (Element commentElement : commentElements) {
                String commentResourceId = commentElement.getElementsByClass("layer").first().attr("href");
                if (commentResourceId.startsWith("#")) {
                    commentResourceId = commentResourceId.substring(1);
{'fraction_non_alphanumeric': 0.10686561145276742, 'fraction_numerical': 0.0056457304163726185, 'mean_word_length': 2.6390315480557596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '142035', 'n_tokens_mistral': 2821, 'n_tokens_neox': 2688, 'n_words': 634}