<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
爬取topswim.net选定帖子指定层数
"""

from lxml import etree
from lxml.html import fromstring, tostring
import requests
import sys
import os
import datetime
from weasyprint import HTML, CSS


try:
    n_layer = int(sys.argv[2])
    tid = int(sys.argv[1])
    flag = int(sys.argv[3])
    if flag == 1:  # update
        FLAG_UPDATE = True
        FLAG_REBUILD = False
    elif flag == 2:  # rebuild
        FLAG_UPDATE = False
        FLAG_REBUILD = True
    else:
        print "NOT IMPLEMENTED"
        sys.exit(0)

except:
    print "[Usage] python topswim tid n_layer flag"


def download(tid, n_layer):
    n_page = 1
    new_html = ''
    header = u"""
    <html>
    <head>
    <title>游泳日报——来自topswim的精选——%s</title>
    <link href="css/style_1.css" type="text/css" rel="stylesheet">
    <link rel="shortcut icon" href="favicon.ico" >
    </head>
    <body>
    <nav id='topbar'>
    <a href="index.html">
    <img id="navtop"
    title="游泳日报" alt="游泳日报" src="image/swimnews.png">
    </a>
    <a href="http://reverland.org">
    <img id="author"
    title="reverland" alt="reverland's blog" src="image/author.png">
    </a>
    </nav>
    """
    # footer
    footer = u"""
    <a id='origin' href='%s'>在topswim上查看原文讨论</a>
    <a id='pdfdownload' href='pdf/%s.pdf'>保存为pdf格式</a>
    </body>
    <footer>© Reverland 2014</footer>
    </html>
    """
    while 1:
        url = "http://www.topswim.net/viewthread.php?tid="\
            + str(tid) + "&page=" + str(n_page)
        print "[I]parsing ", url
        r = requests.get(url)
        assert(r.content.find("charset=gbk"))
        # print "pages got " + str(n_page)
        if n_page == 1:
            title = fromstring(r.content).xpath('//h1/text()')[0].strip()
            new_html = header % title
        else:
            pass
        posts = fromstring(r.content).xpath('//td[@class="postcontent"]')
        # assert(len(posts) == 30)
        # only odd elements matters
        posts = posts[0::2]
        # assert(len(posts) == 15)

        for post in posts:
            postmessages = post.xpath(
                './div[@class="postmessage defaultpost"]')
            assert(len(postmessages) == 1)

            # 尝试下载图片
            images = postmessages[0].xpath('.//img')
            images = download_img(images)
            new_html += tostring(postmessages[0], encoding='unicode')
            if post.xpath(
                './div[@class="postinfo"]/strong/text()'
            )[0] == str(n_layer):
                new_html += footer % (unicode(url.split('&')[-2]),
                                      title)<commit_msg>	modified:   topswim.py 	modified:   html (modified content)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
爬取topswim.net选定帖子指定层数
"""

from lxml import etree
from lxml.html import fromstring, tostring
import requests
import sys
import os
import datetime
from weasyprint import HTML, CSS


try:
    n_layer = int(sys.argv[2])
    tid = int(sys.argv[1])
    flag = int(sys.argv[3])
    if flag == 1:  # update
        FLAG_UPDATE = True
        FLAG_REBUILD = False
    elif flag == 2:  # rebuild
        FLAG_UPDATE = False
        FLAG_REBUILD = True
    else:
        print "NOT IMPLEMENTED"
        sys.exit(0)

except:
    print "[Usage] python topswim tid n_layer flag"


def download(tid, n_layer):
    n_page = 1
    new_html = ''
    header = u"""
    <html>
    <head>
    <title>游泳日报——来自topswim的精选——%s</title>
    <link href="css/style_1.css" type="text/css" rel="stylesheet">
    <link rel="shortcut icon" href="favicon.ico" >
    </head>
    <body>
    <nav id='topbar'>
    <a href="index.html">
    <img id="navtop"
    title="游泳日报" alt="游泳日报" src="image/swimnews.png">
    </a>
    <a href="http://reverland.org">
    <img id="author"
    title="reverland" alt="reverland's blog" src="image/author.png">
    </a>
    </nav>
    """
    # footer
    footer = u"""
    <a id='origin' href='%s'>在topswim上查看原文讨论</a>
    <a id='pdfdownload' href='pdf/%s.pdf'>保存为pdf格式</a>
    <!-- 多说评论框 start -->
    <div class="ds-thread"
    data-thread-key="%s" data-title="%s" data-url="%s"></div>
    <!-- 多说评论框 end -->
    <!-- 多说公共JS代码 start (一个网页只需插入一次) -->
    <script type="text/javascript">
    var duoshuoQuery = {short_name:"reverland"};
    (function() {
    var ds = document.createElement('script');
    ds.type = 'text/javascript';ds.async = true;
    ds.src = (document.location.protocol ==
    'https:' ? 'https:' : 'http:') + '//static.duoshuo.com/embed.js';
    ds.charset = 'UTF-8';
    (document.getElementsByTagName('head')[0]
     || document.getElementsByTagName('body')[0]).appendChild(ds);
    })();
    </script>
<!-- 多说公共JS代码 end -->
    </body>
    <footer>© Reverland 2014</footer>
    </html>
    """
    while 1:
        url = "http://www.topswim.net/viewthread.php?tid="\
            + str(tid) + "&page=" + str(n_page)
        print "[I]parsing ", url
        r = requests.get(url)
        assert(r.content.find("charset=gbk"))
        # print "pages got " + str(n_page)
        if n_page == 1:
            title = fromstring(r.content).xpath('//h1/text()')[0].strip()
            new_html = header % title
        else:
            pass
        posts = fromstring(r.content).xpath('//td[@class="postcontent"]')
        # assert(len(posts) == 30)
        # only odd elements matters
        posts = posts[0::2]
        # assert(len(posts) == 15)

        for post in posts:
            postmessages = post.xpath(
                './div[@class="postmessage defaultpost"]')
            assert(len(postmessages) == 1)

            # 尝试下载图片
            images = postmessages[0].xpath('.//img')
            images = download_img(images)
            new_html += tostring(postmessages[0], encoding='unicode')
            if post.xpath(
                './div[@class="postinfo"]/strong/text()'
            )[0] == str(n_layer):
                c_url = "http://swim.reverland.org/" + title + '.html'
                new_html += footer % (unicode(url.split('&')[-2]),
                                      title,
                                      tid,
                                      title,
                                      c_url)
{'fraction_non_alphanumeric': 0.16663993584603048, 'fraction_numerical': 0.00946271050521251, 'mean_word_length': 2.0749506903353057, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2787265', 'n_tokens_mistral': 2238, 'n_tokens_neox': 2188, 'n_words': 417}