<commit_before># -*- coding: utf-8 -*-


if __name__ == '__main__':
    from lxml.html import parse as parse_html
    from lxml.html import tostring as tostring_html
    from urlparse import urlparse
    import sys

    url = sys.argv[1]
    url_obj = urlparse(url)
    base_url = url_obj.scheme + '://' + url_obj.hostname + '/' + ('/'.join(url_obj.path.split('/')[:-1]))
    target_filename = url_obj.path.split('/')[-2] + '.html'

    print 'Base URL:', base_url
    print 'TOC URL:', url

    data = {}

    toc_doc = parse_html(url)

    title_el = toc_doc.xpath('//div[@id="ct_title"]/h1')[0]
    data['title'] = title_el.text
    data['author'] = title_el.getchildren() and title_el.getchildren()[0].text or 'Anonymous'



    chapters = []
    data['chapters'] = chapters
    for el in toc_doc.xpath('//div[@id="catalog_list"]/ul/li/a'):
        ch_url = el.attrib.get('href')
        if ch_url.startswith('http://vip'):
            continue
        print 'Fetching', ch_url
        ch_data = {}
        ch_data['title'] = el.text
        ch_doc = parse_html(base_url + '/' + ch_url)
        ch_data['content'] = ''.join([tostring_html(p) for p in ch_doc.xpath('//div[@id="zjcontentdiv"]/p')])
        chapters.append(ch_data)

    from jinja2 import Environment, FileSystemLoader
    env = Environment(loader=FileSystemLoader('.'))
    template = env.get_template('novel.html')
    open(target_filename, 'w').write(template.render(data).encode('utf-8'))

# Local Variables: **
# comment-column: 56 **
# indent-tabs-mode: nil **
# python-indent: 4 **
# End: **<commit_msg>Use urllib2 to fetch html,  then parse them to lxml<commit_after># -*- coding: utf-8 -*-

from urllib2 import urlopen, Request

def fetch_html(url):
    req = Request(url)
    req.add_header('user-agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:29.0) Gecko/20100101 Firefox/29.0')
    resp = urlopen(req, None, 10)
    if resp.code != 200:
        raise "HTTP " + resp.code
    return resp


if __name__ == '__main__':
    from lxml.html import parse as parse_html
    from lxml.html import tostring as tostring_html
    from urlparse import urlparse
    import sys

    url = sys.argv[1]
    url_obj = urlparse(url)
    base_url = url_obj.scheme + '://' + url_obj.hostname + '/' + ('/'.join(url_obj.path.split('/')[:-1]))
    target_filename = url_obj.path.split('/')[-2] + '.html'

    print 'Base URL:', base_url
    print 'TOC URL:', url

    data = {}

    toc_doc = parse_html(fetch_html(url))

    title_el = toc_doc.xpath('//div[@id="ct_title"]/h1')[0]
    data['title'] = title_el.text
    data['author'] = title_el.getchildren() and title_el.getchildren()[0].text or 'Anonymous'



    chapters = []
    data['chapters'] = chapters
    for el in toc_doc.xpath('//div[@id="catalog_list"]/ul/li/a'):
        ch_url = el.attrib.get('href')
        if ch_url.startswith('http://vip'):
            continue
        print 'Fetching', ch_url
        ch_data = {}
        ch_data['title'] = el.text
        ch_doc = parse_html(fetch_html(base_url + '/' + ch_url))
        ch_data['content'] = ''.join([tostring_html(p) for p in ch_doc.xpath('//div[@id="zjcontentdiv"]/p')])
        chapters.append(ch_data)

    from jinja2 import Environment, FileSystemLoader
    env = Environment(loader=FileSystemLoader('.'))
    template = env.get_template('novel.html')
    open(target_filename, 'w').write(template.render(data).encode('utf-8'))

# Local Variables: **
# comment-column: 56 **
# indent-tabs-mode: nil **
# python-indent: 4 **
# End: **
{'fraction_non_alphanumeric': 0.17088427637190787, 'fraction_numerical': 0.015069661643446119, 'mean_word_length': 3.1146198830409357, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7066147', 'n_tokens_mistral': 1286, 'n_tokens_neox': 1242, 'n_words': 246}