<commit_before>#!python/bin/python3
import click
import requests
import html5lib
import os
import os.path as osp
from slugify import slugify_filename
from mimetypes import guess_extension
from time import sleep

headers = {
    'Host': 'pururin.com',
    'Referer': 'http://pururin.com/',
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:30.0) '
                   'Gecko/20100101 Firefox/30.0'),
}

ns = {
    'ns': 'http://www.w3.org/1999/xhtml',
}

x_name = '//ns:h1[@class="otitle"]/text()'
x_thumbs = ('//ns:a[starts-with(., "View all") and '
            'contains(., "thumbnails")]/@href')
x_pages = '//ns:ul[@class="thumblist"]/ns:li/ns:a/@href'
x_image = '//ns:a[@class="image-next"]/ns:img/@src'

path = osp.expanduser('~/Pururin')


def get_more(doc, sel):
    res = doc.xpath(sel, namespaces=ns)
    assert res
    return res


def get_one(doc, sel):
    res = get_more(doc, sel)[0]
    assert res
    return res


def rel_url(url):
    return 'http://pururin.com%s' % url


def page_fsname(fsname, i, ext):
    return '%s__%03d%s' % (fsname, i + 1, ext)


def has_page(fsname, i):
    for ext in ('.jpeg', '.png'):
        page_path = osp.join(path, fsname, page_fsname(fsname, i, ext))
        if osp.isfile(page_path):
            return page_path
    return False


@click.command()
@click.argument('url')
def cli(url):
    robot = requests.Session()
    click.echo('loading %s' % url)
    res = robot.get(url, headers=headers)
    assert res.status_code == 200

    html = html5lib.parse(res.text, treebuilder='lxml')
    name = get_one(html, x_name)
    click.echo('name: %s' % name)

    fsname = slugify_filename(name)
    save_path = osp.join(path, fsname)
    click.echo('save to: %s' % save_path)
    os.makedirs(save_path, exist_ok=True)

    thumbs_url = rel_url(get_one(html, x_thumbs))
    click.echo('pause')
    sleep(1)

    click.echo('loading %s' % thumbs_url)
    headers['Referer'] = url
    res = robot.get(thumbs_url, headers=headers)
    assert res.status_code == 200

    html = html5lib.parse(res.text, treebuilder='lxml')
    pages = list(map(rel_url, get_more(html, x_pages)))
    assert pages
    click.echo('pages: %d' % len(pages))

    headers['Referer'] = thumbs_url
    for i, p in enumerate(pages):
        page_path = has_page(fsname, i)
        if page_path:
            click.echo('has page: %s' % page_path)
            continue

        click.echo('pause')
        sleep(1)<commit_msg>--sleep<commit_after>#!python/bin/python3
import click
import requests
import html5lib
import os
import os.path as osp
from slugify import slugify_filename
from mimetypes import guess_extension
from time import sleep as _sleep

headers = {
    'Host': 'pururin.com',
    'Referer': 'http://pururin.com/',
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:30.0) '
                   'Gecko/20100101 Firefox/30.0'),
}

ns = {
    'ns': 'http://www.w3.org/1999/xhtml',
}

x_name = '//ns:h1[@class="otitle"]/text()'
x_thumbs = ('//ns:a[starts-with(., "View all") and '
            'contains(., "thumbnails")]/@href')
x_pages = '//ns:ul[@class="thumblist"]/ns:li/ns:a/@href'
x_image = '//ns:a[@class="image-next"]/ns:img/@src'

path = osp.expanduser('~/Pururin')


def get_more(doc, sel):
    res = doc.xpath(sel, namespaces=ns)
    assert res
    return res


def get_one(doc, sel):
    res = get_more(doc, sel)[0]
    assert res
    return res


def rel_url(url):
    return 'http://pururin.com%s' % url


def page_fsname(fsname, i, ext):
    return '%s__%03d%s' % (fsname, i + 1, ext)


def has_page(fsname, i):
    for ext in ('.jpeg', '.png'):
        page_path = osp.join(path, fsname, page_fsname(fsname, i, ext))
        if osp.isfile(page_path):
            return page_path
    return False


@click.command()
@click.option('--sleep', default=1, help=('Delay requests for a given '
                                          'number of seconds.'))
@click.argument('url')
def cli(sleep, url):
    robot = requests.Session()
    click.echo('loading %s' % url)
    res = robot.get(url, headers=headers)
    assert res.status_code == 200

    html = html5lib.parse(res.text, treebuilder='lxml')
    name = get_one(html, x_name)
    click.echo('name: %s' % name)

    fsname = slugify_filename(name)
    save_path = osp.join(path, fsname)
    click.echo('save to: %s' % save_path)
    os.makedirs(save_path, exist_ok=True)

    thumbs_url = rel_url(get_one(html, x_thumbs))
    click.echo('pause')
    _sleep(sleep)

    click.echo('loading %s' % thumbs_url)
    headers['Referer'] = url
    res = robot.get(thumbs_url, headers=headers)
    assert res.status_code == 200

    html = html5lib.parse(res.text, treebuilder='lxml')
    pages = list(map(rel_url, get_more(html, x_pages)))
    assert pages
    click.echo('pages: %d' % len(pages))

    headers['Referer'] = thumbs_url
    for i, p in enumerate(pages):
        page_path = has_page(fsname, i)
        if page_path:
            click.echo('has page: %s' % page_path)
            continue

        click.echo('pause')
        _sleep(sleep)
{'fraction_non_alphanumeric': 0.15976331360946747, 'fraction_numerical': 0.015976331360946745, 'mean_word_length': 3.034208432776452, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6299707', 'n_tokens_mistral': 1964, 'n_tokens_neox': 1858, 'n_words': 404}