<commit_before>import os
import sys
import re
from cStringIO import StringIO

import requests
from PIL import Image
from lxml import etree

url = sys.argv[1]

# find thread id
m = re.match('thread-(\d+)-.*', url.rsplit('/', 1)[1])
thread_id = m.group(1)

# create required folders
if not os.path.exists('images'):
    os.mkdir('images')

folder = os.path.join('images', thread_id)
if not os.path.exists(folder):
    os.mkdir(folder)

# fetch html and find images
resp = requests.get(url)
html = etree.HTML(resp.content)
image_urls = html.xpath('//img/@file')

# save images
for image_url in image_urls:
    filename = image_url.rsplit('/', 1)[1]

    # ignore useless image
    try:
        resp = requests.get(image_url)
    except requests.exceptions.MissingSchema:
        continue

    # ignore small images
    im = Image.open(StringIO(resp.content))
    width, height = im.size
    if width < 400 or height < 400:
        continue

    # save image
    with open(os.path.join(folder, filename), 'wb+') as f:
        f.write(resp.content)<commit_msg>small fixes<commit_after>import os
import sys
import re
from cStringIO import StringIO

import requests
from PIL import Image
from lxml import etree

url = sys.argv[1]

# find thread id
m = re.match('thread-(\d+)-.*', url.rsplit('/', 1)[1])
thread_id = m.group(1)

# create image folder
if not os.path.exists('images'):
    os.mkdir('images')

# fetch html and find images
resp = requests.get(url)
if resp.status_code != 200:
    sys.exit('Oops')


# parse html
html = etree.HTML(resp.content)

# title
try:
    title = html.find('.//title').text.split(' - ')[0].strip()
except AttributeError:
    assert False, 'There is no content, please try again.'

# create target folder for saving images
folder = os.path.join('images', "%s - %s" % (thread_id, title))
if not os.path.exists(folder):
    os.mkdir(folder)

# iterate and save images
image_urls = html.xpath('//img/@file')
for image_url in image_urls:
    filename = image_url.rsplit('/', 1)[1]

    # ignore useless image
    if not image_url.startswith('http'):
        continue

    # fetch image
    print 'Fetching %s ...' % image_url
    resp = requests.get(image_url)

    # ignore small images
    im = Image.open(StringIO(resp.content))
    width, height = im.size
    if width < 400 or height < 400:
        continue

    # save image
    with open(os.path.join(folder, filename), 'wb+') as f:
        f.write(resp.content)
{'fraction_non_alphanumeric': 0.13636363636363635, 'fraction_numerical': 0.011466011466011465, 'mean_word_length': 3.301056338028169, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1621442', 'n_tokens_mistral': 839, 'n_tokens_neox': 806, 'n_words': 235}