<commit_before>import urllib2
import urlparse

from BeautifulSoup import BeautifulSoup

URL = 'http://www.shalgreetings.com/'
parsed_url = urlparse.urlparse(URL)
global_url = []
visited_url = []

def getimages(node):
    images = []
    try:
        soup = BeautifulSoup(urllib2.urlopen(URL))
        for image in soup.findAll("img"):
            img = image["src"]
            parsed_img = urlparse.urlparse(img)
            if not parsed_img.scheme:
                img = urlparse.urljoin(URL,parsed_img.path)
            images.append(img)
    except (IOError, KeyError):
        pass
    return images

def childrenfun(node):
    if isinstance(node, list):
        return iter(node)
    else:
        links = []
        try:
            soup = BeautifulSoup(urllib2.urlopen(node))
            for l in soup.findAll("a"):
                l = l["href"]
                parsed = urlparse.urlparse(l)
                if (parsed.scheme and (parsed.scheme in ('http','https')) and (parsed.netloc in parsed_url.netloc)):
                    link = urlparse.urlunparse((parsed.scheme,parsed.netloc,parsed.path,'','',''))
                    if not link in global_url:
                        global_url.append(link)
                        links.append(link)
        except (IOError, KeyError):
            pass
        return links

def breadth_first(tree,children=childrenfun):
    """Traverse the nodes of a tree in breadth-first order.
    The first argument should be the tree root; children
    should be a function taking as argument a tree node and
    returning an iterator of the node's children.
    """
    yield tree
    last = tree
    for node in breadth_first(tree,children):
        for child in children(node):
            yield child
            last = child
        if last == node:
            return

node = URL
for n in breadth_first(node):
    if n not in visited_url:
        visited_url.append(n)
        for img in getimages(n):
            print img<commit_msg>checks the size of image<commit_after>import urllib2
import urllib
import urlparse
import sys

try:
    from BeautifulSoup import BeautifulSoup
except ImportError:
    print "Pre-requsite not met - BeautifulSoup library"
    sys.exit(-1)

try:
    import Image
except ImportError:
    print "Pre-requisite not met - Python Imaging library"
    sys.exit(-1)

URL = 'http://www.indochino.com/'
parsed_url = urlparse.urlparse(URL)
global_url = []
visited_url = []

def getimages(node):
    images = []
    try:
        soup = BeautifulSoup(urllib2.urlopen(URL))
        for image in soup.findAll("img"):
            img = image["src"]
            if img.split('.')[-1] in ('jpg','png','jpeg','gif'):
                parsed_img = urlparse.urlparse(img)
                if not parsed_img.scheme:
                    img = urlparse.urljoin(URL,parsed_img.path)
                images.append(img)
    except (IOError, KeyError, IndexError):
        pass
    return images

def childrenfun(node):
    if isinstance(node, list):
        return iter(node)
    else:
        links = []
        try:
            soup = BeautifulSoup(urllib2.urlopen(node))
            for l in soup.findAll("a"):
                l = l["href"]
                parsed = urlparse.urlparse(l)
                if (parsed.scheme and (parsed.scheme in ('http','https')) and (parsed.netloc in parsed_url.netloc)):
                    link = urlparse.urlunparse((parsed.scheme,parsed.netloc,parsed.path,'','',''))
                    if not link in global_url:
                        global_url.append(link)
                        links.append(link)
        except (IOError, KeyError):
            pass
        return links

def breadth_first(tree,children=childrenfun):
    """Traverse the nodes of a tree in breadth-first order.
    The first argument should be the tree root; children
    should be a function taking as argument a tree node and
    returning an iterator of the node's children.
    """
    yield tree
    last = tree
    for node in breadth_first(tree,children):
        for child in children(node):
            yield child
            last = child
        if last == node:
            return

node = URL
for n in breadth_first(node):
    if n not in visited_url:
        visited_url.append(n)
        for img in getimages(n):
            tmp_loc, hdrs = urllib.urlretrieve(img)
            im  = Image.open(tmp_loc)
            x, y = im.size
            if x > 10 and y > 10:
                print img
{'fraction_non_alphanumeric': 0.09727191413237925, 'fraction_numerical': 0.0029069767441860465, 'mean_word_length': 1.9899732620320856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5503521', 'n_tokens_mistral': 1298, 'n_tokens_neox': 1293, 'n_words': 371}