# Path: webalyzer/collected/models.py
# class Screenshot(models.Model):
#     page = models.ForeignKey(Page)
#     file = ImageField(upload_to='screenshots')
#     width = models.PositiveIntegerField()
#     height = models.PositiveIntegerField()
#     engine = models.CharField(max_length=100, default='slimerjs')
#     added = models.DateTimeField(auto_now_add=True)
# 
#     @classmethod
#     def get(cls, page):
#         for each in cls.objects.filter(page=page).order_by('-added'):
#             return each
# 
# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)


import os
import subprocess
import urlparse
import shutil
import tempfile
from contextlib import contextmanager
from lxml import etree
from lxml.cssselect import CSSSelector
from celery import shared_task
from django.core.files import File
from webalyzer.collected.models import Screenshot
from webalyzer.collector.models import Page
    print "Generating screenshot for", page_id
    page = Page.objects.get(id=page_id)
    parser = etree.HTMLParser()
    html = page.html.strip()
    if not html.lower().startswith('<!doctype'):
        html = '<!doctype html>\n' + html
    tree = etree.fromstring(html, parser).getroottree()
    dom = tree.getroot()
    # lxml inserts a doctype if none exists, so only include it in
    # the root if it was in the original html.
    root = tree if html.startswith(tree.docinfo.doctype) else dom

    # remove all scripts
    for script in CSSSelector('script')(dom):
        script.getparent().remove(script)
    for element in CSSSelector('img[src]')(dom):
        element.attrib['src'] = urlparse.urljoin(
            page.url,
            element.attrib['src']
        )
    for element in CSSSelector('link[href]')(dom):
        if not element.attrib.get('rel') == 'stylesheet':
            continue
        element.attrib['href'] = urlparse.urljoin(
            page.url,
            element.attrib['href']
        )

    with tmpdir(str(page.id)) as dir_:
        html_file = os.path.join(

# Path: webalyzer/collected/models.py
# class Screenshot(models.Model):
#     page = models.ForeignKey(Page)
#     file = ImageField(upload_to='screenshots')
#     width = models.PositiveIntegerField()
#     height = models.PositiveIntegerField()
#     engine = models.CharField(max_length=100, default='slimerjs')
#     added = models.DateTimeField(auto_now_add=True)
# 
#     @classmethod
#     def get(cls, page):
#         for each in cls.objects.filter(page=page).order_by('-added'):
#             return each
# 
# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)


# Path: webalyzer/collected/tasks.py
import os
import subprocess
import urlparse
import shutil
import tempfile
from contextlib import contextmanager
from lxml import etree
from lxml.cssselect import CSSSelector
from celery import shared_task
from django.core.files import File
from webalyzer.collected.models import Screenshot
from webalyzer.collector.models import Page

    print "Generating screenshot for", page_id
    page = Page.objects.get(id=page_id)
    parser = etree.HTMLParser()
    html = page.html.strip()
    if not html.lower().startswith('<!doctype'):
        html = '<!doctype html>\n' + html
    tree = etree.fromstring(html, parser).getroottree()
    dom = tree.getroot()
    # lxml inserts a doctype if none exists, so only include it in
    # the root if it was in the original html.
    root = tree if html.startswith(tree.docinfo.doctype) else dom

    # remove all scripts
    for script in CSSSelector('script')(dom):
        script.getparent().remove(script)
    for element in CSSSelector('img[src]')(dom):
        element.attrib['src'] = urlparse.urljoin(
            page.url,
            element.attrib['src']
        )
    for element in CSSSelector('link[href]')(dom):
        if not element.attrib.get('rel') == 'stylesheet':
            continue
        element.attrib['href'] = urlparse.urljoin(
            page.url,
            element.attrib['href']
        )

    with tmpdir(str(page.id)) as dir_:
        html_file = os.path.join(

            dir_,