# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)
# 
# Path: webalyzer/collected/models.py
# class Screenshot(models.Model):
#     page = models.ForeignKey(Page)
#     file = ImageField(upload_to='screenshots')
#     width = models.PositiveIntegerField()
#     height = models.PositiveIntegerField()
#     engine = models.CharField(max_length=100, default='slimerjs')
#     added = models.DateTimeField(auto_now_add=True)
# 
#     @classmethod
#     def get(cls, page):
#         for each in cls.objects.filter(page=page).order_by('-added'):
#             return each
# 
# Path: webalyzer/collected/tasks.py
# @shared_task
# def find_page_title(page_id):
#     page = Page.objects.get(id=page_id)
#     parser = etree.HTMLParser()
#     html = page.html.strip()
#     tree = etree.fromstring(html, parser).getroottree()
#     dom = tree.getroot()
# 
#     for element in CSSSelector('title')(dom):
#         page.title = element.text.strip()
#         page.save()
#         return
# 
# @shared_task
# def generate_screenshot(page_id):
#     print "Generating screenshot for", page_id
#     page = Page.objects.get(id=page_id)
#     parser = etree.HTMLParser()
#     html = page.html.strip()
#     if not html.lower().startswith('<!doctype'):
#         html = '<!doctype html>\n' + html
#     tree = etree.fromstring(html, parser).getroottree()
#     dom = tree.getroot()
#     # lxml inserts a doctype if none exists, so only include it in
#     # the root if it was in the original html.
#     root = tree if html.startswith(tree.docinfo.doctype) else dom
# 
#     # remove all scripts
#     for script in CSSSelector('script')(dom):
#         script.getparent().remove(script)
#     for element in CSSSelector('img[src]')(dom):
#         element.attrib['src'] = urlparse.urljoin(
#             page.url,
#             element.attrib['src']
#         )
#     for element in CSSSelector('link[href]')(dom):
#         if not element.attrib.get('rel') == 'stylesheet':
#             continue
#         element.attrib['href'] = urlparse.urljoin(
#             page.url,
#             element.attrib['href']
#         )
# 
#     with tmpdir(str(page.id)) as dir_:
#         html_file = os.path.join(
#             dir_,
#             'page-{}.html'.format(page.id)
#         )
#         png_file = os.path.join(
#             dir_,
#             'screenshot-{}.png'.format(page.id)
#         )
#         with open(html_file, 'w') as f:
#             f.write('<!doctype html>\n')
#             f.write(etree.tostring(
#                 root,
#                 pretty_print=False,
#                 method='html',
#                 encoding='utf-8',
#             ))
# 
#         # that 'screenshot.js' file is in the same directory as this file
#         _here = os.path.dirname(__file__)
#         width, height = 1400, 900
#         cmd = [
#             'phantomjs',
#             '--ignore-ssl-errors=yes',
#             os.path.join(_here, 'screenshot.js'),
#             html_file,
#             png_file,
#             str(width),
#             str(height)
#         ]
#         # print cmd
#         process = subprocess.Popen(
#             cmd,
#             stdout=subprocess.PIPE,
#             stderr=subprocess.PIPE
#         )
#         stdout, stderr = process.communicate()
#         # print (stdout, stderr)
#         if not os.path.isfile(png_file):
#             raise Exception("No PNG file created")
# 
#         with open(png_file, 'rb') as f:
#             Screenshot.objects.create(
#                 page=page,
#                 file=File(f),
#                 width=width,
#                 height=height,
#                 engine='phantomjs'
#             )


import time
from jsonview.decorators import json_view
from sorl.thumbnail import get_thumbnail
from django import http
from django.shortcuts import render, get_object_or_404
from django.db.models import Sum, Max
from django.core.cache import cache
from django.core.paginator import Paginator
from webalyzer.collector.models import Page
from webalyzer.collected.models import Screenshot
from .tasks import find_page_title, generate_screenshot






def index(request, *args, **kwargs):
    # the *args, **kwargs is because this view is used in urls.py
    # as a catch-all for the sake of pushstate
    context = {}
    # html_formatter = HtmlFormatter(linenos=True)
    # context['pygments_css'] = html_formatter.get_style_defs('.highlight')
    return render(request, 'collected/index.html', context)


@json_view
def recently_collected_domains(request):
    # TODO this'll be based on auth or something instead
    context = {}
    results = (
        Page.objects.values('domain')
        .annotate(m=Max('modified'))
        .order_by('-m')
    )
    context['domains'] = [x['domain'] for x in results]
    return context


@json_view
def collected_pages(request, domain):

# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)
# 
# Path: webalyzer/collected/models.py
# class Screenshot(models.Model):
#     page = models.ForeignKey(Page)
#     file = ImageField(upload_to='screenshots')
#     width = models.PositiveIntegerField()
#     height = models.PositiveIntegerField()
#     engine = models.CharField(max_length=100, default='slimerjs')
#     added = models.DateTimeField(auto_now_add=True)
# 
#     @classmethod
#     def get(cls, page):
#         for each in cls.objects.filter(page=page).order_by('-added'):
#             return each
# 
# Path: webalyzer/collected/tasks.py
# @shared_task
# def find_page_title(page_id):
#     page = Page.objects.get(id=page_id)
#     parser = etree.HTMLParser()
#     html = page.html.strip()
#     tree = etree.fromstring(html, parser).getroottree()
#     dom = tree.getroot()
# 
#     for element in CSSSelector('title')(dom):
#         page.title = element.text.strip()
#         page.save()
#         return
# 
# @shared_task
# def generate_screenshot(page_id):
#     print "Generating screenshot for", page_id
#     page = Page.objects.get(id=page_id)
#     parser = etree.HTMLParser()
#     html = page.html.strip()
#     if not html.lower().startswith('<!doctype'):
#         html = '<!doctype html>\n' + html
#     tree = etree.fromstring(html, parser).getroottree()
#     dom = tree.getroot()
#     # lxml inserts a doctype if none exists, so only include it in
#     # the root if it was in the original html.
#     root = tree if html.startswith(tree.docinfo.doctype) else dom
# 
#     # remove all scripts
#     for script in CSSSelector('script')(dom):
#         script.getparent().remove(script)
#     for element in CSSSelector('img[src]')(dom):
#         element.attrib['src'] = urlparse.urljoin(
#             page.url,
#             element.attrib['src']
#         )
#     for element in CSSSelector('link[href]')(dom):
#         if not element.attrib.get('rel') == 'stylesheet':
#             continue
#         element.attrib['href'] = urlparse.urljoin(
#             page.url,
#             element.attrib['href']
#         )
# 
#     with tmpdir(str(page.id)) as dir_:
#         html_file = os.path.join(
#             dir_,
#             'page-{}.html'.format(page.id)
#         )
#         png_file = os.path.join(
#             dir_,
#             'screenshot-{}.png'.format(page.id)
#         )
#         with open(html_file, 'w') as f:
#             f.write('<!doctype html>\n')
#             f.write(etree.tostring(
#                 root,
#                 pretty_print=False,
#                 method='html',
#                 encoding='utf-8',
#             ))
# 
#         # that 'screenshot.js' file is in the same directory as this file
#         _here = os.path.dirname(__file__)
#         width, height = 1400, 900
#         cmd = [
#             'phantomjs',
#             '--ignore-ssl-errors=yes',
#             os.path.join(_here, 'screenshot.js'),
#             html_file,
#             png_file,
#             str(width),
#             str(height)
#         ]
#         # print cmd
#         process = subprocess.Popen(
#             cmd,
#             stdout=subprocess.PIPE,
#             stderr=subprocess.PIPE
#         )
#         stdout, stderr = process.communicate()
#         # print (stdout, stderr)
#         if not os.path.isfile(png_file):
#             raise Exception("No PNG file created")
# 
#         with open(png_file, 'rb') as f:
#             Screenshot.objects.create(
#                 page=page,
#                 file=File(f),
#                 width=width,
#                 height=height,
#                 engine='phantomjs'
#             )


# Path: webalyzer/collected/views.py
import time
from jsonview.decorators import json_view
from sorl.thumbnail import get_thumbnail
from django import http
from django.shortcuts import render, get_object_or_404
from django.db.models import Sum, Max
from django.core.cache import cache
from django.core.paginator import Paginator
from webalyzer.collector.models import Page
from webalyzer.collected.models import Screenshot
from .tasks import find_page_title, generate_screenshot







def index(request, *args, **kwargs):
    # the *args, **kwargs is because this view is used in urls.py
    # as a catch-all for the sake of pushstate
    context = {}
    # html_formatter = HtmlFormatter(linenos=True)
    # context['pygments_css'] = html_formatter.get_style_defs('.highlight')
    return render(request, 'collected/index.html', context)


@json_view
def recently_collected_domains(request):
    # TODO this'll be based on auth or something instead
    context = {}
    results = (
        Page.objects.values('domain')
        .annotate(m=Max('modified'))
        .order_by('-m')
    )
    context['domains'] = [x['domain'] for x in results]
    return context


@json_view
def collected_pages(request, domain):

    pages = Page.objects.filter(domain=domain)