# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)
# 
# Path: webalyzer/analyzer/models.py
# class Result(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     line = models.PositiveIntegerField(null=True)  # link's don't have this
#     before = models.TextField()
#     after = models.TextField()
#     ignored = models.BooleanField(default=False)
#     modified = models.DateTimeField(auto_now=True)
# 
#     @property
#     def suspects(self):
#         return Suspect.objects.filter(result=self)
# 
# Path: webalyzer/analyzer/tasks.py
# @shared_task
# def start_analysis(domain):
#     pages = Page.objects.filter(domain=domain)
#     processor = ExtendedProcessor()
#     t0 = time.time()
#     for page in pages:
#         try:
#             processor.process_html(page.html, page.url)
#         except DownloadError as err:
#             print "Failed to downlod on", page.url
#             print err
#             raise
#     t1 = time.time()
#     processor.process()
#     t2 = time.time()
#     print t2-t1
#     print t1-t0
#     print
#     # delete all old analysises
#     Result.objects.filter(domain=domain).delete()
# 
#     for inline in processor.inlines:
#         result = Result.objects.create(
#             domain=domain,
#             url=inline.url,
#             line=inline.line,
#             before=smart_unicode(inline.before),
#             after=smart_unicode(inline.after),
#         )
#         selectors = OrderedDict(get_selectors(result.before))
#         after = set(
#             x[0] for x in get_selectors(
#                 result.after
#             )
#         )
#         removed_keys = set(selectors.keys()) - after
#         lines = result.before.splitlines()
# 
#         for key in removed_keys:
#             selector, style = selectors[key]
#             line = None
#             for i, line_ in enumerate(lines):
#                 if line_.startswith(selector):
#                     line = i + 1
#                     break
# 
#             Suspect.objects.create(
#                 result=result,
#                 selector=key,
#                 selector_full=selector,
#                 style=style,
#                 line=line
#             )
# 
#     for link in processor.links:
#         result = Result.objects.create(
#             domain=domain,
#             url=link.href,
#             before=smart_unicode(link.before),
#             after=smart_unicode(link.after),
#         )
#         selectors = OrderedDict(get_selectors(
#             result.before, result.url
#         ))
#         after = set(
#             x[0] for x in get_selectors(
#                 result.after, result.url
#             )
#         )
#         removed_keys = set(selectors.keys()) - after
#         lines = result.before.splitlines()
# 
#         for key in removed_keys:
#             selector, style = selectors[key]
#             line = None
#             for i, line_ in enumerate(lines):
#                 if line_.startswith(selector):
#                     line = i + 1
#                     break
# 
#             Suspect.objects.create(
#                 result=result,
#                 selector=key,
#                 selector_full=selector,
#                 style=style,
#                 line=line
#             )


import os
import json
import difflib
import hashlib
import tempfile
import subprocess
import shutil
from contextlib import contextmanager
from pygments import highlight
from pygments.lexers import CssLexer, DiffLexer
from pygments.formatters import HtmlFormatter
from jsonview.decorators import json_view
from django import http
from django.shortcuts import render, redirect, get_object_or_404
from django.core.urlresolvers import reverse
from django.db.models import Max
from django.db import transaction
from django.core.cache import cache
from webalyzer.collector.models import Page
from webalyzer.analyzer.models import Result
from .tasks import start_analysis
def recent_submissions(request):
    context = {}
    results = (
        Result.objects.values('domain')
        .annotate(m=Max('modified'))
        .order_by('-m')
    )
    context['domains'] = [x['domain'] for x in results]
    return context


@json_view
@transaction.atomic
def submit(request):
    if request.method != 'POST':  # required_P
        return redirect(reverse('analyzer:start'))

    POST = json.loads(request.body)
    domain = POST['domain']
    start_analysis.delay(domain)
    jobs_ahead = cache.get('jobs_ahead', 0)
    return {
        'jobs_ahead': jobs_ahead,
    }


@json_view
def analyzed(request, domain):
    # from time import sleep
    # sleep(3)

# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)
# 
# Path: webalyzer/analyzer/models.py
# class Result(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     line = models.PositiveIntegerField(null=True)  # link's don't have this
#     before = models.TextField()
#     after = models.TextField()
#     ignored = models.BooleanField(default=False)
#     modified = models.DateTimeField(auto_now=True)
# 
#     @property
#     def suspects(self):
#         return Suspect.objects.filter(result=self)
# 
# Path: webalyzer/analyzer/tasks.py
# @shared_task
# def start_analysis(domain):
#     pages = Page.objects.filter(domain=domain)
#     processor = ExtendedProcessor()
#     t0 = time.time()
#     for page in pages:
#         try:
#             processor.process_html(page.html, page.url)
#         except DownloadError as err:
#             print "Failed to downlod on", page.url
#             print err
#             raise
#     t1 = time.time()
#     processor.process()
#     t2 = time.time()
#     print t2-t1
#     print t1-t0
#     print
#     # delete all old analysises
#     Result.objects.filter(domain=domain).delete()
# 
#     for inline in processor.inlines:
#         result = Result.objects.create(
#             domain=domain,
#             url=inline.url,
#             line=inline.line,
#             before=smart_unicode(inline.before),
#             after=smart_unicode(inline.after),
#         )
#         selectors = OrderedDict(get_selectors(result.before))
#         after = set(
#             x[0] for x in get_selectors(
#                 result.after
#             )
#         )
#         removed_keys = set(selectors.keys()) - after
#         lines = result.before.splitlines()
# 
#         for key in removed_keys:
#             selector, style = selectors[key]
#             line = None
#             for i, line_ in enumerate(lines):
#                 if line_.startswith(selector):
#                     line = i + 1
#                     break
# 
#             Suspect.objects.create(
#                 result=result,
#                 selector=key,
#                 selector_full=selector,
#                 style=style,
#                 line=line
#             )
# 
#     for link in processor.links:
#         result = Result.objects.create(
#             domain=domain,
#             url=link.href,
#             before=smart_unicode(link.before),
#             after=smart_unicode(link.after),
#         )
#         selectors = OrderedDict(get_selectors(
#             result.before, result.url
#         ))
#         after = set(
#             x[0] for x in get_selectors(
#                 result.after, result.url
#             )
#         )
#         removed_keys = set(selectors.keys()) - after
#         lines = result.before.splitlines()
# 
#         for key in removed_keys:
#             selector, style = selectors[key]
#             line = None
#             for i, line_ in enumerate(lines):
#                 if line_.startswith(selector):
#                     line = i + 1
#                     break
# 
#             Suspect.objects.create(
#                 result=result,
#                 selector=key,
#                 selector_full=selector,
#                 style=style,
#                 line=line
#             )


# Path: webalyzer/analyzer/views.py
import os
import json
import difflib
import hashlib
import tempfile
import subprocess
import shutil
from contextlib import contextmanager
from pygments import highlight
from pygments.lexers import CssLexer, DiffLexer
from pygments.formatters import HtmlFormatter
from jsonview.decorators import json_view
from django import http
from django.shortcuts import render, redirect, get_object_or_404
from django.core.urlresolvers import reverse
from django.db.models import Max
from django.db import transaction
from django.core.cache import cache
from webalyzer.collector.models import Page
from webalyzer.analyzer.models import Result
from .tasks import start_analysis

def recent_submissions(request):
    context = {}
    results = (
        Result.objects.values('domain')
        .annotate(m=Max('modified'))
        .order_by('-m')
    )
    context['domains'] = [x['domain'] for x in results]
    return context


@json_view
@transaction.atomic
def submit(request):
    if request.method != 'POST':  # required_P
        return redirect(reverse('analyzer:start'))

    POST = json.loads(request.body)
    domain = POST['domain']
    start_analysis.delay(domain)
    jobs_ahead = cache.get('jobs_ahead', 0)
    return {
        'jobs_ahead': jobs_ahead,
    }


@json_view
def analyzed(request, domain):
    # from time import sleep
    # sleep(3)

    results = Result.objects.filter(domain=domain)