# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)
# 
# Path: webalyzer/analyzer/models.py
# class Result(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     line = models.PositiveIntegerField(null=True)  # link's don't have this
#     before = models.TextField()
#     after = models.TextField()
#     ignored = models.BooleanField(default=False)
#     modified = models.DateTimeField(auto_now=True)
# 
#     @property
#     def suspects(self):
#         return Suspect.objects.filter(result=self)
# 
# class Suspect(models.Model):
#     result = models.ForeignKey(Result)
#     selector = models.TextField()
#     # useful when the selector appears in a bundle with other selectors
#     selector_full = models.TextField()
#     style = models.TextField()
#     line = models.PositiveIntegerField(null=True)  # null if it goes wrong
#     modified = models.DateTimeField(auto_now=True)
# 
#     class Meta:
#         ordering = ('modified',)
# 
#     @property
#     def size(self):
#         return len(self.style)


import os
import stat
import codecs
import hashlib
import tempfile
import time
import requests
import cssutils
from collections import OrderedDict
from django.utils.encoding import smart_unicode
from django.conf import settings
from celery import shared_task
from mincss.processor import Processor, DownloadError
from webalyzer.collector.models import Page
from webalyzer.analyzer.models import Result, Suspect
                style=style,
                line=line
            )

    for link in processor.links:
        result = Result.objects.create(
            domain=domain,
            url=link.href,
            before=smart_unicode(link.before),
            after=smart_unicode(link.after),
        )
        selectors = OrderedDict(get_selectors(
            result.before, result.url
        ))
        after = set(
            x[0] for x in get_selectors(
                result.after, result.url
            )
        )
        removed_keys = set(selectors.keys()) - after
        lines = result.before.splitlines()

        for key in removed_keys:
            selector, style = selectors[key]
            line = None
            for i, line_ in enumerate(lines):
                if line_.startswith(selector):
                    line = i + 1
                    break


# Path: webalyzer/collector/models.py
# class Page(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     html = models.TextField()
#     size = models.PositiveIntegerField(default=0)
#     title = models.TextField(null=True)
#     added = models.DateTimeField(auto_now_add=True)
#     modified = models.DateTimeField(auto_now=True)
#     source_hash = models.IntegerField(default=0, db_index=True)
# 
#     def __unicode__(self):
#         return self.url
# 
#     def __repr__(self):
#         return '<%s: %s %.2fKb>' % (
#             self.__class__.__name__,
#             self.url,
#             len(self.html) / 1024.0
#         )
# 
#     def __len__(self):
#         return len(self.html)
# 
# Path: webalyzer/analyzer/models.py
# class Result(models.Model):
#     domain = models.CharField(max_length=100, db_index=True)
#     url = models.URLField()
#     line = models.PositiveIntegerField(null=True)  # link's don't have this
#     before = models.TextField()
#     after = models.TextField()
#     ignored = models.BooleanField(default=False)
#     modified = models.DateTimeField(auto_now=True)
# 
#     @property
#     def suspects(self):
#         return Suspect.objects.filter(result=self)
# 
# class Suspect(models.Model):
#     result = models.ForeignKey(Result)
#     selector = models.TextField()
#     # useful when the selector appears in a bundle with other selectors
#     selector_full = models.TextField()
#     style = models.TextField()
#     line = models.PositiveIntegerField(null=True)  # null if it goes wrong
#     modified = models.DateTimeField(auto_now=True)
# 
#     class Meta:
#         ordering = ('modified',)
# 
#     @property
#     def size(self):
#         return len(self.style)


# Path: webalyzer/analyzer/tasks.py
import os
import stat
import codecs
import hashlib
import tempfile
import time
import requests
import cssutils
from collections import OrderedDict
from django.utils.encoding import smart_unicode
from django.conf import settings
from celery import shared_task
from mincss.processor import Processor, DownloadError
from webalyzer.collector.models import Page
from webalyzer.analyzer.models import Result, Suspect

                style=style,
                line=line
            )

    for link in processor.links:
        result = Result.objects.create(
            domain=domain,
            url=link.href,
            before=smart_unicode(link.before),
            after=smart_unicode(link.after),
        )
        selectors = OrderedDict(get_selectors(
            result.before, result.url
        ))
        after = set(
            x[0] for x in get_selectors(
                result.after, result.url
            )
        )
        removed_keys = set(selectors.keys()) - after
        lines = result.before.splitlines()

        for key in removed_keys:
            selector, style = selectors[key]
            line = None
            for i, line_ in enumerate(lines):
                if line_.startswith(selector):
                    line = i + 1
                    break


            Suspect.objects.create(