<commit_before>import re
import imp
import os.path
import sys

from datetime import datetime
from datetime import timedelta
from httplib import BadStatusLine
from HTMLParser import HTMLParseError
import logging
import urllib2

from django.conf import settings
from django.utils.importlib import import_module
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models import signals as model_signals
from django.test.client import Client
try:
    from django.utils.timezone import now
except ImportError:
    now = datetime.now 

from linkcheck_settings import MAX_URL_LENGTH
from linkcheck_settings import MEDIA_PREFIX
from linkcheck_settings import SITE_DOMAINS
from linkcheck_settings import EXTERNAL_REGEX_STRING
from linkcheck_settings import EXTERNAL_RECHECK_INTERVAL
from linkcheck_settings import LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT

logger = logging.getLogger('linkcheck')

TIMEOUT_SUPPORT = False
if sys.version_info >= (2,6): #timeout arg of urlopen is available
    TIMEOUT_SUPPORT = True

EXTERNAL_REGEX = re.compile(EXTERNAL_REGEX_STRING)
METHOD_NOT_ALLOWED = 405

class HeadRequest(urllib2.Request):
    def get_method(self):
        return "HEAD"

class GetRequest(urllib2.Request):
    def get_method(self):
        return "GET"

def html_decode(s):
    """
    Returns the ASCII decoded version of the given HTML string. This does
    NOT remove normal HTML tags like <p>.
    """
    html_codes = (
            ("'", '&#39;'),
            ('"', '&quot;'),
            ('>', '&gt;'),
            ('<', '&lt;'),
            ('&', '&amp;')
        )
    for code in html_codes:
        s = s.replace(code[1], code[0])
    return s

class Url(models.Model):
    # A URL represents a distinct URL.
    # A single Url can have multiple Links associated with it
    url = models.CharField(max_length=MAX_URL_LENGTH, unique=True) # See http://www.boutell.com/newfaq/misc/urllength.html
    last_checked = models.DateTimeField(blank=True, null=True)
    status = models.NullBooleanField()
    message = models.CharField(max_length=1024, blank=True, null=True)
    still_exists = models.BooleanField(default=False)

    @property
    def type(self):
        if EXTERNAL_REGEX.match(self.url):
            return 'external'
        if self.url.startswith('mailto'):
            return 'mailto'
        elif str(self.url)=='':
            return 'empty'
        elif self.url.startswith('#'):
            return 'anchor'
        elif self.url.startswith(MEDIA_PREFIX):
            return 'file'
        else:
            return 'unknown'

    @property
    def get_message(self):
        if self.last_checked:
            return self.message
        else:
            return "URL Not Yet Checked"

    @property
    def colour(self):
        if not self.last_checked:
            return 'blue'
        elif self.status==True:
            return 'green'
        else:
            return 'red'

    def __unicode__(self):
        return self.url

    @property
    def external(self):
        return EXTERNAL_REGEX.match(self.url)

    def url_unquoted(self):
        try:
            # URLs should be ascii encodable
            url = self.url.encode('ascii')
        except UnicodeEncodeError:
            url = self.url
        return urllib2.unquote(url).decode('utf8')

    def check(self, check_internal=True, check_external=True, external_recheck_interval=EXTERNAL_RECHECK_INTERVAL):

        from linkcheck.utils import LinkCheckHandler
        external_recheck_datetime = now() - timedelta(minutes=external_recheck_interval)
        self.status  = False

        # Remove current domain from URLs as the test client chokes when trying to test them during a page save
        # They shouldn't generally exist but occasionally slip through
        # If settings.SITE_DOMAINS isn't set then use settings.SITE_DOMAIN
        # but also check for variants: example.org, www.example.org, test.example.org

        original_url = None # used to restore the original url afterwards

        if SITE_DOMAINS: #if the setting is present
            internal_exceptions = SITE_DOMAINS

        else: # try using SITE_DOMAIN
            root_domain = settings.SITE_DOMAIN
            if root_domain.startswith('www.'):
                root_domain = root_domain[4:]
            elif root_domain.startswith('test.'):
                root_domain = root_domain[5:]
            internal_exceptions = ['http://'+root_domain, 'http://www.'+root_domain, 'http://test.'+root_domain]

        for ex in internal_exceptions:
            if ex and self.url.startswith(ex):
                original_url = self.url
                self.url = self.url.replace(ex, '', 1)

        if check_internal and (not self.external):
            if not(self.url):
                self.message = 'Empty link'

            elif self.url.startswith('mailto:'):
                self.status = None
                self.message = 'Email link (not automatically checked)'

            elif self.url.startswith('#'):
                self.status = None
                self.message = 'Link to within the same page (not automatically checked)'

            elif self.url.startswith(MEDIA_PREFIX):
                #TODO Assumes a direct mapping from media url to local filesystem path. This will break quite easily for alternate setups
                path = settings.MEDIA_ROOT + self.url_unquoted()[len(MEDIA_PREFIX)-1:]
                decoded_path = html_decode(path)
                if os.path.exists(path) or os.path.exists(decoded_path):
                    self.message = 'Working file link'
                    self.status = True
                else:
                    self.message = 'Missing Document'

            elif getattr(self, '_internal_hash', False) and getattr(self, '_instance', None):
                # This is a hash link pointing to itself
                from linkcheck import parse_anchors

                hash = self._internal_hash
                instance = self._instance
                if hash == '#': # special case, point to #
                    self.message = 'Working internal hash anchor'
                    self.status = True
                else:
                    hash = hash[1:] #'#something' => 'something'
                    html_content = ''
                    for field in instance._linklist.html_fields:
                        html_content += getattr(instance, field, '')
                    names = parse_anchors(html_content)
                    if hash in names:
                        self.message = 'Working internal hash anchor'
                        self.status = True
                    else:
                        self.message = 'Broken internal hash anchor'

            elif self.url.startswith('/'):
                old_prepend_setting = settings.PREPEND_WWW
                settings.PREPEND_WWW = False
                c = Client()
                c.handler = LinkCheckHandler()
                response = c.get(self.url, follow=True)
                if response.status_code == 200:<commit_msg>make linkcheck compatible with new version of djanog-reversion<commit_after>import re
import imp
import os.path
import sys

from datetime import datetime
from datetime import timedelta
from httplib import BadStatusLine
from HTMLParser import HTMLParseError
import logging
import urllib2

from django.conf import settings
from django.utils.importlib import import_module
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models import signals as model_signals
from django.test.client import Client
try:
    from django.utils.timezone import now
except ImportError:
    now = datetime.now 

from linkcheck_settings import MAX_URL_LENGTH
from linkcheck_settings import MEDIA_PREFIX
from linkcheck_settings import SITE_DOMAINS
from linkcheck_settings import EXTERNAL_REGEX_STRING
from linkcheck_settings import EXTERNAL_RECHECK_INTERVAL
from linkcheck_settings import LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT

logger = logging.getLogger('linkcheck')

TIMEOUT_SUPPORT = False
if sys.version_info >= (2,6): #timeout arg of urlopen is available
    TIMEOUT_SUPPORT = True

EXTERNAL_REGEX = re.compile(EXTERNAL_REGEX_STRING)
METHOD_NOT_ALLOWED = 405

class HeadRequest(urllib2.Request):
    def get_method(self):
        return "HEAD"

class GetRequest(urllib2.Request):
    def get_method(self):
        return "GET"

def html_decode(s):
    """
    Returns the ASCII decoded version of the given HTML string. This does
    NOT remove normal HTML tags like <p>.
    """
    html_codes = (
            ("'", '&#39;'),
            ('"', '&quot;'),
            ('>', '&gt;'),
            ('<', '&lt;'),
            ('&', '&amp;')
        )
    for code in html_codes:
        s = s.replace(code[1], code[0])
    return s

class Url(models.Model):
    # A URL represents a distinct URL.
    # A single Url can have multiple Links associated with it
    url = models.CharField(max_length=MAX_URL_LENGTH, unique=True) # See http://www.boutell.com/newfaq/misc/urllength.html
    last_checked = models.DateTimeField(blank=True, null=True)
    status = models.NullBooleanField()
    message = models.CharField(max_length=1024, blank=True, null=True)
    still_exists = models.BooleanField(default=False)

    @property
    def type(self):
        if EXTERNAL_REGEX.match(self.url):
            return 'external'
        if self.url.startswith('mailto'):
            return 'mailto'
        elif str(self.url)=='':
            return 'empty'
        elif self.url.startswith('#'):
            return 'anchor'
        elif self.url.startswith(MEDIA_PREFIX):
            return 'file'
        else:
            return 'unknown'

    @property
    def get_message(self):
        if self.last_checked:
            return self.message
        else:
            return "URL Not Yet Checked"

    @property
    def colour(self):
        if not self.last_checked:
            return 'blue'
        elif self.status==True:
            return 'green'
        else:
            return 'red'

    def __unicode__(self):
        return self.url

    @property
    def external(self):
        return EXTERNAL_REGEX.match(self.url)

    def url_unquoted(self):
        try:
            # URLs should be ascii encodable
            url = self.url.encode('ascii')
        except UnicodeEncodeError:
            url = self.url
        return urllib2.unquote(url).decode('utf8')

    def check(self, check_internal=True, check_external=True, external_recheck_interval=EXTERNAL_RECHECK_INTERVAL):

        from linkcheck.utils import LinkCheckHandler
        external_recheck_datetime = now() - timedelta(minutes=external_recheck_interval)
        self.status  = False

        # Remove current domain from URLs as the test client chokes when trying to test them during a page save
        # They shouldn't generally exist but occasionally slip through
        # If settings.SITE_DOMAINS isn't set then use settings.SITE_DOMAIN
        # but also check for variants: example.org, www.example.org, test.example.org

        original_url = None # used to restore the original url afterwards

        if SITE_DOMAINS: #if the setting is present
            internal_exceptions = SITE_DOMAINS

        else: # try using SITE_DOMAIN
            root_domain = settings.SITE_DOMAIN
            if root_domain.startswith('www.'):
                root_domain = root_domain[4:]
            elif root_domain.startswith('test.'):
                root_domain = root_domain[5:]
            internal_exceptions = ['http://'+root_domain, 'http://www.'+root_domain, 'http://test.'+root_domain]

        for ex in internal_exceptions:
            if ex and self.url.startswith(ex):
                original_url = self.url
                self.url = self.url.replace(ex, '', 1)

        if check_internal and (not self.external):
            if not(self.url):
                self.message = 'Empty link'

            elif self.url.startswith('mailto:'):
                self.status = None
                self.message = 'Email link (not automatically checked)'

            elif self.url.startswith('#'):
                self.status = None
                self.message = 'Link to within the same page (not automatically checked)'

            elif self.url.startswith(MEDIA_PREFIX):
                #TODO Assumes a direct mapping from media url to local filesystem path. This will break quite easily for alternate setups
                path = settings.MEDIA_ROOT + self.url_unquoted()[len(MEDIA_PREFIX)-1:]
                decoded_path = html_decode(path)
                if os.path.exists(path) or os.path.exists(decoded_path):
                    self.message = 'Working file link'
                    self.status = True
                else:
                    self.message = 'Missing Document'

            elif getattr(self, '_internal_hash', False) and getattr(self, '_instance', None):
                # This is a hash link pointing to itself
                from linkcheck import parse_anchors

                hash = self._internal_hash
                instance = self._instance
                if hash == '#': # special case, point to #
                    self.message = 'Working internal hash anchor'
                    self.status = True
                else:
                    hash = hash[1:] #'#something' => 'something'
                    html_content = ''
                    for field in instance._linklist.html_fields:
                        html_content += getattr(instance, field, '')
                    names = parse_anchors(html_content)
                    if hash in names:
                        self.message = 'Working internal hash anchor'
                        self.status = True
                    else:
                        self.message = 'Broken internal hash anchor'

            elif self.url.startswith('/'):
                old_prepend_setting = settings.PREPEND_WWW
                settings.PREPEND_WWW = False
                c = Client()
                c.handler = LinkCheckHandler()
                response = c.get(self.url, follow=True)
                #using test client will clear the RevisionContextManager stack. 
                from reversion.revisions import revision_context_manager
                revision_context_manager.start()

                if response.status_code == 200:
{'fraction_non_alphanumeric': 0.08367192342125197, 'fraction_numerical': 0.0035810205908683975, 'mean_word_length': 2.157642965862144, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '70172', 'n_tokens_mistral': 4023, 'n_tokens_neox': 3831, 'n_words': 1091}