<commit_before>import re
import imp
import os.path
import sys

from datetime import datetime
from datetime import timedelta
from httplib import BadStatusLine
from HTMLParser import HTMLParseError
import logging
import urllib2

from django.conf import settings
from django.utils.importlib import import_module
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models import signals as model_signals
from django.test.client import Client
try:
    from django.utils.timezone import now
except ImportError:
    now = datetime.now 

from linkcheck_settings import MAX_URL_LENGTH
from linkcheck_settings import MEDIA_PREFIX
from linkcheck_settings import SITE_DOMAINS
from linkcheck_settings import EXTERNAL_REGEX_STRING
from linkcheck_settings import EXTERNAL_RECHECK_INTERVAL
from linkcheck_settings import LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT

logger = logging.getLogger('linkcheck')

TIMEOUT_SUPPORT = False
if sys.version_info >= (2,6): #timeout arg of urlopen is available
    TIMEOUT_SUPPORT = True

EXTERNAL_REGEX = re.compile(EXTERNAL_REGEX_STRING)
METHOD_NOT_ALLOWED = 405

class HeadRequest(urllib2.Request):
    def get_method(self):
        return "HEAD"

class GetRequest(urllib2.Request):
    def get_method(self):
        return "GET"

class Url(models.Model):
    # A URL represents a distinct URL.
    # A single Url can have multiple Links associated with it
    url = models.CharField(max_length=MAX_URL_LENGTH, unique=True) # See http://www.boutell.com/newfaq/misc/urllength.html
    last_checked = models.DateTimeField(blank=True, null=True)
    status = models.NullBooleanField()
    message = models.CharField(max_length=1024, blank=True, null=True)
    still_exists = models.BooleanField()
<commit_msg>bug fix: set default value for still_exists<commit_after>import re
import imp
import os.path
import sys

from datetime import datetime
from datetime import timedelta
from httplib import BadStatusLine
from HTMLParser import HTMLParseError
import logging
import urllib2

from django.conf import settings
from django.utils.importlib import import_module
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models import signals as model_signals
from django.test.client import Client
try:
    from django.utils.timezone import now
except ImportError:
    now = datetime.now 

from linkcheck_settings import MAX_URL_LENGTH
from linkcheck_settings import MEDIA_PREFIX
from linkcheck_settings import SITE_DOMAINS
from linkcheck_settings import EXTERNAL_REGEX_STRING
from linkcheck_settings import EXTERNAL_RECHECK_INTERVAL
from linkcheck_settings import LINKCHECK_CONNECTION_ATTEMPT_TIMEOUT

logger = logging.getLogger('linkcheck')

TIMEOUT_SUPPORT = False
if sys.version_info >= (2,6): #timeout arg of urlopen is available
    TIMEOUT_SUPPORT = True

EXTERNAL_REGEX = re.compile(EXTERNAL_REGEX_STRING)
METHOD_NOT_ALLOWED = 405

class HeadRequest(urllib2.Request):
    def get_method(self):
        return "HEAD"

class GetRequest(urllib2.Request):
    def get_method(self):
        return "GET"

class Url(models.Model):
    # A URL represents a distinct URL.
    # A single Url can have multiple Links associated with it
    url = models.CharField(max_length=MAX_URL_LENGTH, unique=True) # See http://www.boutell.com/newfaq/misc/urllength.html
    last_checked = models.DateTimeField(blank=True, null=True)
    status = models.NullBooleanField()
    message = models.CharField(max_length=1024, blank=True, null=True)
    still_exists = models.BooleanField(default=False)

{'fraction_non_alphanumeric': 0.060211554109031735, 'fraction_numerical': 0.006509357200976403, 'mean_word_length': 5.4588441330998245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5481420', 'n_tokens_mistral': 1082, 'n_tokens_neox': 1065, 'n_words': 318}