<commit_before>from django.db import models
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator
from django.conf import settings
from django.db.models.signals import post_delete
from django.contrib.auth.hashers import make_password
from main import dnstools
import dns.resolver
from datetime import datetime

import re
import base64
import binascii

class BaseModel(models.Model):
    last_update = models.DateTimeField(auto_now=True)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        abstract = True


class BaseModelCreatedBy(BaseModel):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True)

    class Meta:
        abstract = True


class BaseModelRequiredCreatedBy(BaseModel):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL)

    class Meta:
        abstract = True


class BlacklistedDomain(BaseModelCreatedBy):
    domain = models.CharField(
        max_length=256,
        unique=True,
        help_text='Blacklisted domain. Evaluated as regex (search).')

    def __unicode__(self):
        return u"%s" % (self.domain, )


def domain_blacklist_validator(value):
    for bd in BlacklistedDomain.objects.all():
        if re.search(bd.domain, value):
            raise ValidationError(u'This domain is not allowed')


def ns_update_key_validator(value):
    try:
        base64.decodestring(value)
    except binascii.Error as e:
        raise ValidationError(u'Invalid update key: Must be base64 (%s)' % e)


UPDATE_ALGORITHMS = (
    ('HMAC_SHA512', 'HMAC_SHA512'),
)


class Domain(BaseModelCreatedBy):
    domain = models.CharField(max_length=256, unique=True)
    nameserver_ip = models.IPAddressField(
        max_length=256)
    nameserver_update_key = models.CharField(max_length=256, validators=[
        ns_update_key_validator])
    nameserver_update_algorithm = models.CharField(
        max_length=256, choices=UPDATE_ALGORITHMS)
    available_for_everyone = models.BooleanField(default=False)

    def __unicode__(self):
        return u"%s" % (self.domain, )


class Host(BaseModelRequiredCreatedBy):
    subdomain = models.CharField(max_length=256, validators=[
        RegexValidator(
            regex=r'^(([a-z0-9][a-z0-9\-]*[a-z0-9])|[a-z0-9])$',
            message='Invalid subdomain: only "a-z", "0-9" and "-" is allowed'
        ),
        domain_blacklist_validator])
    domain = models.ForeignKey(Domain)
    update_secret = models.CharField(max_length=256)
    comment = models.CharField(
        max_length=256, default='', blank=True, null=True)

    last_api_update = models.DateTimeField(blank=True, null=True)

    def __unicode__(self):
        return u"%s.%s - %s" % (
            self.subdomain, self.domain.domain, self.comment)

    class Meta:
        unique_together = (('subdomain', 'domain'),)

    def get_fqdn(self):
        return '%s.%s' % (self.subdomain, self.domain.domain)

    @classmethod
    def filter_by_fqdn(cls, fqdn, **kwargs):
        # Assuming subdomain has no dots (.) the fqdn is split at the first dot
        if not '.' in fqdn:
            return Host.objects.none()
        splitted = fqdn.split('.', 1)
        return Host.objects.filter(
            subdomain=splitted[0], domain__domain=splitted[1], **kwargs)

    def getIPv4(self):
        try:
            return dnstools.query_ns(self.get_fqdn(), 'A', origin=self.domain.domain)
        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers):
            return ''

    def getIPv6(self):
        try:
            return dnstools.query_ns(self.get_fqdn(), 'AAAA', origin=self.domain.domain)
        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers):
            return ''
<commit_msg>don't 500 if nameserver times out<commit_after>from django.db import models
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator
from django.conf import settings
from django.db.models.signals import post_delete
from django.contrib.auth.hashers import make_password
from main import dnstools
import dns.resolver
from datetime import datetime

import re
import base64
import binascii

class BaseModel(models.Model):
    last_update = models.DateTimeField(auto_now=True)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        abstract = True


class BaseModelCreatedBy(BaseModel):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, blank=True, null=True)

    class Meta:
        abstract = True


class BaseModelRequiredCreatedBy(BaseModel):
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL)

    class Meta:
        abstract = True


class BlacklistedDomain(BaseModelCreatedBy):
    domain = models.CharField(
        max_length=256,
        unique=True,
        help_text='Blacklisted domain. Evaluated as regex (search).')

    def __unicode__(self):
        return u"%s" % (self.domain, )


def domain_blacklist_validator(value):
    for bd in BlacklistedDomain.objects.all():
        if re.search(bd.domain, value):
            raise ValidationError(u'This domain is not allowed')


def ns_update_key_validator(value):
    try:
        base64.decodestring(value)
    except binascii.Error as e:
        raise ValidationError(u'Invalid update key: Must be base64 (%s)' % e)


UPDATE_ALGORITHMS = (
    ('HMAC_SHA512', 'HMAC_SHA512'),
)


class Domain(BaseModelCreatedBy):
    domain = models.CharField(max_length=256, unique=True)
    nameserver_ip = models.IPAddressField(
        max_length=256)
    nameserver_update_key = models.CharField(max_length=256, validators=[
        ns_update_key_validator])
    nameserver_update_algorithm = models.CharField(
        max_length=256, choices=UPDATE_ALGORITHMS)
    available_for_everyone = models.BooleanField(default=False)

    def __unicode__(self):
        return u"%s" % (self.domain, )


class Host(BaseModelRequiredCreatedBy):
    subdomain = models.CharField(max_length=256, validators=[
        RegexValidator(
            regex=r'^(([a-z0-9][a-z0-9\-]*[a-z0-9])|[a-z0-9])$',
            message='Invalid subdomain: only "a-z", "0-9" and "-" is allowed'
        ),
        domain_blacklist_validator])
    domain = models.ForeignKey(Domain)
    update_secret = models.CharField(max_length=256)
    comment = models.CharField(
        max_length=256, default='', blank=True, null=True)

    last_api_update = models.DateTimeField(blank=True, null=True)

    def __unicode__(self):
        return u"%s.%s - %s" % (
            self.subdomain, self.domain.domain, self.comment)

    class Meta:
        unique_together = (('subdomain', 'domain'),)

    def get_fqdn(self):
        return '%s.%s' % (self.subdomain, self.domain.domain)

    @classmethod
    def filter_by_fqdn(cls, fqdn, **kwargs):
        # Assuming subdomain has no dots (.) the fqdn is split at the first dot
        if not '.' in fqdn:
            return Host.objects.none()
        splitted = fqdn.split('.', 1)
        return Host.objects.filter(
            subdomain=splitted[0], domain__domain=splitted[1], **kwargs)

    def getIPv4(self):
        try:
            return dnstools.query_ns(self.get_fqdn(), 'A', origin=self.domain.domain)
        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout):
            return ''

    def getIPv6(self):
        try:
            return dnstools.query_ns(self.get_fqdn(), 'AAAA', origin=self.domain.domain)
        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.resolver.Timeout):
            return ''

{'fraction_non_alphanumeric': 0.10885847107438017, 'fraction_numerical': 0.013558884297520661, 'mean_word_length': 3.3171683389074693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '202722', 'n_tokens_mistral': 2549, 'n_tokens_neox': 2427, 'n_words': 509}