<commit_before>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain


class IndicatorManager(models.Manager):

    LOGGER = logging.getLogger(__name__)

    def host_records(self, indicator):
        record_type = 'HR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_cert(self, indicator):
        record_type = 'CE'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
        
        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_indicator__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        IndicatorManager.LOGGER.info("Failed to retrieve certificate data for indicator %s" % indicator)
        return records

    def recent_tc(self, indicator):
        record_type = 'TR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__exact=indicator) |
                                             Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        IndicatorManager.LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source="PTO"),
                                                 ~Q(info_source="IID"),
                                                 Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))
        return records

    def malware_records(self, indicator):
        record_type = 'MR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__contains=indicator))
        return records

    def recent_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__lt=time_frame),
                                             Q(info__contains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = 'WR'

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        record = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_query__endswith=indicator) |
                                                 Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                                                                     'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                new_record = {'latest': record['latest'], 'earliest': record['earliest'], 'info': record_info}
                unique_records.append(new_record)
                tracking.append(hash_value)

        return unique_records

    def safebrowsing_record(self, indicator):
        record_type = 'SB'
        
        records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info__at_indicator__exact=indicator))
        return records<commit_msg>Clear out timeframe lookup after client feedback.<commit_after>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain


class IndicatorManager(models.Manager):

    LOGGER = logging.getLogger(__name__)

    def host_records(self, indicator):
        record_type = 'HR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_cert(self, indicator):
        record_type = 'CE'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
        
        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_indicator__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        IndicatorManager.LOGGER.info("Failed to retrieve certificate data for indicator %s" % indicator)
        return records

    def recent_tc(self, indicator):
        record_type = 'TR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__exact=indicator) |
                                             Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        IndicatorManager.LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        record_type = 'HR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source="PTO"),
                                                 ~Q(info_source="IID"),
                                                 Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_domain__endswith=indicator) |
                                                 Q(info__at_ip__endswith=indicator))
        return records

    def malware_records(self, indicator):
        record_type = 'MR'

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__contains=indicator))
        return records

    def recent_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = 'MR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info_date__lt=time_frame),
                                             Q(info__contains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = 'WR'

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        record = self.get_queryset().filter(Q(record_type=record_type),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        record_type = 'WR'
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_query__endswith=indicator) |
                                                 Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                                                                     'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                new_record = {'latest': record['latest'], 'earliest': record['earliest'], 'info': record_info}
                unique_records.append(new_record)
                tracking.append(hash_value)

        return unique_records

    def safebrowsing_record(self, indicator):
        record_type = 'SB'
        records = self.get_queryset().filter(Q(record_type=record_type),
                                             Q(info__at_indicator__exact=indicator))
        return records
{'fraction_non_alphanumeric': 0.086175083714891, 'fraction_numerical': 0.002323515342035126, 'mean_word_length': 1.6573451970219721, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 136, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5767702', 'n_tokens_mistral': 3676, 'n_tokens_neox': 3533, 'n_words': 559}