<commit_before>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain
from pivoteer.records import RecordType, RecordSource

LOGGER = logging.getLogger(__name__)

class IndicatorManager(models.Manager):

    def host_records(self, indicator):
        record_type = RecordType.HR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_cert(self, indicator):
        """Retrieve the most recent censys.io certificate result for the provided indicator
        
            Args:
                indicator (str): The indicator to search for
            
            Returns (IndicatorRecord): The IndicatorRecord for the most recently saved
                result for the provided indicator or an empty query set if no record was found.
        """
        
        # TODO: Why are we returning empty query sets hiversus None when there are no results?
        
        record_type = RecordType.CE
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_indicator__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve certificate data for indicator %s" % indicator)
        return records

    def recent_tc(self, indicator):
        """Retrieve the most recent ThreatCrowd record for the provided indicator
        
            Args:
                indicator (str): The indicator to search for
            
            Returns (IndicatorRecord): The indicator record for the most recently saved
                result for the provided indicator.
        """
        record_type = RecordType.TR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__exact=indicator) |
                                             Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        record_type = RecordType.HR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))

        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_domain__endswith=indicator) |
        #                                      Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        record_type = RecordType.HR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__contains=indicator))

            # records = self.get_queryset().filter(Q(record_type=record_type.name),
            #                                        Q(info_date__lt=time_frame),
            #                                        Q(info__at_domain__endswith)=indicator) |
            #                                        Q(info__at_ip__endswith=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source=RecordSource.PTO.name),
                                                 ~Q(info_source=RecordSource.IID.name),
                                                 Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__contains=indicator))

            # records = self.get_queryset().filter(~Q(info_source=RecordSource.PTO.name),
            #                                      ~Q(info_source=RecordSource.IID.name),
            #                                      Q(record_type=record_type.name),
            #                                      Q(info_date__lt=time_frame),
            #                                      Q(info__at_domain__endswith=indicator) |
            #                                      Q(info__at_ip__endswith=indicator))
        return records

    def malware_records(self, indicator):
        record_type = RecordType.MR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__contains=indicator))
        return records

    def recent_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__lt=time_frame),
                                             Q(info__contains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = RecordType.WR

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        record = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info_date__gte=time_frame),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')

        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__at_query__endswith=indicator) |
                                                 Q(info__at_domain_name__endswith=indicator)).values('info_hash',
                                                                                                     'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                span = str(record['earliest']) + " / " + str(record['latest'])
                new_record = {'latest': record['latest'],
                              'earliest': record['earliest'],
                              'info_date': span,
                              'info': record_info}
                unique_records.append(new_record)<commit_msg>Update to resolve Whois data in Recent and Historical tabs<commit_after>import pickle
import hashlib
import datetime
import logging
from django.db import models
from django.db.models import Q
from django.db.models import Max, Min
from django_pgjson.fields import JsonField
from core.utilities import check_domain_valid, get_base_domain
from pivoteer.records import RecordType, RecordSource

LOGGER = logging.getLogger(__name__)

class IndicatorManager(models.Manager):

    def host_records(self, indicator):
        record_type = RecordType.HR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__at_domain__endswith=indicator) |
                                             Q(info__at_ip__endswith=indicator))
        return records

    def recent_cert(self, indicator):
        """Retrieve the most recent censys.io certificate result for the provided indicator
        
            Args:
                indicator (str): The indicator to search for
            
            Returns (IndicatorRecord): The IndicatorRecord for the most recently saved
                result for the provided indicator or an empty query set if no record was found.
        """
        
        # TODO: Why are we returning empty query sets hiversus None when there are no results?
        
        record_type = RecordType.CE
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_indicator__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve certificate data for indicator %s" % indicator)
        return records

    def recent_tc(self, indicator):
        """Retrieve the most recent ThreatCrowd record for the provided indicator
        
            Args:
                indicator (str): The indicator to search for
            
            Returns (IndicatorRecord): The indicator record for the most recently saved
                result for the provided indicator.
        """
        record_type = RecordType.TR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__at_domain__exact=indicator) |
                                             Q(info__at_ip__exact=indicator)).values('info', 'info_date')
        if records:
            return records.latest('info_date')
        LOGGER.info("Failed to retrieve ThreatCrowd data for indicator %s" % indicator)
        return records

    def recent_hosts(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        record_type = RecordType.HR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))

        # records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                      Q(info_date__gte=time_frame),
        #                                      Q(info__at_domain__endswith=indicator) |
        #                                      Q(info__at_ip__endswith=indicator))
        return records

    def historical_hosts(self, indicator, request):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains
        record_type = RecordType.HR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if request.user.is_staff:
            records = self.get_queryset().filter(Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__contains=indicator))

            # records = self.get_queryset().filter(Q(record_type=record_type.name),
            #                                        Q(info_date__lt=time_frame),
            #                                        Q(info__at_domain__endswith)=indicator) |
            #                                        Q(info__at_ip__endswith=indicator))

        else:
            records = self.get_queryset().filter(~Q(info_source=RecordSource.PTO.name),
                                                 ~Q(info_source=RecordSource.IID.name),
                                                 Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__contains=indicator))

            # records = self.get_queryset().filter(~Q(info_source=RecordSource.PTO.name),
            #                                      ~Q(info_source=RecordSource.IID.name),
            #                                      Q(record_type=record_type.name),
            #                                      Q(info_date__lt=time_frame),
            #                                      Q(info__at_domain__endswith=indicator) |
            #                                      Q(info__at_ip__endswith=indicator))
        return records

    def malware_records(self, indicator):
        record_type = RecordType.MR

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info__contains=indicator))
        return records

    def recent_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__gte=time_frame),
                                             Q(info__contains=indicator))
        return records

    def historical_malware(self, indicator):
        record_type = RecordType.MR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(days=-30)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                             Q(info_date__lt=time_frame),
                                             Q(info__contains=indicator))
        return records

    def whois_records(self, indicator):
        record_type = RecordType.WR

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        records = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info__at_query__endswith=indicator) |
                                            Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        return records

    def recent_whois(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        record = self.get_queryset().filter(Q(record_type=record_type.name),
                                            Q(info_date__gte=time_frame),
                                            Q(info__contains=indicator)).values('info', 'info_date')

        # record = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                     Q(info_date__gte=time_frame),
        #                                     Q(info__at_query__endswith=indicator) |
        #                                     Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
        if record:
            return record.latest('info_date')

        return record

    def historical_whois(self, indicator):
        # Updated by LNguyen
        # Date: 26April2017
        # Description: Former query was not correctly handling unicode characters in the info field so had to update where condition to use wildcard contains

        record_type = RecordType.WR
        time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)

        if check_domain_valid(indicator):
            indicator = get_base_domain(indicator)

        raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
                                                 Q(info_date__lt=time_frame),
                                                 Q(info__contains=indicator)).values('info_hash', 'info_date')

        # raw_records = self.get_queryset().filter(Q(record_type=record_type.name),
        #                                          Q(info_date__lt=time_frame),
        #                                          Q(info__at_query__endswith=indicator) |
        #                                          Q(info__at_domain_name__endswith=indicator)).values('info_hash',
        #                                                                                              'info_date')

        tracking = []
        unique_records = []
        annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))

        for record in annotated_records:
            hash_value = record['info_hash']

            if hash_value not in tracking:
                record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
                span = str(record['earliest']) + " / " + str(record['latest'])
                new_record = {'latest': record['latest'],
                              'earliest': record['earliest'],
                              'info_date': span,
                              'info': record_info}
                unique_records.append(new_record)
{'fraction_non_alphanumeric': 0.07903780068728522, 'fraction_numerical': 0.0034862293938941183, 'mean_word_length': 1.6220945416557848, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 172, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3903813', 'n_tokens_mistral': 4939, 'n_tokens_neox': 4712, 'n_words': 944}