<commit_before>#!/usr/bin/env python
# encoding: utf-8
import datetime
from urllib2 import HTTPError
from dnsdb_query import DnsdbClient, QueryError
from cortexutils.analyzer import Analyzer


class DnsDbAnalyzer(Analyzer):

    def __init__(self):
        Analyzer.__init__(self)
        self.service = self.get_param(
            'config.service', None, 'Service parameter is missing')
        self.dnsdb_server = self.get_param(
            'config.server', None, 'Missing DNSDB server name')
        self.dnsdb_key = self.get_param(
            'config.key', None, 'Missing DNSDB API key')

    def execute_dnsdb_service(self, client):
        data = self.get_data()

        if self.service == 'domain_name' and self.data_type in ['domain', 'fqdn']:
            return client.query_rrset(data)
        elif self.service == 'ip_history' and self.data_type == 'ip':
            return client.query_rdata_ip(data)
        elif self.service == 'name_history' and self.data_type in ['domain', 'fqdn']:
            return client.query_rdata_name(data)
        else:
            self.error('Unknown DNSDB service or invalid data type')

    def update_date(self, field, row):
        if field in row:
            row[field] = datetime.datetime.utcfromtimestamp(
                row[field]).strftime('%Y%m%dT%H%M%S') + '+0000'
        return row

    def summary(self, raw):
        # taxonomy = {"level": "info", "namespace": "Farsight", "predicate": "DNSDB", "value": 0}
        taxonomies = []
        level = "info"
        namespace = "Farsight"
        predicate = "DNSDB"

        if "records" in raw:
            r = len(raw["records"])

            if r == 0 or r == 1:
                value = "{} record".format(r)
            else:
                value = "{} records".format(r)

            taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))

        return {'taxonomies': taxonomies}

    def run(self):<commit_msg>Add DNSDB API parameters (#319)<commit_after>#!/usr/bin/env python
# encoding: utf-8
import datetime
from urllib2 import HTTPError
from dnsdb_query import DnsdbClient, QueryError
from cortexutils.analyzer import Analyzer


class DnsDbAnalyzer(Analyzer):

    def __init__(self):
        Analyzer.__init__(self)
        self.service = self.get_param(
            'config.service', None, 'Service parameter is missing')
        self.dnsdb_server = self.get_param(
            'config.server', None, 'Missing DNSDB server name')
        self.dnsdb_key = self.get_param(
            'config.key', None, 'Missing DNSDB API key')

    def execute_dnsdb_service(self, client):
        data = self.get_data()
        rrtype = self.getParam('parameters.rrtype', None, None)
        bailiwick = self.getParam('parameters.bailiwick', None, None)
        before = self.getParam('parameters.before', None, None)
        after = self.getParam('parameters.after', None, None)

        if self.service == 'domain_name' and self.data_type in ['domain', 'fqdn']:
            return client.query_rrset(data, rrtype=rrtype, bailiwick=bailiwick, before=before, after=after)
        elif self.service == 'ip_history' and self.data_type == 'ip':
            return client.query_rdata_ip(data, before=before, after=after)
        elif self.service == 'name_history' and self.data_type in ['domain', 'fqdn']:
            return client.query_rdata_name(data, rrtype=rrtype, before=before, after=after)
        else:
            self.error('Unknown DNSDB service or invalid data type')

    def update_date(self, field, row):
        if field in row:
            row[field] = datetime.datetime.utcfromtimestamp(
                row[field]).strftime('%Y%m%dT%H%M%S') + '+0000'
        return row

    def summary(self, raw):
        # taxonomy = {"level": "info", "namespace": "Farsight", "predicate": "DNSDB", "value": 0}
        taxonomies = []
        level = "info"
        namespace = "Farsight"
        predicate = "DNSDB"

        if "records" in raw:
            r = len(raw["records"])

            if r == 0 or r == 1:
                value = "{} record".format(r)
            else:
                value = "{} records".format(r)

            taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))

        return {'taxonomies': taxonomies}

    def run(self):
{'fraction_non_alphanumeric': 0.12456506610995129, 'fraction_numerical': 0.0048712595685455815, 'mean_word_length': 2.5344262295081967, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3185190', 'n_tokens_mistral': 1365, 'n_tokens_neox': 1310, 'n_words': 324}