<commit_before>#add your custom response handler class to this module
import json
import datetime
#the default handler , does nothing , just passes the raw output directly to STDOUT
class DefaultResponseHandler:

    def __init__(self,**args):
        pass

    def __call__(self, response_object,raw_response_output,response_type,req_args,endpoint):
        print_xml_stream(raw_response_output)


class OpenDNSErrorsEventHandler:

    def __init__(self,**args):
        pass

    def __call__(self, response_object,raw_response_output,response_type,req_args,endpoint):
        if response_type == "json":
            output = json.loads(raw_response_output)

            #perform any custom processing of the JSON response
            for line in output:
                print_xml_stream(json.dumps(line))
        else:
            print_xml_stream(raw_response_output)

class OpenDNSHandler:

    def __init__(self,**args):
        pass

    def __call__(self, response_object,raw_response_output,response_type,req_args,endpoint):
        if response_type == "json":
            output = json.loads(raw_response_output)

            #perform any custom processing of the JSON response
            for line in output:
                print_xml_stream(json.dumps(line))
        else:
            print_xml_stream(raw_response_output)



#HELPER FUNCTIONS

# prints XML stream
def print_xml_stream(s):
    print "<stream><event unbroken=\"1\"><data>%s</data><done/></event></stream>" % encodeXMLText(s)



def encodeXMLText(text):
    text = text.replace("&", "&amp;")
    text = text.replace("\"", "&quot;")
    text = text.replace("'", "&apos;")
    text = text.replace("<", "&lt;")
    text = text.replace(">", "&gt;")
    text = text.replace("\n", "")
    return text<commit_msg>convert timestamp to UTC for automatic parsing by Splunk<commit_after># add your custom response handler class to this module
import json
import datetime
# the default handler , does nothing , just passes the raw output directly to STDOUT
class DefaultResponseHandler:

    def __init__(self,**args):
        pass

    def __call__(self, response_object,raw_response_output,response_type,req_args,endpoint):
        print_xml_stream(raw_response_output)


class OpenDNSErrorsEventHandler:

    def __init__(self,**args):
        pass

    def __call__(self, response_object,raw_response_output,response_type,req_args,endpoint):
        if response_type == "json":
            output = json.loads(raw_response_output, object_hook=convert_timestamp)
            #output = json.loads(raw_response_output)

            # perform any custom processing of the JSON response
            for line in output:
                print_xml_stream(json.dumps(line))

        else:
            print_xml_stream(raw_response_output)

class OpenDNSHandler:

    def __init__(self,**args):
        pass

    def __call__(self, response_object,raw_response_output,response_type,req_args,endpoint):
        if response_type == "json":
            output = json.loads(raw_response_output, object_hook=convert_timestamp)
            # output = json.loads(raw_response_output)

            # perform any custom processing of the JSON response
            for line in output:
                print_xml_stream(json.dumps(line))
        else:
            print_xml_stream(raw_response_output)



# convert unix timestamp to UTC
def convert_timestamp(json):
    # number of milliseconds in a second
    ms_in_seconds = 1000.0
    # PST is -8 hours from UTC
    PST = 8
    unix_format = '%s%f'
    unix_timestamp = json['timestamp']
    datetime_obj = datetime.datetime.fromtimestamp(unix_timestamp/ms_in_seconds)
    utc_datetime = datetime_obj + datetime.timedelta(hours=PST)
    json['timestamp'] = utc_datetime.strftime(unix_format)[:-3]
    return json

# prints XML stream
def print_xml_stream(s):
    print '<stream><event unbroken="1"><data>%s</data><done/></event></stream>' % encodeXMLText(s)



def encodeXMLText(text):
    text = text.replace('&', '&amp;')
    text = text.replace('"', '&quot;')
    text = text.replace("'", '&apos;')
    text = text.replace('<', '&lt;')
    text = text.replace('>', '&gt;')
    text = text.replace("\n", '')
    return text
{'fraction_non_alphanumeric': 0.10745562130177515, 'fraction_numerical': 0.002366863905325444, 'mean_word_length': 3.307849133537207, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3010153', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1224, 'n_words': 294}