<commit_before>#!/usr/bin/env python
#
# Copyright 2012 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import urllib
import httplib2

import logging
logger = logging.getLogger('splunk.xml2json.forward')

SPLUNKD_CONNECTION_TIMEOUT = 30

def safe_urlquote(string, safe='/', decodeFrom='utf-8', encodeFrom='utf-8'):
    '''
    Safely encode high byte characters from unicode or
    some other encoding to UTF-8 url strings.

    For some reason urllib.quote can't handle high byte unicode strings,
    although urllib.unquote can unquote anything. Awesome.

    Always returns STR objects!
    '''
    return urllib.quote(to_utf8(string, decodeFrom, encodeFrom), safe)

def url_encodedict(query):
    '''
    Convert a dictionary to a url-encoded" string.
    Multi-values keys can be assigned using a list (eg., {"foo": ["bar1", "bar2"]}.
    
    Note: None type values are removed.
    '''
    qargs = []
    [ qargs.extend([(k, e) for e in v]) for k,v in [ (k, v if isinstance(v, (list, tuple)) else (v,) ) for k, v in query.iteritems() if v != None ] ]
    return '&'.join( [ '%s=%s' % ( safe_urlquote(unicode(k)),safe_urlquote(unicode(v)) ) for k,v in qargs ] )

def to_utf8(obj, decodeFrom='utf-8', encodeTo='utf-8'):
    '''
    Attempts to return a utf-8 encoded str object if obj is an instance of basestring,
    otherwise just returns obj.
    
    Can be used to safely print out high byte unicode characters.
    Example:
    '''
    if isinstance(obj, unicode):
        return obj.encode(encodeTo)

    elif isinstance(obj, str):
        return obj.decode(decodeFrom).encode(encodeTo)
        
    elif '__str__' in dir(obj):
        return to_utf8(unicode(obj))

    return obj

def make_request(url, headers={}, get=None, post=None, payload=None, method={}, 
                session_key=None, basic_auth=None, *args, **kwargs):
    try:
        h = httplib2.Http(<commit_msg>Add a comment per Fred's review about why we have try/except around httplib2<commit_after>#!/usr/bin/env python
#
# Copyright 2012 Splunk, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"): you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import urllib
import httplib2

import logging
logger = logging.getLogger('splunk.xml2json.forward')

SPLUNKD_CONNECTION_TIMEOUT = 30

def safe_urlquote(string, safe='/', decodeFrom='utf-8', encodeFrom='utf-8'):
    '''
    Safely encode high byte characters from unicode or
    some other encoding to UTF-8 url strings.

    For some reason urllib.quote can't handle high byte unicode strings,
    although urllib.unquote can unquote anything. Awesome.

    Always returns STR objects!
    '''
    return urllib.quote(to_utf8(string, decodeFrom, encodeFrom), safe)

def url_encodedict(query):
    '''
    Convert a dictionary to a url-encoded" string.
    Multi-values keys can be assigned using a list (eg., {"foo": ["bar1", "bar2"]}.
    
    Note: None type values are removed.
    '''
    qargs = []
    [ qargs.extend([(k, e) for e in v]) for k,v in [ (k, v if isinstance(v, (list, tuple)) else (v,) ) for k, v in query.iteritems() if v != None ] ]
    return '&'.join( [ '%s=%s' % ( safe_urlquote(unicode(k)),safe_urlquote(unicode(v)) ) for k,v in qargs ] )

def to_utf8(obj, decodeFrom='utf-8', encodeTo='utf-8'):
    '''
    Attempts to return a utf-8 encoded str object if obj is an instance of basestring,
    otherwise just returns obj.
    
    Can be used to safely print out high byte unicode characters.
    Example:
    '''
    if isinstance(obj, unicode):
        return obj.encode(encodeTo)

    elif isinstance(obj, str):
        return obj.decode(decodeFrom).encode(encodeTo)
        
    elif '__str__' in dir(obj):
        return to_utf8(unicode(obj))

    return obj

def make_request(url, headers={}, get=None, post=None, payload=None, method={}, 
                session_key=None, basic_auth=None, *args, **kwargs):
    try:
        # In Splunk 4.3's version of httplib, we have the new
        # disable_ssl_certificate_validation kwarg. However,
        # it will throw an error if it isn't available, so we try
        # without it.
        h = httplib2.Http(
{'fraction_non_alphanumeric': 0.10861865407319952, 'fraction_numerical': 0.010035419126328217, 'mean_word_length': 3.374354561101549, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4298147', 'n_tokens_mistral': 1575, 'n_tokens_neox': 1520, 'n_words': 572}