<commit_before>#!/usr/bin/env python
# 
# Copyright (c) 2014 <NAME>
# 
# This file is part of gl2toolkit
#
# This program is free software under the terms of the MIT License 
# You should have received a copy of the MIT License
# along with this program; if not, see <http://opensource.org/licenses/MIT>.
#

import json
import urllib2
import getpass
import os
import base64

from os.path import expanduser

# curl data types
def curl_get_json(url):
    return json.loads(curl_get(url))

def curl_get_plain(url):
    return curl_get(url)


# Curl methods
def curl_get(url):
    linkdata = get_linkdata()
    
    request = urllib2.Request( linkdata['url'] + url )
    base64string = base64.encodestring('%s:%s' % (linkdata['user'], linkdata['passwd'])).replace('\n', '')
    request.add_header("Authorization", "Basic %s" % base64string)
    request.get_method = lambda: 'GET'
    
    result = urllib2.urlopen(request)
    cnt = result.read()
    result.close()
    
    return cnt
    
# Credentials via manual input if not saved
def get_linkdata():
    linkdatafile = expanduser("~") + "/.gl2t_linkdata"
    
    if os.access(linkdatafile, os.R_OK):
        with open(linkdatafile) as data_file:    
            data = json.load(data_file)
            return data
    else:
        return read_link_data()

# Query linkdata on commandline 
def read_link_data():
    url = raw_input("Graylog connection url: ")
    user = raw_input("Username: ")
    passwd = getpass.getpass("Password (no visible output ): ")
    
    linkdata = { "url" : url, "user" : user, "passwd" : <PASSWORD> }
    
    return linkdata

# JSON Formated prints
def print_json_oneline(data):
    print json.dumps(data)
    
def print_json_nosort(data):
    print json.dumps(data, indent=4)
    
def print_json_nice(data):
    print json.dumps(data, indent=4, sort_keys=True)<commit_msg>Switched web service handling to httplib<commit_after>#!/usr/bin/env python
# 
# Copyright (c) 2014 <NAME>
# 
# This file is part of gl2toolkit
#
# This program is free software under the terms of the MIT License 
# You should have received a copy of the MIT License
# along with this program; if not, see <http://opensource.org/licenses/MIT>.
#

import json
import getpass
import os
import base64
import httplib

from os.path import expanduser

# curl data types
def curl_get_json(url):
    return json.loads(curl_get(url)["data"])

def curl_get_plain(url):
    return curl_get(url)["data"]


# Curl methods
def curl_get(url):
    linkdata = get_linkdata()
    base64string = base64.encodestring('%s:%s' % (linkdata['user'], linkdata['passwd'])).replace('\n', '')    
    
    conn = httplib.HTTPConnection( linkdata['host'], int(linkdata['port']) )
    conn.putrequest("GET", url)
    conn.putheader("Host", linkdata['host'])
    conn.putheader("Authorization", "Basic %s" % base64string)
    conn.endheaders()
    conn.send("")
    
    response = conn.getresponse()
    res_dict = { "status" : response.status , "reason" : response.reason , "data" : response.read() }
    
    return res_dict
    
# Credentials via manual input if not saved
def get_linkdata():
    linkdatafile = expanduser("~") + "/.gl2t_linkdata"
    
    if os.access(linkdatafile, os.R_OK):
        with open(linkdatafile) as data_file:    
            data = json.load(data_file)
            return data
    else:
        return read_link_data()

# Query linkdata on commandline 
def read_link_data():
    host = raw_input("Graylog host: ")
    port = raw_input("Graylog api port: ")
    user = raw_input("Username: ")
    passwd = <PASSWORD>("Password (no visible output ): ")
    
    linkdata = { "host" : host, "port" : port, "user" : user, "passwd" : <PASSWORD> }
    
    return linkdata

# JSON Formated prints
def print_json_oneline(data):
    print json.dumps(data)
    
def print_json_nosort(data):
    print json.dumps(data, indent=4)
    
def print_json_nice(data):
    print json.dumps(data, indent=4, sort_keys=True)
    
{'fraction_non_alphanumeric': 0.12292191435768261, 'fraction_numerical': 0.008816120906801008, 'mean_word_length': 3.431919642857143, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707850', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1317, 'n_words': 346}