<commit_before>import os
import socket
import urlparse
import urllib2
import httplib
import wget as wget_wget
import smtplib
import SimpleHTTPServer
import SocketServer

import logging
log = logging.getLogger(__file__)

import file_utils
import exe_utils


def get_hostname():
    return socket.gethostname()

def basic_web_server(port="50080"):
    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
    httpd = SocketServer.TCPServer(("", port), Handler)
    print "serving at port", port
    httpd.serve_forever()

def background_web_server(dir=None, port="50080"):
    file_utils.cd(dir)
    ret_val = exe_utils.run_python("-m SimpleHTTPServer " + port, fork=True, pid_file="py_server.pid")
    return ret_val

def my_wget(url, file_path, delete_first=True):
    """ wget a file from url
        IMPORTANT NOTE: this will *not* work if the URL is a redirect, etc...
    """
    is_success = True
    try:
        # get gtfs file from url
        req = urllib2.Request(url)
        res = urllib2.urlopen(req)

        if delete_first:
            file_utils.rm(file_path)

        # write it out
        f = open(file_path, 'wb')
        f.write(res.read())
        f.flush()
        f.close()
        res.close()

        log.info("wget: downloaded {} into file {}".format(url, file_path))
    except Exception, e:
        log.warn(e)
        is_success = False
    return is_success

def wget(url, file_path, delete_first=True):
    """ wget a file from url
        IMPORTANT NOTE: this will *not* work if the URL is a redirect, etc...
    """
    is_success = True
    if delete_first:
        file_utils.rm(file_path)
    wget_wget.download(url, file_path)
    log.info("wget: downloaded {} into file {}".format(url, file_path))
    return is_success

def post(hostname, port, path, data):
    '''
    '''
    statuscode = -111
    try:
        webservice = httplib.HTTP("{}:{}".format(hostname, port))
        webservice.putrequest("POST", path)
        webservice.putheader("Host", hostname)
        webservice.putheader("Content-type", "text/xml")
        webservice.putheader("Content-length", "%d" % len(data))
        webservice.endheaders()
        webservice.send(data)
        statuscode, statusmessage, header = webservice.getreply()
        log.info("{} :: {} :: {}".format(statuscode, statusmessage, header ))
        #result = webservice.getfile().read()
        #log.info("{}".format(result))
    except Exception, e:
        log.info(e)
    return statuscode

def post_data(url, data):
    '''
    '''<commit_msg>add a get util that will capture exceptions<commit_after>import os
import socket
import urlparse
import urllib2
import httplib
import wget as wget_wget
import smtplib
import SimpleHTTPServer
import SocketServer

import logging
log = logging.getLogger(__file__)

import file_utils
import exe_utils


def get_hostname():
    return socket.gethostname()

def basic_web_server(port="50080"):
    Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
    httpd = SocketServer.TCPServer(("", port), Handler)
    print "serving at port", port
    httpd.serve_forever()

def background_web_server(dir=None, port="50080"):
    file_utils.cd(dir)
    ret_val = exe_utils.run_python("-m SimpleHTTPServer " + port, fork=True, pid_file="py_server.pid")
    return ret_val

def my_wget(url, file_path, delete_first=True):
    """ wget a file from url
        IMPORTANT NOTE: this will *not* work if the URL is a redirect, etc...
    """
    is_success = True
    try:
        # get gtfs file from url
        req = urllib2.Request(url)
        res = urllib2.urlopen(req)

        if delete_first:
            file_utils.rm(file_path)

        # write it out
        f = open(file_path, 'wb')
        f.write(res.read())
        f.flush()
        f.close()
        res.close()

        log.info("wget: downloaded {} into file {}".format(url, file_path))
    except Exception, e:
        log.warn(e)
        is_success = False
    return is_success

def wget(url, file_path, delete_first=True):
    """ wget a file from url
        IMPORTANT NOTE: this will *not* work if the URL is a redirect, etc...
    """
    is_success = True
    if delete_first:
        file_utils.rm(file_path)
    wget_wget.download(url, file_path)
    log.info("wget: downloaded {} into file {}".format(url, file_path))
    return is_success

def get(url):
    ''' safe http get of url, with return of
    '''
    success = True
    response = None
    try:
        response = urllib2.urlopen(url)
    except Exception, e:
        log.info(e)
        success = False
    return success, response

def post(hostname, port, path, data):
    '''
    '''
    statuscode = -111
    try:
        webservice = httplib.HTTP("{}:{}".format(hostname, port))
        webservice.putrequest("POST", path)
        webservice.putheader("Host", hostname)
        webservice.putheader("Content-type", "text/xml")
        webservice.putheader("Content-length", "%d" % len(data))
        webservice.endheaders()
        webservice.send(data)
        statuscode, statusmessage, header = webservice.getreply()
        log.info("{} :: {} :: {}".format(statuscode, statusmessage, header ))
        #result = webservice.getfile().read()
        #log.info("{}".format(result))
    except Exception, e:
        log.info(e)
    return statuscode

def post_data(url, data):
    '''
    '''
{'fraction_non_alphanumeric': 0.12098581030619865, 'fraction_numerical': 0.0061613144137415985, 'mean_word_length': 2.867870036101083, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249521', 'n_tokens_mistral': 1720, 'n_tokens_neox': 1711, 'n_words': 429}