<commit_before># Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright (C) 2006 Fluendo, S.A. (www.fluendo.com).
# Copyright 2006, <NAME> <<EMAIL>>

import urlparse

from coherence.extern.et import parse_xml as et_parse_xml

from coherence import SERVER_ID


from twisted.web import server, http, static
from twisted.web import client, error
from twisted.web import proxy, resource, server
from twisted.internet import reactor,protocol,defer
from twisted.python import failure

try:
    from twisted.protocols._c_urlarg import unquote
except ImportError:
    from urllib import unquote

try:
    import netifaces
    have_netifaces = True
except ImportError:
    have_netifaces = False

def parse_xml(data, encoding="utf-8"):
    return et_parse_xml(data,encoding)

def parse_http_response(data):

    """ don't try to get the body, there are reponses without """
    header = data.split('\r\n\r\n')[0]

    lines = header.split('\r\n')
    cmd = lines[0].split(' ')
    lines = map(lambda x: x.replace(': ', ':', 1), lines[1:])
    lines = filter(lambda x: len(x) > 0, lines)

    headers = [x.split(':', 1) for x in lines]
    headers = dict(map(lambda x: (x[0].lower(), x[1]), headers))

    return cmd, headers


def get_ip_address(ifname):
    """
    determine the IP address by interface name

    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/439094
    (c) <NAME>
    Uses the Linux SIOCGIFADDR ioctl to find the IP address associated
    with a network interface, given the name of that interface, e.g. "eth0".
    The address is returned as a string containing a dotted quad.

    Updated to work on BSD. OpenBSD and OSX share the same value for
    SIOCGIFADDR, and its likely that other BSDs do too.

    Updated to work on Windows,
    using the optional Python module netifaces
    http://alastairs-place.net/netifaces/

    Thx Lawrence for that patch!
    """

    import sys
    if sys.platform == 'win32':
        if have_netifaces:
            if ifname in netifaces.interfaces():
                iface = netifaces.ifaddresses(ifname)
                ifaceadr = iface(netifaces.AF_INET)
                # we now have a list of address dictionaries, there may be multiple addresses bound
                return ifaceadr[0]['addr']
        return '127.0.0.1'

    from os import uname
    import socket
    import fcntl
    import struct

    system_type = uname()[0]

    if system_type == "Linux":
        SIOCGIFADDR = 0x8915
    else:
        SIOCGIFADDR = 0xc0206921

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        SIOCGIFADDR,
        struct.pack('256s', ifname[:15])
    )[20:24])

def get_host_address():
    """ try to get determine the interface used for
        the default route, as this is most likely
        the interface we should bind to (on a single homed host!)
    """

    import sys
    if sys.platform == 'win32':
        if have_netifaces:
            return get_ip_address(netifaces.interfaces()[0])    # on windows assume first interface is primary

    try:
        route_file = '/proc/net/route'
        route = open(route_file)
        if(route):
            tmp = route.readline() #skip first line
            while (tmp != ''):
                tmp = route.readline()
                l = tmp.split('\t')
                if (len(l) > 2):
                    if l[2] != '00000000': #default gateway...
                        route.close()
                        return get_ip_address(l[0])
    except IOError, msg:
        """ fallback to parsing the output of netstat """
        from os import uname
        import posix
        (osname,_, _, _,_) = uname()
        osname = osname.lower()
        f = posix.popen('netstat -rn')
        lines = f.readlines()
        f.close()
        for l in lines:
            parts = [x.strip() for x in l.split(' ') if len(x) > 0]
            if parts[0] in ('0.0.0.0','default'):
                if osname[:6] == 'darwin':
                    return get_ip_address(parts[5])
                else:
                    return get_ip_address(parts[-1])
    except Exception, msg:
        import traceback
        traceback.print_exc()<commit_msg>do a bit better error catching, when interface detection on win32 fails<commit_after># Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright (C) 2006 Fluendo, S.A. (www.fluendo.com).
# Copyright 2006, <NAME> <<EMAIL>>

import urlparse

from coherence.extern.et import parse_xml as et_parse_xml

from coherence import SERVER_ID


from twisted.web import server, http, static
from twisted.web import client, error
from twisted.web import proxy, resource, server
from twisted.internet import reactor,protocol,defer
from twisted.python import failure

try:
    from twisted.protocols._c_urlarg import unquote
except ImportError:
    from urllib import unquote

try:
    import netifaces
    have_netifaces = True
except ImportError:
    have_netifaces = False

def parse_xml(data, encoding="utf-8"):
    return et_parse_xml(data,encoding)

def parse_http_response(data):

    """ don't try to get the body, there are reponses without """
    header = data.split('\r\n\r\n')[0]

    lines = header.split('\r\n')
    cmd = lines[0].split(' ')
    lines = map(lambda x: x.replace(': ', ':', 1), lines[1:])
    lines = filter(lambda x: len(x) > 0, lines)

    headers = [x.split(':', 1) for x in lines]
    headers = dict(map(lambda x: (x[0].lower(), x[1]), headers))

    return cmd, headers


def get_ip_address(ifname):
    """
    determine the IP address by interface name

    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/439094
    (c) <NAME>
    Uses the Linux SIOCGIFADDR ioctl to find the IP address associated
    with a network interface, given the name of that interface, e.g. "eth0".
    The address is returned as a string containing a dotted quad.

    Updated to work on BSD. OpenBSD and OSX share the same value for
    SIOCGIFADDR, and its likely that other BSDs do too.

    Updated to work on Windows,
    using the optional Python module netifaces
    http://alastairs-place.net/netifaces/

    Thx Lawrence for that patch!
    """

    import sys
    if sys.platform == 'win32':
        if have_netifaces:
            if ifname in netifaces.interfaces():
                iface = netifaces.ifaddresses(ifname)
                ifaceadr = iface(netifaces.AF_INET)
                # we now have a list of address dictionaries, there may be multiple addresses bound
                return ifaceadr[0]['addr']
        return '127.0.0.1'

    from os import uname
    import socket
    import fcntl
    import struct

    system_type = uname()[0]

    if system_type == "Linux":
        SIOCGIFADDR = 0x8915
    else:
        SIOCGIFADDR = 0xc0206921

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    return socket.inet_ntoa(fcntl.ioctl(
        s.fileno(),
        SIOCGIFADDR,
        struct.pack('256s', ifname[:15])
    )[20:24])

def get_host_address():
    """ try to get determine the interface used for
        the default route, as this is most likely
        the interface we should bind to (on a single homed host!)
    """

    import sys
    if sys.platform == 'win32':
        if have_netifaces:
            interfaces = netifaces.interfaces()
            if len(interfaces):
                return get_ip_address(interfaces[0])    # on windows assume first interface is primary
    else:
        try:
            route_file = '/proc/net/route'
            route = open(route_file)
            if(route):
                tmp = route.readline() #skip first line
                while (tmp != ''):
                    tmp = route.readline()
                    l = tmp.split('\t')
                    if (len(l) > 2):
                        if l[2] != '00000000': #default gateway...
                            route.close()
                            return get_ip_address(l[0])
        except IOError, msg:
            """ fallback to parsing the output of netstat """
            from os import uname
            import posix
            (osname,_, _, _,_) = uname()
            osname = osname.lower()
            f = posix.popen('netstat -rn')
            lines = f.readlines()
            f.close()
            for l in lines:
                parts = [x.strip() for x in l.split(' ') if len(x) > 0]
                if parts[0] in ('0.0.0.0','default'):
                    if osname[:6] == 'darwin':
                        return get_ip_address(parts[5])
                    else:
                        return get_ip_address(parts[-1])
        except Exception, msg:
            import traceback
            traceback.print_exc()
{'fraction_non_alphanumeric': 0.10338153250597745, 'fraction_numerical': 0.018217010133211888, 'mean_word_length': 2.193020719738277, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 4, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3801343', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2597, 'n_words': 821}