<commit_before># -*- coding: utf-8 -*-

import httplib2
import logging
import os

from urlparse import urlparse, urlunparse

from . import settings, xsd


logger = logging.getLogger('soapbox')
logger.addHandler(logging.NullHandler())


# --- File Functions ----------------------------------------------------------
def open_document(path):
    logger.info('Opening document \'%s\'...' % path)
    # Handle documents available on the Internet:
    if path.startswith('http:'):
        disable_validation = not os.path.exists(settings.CA_CERTIFICATE_FILE)
        http = httplib2.Http(
            ca_certs=settings.CA_CERTIFICATE_FILE,
            disable_ssl_certificate_validation=disable_validation,
            timeout=settings.REQUEST_TIMEOUT,
        )
        _, content = http.request(path)
        return content

    # Attempt to open the document from the filesystem:
    else:
        return open(path, 'r').read()


# --- Template Filters --------------------------------------------------------
def remove_namespace(full_typename):
    if not full_typename:
        return None
    return full_typename.split(':')[-1]


def capitalize(value):
    return value[0].upper() + value[1:]


def uncapitalize(value):
    if value == 'QName':
        return value
    return value[0].lower() + value[1:]


def get_get_type(xsd_namespaces):
    def get_type(full_typename):
        if not full_typename:
            return None
        typename = full_typename.split(':')
        if len(typename) < 2:
            typename.insert(0, None)
        ns, typename = typename
        if ns in xsd_namespaces:
            return 'xsd.%s' % capitalize(typename)
        else:
            return '\'%s\'' % capitalize(typename)
    return get_type


def use(usevalue):
    if usevalue == xsd.Use.OPTIONAL:
        return 'xsd.Use.OPTIONAL'
    elif usevalue == xsd.Use.REQUIRED:
        return 'xsd.Use.REQUIRED'
    elif usevalue == xsd.Use.PROHIBITED:
        return 'xsd.Use.PROHIBITED'
    else:
        raise ValueError


def url_regex(url):
    '''
    http://example.net/ws/endpoint --> ^ws/endpoint$
    '''
    o = urlparse(url)
    return r'^%s$' % o.path.lstrip('/')


def url_component(url, item):<commit_msg>move import of soapbox.xsd (in soapbox.utils) into method as the utils package from my point of view provides basic utilities for all other modules. With a class-level import of "xsd" that module can not use any functionality from the utils module.<commit_after># -*- coding: utf-8 -*-

import httplib2
import logging
import os

from urlparse import urlparse, urlunparse

from . import settings


logger = logging.getLogger('soapbox')
logger.addHandler(logging.NullHandler())


# --- File Functions ----------------------------------------------------------
def open_document(path):
    logger.info('Opening document \'%s\'...' % path)
    # Handle documents available on the Internet:
    if path.startswith('http:'):
        disable_validation = not os.path.exists(settings.CA_CERTIFICATE_FILE)
        http = httplib2.Http(
            ca_certs=settings.CA_CERTIFICATE_FILE,
            disable_ssl_certificate_validation=disable_validation,
            timeout=settings.REQUEST_TIMEOUT,
        )
        _, content = http.request(path)
        return content

    # Attempt to open the document from the filesystem:
    else:
        return open(path, 'r').read()


# --- Template Filters --------------------------------------------------------
def remove_namespace(full_typename):
    if not full_typename:
        return None
    return full_typename.split(':')[-1]


def capitalize(value):
    return value[0].upper() + value[1:]


def uncapitalize(value):
    if value == 'QName':
        return value
    return value[0].lower() + value[1:]


def get_get_type(xsd_namespaces):
    def get_type(full_typename):
        if not full_typename:
            return None
        typename = full_typename.split(':')
        if len(typename) < 2:
            typename.insert(0, None)
        ns, typename = typename
        if ns in xsd_namespaces:
            return 'xsd.%s' % capitalize(typename)
        else:
            return '\'%s\'' % capitalize(typename)
    return get_type


def use(usevalue):
    from .xsd import Use
    if usevalue == Use.OPTIONAL:
        return 'xsd.Use.OPTIONAL'
    elif usevalue == Use.REQUIRED:
        return 'xsd.Use.REQUIRED'
    elif usevalue == Use.PROHIBITED:
        return 'xsd.Use.PROHIBITED'
    else:
        raise ValueError


def url_regex(url):
    '''
    http://example.net/ws/endpoint --> ^ws/endpoint$
    '''
    o = urlparse(url)
    return r'^%s$' % o.path.lstrip('/')


def url_component(url, item):
{'fraction_non_alphanumeric': 0.1577260098311605, 'fraction_numerical': 0.004274417610600555, 'mean_word_length': 2.2522585128561503, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2799471', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1352, 'n_words': 344}