<commit_before>from models import EntityAnnotation, TextAnnotation

class Enhancer():

    OUTPUTFORMAT = {
        'jsonld':'application/json',
        'xml':'application/rdf+xml',
        'rdfjson':'application/rdf+json',
        'nt' :'text/rdf+nt',
        'turtle':'text/turtle',
    }

    STANBOL_ENHANCER_PATH = "enhancer"
    STANBOL_CHAIN_PATH = "chain"
    STANBOL_DEFAULT_CHAIN = "default"

    ENHANCER_CHAIN = "chain"
    ENHANCER_FORMAT = "format"
    ENHANCER_DEREFERENCING_FIELDS = "fields"
    ENHANCER_LDPATH_PREFIXES = "prefixes"
    ENHANCER_LDPATH_FIELDS = "ldpath"

    __ENHANCER_DEFAULT_FORMAT = 'turtle'

    __ENHANCER_PARAM_LDPATH = "enhancer.engines.dereference.ldpath"
    __ENHANCER_PARAM_DEREF_FIELDS = "enhancer.engines.dereference.fields"

    def __init__(self, rest_client):
        self.__rest_client = rest_client
        """:type : RestClient"""

    def enhance(self, content, parameters=None):
        import urllib
        import parser

        if not parameters:
            parameters = {}

        enhancer_endpoint = self.__rest_client.endpoint + self.STANBOL_ENHANCER_PATH

        if self.ENHANCER_CHAIN in parameters:
            enhancer_endpoint += "/" + self.STANBOL_CHAIN_PATH + "/" + parameters[self.ENHANCER_CHAIN]

        deref_fields = False
        if self.ENHANCER_DEREFERENCING_FIELDS in parameters:
            fields = parameters[self.ENHANCER_DEREFERENCING_FIELDS]
            if isinstance(fields, dict):
                fields = fields.values()
                if len(fields) > 0:
                    deref_fields = True
                    for i, field in enumerate(fields):
                        if i == 0:
                            enhancer_endpoint += "?"
                        else:
                            enhancer_endpoint += "&"
                        enhancer_endpoint += self.__ENHANCER_PARAM_DEREF_FIELDS + "=" + urllib.quote(field)

        if self.ENHANCER_LDPATH_FIELDS in parameters:
            ldpath = self.__build_ldpath(parameters.get(self.ENHANCER_LDPATH_PREFIXES),
                                         parameters[self.ENHANCER_LDPATH_FIELDS])
            if ldpath and len(ldpath) > 0:
                if deref_fields:
                    enhancer_endpoint += "&"
                else:
                    enhancer_endpoint += "?"
                enhancer_endpoint += self.__ENHANCER_PARAM_LDPATH + "=" + urllib.quote(ldpath)

        headers = {'Content-Type':'text/plain'}<commit_msg>used default arguments instead of custom code int he enhancer client<commit_after>from models import EntityAnnotation, TextAnnotation

class Enhancer():

    OUTPUTFORMAT = {
        'jsonld':'application/json',
        'xml':'application/rdf+xml',
        'rdfjson':'application/rdf+json',
        'nt' :'text/rdf+nt',
        'turtle':'text/turtle',
    }

    STANBOL_ENHANCER_PATH = "enhancer"
    STANBOL_CHAIN_PATH = "chain"
    STANBOL_DEFAULT_CHAIN = "default"

    ENHANCER_CHAIN = "chain"
    ENHANCER_FORMAT = "format"
    ENHANCER_DEREFERENCING_FIELDS = "fields"
    ENHANCER_LDPATH_PREFIXES = "prefixes"
    ENHANCER_LDPATH_FIELDS = "ldpath"

    __ENHANCER_DEFAULT_FORMAT = 'turtle'

    __ENHANCER_PARAM_LDPATH = "enhancer.engines.dereference.ldpath"
    __ENHANCER_PARAM_DEREF_FIELDS = "enhancer.engines.dereference.fields"

    def __init__(self, rest_client):
        self.__rest_client = rest_client
        """:type : RestClient"""

    def enhance(self, content, parameters={}):
        import urllib
        import parser

        enhancer_endpoint = self.__rest_client.endpoint + self.STANBOL_ENHANCER_PATH

        if self.ENHANCER_CHAIN in parameters:
            enhancer_endpoint += "/" + self.STANBOL_CHAIN_PATH + "/" + parameters[self.ENHANCER_CHAIN]

        deref_fields = False
        if self.ENHANCER_DEREFERENCING_FIELDS in parameters:
            fields = parameters[self.ENHANCER_DEREFERENCING_FIELDS]
            if isinstance(fields, dict):
                fields = fields.values()
                if len(fields) > 0:
                    deref_fields = True
                    for i, field in enumerate(fields):
                        if i == 0:
                            enhancer_endpoint += "?"
                        else:
                            enhancer_endpoint += "&"
                        enhancer_endpoint += self.__ENHANCER_PARAM_DEREF_FIELDS + "=" + urllib.quote(field)

        if self.ENHANCER_LDPATH_FIELDS in parameters:
            ldpath = self.__build_ldpath(parameters.get(self.ENHANCER_LDPATH_PREFIXES),
                                         parameters[self.ENHANCER_LDPATH_FIELDS])
            if ldpath and len(ldpath) > 0:
                if deref_fields:
                    enhancer_endpoint += "&"
                else:
                    enhancer_endpoint += "?"
                enhancer_endpoint += self.__ENHANCER_PARAM_LDPATH + "=" + urllib.quote(ldpath)

        headers = {'Content-Type':'text/plain'}
{'fraction_non_alphanumeric': 0.09322885272252361, 'fraction_numerical': 0.0012055455093429777, 'mean_word_length': 2.119047619047619, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5314991', 'n_tokens_mistral': 1636, 'n_tokens_neox': 1551, 'n_words': 243}