<commit_before>'''
This module provides a wrapper for the TagMe API.
'''

from __future__ import absolute_import, division, print_function, unicode_literals

import dateutil.parser
import json
import logging
import requests
import six

from html.parser import HTMLParser

__all__ = [
    'annotate', 'mentions', 'relatedness_wid', 'relatedness_title', 'Annotation',
    'AnnotateResponse', 'Mention', 'MentionsResponse', 'Relatedness', 'RelatednessResponse',
    'normalize_title', 'title_to_uri',
    ]

__author__ = '<NAME> <<EMAIL>>'

DEFAULT_TAG_API = "https://tagme.d4science.org/tagme/tag"
DEFAULT_SPOT_API = "https://tagme.d4science.org/tagme/spot"
DEFAULT_REL_API = "https://tagme.d4science.org/tagme/rel"
DEFAULT_LANG = "en"
DEFAULT_LONG_TEXT = 3
WIKIPEDIA_URI_BASE = u"https://{}.wikipedia.org/wiki/{}"
MAX_RELATEDNESS_PAIRS_PER_REQUEST = 100
GCUBE_TOKEN = None
HTML_PARSER = HTMLParser()

class Annotation(object):
    '''
    An annotation, i.e. a link of a part of text to an entity.
    '''
    def __init__(self, ann_json):
        self.begin = int(ann_json.get("start"))
        self.end = int(ann_json.get("end"))
        self.entity_id = int(ann_json.get("id"))
        self.entity_title = ann_json.get("title")
        self.score = float(ann_json.get("rho"))
        self.mention = ann_json.get("spot")

    def __str__(self):
        return u"{} -> {} (score: {})".format(self.mention, self.entity_title, self.score)

    def uri(self, lang=DEFAULT_LANG):
        '''
        Get the URI of this annotation entity.
        :param lang: the Wikipedia language.
        '''
        return title_to_uri(self.entity_title, lang)


class AnnotateResponse(object):
    '''
    A response to a call to the annotation (/tag) service. It contains the list of annotations
    found.
    '''
    def __init__(self, json_content):
        self.annotations = [Annotation(ann_json) for ann_json in json_content["annotations"] if "title" in ann_json]
        self.time = int(json_content["time"])
        self.lang = json_content["lang"]
        self.timestamp = dateutil.parser.parse(json_content["timestamp"])

    def get_annotations(self, min_rho=None):
        '''<commit_msg>Add the original json returned by the API to AnnotateResponse objects.<commit_after>'''
This module provides a wrapper for the TagMe API.
'''

from __future__ import absolute_import, division, print_function, unicode_literals

import dateutil.parser
import json
import logging
import requests
import six

from html.parser import HTMLParser

__all__ = [
    'annotate', 'mentions', 'relatedness_wid', 'relatedness_title', 'Annotation',
    'AnnotateResponse', 'Mention', 'MentionsResponse', 'Relatedness', 'RelatednessResponse',
    'normalize_title', 'title_to_uri',
    ]

__author__ = '<NAME> <<EMAIL>>'

DEFAULT_TAG_API = "https://tagme.d4science.org/tagme/tag"
DEFAULT_SPOT_API = "https://tagme.d4science.org/tagme/spot"
DEFAULT_REL_API = "https://tagme.d4science.org/tagme/rel"
DEFAULT_LANG = "en"
DEFAULT_LONG_TEXT = 3
WIKIPEDIA_URI_BASE = u"https://{}.wikipedia.org/wiki/{}"
MAX_RELATEDNESS_PAIRS_PER_REQUEST = 100
GCUBE_TOKEN = None
HTML_PARSER = HTMLParser()

class Annotation(object):
    '''
    An annotation, i.e. a link of a part of text to an entity.
    '''
    def __init__(self, ann_json):
        self.begin = int(ann_json.get("start"))
        self.end = int(ann_json.get("end"))
        self.entity_id = int(ann_json.get("id"))
        self.entity_title = ann_json.get("title")
        self.score = float(ann_json.get("rho"))
        self.mention = ann_json.get("spot")

    def __str__(self):
        return u"{} -> {} (score: {})".format(self.mention, self.entity_title, self.score)

    def uri(self, lang=DEFAULT_LANG):
        '''
        Get the URI of this annotation entity.
        :param lang: the Wikipedia language.
        '''
        return title_to_uri(self.entity_title, lang)


class AnnotateResponse(object):
    '''
    A response to a call to the annotation (/tag) service. It contains the list of annotations
    found.
    '''
    def __init__(self, json_content):
        self.annotations = [Annotation(ann_json) for ann_json in json_content["annotations"] if "title" in ann_json]
        self.time = int(json_content["time"])
        self.lang = json_content["lang"]
        self.timestamp = dateutil.parser.parse(json_content["timestamp"])
        self.original_json = json_content

    def get_annotations(self, min_rho=None):
        '''
{'fraction_non_alphanumeric': 0.12950123014985462, 'fraction_numerical': 0.0031312905390293, 'mean_word_length': 3.6876310272536688, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5065986', 'n_tokens_mistral': 1453, 'n_tokens_neox': 1420, 'n_words': 336}