<commit_before>import requests

from pp_api import utils as u


def extract(text, pid, server, auth_data=None, session=None, **kwargs):
    """
    Make extract call using project determined by pid.

    :param auth_data: 
    :param session: 
    :param text: text
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'numberOfConcepts': 100000,
        'numberOfTerms': 100000,
        'text': text,
        'projectId': pid,
        'language': 'en',
        'useTransitiveBroaderConcepts': True,
        'useRelatedConcepts': True,
        'sentimentAnalysis': True
    }
    data.update(kwargs)
    session = u.get_session(session, auth_data)
    r = session.post(
        server + '/extractor/api/extract',
        data=data,
        files={'file': ('.txt', text)}
    )
    r.raise_for_status()
    return r


def get_cpts_from_response(r):
    attributes = ['prefLabel', 'frequencyInDocument', 'uri',
                  'transitiveBroaderConcepts', 'relatedConcepts']

    extr_cpts = []
    concept_container = r.json()

    if not 'concepts' in concept_container:
        if not 'document' in concept_container:
            return extr_cpts
        if not 'concepts' in concept_container['document']:
            #no mention of concepts either in the json directly or document inside
            return extr_cpts
        else:
            #concepts are mentioned inside 'document'
            concept_container = concept_container['document']

    for cpt_json in concept_container['concepts']:
        cpt = dict()
        for attr in attributes:
            if attr in cpt_json:
                cpt[attr] = cpt_json[attr]
            else:
                cpt[attr] = []
        extr_cpts.append(cpt)

    return extr_cpts


def get_shadow_cpts_from_response(r):
    attributes = ['prefLabel', 'uri',
                  'transitiveBroaderConcepts', 'corporaScore']

    shadow_cpts = []
    concept_container = r.json()

    if not 'shadowConcepts' in concept_container:
        if not 'document' in concept_container:
            return shadow_cpts
        if not 'shadowConcepts' in concept_container['document']:
            #no mention of concepts either in the json directly or document inside
            return shadow_cpts
        else:
            #concepts are mentioned inside 'document'
            concept_container = concept_container['document']

    for cpt_json in concept_container['shadowConcepts']:
        cpt = dict()
        for attr in attributes:
            if attr in cpt_json:
                cpt[attr] = cpt_json[attr]
            else:
                cpt[attr] = []
        shadow_cpts.append(cpt)

    return shadow_cpts


def get_terms_from_response(r):
    attributes = ['textValue', 'frequencyInDocument', 'score']
    extr_terms = []
    term_container = r.json()

    found = False
    for term_key_word in ['freeTerms', 'extractedTerms']:
        if term_key_word in term_container:
            found = True
            break
        elif 'document' in term_container and term_key_word in term_container['document']:
            term_container = term_container['document']
            found = True
            break

    assert found, [extr_terms, list(term_container.keys())]

    for term_json in term_container[term_key_word]:
        term = dict()
        for attr in attributes:
            if attr in term_json:
                term[attr] = term_json[attr]
            else:
                term[attr] = []
        extr_terms.append(term)

    return extr_terms


def get_sentiment_from_response(r):
    return r.json()["sentiments"][0]["score"]


def get_pref_labels(uris, pid, server, auth_data=None, session=None):
    """
    Make extract call using project determined by pid.

    :param uris: 
    :param auth_data: 
    :param session: 
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'concepts': uris,
        'projectId': pid,
        'language': 'en',
    }
    session = u.get_session(session, auth_data)
    r = session.get(
        server + '/PoolParty/api/thesaurus/{}/concepts'.format(pid),
        params=data
    )
    r.raise_for_status()
    return [x['prefLabel'] for x in r.json()]


def get_cpt_corpus_freqs(corpus_id, server, pid, auth_data=None, session=None):
    """
    Make call to PP to extract frequencies of concepts in a corpus.

    :param auth_data: 
    :param session: 
    :param corpus_id: corpus id
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'corpusId': corpus_id,
        'startIndex': 0
    }
    suffix = '/PoolParty/api/corpusmanagement/{pid}/results/concepts'.format(
        pid=pid
    )
    results = []
    session = u.get_session(session, auth_data)
    while True:
        r = session.get(server + suffix, params=data)
        r.raise_for_status()
        data['startIndex'] += 20
        results += r.json()
        if not len(r.json()):
            break
    return results


def get_cpt_path(cpt_uri, server, pid, auth_data=None, session=None):
    """
    Make call to PP to extract path of concept.

    :param cpt_uri: 
    :param auth_data: 
    :param session: 
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'concept': cpt_uri
    }
    suffix = '/PoolParty/api/thesaurus/{pid}/getPaths'.format(
        pid=pid
    )
    session = u.get_session(session, auth_data)
    r = session.get(server + suffix, params=data)
    r.raise_for_status()
    broaders = [(x['uri'], x['prefLabel']) for x in r.json()[0]['conceptPath']]
    cpt_scheme = r.json()[0]['conceptScheme']
    result = [(cpt_scheme['uri'], cpt_scheme['title'])] + broaders
    return result


def get_term_coocs(term_str, corpus_id, server, pid,
                   auth_data=None, session=None):
    suffix = '/PoolParty/api/corpusmanagement/' \
             '{pid}/results/cooccurrence/term'.format(
        pid=pid
    )
    data = {
        'corpusId': corpus_id,
        'term': term_str,
        'startIndex': 0,
        'limit': 2**15, #int(sys.maxsize)<commit_msg>added a check that allows us to to pass a rdflib.URIRef object to the get_cpt_path, not only a string<commit_after>import requests

from pp_api import utils as u


def extract(text, pid, server, auth_data=None, session=None, **kwargs):
    """
    Make extract call using project determined by pid.

    :param auth_data: 
    :param session: 
    :param text: text
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'numberOfConcepts': 100000,
        'numberOfTerms': 100000,
        'text': text,
        'projectId': pid,
        'language': 'en',
        'useTransitiveBroaderConcepts': True,
        'useRelatedConcepts': True,
        'sentimentAnalysis': True
    }
    data.update(kwargs)
    session = u.get_session(session, auth_data)
    r = session.post(
        server + '/extractor/api/extract',
        data=data,
        files={'file': ('.txt', text)}
    )
    r.raise_for_status()
    return r


def get_cpts_from_response(r):
    attributes = ['prefLabel', 'frequencyInDocument', 'uri',
                  'transitiveBroaderConcepts', 'relatedConcepts']

    extr_cpts = []
    concept_container = r.json()

    if not 'concepts' in concept_container:
        if not 'document' in concept_container:
            return extr_cpts
        if not 'concepts' in concept_container['document']:
            #no mention of concepts either in the json directly or document inside
            return extr_cpts
        else:
            #concepts are mentioned inside 'document'
            concept_container = concept_container['document']

    for cpt_json in concept_container['concepts']:
        cpt = dict()
        for attr in attributes:
            if attr in cpt_json:
                cpt[attr] = cpt_json[attr]
            else:
                cpt[attr] = []
        extr_cpts.append(cpt)

    return extr_cpts


def get_shadow_cpts_from_response(r):
    attributes = ['prefLabel', 'uri',
                  'transitiveBroaderConcepts', 'corporaScore']

    shadow_cpts = []
    concept_container = r.json()

    if not 'shadowConcepts' in concept_container:
        if not 'document' in concept_container:
            return shadow_cpts
        if not 'shadowConcepts' in concept_container['document']:
            #no mention of concepts either in the json directly or document inside
            return shadow_cpts
        else:
            #concepts are mentioned inside 'document'
            concept_container = concept_container['document']

    for cpt_json in concept_container['shadowConcepts']:
        cpt = dict()
        for attr in attributes:
            if attr in cpt_json:
                cpt[attr] = cpt_json[attr]
            else:
                cpt[attr] = []
        shadow_cpts.append(cpt)

    return shadow_cpts


def get_terms_from_response(r):
    attributes = ['textValue', 'frequencyInDocument', 'score']
    extr_terms = []
    term_container = r.json()

    found = False
    for term_key_word in ['freeTerms', 'extractedTerms']:
        if term_key_word in term_container:
            found = True
            break
        elif 'document' in term_container and term_key_word in term_container['document']:
            term_container = term_container['document']
            found = True
            break

    assert found, [extr_terms, list(term_container.keys())]

    for term_json in term_container[term_key_word]:
        term = dict()
        for attr in attributes:
            if attr in term_json:
                term[attr] = term_json[attr]
            else:
                term[attr] = []
        extr_terms.append(term)

    return extr_terms


def get_sentiment_from_response(r):
    return r.json()["sentiments"][0]["score"]


def get_pref_labels(uris, pid, server, auth_data=None, session=None):
    """
    Make extract call using project determined by pid.

    :param uris: 
    :param auth_data: 
    :param session: 
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'concepts': uris,
        'projectId': pid,
        'language': 'en',
    }
    session = u.get_session(session, auth_data)
    r = session.get(
        server + '/PoolParty/api/thesaurus/{}/concepts'.format(pid),
        params=data
    )
    r.raise_for_status()
    return [x['prefLabel'] for x in r.json()]


def get_cpt_corpus_freqs(corpus_id, server, pid, auth_data=None, session=None):
    """
    Make call to PP to extract frequencies of concepts in a corpus.

    :param auth_data: 
    :param session: 
    :param corpus_id: corpus id
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    data = {
        'corpusId': corpus_id,
        'startIndex': 0
    }
    suffix = '/PoolParty/api/corpusmanagement/{pid}/results/concepts'.format(
        pid=pid
    )
    results = []
    session = u.get_session(session, auth_data)
    while True:
        r = session.get(server + suffix, params=data)
        r.raise_for_status()
        data['startIndex'] += 20
        results += r.json()
        if not len(r.json()):
            break
    return results


def get_cpt_path(cpt_uri, server, pid, auth_data=None, session=None):
    """
    Make call to PP to extract path of concept.

    :param cpt_uri: 
    :param auth_data: 
    :param session: 
    :param pid: id of project
    :param server: server url
    :return: response object
    """
    if cpt_uri.__class__.__name__ == 'URIRef':
        cpt_uri = str(cpt_uri)
    data = {
        'concept': cpt_uri
    }
    suffix = '/PoolParty/api/thesaurus/{pid}/getPaths'.format(
        pid=pid
    )
    session = u.get_session(session, auth_data)
    r = session.get(server + suffix, params=data)
    r.raise_for_status()
    broaders = [(x['uri'], x['prefLabel']) for x in r.json()[0]['conceptPath']]
    cpt_scheme = r.json()[0]['conceptScheme']
    result = [(cpt_scheme['uri'], cpt_scheme['title'])] + broaders
    return result


def get_term_coocs(term_str, corpus_id, server, pid,
                   auth_data=None, session=None):
    suffix = '/PoolParty/api/corpusmanagement/' \
             '{pid}/results/cooccurrence/term'.format(
        pid=pid
    )
    data = {
        'corpusId': corpus_id,
        'term': term_str,
        'startIndex': 0,
        'limit': 2**15, #int(sys.maxsize)
{'fraction_non_alphanumeric': 0.10827520608750793, 'fraction_numerical': 0.0034876347495244133, 'mean_word_length': 2.354692900824249, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3555292', 'n_tokens_mistral': 3969, 'n_tokens_neox': 3802, 'n_words': 990}