<commit_before>import hypothesisapi
from pycproject.readctree import CProject
import argparse
from SPARQLWrapper import SPARQLWrapper, JSON

def payload_builder(url, annotation_content, annotation_pre=None, annotation_exact=None, annotation_suffix=None):
    if len(annotation_pre)>32:
        annotation_pre=annotation_pre[-32:]
    if len(annotation_pre)>32:
        annotation_suffix=annotation_suffix[:32]
    payload = {
        "text": annotation_content,
        "group": "__world__",
        "uri": url,
        "target": [{"source": url,
                    "selector": [{"type":"TextQuoteSelector",
                                  "prefix": annotation_pre,
                                  "exact": annotation_exact,
                                  "suffix": annotation_suffix
                                  },
                                 {"type": "TestPositionSelector", "start": None, "end": None}]
                    }]
    }
    return payload

def icun_status_from_wikidata(string):
    sparql = SPARQLWrapper("http://query.wikidata.org/sparql")
    sparql.setQuery("""
    SELECT ?statusLabel WHERE {{
    ?item wdt:P141 ?status.
    ?item rdfs:label "{}"@en
        SERVICE wikibase:label {{bd:serviceParam wikibase:language "en" .}}
}}
    """.format(string))
    sparql.setReturnFormat(JSON)
    result = sparql.query().convert()
    if result['results']['bindings'] and len(result['results']['bindings'])==1:
        return result['results']['bindings'][0]['statusLabel']['value']
    return None

def annotate_EuropePMC_from_CTree(ctree, H):
    if ctree.ID[0:3] == "PMC":
        url = "http://europepmc.org/articles/{}/".format(ctree.ID)
    else:
        return
    results = (ctree.show_results("species"))
    if results and "binomial" in results and results['binomial']:
        for result in results['binomial']:
            status = icun_status_from_wikidata(result['exact'])
            if status:
                annotation = "according to wikidata " + result['exact'] + " has IUCN redlist status: " + status
                pre_tidy = result['pre'].replace("( ", "(")
                post_tidy = result['post'].replace("( ", "(")
                payload=payload_builder(url, annotation, pre_tidy, result['exact'], post_tidy)
                H.create(payload)



parser = argparse.ArgumentParser(description='Read Mined Terms from ContentMine and'
                                             ' Annotate documents online with Hypothesis')
parser.add_argument("cproject", help="name of the cproject to work on")
parser.add_argument("-u", help="your hypothes.is username", dest="user", action="store", required=True)
parser.add_argument("-k", help="your hypothes.is apikey", dest="api_key", action="store", required=True)
args=parser.parse_args()

print("Writing annotations to hypothes.is with user: {} and api key: {}". format(
    args.user, args.api_key
))

H = hypothesisapi.API(args.user, args.api_key)

cproject = CProject("./", args.cproject)


for ctree in cproject.get_ctrees():
    annotate_EuropePMC_from_CTree(ctree, H)
<commit_msg>Use altLabel on wikidata too for more species<commit_after>import hypothesisapi
from pycproject.readctree import CProject
import argparse
from SPARQLWrapper import SPARQLWrapper, JSON


def payload_builder(url, annotation_content, annotation_pre=None, annotation_exact=None, annotation_suffix=None):
    if len(annotation_pre)>32:
        annotation_pre=annotation_pre[-32:]
    if len(annotation_pre)>32:
        annotation_suffix=annotation_suffix[:32]
    payload = {
        "text": annotation_content,
        "group": "__world__",
        "uri": url,
        "target": [{"source": url,
                    "selector": [{"type":"TextQuoteSelector",
                                  "prefix": annotation_pre,
                                  "exact": annotation_exact,
                                  "suffix": annotation_suffix
                                  }]
                    }]
    }
    return payload


def icun_status_from_wikidata(string):
    sparql = SPARQLWrapper("http://query.wikidata.org/sparql")
    sparql.setQuery("""
    SELECT ?statusLabel WHERE {{
    ?item wdt:P141 ?status.
    {{ ?item rdfs:label "{}"@en }} UNION {{ ?item skos:altLabel "{}"@en }}
        SERVICE wikibase:label {{bd:serviceParam wikibase:language "en" .}}
}}
    """.format(string, string))
    sparql.setReturnFormat(JSON)
    result = sparql.query().convert()
    if result['results']['bindings'] and len(result['results']['bindings'])==1:
        return result['results']['bindings'][0]['statusLabel']['value']
    return None


def annotate_europepmc_from_ctree(ctree, H):
    if ctree.ID[0:3] == "PMC":
        url = "http://europepmc.org/articles/{}/".format(ctree.ID)
    else:
        return
    results = (ctree.show_results("species"))
    if results and "binomial" in results and results['binomial']:
        for result in results['binomial']:
            status = icun_status_from_wikidata(result['exact'])
            if status:
                annotation = "according to wikidata " + result['exact'] + " has IUCN redlist status: " + status
                pre_tidy = result['pre'].replace("( ", "(")
                post_tidy = result['post'].replace("( ", "(")
                payload=payload_builder(url, annotation, pre_tidy, result['exact'], post_tidy)
                H.create(payload)


parser = argparse.ArgumentParser(description='Read Mined Terms from ContentMine and'
                                             ' Annotate documents online with Hypothesis')
parser.add_argument("cproject", help="name of the cproject to work on")
parser.add_argument("-u", help="your hypothes.is username", dest="user", action="store", required=True)
parser.add_argument("-k", help="your hypothes.is apikey", dest="api_key", action="store", required=True)
args=parser.parse_args()

print("Writing annotations to hypothes.is with user: {} and api key: {}". format(
    args.user, args.api_key
))

H = hypothesisapi.API(args.user, args.api_key)

cproject = CProject("./", args.cproject)

for ctree in cproject.get_ctrees():
    annotate_europepmc_from_ctree(ctree, H)

{'fraction_non_alphanumeric': 0.13618677042801555, 'fraction_numerical': 0.0048638132295719845, 'mean_word_length': 2.853216739537789, 'pattern_counts': {'":': 25, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6152286', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1787, 'n_words': 384}