<commit_before>import logging
from collections import defaultdict

import nltk
from SPARQLWrapper import SPARQLWrapper, JSON

from mies.celery import app


def extract_entity_names(t):
    entity_names = []

    if hasattr(t, 'label') and t.label:
        if t.label() == 'NE':
            entity_names.append(' '.join([child[0] for child in t]))
        else:
            for child in t:
                entity_names.extend(extract_entity_names(child))

    return entity_names


def extract_named_entities(text):
    sentences = nltk.sent_tokenize(text)
    tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
    tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
    chunked_sentences = nltk.ne_chunk_sents(tagged_sentences, binary=True)
    entity_names = []
    for tree in chunked_sentences:
        entity_names.extend(extract_entity_names(tree))
    return list(set(entity_names))


def get_entries_in_wikipedia(name):
    entries = []
    sparql = SPARQLWrapper("http://dbpedia.org/sparql")
    name = name.replace(" ", "_")
    query = """
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
        SELECT ?label
        WHERE { <http://dbpedia.org/resource/%s> rdfs:label ?label }
    """ % name
    sparql.setReturnFormat(JSON)
    sparql.setQuery(query)
    results = sparql.query().convert()
    for result in results["results"]["bindings"]:
        entries.append(result["label"]["value"])
    return entries


def get_concept_features_from_wikipedia(name):
    entries = []
    sparql = SPARQLWrapper("http://dbpedia.org/sparql")
    name = name.replace(" ", "_")
    query = """
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    SELECT ?label, ?pic, ?wiki
    WHERE {
        <http://dbpedia.org/resource/%s> rdfs:label ?label .
        <http://dbpedia.org/resource/%s> dbo:thumbnail ?pic .
        <http://dbpedia.org/resource/%s> foaf:isPrimaryTopicOf ?wiki
    }
    """ % (name, name, name)
    sparql.setReturnFormat(JSON)
    sparql.setQuery(query)
    results = sparql.query().convert()
    for result in results["results"]["bindings"]:
        entries.append(dict(label=result["label"]["value"],
                            picture=result["picture"]["value"],
                            wiki=result["wiki"]["value"]))
    return entries


# TODO template function for filtering

def filter_named_entities_by_appearance_in_metadata(named_entities, metadata):
    RANK_THRESHOLD = 2
    all_metadata = metadata.values()
    concept_rank = defaultdict(int)
    for concept in named_entities:
        score = 0
        for value in all_metadata:
            if value is not None and concept in value:
                score += 1
        concept_rank[concept] = score
    results = [concept for concept in named_entities if concept_rank[concept] > RANK_THRESHOLD]
    logging.info("Metadata ditched" + ":"*80)<commit_msg>bug fixes<commit_after>import logging
from collections import defaultdict

import nltk
from SPARQLWrapper import SPARQLWrapper, JSON

from mies.celery import app


def extract_entity_names(t):
    entity_names = []

    if hasattr(t, 'label') and t.label:
        if t.label() == 'NE':
            entity_names.append(' '.join([child[0] for child in t]))
        else:
            for child in t:
                entity_names.extend(extract_entity_names(child))

    return entity_names


def extract_named_entities(text):
    sentences = nltk.sent_tokenize(text)
    tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
    tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
    chunked_sentences = nltk.ne_chunk_sents(tagged_sentences, binary=True)
    entity_names = []
    for tree in chunked_sentences:
        entity_names.extend(extract_entity_names(tree))
    return list(set(entity_names))


def get_entries_in_wikipedia(name):
    entries = []
    sparql = SPARQLWrapper("http://dbpedia.org/sparql")
    name = name.replace(" ", "_")
    query = """
        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
        SELECT ?label
        WHERE { <http://dbpedia.org/resource/%s> rdfs:label ?label }
    """ % name
    sparql.setReturnFormat(JSON)
    sparql.setQuery(query)
    results = sparql.query().convert()
    for result in results["results"]["bindings"]:
        entries.append(result["label"]["value"])
    return entries


def get_concept_features_from_wikipedia(name):
    entries = []
    sparql = SPARQLWrapper("http://dbpedia.org/sparql")
    name = name.replace(" ", "_")
    query = """
    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
    SELECT ?label, ?pic, ?wiki
    WHERE {
        <http://dbpedia.org/resource/%s> rdfs:label ?label .
        <http://dbpedia.org/resource/%s> dbo:thumbnail ?pic .
        <http://dbpedia.org/resource/%s> foaf:isPrimaryTopicOf ?wiki
    }
    """ % (name, name, name)
    sparql.setReturnFormat(JSON)
    sparql.setQuery(query)
    results = sparql.query().convert()
    for result in results["results"]["bindings"]:
        entries.append(dict(label=result["label"]["value"],
                            picture=result["pic"]["value"],
                            wiki=result["wiki"]["value"]))
    return entries


# TODO template function for filtering

def filter_named_entities_by_appearance_in_metadata(named_entities, metadata):
    RANK_THRESHOLD = 2
    all_metadata = metadata.values()
    concept_rank = defaultdict(int)
    for concept in named_entities:
        score = 0
        for value in all_metadata:
            if value is not None and isinstance(value, basestring) and concept in value:
                score += 1
        concept_rank[concept] = score
    results = [concept for concept in named_entities if concept_rank[concept] > RANK_THRESHOLD]
    logging.info("Metadata ditched" + ":"*80)
{'fraction_non_alphanumeric': 0.12397680763983629, 'fraction_numerical': 0.0068212824010914054, 'mean_word_length': 3.081419624217119, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6744662', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1859, 'n_words': 379}