<commit_before>import collections
import copy
from rdflib.namespace import SKOS
from magpie.candidates.keyword_token import KeywordToken
from magpie.candidates.utils import get_anchors


def generate_subgraph_candidates(document, ontology):
    all_words = document.get_meaningful_words()
    anchors = get_anchors(all_words, ontology)

    # TODO Filter no standalone ones
    candidates = set(copy.copy(anchors))

    # Broader
    broader_nodes = get_related_concepts(
        anchors,
        SKOS.broader,
        ontology,
        depth=2
    )
    candidates.update(broader_nodes)

    # Narrower
    narrower_nodes = get_related_concepts(
        anchors,
        SKOS.narrower,
        ontology,
        depth=2
    )
    candidates.update(narrower_nodes)

    # Composite
    composite_nodes = get_related_concepts(
        anchors,
        SKOS.composite,
        ontology,
        depth=1
    )
    candidates.update(composite_nodes)

    # CompositeOf
    compositeOf_nodes = get_related_concepts(
        anchors,
        SKOS.compositeOf,
        ontology,
        # depth=1
    )
    candidates.update(compositeOf_nodes)

    # Related
    related_nodes = get_related_concepts(
        anchors,
        SKOS.related,
        ontology,
        depth=2
    )
    candidates.update(related_nodes)

    for kw in candidates:
        kw.canonical_label = ontology.get_canonical_label_from_uri(kw.uri)
        kw.parsed_label = ontology.get_parsed_label_from_uri(kw.uri)

    return candidates


def get_related_concepts(anchors, relation, ontology, depth=None):
    """ Uses BFS to walk the ontology graph through a certain relation edges
    and gets nodes up to a certain depth. Not specifying depth means unlimited
    depth. """
    nodes = set()
    queue = collections.deque([(0, a) for a in anchors])

    while len(queue) != 0:
        distance, node = queue.popleft()
        if not depth or distance < depth:
            for child_uri in ontology.get_children_of_node(node.uri, relation):
                token = KeywordToken(child_uri)
                if token not in nodes:
                    queue.append((distance + 1, token))
                    nodes.add(token)<commit_msg>style: use deque as a boolean in a while loop<commit_after>import collections
import copy
from rdflib.namespace import SKOS
from magpie.candidates.keyword_token import KeywordToken
from magpie.candidates.utils import get_anchors


def generate_subgraph_candidates(document, ontology):
    all_words = document.get_meaningful_words()
    anchors = get_anchors(all_words, ontology)

    # TODO Filter no standalone ones
    candidates = set(copy.copy(anchors))

    # Broader
    broader_nodes = get_related_concepts(
        anchors,
        SKOS.broader,
        ontology,
        depth=2
    )
    candidates.update(broader_nodes)

    # Narrower
    narrower_nodes = get_related_concepts(
        anchors,
        SKOS.narrower,
        ontology,
        depth=2
    )
    candidates.update(narrower_nodes)

    # Composite
    composite_nodes = get_related_concepts(
        anchors,
        SKOS.composite,
        ontology,
        depth=1
    )
    candidates.update(composite_nodes)

    # CompositeOf
    compositeOf_nodes = get_related_concepts(
        anchors,
        SKOS.compositeOf,
        ontology,
        # depth=1
    )
    candidates.update(compositeOf_nodes)

    # Related
    related_nodes = get_related_concepts(
        anchors,
        SKOS.related,
        ontology,
        depth=2
    )
    candidates.update(related_nodes)

    for kw in candidates:
        kw.canonical_label = ontology.get_canonical_label_from_uri(kw.uri)
        kw.parsed_label = ontology.get_parsed_label_from_uri(kw.uri)

    return candidates


def get_related_concepts(anchors, relation, ontology, depth=None):
    """ Uses BFS to walk the ontology graph through a certain relation edges
    and gets nodes up to a certain depth. Not specifying depth means unlimited
    depth. """
    nodes = set()
    queue = collections.deque([(0, a) for a in anchors])

    while queue:
        distance, node = queue.popleft()
        if not depth or distance < depth:
            for child_uri in ontology.get_children_of_node(node.uri, relation):
                token = KeywordToken(child_uri)
                if token not in nodes:
                    queue.append((distance + 1, token))
                    nodes.add(token)
{'fraction_non_alphanumeric': 0.07165461121157324, 'fraction_numerical': 0.0033905967450271247, 'mean_word_length': 2.5063391442155307, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3601615', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1305, 'n_words': 305}