<commit_before>from spacy.en import English

nlp = English()

# TODO:  Move behind a shared NER abstract wrapper(?)

"""Map to consistent names """
normalize_types = {"ORG": "<ORGANIZATION>",
                   "GPE": "<LOCATION>",
                   "PERSON": "<PERSON>"}


def ner(text, types=["<PERSON>", "<ORGANIZATION>", "<LOCATION>"]):
    """Finds named entities in given text string. Only returns
    requested types (and the text of other tokens). """
    doc = nlp(" ".join(text.split()))

    # merge adjacent tokens of same type. E.g. [<PERSON>, <PERSON>] -> [<PERSON>]. I.e. just delete second one of each pair.
    entities = [{normalize_types.get(e.label_, e.label_): e.text} for e in doc.ents if normalize_types.get(e.label_, e.label_) in types]
    tokens = [normalize_types.get(t.ent_type_, t.text) for t in doc]
    tokens_filtered = [token for idx, token in enumerate(tokens)
                       if idx == 0 or (token != tokens[idx - 1] and token[0] == "<")]
    return {"tokens": tokens_filtered, "entities": entities}


def dev():
    text = "What the BBC thinks about London  and  <NAME>."
    print(ner(text))<commit_msg>Removed adjacent pairs of identical NER tokens (fix)<commit_after>from spacy.en import English

nlp = English()

# TODO:  Move behind a shared NER abstract wrapper(?)

"""Map to consistent names """
normalize_types = {"ORG": "<ORGANIZATION>",
                   "GPE": "<LOCATION>",
                   "PERSON": "<PERSON>"}


def ner(text, types=["<PERSON>", "<ORGANIZATION>", "<LOCATION>"]):
    """Finds named entities in given text string. Only returns
    requested types (and the text of other tokens). """
    doc = nlp(" ".join(text.split()))

    # merge adjacent tokens of same type. E.g. [<PERSON>, <PERSON>] -> [<PERSON>]. I.e. just delete second one of each pair.
    entities = [{normalize_types.get(e.label_, e.label_): e.text} for e in doc.ents if normalize_types.get(e.label_, e.label_) in types]
    tokens = [normalize_types.get(t.ent_type_, t.text) for t in doc]
    tokens_filtered = [token for idx, token in enumerate(tokens)
                       if idx == 0 or token != tokens[idx - 1] or token[0] != "<"]
    return {"tokens": tokens_filtered, "entities": entities}


def dev():
    text = "What the BBC thinks about London  and  <NAME>."
    print(ner(text))
{'fraction_non_alphanumeric': 0.15437392795883362, 'fraction_numerical': 0.002572898799313894, 'mean_word_length': 3.1810035842293907, 'pattern_counts': {'":': 10, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080051', 'n_tokens_mistral': 751, 'n_tokens_neox': 721, 'n_words': 248}