<commit_before>#!/usr/bin/env python -*- coding: utf-8 -*-
#
# Python Word Sense Disambiguation (pyWSD)
#
# Copyright (C) 2014-2018 alvations
# URL:
# For license information, see LICENSE.md

import string
from itertools import chain

from nltk.corpus import wordnet as wn
from nltk.corpus import stopwords
from nltk import word_tokenize, pos_tag

from pywsd.cosine import cosine_similarity as cos_sim
from pywsd.utils import lemmatize, porter, lemmatize_sentence, synset_properties

pywsd_stopwords = ["'s", "``", "`"]
EN_STOPWORDS = set(stopwords.words('english') + list(string.punctuation) + pywsd_stopwords)


def signatures(ambiguous_word, pos=None, hyperhypo=True, adapted=False,
               remove_stopwords=True, to_lemmatize=True, remove_numbers=True,
               lowercase=True, original_lesk=False):
    """
    :param ambiguous_word:
    :type ambiguous_word:

    """
    # Ensure that the POS is supported.
    pos = pos if pos in ['a', 'r', 's', 'n', 'v', None] else None
    # Holds the synset->signature dictionary.
    synsets_signatures = {}

    # Collects the signatures from WordNet.
    for ss in wn.synsets(ambiguous_word, pos=pos):
        signature = []
        # Adds the definition, example sentences and lemma_names.
        signature += word_tokenize(ss.definition())
        # If the original lesk signature is requested.
        # Skip the other signatures.
        if original_lesk:
            synsets_signatures[ss] = signature
            continue
        # Adds the examples and lemma names.
        signature += chain(*[word_tokenize(eg) for eg in ss.examples()])
        signature += ss.lemma_names()

        # Includes lemma_names of hyper-/hyponyms.
        if hyperhypo:
            hyperhyponyms = set(ss.hyponyms() + ss.hypernyms() + ss.instance_hyponyms() + ss.instance_hypernyms())
            signature += set(chain(*[i.lemma_names() for i in hyperhyponyms]))

        # Includes signatures from related senses as in Adapted Lesk.
        if adapted:
            # Includes lemma_names from holonyms, meronyms and similar_tos
            related_senses = set(ss.member_holonyms() + ss.part_holonyms() + ss.substance_holonyms() + \
                                 ss.member_meronyms() + ss.part_meronyms() + ss.substance_meronyms() + \
                                 ss.similar_tos())
            signature += set(chain(*[i.lemma_names() for i in related_senses]))

        # Keep only the unique bag-of-words
        synsets_signatures[ss] = set(signature)

    # Process the signatures as requested.
    for ss, signature in synsets_signatures.items():
        # Removes stopwords.
        if remove_stopwords:
            signature = signature.difference(EN_STOPWORDS)
        # Lemmatized context is preferred over stemmed context.
        if to_lemmatize:
            signature = [lemmatize(s.lower()) if lowercase else lemmatize(s) # Lowercasing checks here.
                         for s in signature
                         # We only throw away if both remove_numbers and s is a digit are true.
                         if not (remove_numbers and s.isdigit())
                         ]
        synsets_signatures[ss] = signature

    return synsets_signatures


def compare_overlaps_greedy(context, synsets_signatures):
    """
    Calculate overlaps between the context sentence and the synset_signature
    and returns the synset with the highest overlap.

    Note: Greedy algorithm only keeps the best sense,
    see https://en.wikipedia.org/wiki/Greedy_algorithm

    Only used by original_lesk(). Keeping greedy algorithm for documentary sake,
    because original_lesks is greedy.
    """
    max_overlaps = 0; lesk_sense = None
    for ss in synsets_signatures:
        overlaps = set(synsets_signatures[ss]).intersection(context)
        if len(overlaps) > max_overlaps:
            lesk_sense = ss
            max_overlaps = len(overlaps)<commit_msg>tease apart synset signatures and surface word signatures<commit_after>#!/usr/bin/env python -*- coding: utf-8 -*-
#
# Python Word Sense Disambiguation (pyWSD)
#
# Copyright (C) 2014-2018 alvations
# URL:
# For license information, see LICENSE.md

import string
import pickle
from itertools import chain

from nltk.corpus import wordnet as wn
from nltk.corpus import stopwords
from nltk import word_tokenize, pos_tag

from pywsd.cosine import cosine_similarity as cos_sim
from pywsd.utils import lemmatize, porter, lemmatize_sentence, synset_properties

pywsd_stopwords = ["'s", "``", "`"]
EN_STOPWORDS = set(stopwords.words('english') + list(string.punctuation) + pywsd_stopwords)


def synset_signatures(ss, pos=None, hyperhypo=True, adapted=False,
                      remove_stopwords=True, to_lemmatize=True, remove_numbers=True,
                      lowercase=True, original_lesk=False):
    """
    :param ss: A WordNet synset.
    :type ss: nltk.corpus.wordnet.Synset
    """
    # Collects the signatures from WordNet.
    signature = []
    # Adds the definition, example sentences and lemma_names.
    signature += word_tokenize(ss.definition())
    # If the original lesk signature is requested, skip the other signatures.
    if original_lesk:
        synsets_signatures[ss] = signature
        return set(signature)
    # Adds the examples and lemma names.
    signature += chain(*[word_tokenize(eg) for eg in ss.examples()])
    signature += ss.lemma_names()

    # Includes lemma_names of hyper-/hyponyms.
    if hyperhypo:
        hyperhyponyms = set(ss.hyponyms() + ss.hypernyms() + ss.instance_hyponyms() + ss.instance_hypernyms())
        signature += set(chain(*[i.lemma_names() for i in hyperhyponyms]))

    # Includes signatures from related senses as in Adapted Lesk.
    if adapted:
        # Includes lemma_names from holonyms, meronyms and similar_tos
        related_senses = set(ss.member_holonyms() + ss.part_holonyms() + ss.substance_holonyms() + \
                             ss.member_meronyms() + ss.part_meronyms() + ss.substance_meronyms() + \
                             ss.similar_tos())
        signature += set(chain(*[i.lemma_names() for i in related_senses]))

    # Removes stopwords.
    if remove_stopwords:
        signature = set(signature).difference(EN_STOPWORDS)

    # Lemmatized context is preferred over stemmed context.
    if to_lemmatize:
        signature = [lemmatize(s.lower()) if lowercase else lemmatize(s) # Lowercasing checks here.
                     for s in signature
                     # We only throw away if both remove_numbers and s is a digit are true.
                     if not (remove_numbers and s.isdigit())
                     ]
    # Keep only the unique bag-of-words
    return signature


def signatures(ambiguous_word, pos=None, hyperhypo=True, adapted=False,
               remove_stopwords=True, to_lemmatize=True, remove_numbers=True,
               lowercase=True, original_lesk=False):
    """
    :param ambiguous_word: The ambiguous word.
    :type ambiguous_word: str
    """
    # Ensure that the POS is supported.
    pos = pos if pos in ['a', 'r', 's', 'n', 'v', None] else None
    # Holds the synset->signature dictionary.
    synsets_signatures = {}
    for ss in wn.synsets(ambiguous_word):
        synsets_signatures[ss] = synset_signatures(ss, pos=None,
                                    hyperhypo=True, adapted=False,
                                    remove_stopwords=True, to_lemmatize=True,
                                    remove_numbers=True, lowercase=True,
                                    original_lesk=False)

    return synsets_signatures


def compare_overlaps_greedy(context, synsets_signatures):
    """
    Calculate overlaps between the context sentence and the synset_signature
    and returns the synset with the highest overlap.

    Note: Greedy algorithm only keeps the best sense,
    see https://en.wikipedia.org/wiki/Greedy_algorithm

    Only used by original_lesk(). Keeping greedy algorithm for documentary sake,
    because original_lesks is greedy.
    """
    max_overlaps = 0; lesk_sense = None
    for ss in synsets_signatures:
        overlaps = set(synsets_signatures[ss]).intersection(context)
        if len(overlaps) > max_overlaps:
            lesk_sense = ss
            max_overlaps = len(overlaps)
{'fraction_non_alphanumeric': 0.08849450682120005, 'fraction_numerical': 0.002414584087890861, 'mean_word_length': 2.706487695749441, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6459022', 'n_tokens_mistral': 2541, 'n_tokens_neox': 2407, 'n_words': 678}