natural language toolkit lin s thesaurus c 20012023 nltk project dan blanchard dblanchardets org url https www nltk org for license information see license txt wrapper for the lispformatted thesauruses distributed by dekang lin compiled regular expression for extracting the key from the first line of each thesaurus entry keyre re compiler desc 09 staticmethod def defaultdictfactory initialize the thesaurus param root root directory containing thesaurus lisp files type root cstring param badscore the score to give to words which do not appear in each other s sets of synonyms type badscore cfloat start of entry end of entry lines with pairs of ngrams and scores returns the similarity score for two ngrams param ngram1 first ngram to compare type ngram1 cstring param ngram2 second ngram to compare type ngram2 cstring param fileid thesaurus fileid to search in if none search all fileids type fileid cstring return if fileid is specified just the score for the two ngrams otherwise list of tuples of fileids and scores entries don t contain themselves so make sure similarity between item and itself is 1 0 returns a list of scored synonyms tuples of synonyms and scores for the current ngram param ngram ngram to lookup type ngram cstring param fileid thesaurus fileid to search in if none search all fileids type fileid cstring return if fileid is specified list of tuples of scores and synonyms otherwise list of tuples of fileids and lists where inner lists consist of tuples of scores and synonyms returns a list of synonyms for the current ngram param ngram ngram to lookup type ngram cstring param fileid thesaurus fileid to search in if none search all fileids type fileid cstring return if fileid is specified list of synonyms otherwise list of tuples of fileids and lists where inner lists contain synonyms determines whether or not the given ngram is in the thesaurus param ngram ngram to lookup type ngram cstring return whether the given ngram is in the thesaurus demo natural language toolkit lin s thesaurus c 2001 2023 nltk project dan blanchard dblanchard ets org url https www nltk org for license information see license txt wrapper for the lisp formatted thesauruses distributed by dekang lin compiled regular expression for extracting the key from the first line of each thesaurus entry factory for creating defaultdict of defaultdict dict s initialize the thesaurus param root root directory containing thesaurus lisp files type root c string param badscore the score to give to words which do not appear in each other s sets of synonyms type badscore c float start of entry end of entry lines with pairs of ngrams and scores returns the similarity score for two ngrams param ngram1 first ngram to compare type ngram1 c string param ngram2 second ngram to compare type ngram2 c string param fileid thesaurus fileid to search in if none search all fileids type fileid c string return if fileid is specified just the score for the two ngrams otherwise list of tuples of fileids and scores entries don t contain themselves so make sure similarity between item and itself is 1 0 returns a list of scored synonyms tuples of synonyms and scores for the current ngram param ngram ngram to lookup type ngram c string param fileid thesaurus fileid to search in if none search all fileids type fileid c string return if fileid is specified list of tuples of scores and synonyms otherwise list of tuples of fileids and lists where inner lists consist of tuples of scores and synonyms returns a list of synonyms for the current ngram param ngram ngram to lookup type ngram c string param fileid thesaurus fileid to search in if none search all fileids type fileid c string return if fileid is specified list of synonyms otherwise list of tuples of fileids and lists where inner lists contain synonyms determines whether or not the given ngram is in the thesaurus param ngram ngram to lookup type ngram c string return whether the given ngram is in the thesaurus demo
import re
from collections import defaultdict
from functools import reduce

from nltk.corpus.reader import CorpusReader


class LinThesaurusCorpusReader(CorpusReader):
    

    
    
    _key_re = re.compile(r'\("?([^"]+)"? \(desc [0-9.]+\).+')

    @staticmethod
    def __defaultdict_factory():
        
        return defaultdict(dict)

    def __init__(self, root, badscore=0.0):
        

        super().__init__(root, r"sim[A-Z]\.lsp")
        self._thesaurus = defaultdict(LinThesaurusCorpusReader.__defaultdict_factory)
        self._badscore = badscore
        for path, encoding, fileid in self.abspaths(
            include_encoding=True, include_fileid=True
        ):
            with open(path) as lin_file:
                first = True
                for line in lin_file:
                    line = line.strip()
                    
                    if first:
                        key = LinThesaurusCorpusReader._key_re.sub(r"\1", line)
                        first = False
                    
                    elif line == "))":
                        first = True
                    
                    else:
                        split_line = line.split("\t")
                        if len(split_line) == 2:
                            ngram, score = split_line
                            self._thesaurus[fileid][key][ngram.strip('"')] = float(
                                score
                            )

    def similarity(self, ngram1, ngram2, fileid=None):
        
        
        if ngram1 == ngram2:
            if fileid:
                return 1.0
            else:
                return [(fid, 1.0) for fid in self._fileids]
        else:
            if fileid:
                return (
                    self._thesaurus[fileid][ngram1][ngram2]
                    if ngram2 in self._thesaurus[fileid][ngram1]
                    else self._badscore
                )
            else:
                return [
                    (
                        fid,
                        (
                            self._thesaurus[fid][ngram1][ngram2]
                            if ngram2 in self._thesaurus[fid][ngram1]
                            else self._badscore
                        ),
                    )
                    for fid in self._fileids
                ]

    def scored_synonyms(self, ngram, fileid=None):
        
        if fileid:
            return self._thesaurus[fileid][ngram].items()
        else:
            return [
                (fileid, self._thesaurus[fileid][ngram].items())
                for fileid in self._fileids
            ]

    def synonyms(self, ngram, fileid=None):
        
        if fileid:
            return self._thesaurus[fileid][ngram].keys()
        else:
            return [
                (fileid, self._thesaurus[fileid][ngram].keys())
                for fileid in self._fileids
            ]

    def __contains__(self, ngram):
        
        return reduce(
            lambda accum, fileid: accum or (ngram in self._thesaurus[fileid]),
            self._fileids,
            False,
        )







def demo():
    from nltk.corpus import lin_thesaurus as thes

    word1 = "business"
    word2 = "enterprise"
    print("Getting synonyms for " + word1)
    print(thes.synonyms(word1))

    print("Getting scored synonyms for " + word1)
    print(thes.scored_synonyms(word1))

    print("Getting synonyms from simN.lsp (noun subsection) for " + word1)
    print(thes.synonyms(word1, fileid="simN.lsp"))

    print("Getting synonyms from simN.lsp (noun subsection) for " + word1)
    print(thes.synonyms(word1, fileid="simN.lsp"))

    print(f"Similarity score for {word1} and {word2}:")
    print(thes.similarity(word1, word2))


if __name__ == "__main__":
    demo()