<commit_before>
no_wordnet = False
def wordnet_available():
    '''
    Returns True if wordnet is available, False otherwise.
    With poor-mans memoization.
    '''
    global no_wordnet
    if no_wordnet:
        return False

    try:
        from nltk.corpus import wordnet as wn
    except ModuleError:
        no_wordnet = True
        print "not expanding tags: couldn't load nltk"
        return False

    try:
        wn.synsets("dog")
    except LookupError as e:
        no_wordnet = True
        print"not expanding tags: found nltk, but couldn't load wordnet " +\
                "corpus. nltk error message follows"
        print str(e)
        return False
    return True

def expand_tags(tags):
    if not wordnet_available():
        return tags
    
    from nltk.corpus import wordnet as wn
    # OK, let's get stupid
    expanded_tags = [t for t in tags]
    for tag in tags:
        for synset in wn.synsets(tag):
            for lemma in synset.lemma_names():
                expanded_tags.append(tag)
    expanded_tags = list(set(expanded_tags))
    if len(expanded_tags) == len(tags):
        print "no expansion"
    else:
        print "expanded", tags, "-->", expanded_tags
        import pdb; pdb.set_trace()
    return expanded_tags
<commit_msg>got tag expansion working; now to tie it into search...<commit_after>
no_wordnet = False
def wordnet_available():
    '''
    Returns True if wordnet is available, False otherwise.
    With poor-mans memoization.
    '''
    global no_wordnet
    if no_wordnet:
        return False

    try:
        from nltk.corpus import wordnet as wn
    except ModuleError:
        no_wordnet = True
        print "not expanding tags: couldn't load nltk"
        return False

    try:
        wn.synsets("dog")
    except LookupError as e:
        no_wordnet = True
        print"not expanding tags: found nltk, but couldn't load wordnet " +\
                "corpus. nltk error message follows"
        print str(e)
        return False
    return True

def expand_tags(tags):
    if not wordnet_available():
        return tags
    
    from nltk.corpus import wordnet as wn
    # OK, let's get stupid
    expanded_tags = [t for t in tags]
    for tag in tags:
        for synset in wn.synsets(tag):
            for lemma in synset.lemma_names():
                expanded_tags.append(lemma)
    expanded_tags = list(set(expanded_tags))
    if len(expanded_tags) == len(tags):
        print "no expansion"
    else:
        print "expanded", tags, "-->", expanded_tags
    return expanded_tags

{'fraction_non_alphanumeric': 0.078064012490242, 'fraction_numerical': 0.0, 'mean_word_length': 2.1838509316770187, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3266457', 'n_tokens_mistral': 775, 'n_tokens_neox': 764, 'n_words': 254}