natural language toolkit tagger utilities c 20012023 nltk project edward loper edlopergmail com steven bird stevenbird1gmail com url https www nltk org for license information see license txt given the string representation of a tagged token return the corresponding tuple representation the rightmost occurrence of sep in s will be used to divide s into a word string and a tag string if sep does not occur in s return s none from nltk tag util import str2tuple str2tuple flynn fly nn type s str param s the string representation of a tagged token type sep str param sep the separator string used to separate word strings from tags given the tuple representation of a tagged token return the corresponding string representation this representation is formed by concatenating the token s word string followed by the separator followed by the token s tag if the tag is none then just return the bare word string from nltk tag util import tuple2str taggedtoken fly nn tuple2strtaggedtoken flynn type taggedtoken tuplestr str param taggedtoken the tuple representation of a tagged token type sep str param sep the separator string used to separate word strings from tags given a tagged sentence return an untagged version of that sentence i e return a list containing the first element of each tuple in taggedsentence from nltk tag util import untag untag john nnp saw vbd mary nnp john saw mary natural language toolkit tagger utilities c 2001 2023 nltk project edward loper edloper gmail com steven bird stevenbird1 gmail com url https www nltk org for license information see license txt given the string representation of a tagged token return the corresponding tuple representation the rightmost occurrence of sep in s will be used to divide s into a word string and a tag string if sep does not occur in s return s none from nltk tag util import str2tuple str2tuple fly nn fly nn type s str param s the string representation of a tagged token type sep str param sep the separator string used to separate word strings from tags given the tuple representation of a tagged token return the corresponding string representation this representation is formed by concatenating the token s word string followed by the separator followed by the token s tag if the tag is none then just return the bare word string from nltk tag util import tuple2str tagged_token fly nn tuple2str tagged_token fly nn type tagged_token tuple str str param tagged_token the tuple representation of a tagged token type sep str param sep the separator string used to separate word strings from tags given a tagged sentence return an untagged version of that sentence i e return a list containing the first element of each tuple in tagged_sentence from nltk tag util import untag untag john nnp saw vbd mary nnp john saw mary
def str2tuple(s, sep="/"):
    
    loc = s.rfind(sep)
    if loc >= 0:
        return (s[:loc], s[loc + len(sep) :].upper())
    else:
        return (s, None)


def tuple2str(tagged_token, sep="/"):
    
    word, tag = tagged_token
    if tag is None:
        return word
    else:
        assert sep not in tag, "tag may not contain sep!"
        return f"{word}{sep}{tag}"


def untag(tagged_sentence):
    
    return [w for (w, t) in tagged_sentence]