natural language toolkit combinatory categorial grammar c 20012023 nltk project graeme gange ggangecsse unimelb edu au url https www nltk org for license information see license txt ccg lexicons regular expressions used for parsing components of the lexicon parses a primitive category and subscripts separates the next primitive category from the remainder of the string separates the next application operator from the remainder parses the definition of the righthand side rhs of either a word or a family parses the right hand side that contains category and maybe semantic predicate parses the semantic predicate strips comments from a line class representing a token token category semantics e g eat svarplvar x y eatx y token string categ string semantics expression class representing a lexicon for ccg grammars primitives the list of primitive categories for the lexicon families families of categories entries a mapping of words to possible categories returns all the possible categories for a word return the target category for the parser string representation of the lexicon used for debugging parsing lexicons separate the contents matching the first set of brackets from the rest of the input separate the string for the next portion of the category from the rest of the string parse an application operator parse the subscripts for a primitive category parse a primitive category if the primitive is the special category var replace it with the correct ccgvar parse a string representing a category and returns a tuple with possibly the ccg variable for the category convert string representation into a lexicon for ccgs strip comments and leadingtrailing whitespace a line of primitive categories the first one is the target category ie s n np vp either a family definition or a word definition family definition ie det npn word definition ie which nnsnp rather minimal lexicon based on the openccg tinytiny grammar only incorporates a subset of the morphological subcategories however s np n primitive categories det npn determiners pro np intransvsg snpsg tensed intransitive verbs singular intransvpl snppl plural transvsg snpsgnp tensed transitive verbs singular transvpl snpplnp plural the npsgnsg the npplnpl i pro me pro we pro us pro book nsg books npl peach nsg peaches npl policeman nsg policemen npl boy nsg boys npl sleep intransvsg sleep intransvpl eat intransvpl eat transvpl eats intransvsg eats transvsg see transvpl sees transvsg natural language toolkit combinatory categorial grammar c 2001 2023 nltk project graeme gange ggange csse unimelb edu au url https www nltk org for license information see license txt ccg lexicons regular expressions used for parsing components of the lexicon parses a primitive category and subscripts a za z a za z separates the next primitive category from the remainder of the string a za z a za z separates the next application operator from the remainder parses the definition of the right hand side rhs of either a word or a family s_ s s parses the right hand side that contains category and maybe semantic predicate s parses the semantic predicate strips comments from a line class representing a token token category semantics e g eat s var pl var x y eat x y token string categ string semantics expression class representing a lexicon for ccg grammars primitives the list of primitive categories for the lexicon families families of categories entries a mapping of words to possible categories returns all the possible categories for a word return the target category for the parser string representation of the lexicon used for debugging parsing lexicons separate the contents matching the first set of brackets from the rest of the input separate the string for the next portion of the category from the rest of the string parse an application operator parse the subscripts for a primitive category parse a primitive category if the primitive is the special category var replace it with the correct ccgvar parse a string representing a category and returns a tuple with possibly the ccg variable for the category convert string representation into a lexicon for ccgs strip comments and leading trailing whitespace a line of primitive categories the first one is the target category ie s n np vp either a family definition or a word definition family definition ie det np n word definition ie which n n s np rather minimal lexicon based on the openccg tinytiny grammar only incorporates a subset of the morphological subcategories however s np n primitive categories det np n determiners pro np intransvsg s np sg tensed intransitive verbs singular intransvpl s np pl plural transvsg s np sg np tensed transitive verbs singular transvpl s np pl np plural the np sg n sg the np pl n pl i pro me pro we pro us pro book n sg books n pl peach n sg peaches n pl policeman n sg policemen n pl boy n sg boys n pl sleep intransvsg sleep intransvpl eat intransvpl eat transvpl eats intransvsg eats transvsg see transvpl sees transvsg
import re
from collections import defaultdict

from nltk.ccg.api import CCGVar, Direction, FunctionalCategory, PrimitiveCategory
from nltk.internals import deprecated
from nltk.sem.logic import Expression






PRIM_RE = re.compile(r)



NEXTPRIM_RE = re.compile(r)


APP_RE = re.compile(r)


LEX_RE = re.compile(r, re.UNICODE)


RHS_RE = re.compile(r, re.UNICODE)


SEMANTICS_RE = re.compile(r, re.UNICODE)


COMMENTS_RE = re.compile()


class Token:
    

    def __init__(self, token, categ, semantics=None):
        self._token = token
        self._categ = categ
        self._semantics = semantics

    def categ(self):
        return self._categ

    def semantics(self):
        return self._semantics

    def __str__(self):
        semantics_str = ""
        if self._semantics is not None:
            semantics_str = " {" + str(self._semantics) + "}"
        return "" + str(self._categ) + semantics_str

    def __cmp__(self, other):
        if not isinstance(other, Token):
            return -1
        return cmp((self._categ, self._semantics), other.categ(), other.semantics())


class CCGLexicon:
    

    def __init__(self, start, primitives, families, entries):
        self._start = PrimitiveCategory(start)
        self._primitives = primitives
        self._families = families
        self._entries = entries

    def categories(self, word):
        
        return self._entries[word]

    def start(self):
        
        return self._start

    def __str__(self):
        
        string = ""
        first = True
        for ident in sorted(self._entries):
            if not first:
                string = string + "\n"
            string = string + ident + " => "

            first = True
            for cat in self._entries[ident]:
                if not first:
                    string = string + " | "
                else:
                    first = False
                string = string + "%s" % cat
        return string







def matchBrackets(string):
    
    rest = string[1:]
    inside = "("

    while rest != "" and not rest.startswith(")"):
        if rest.startswith("("):
            (part, rest) = matchBrackets(rest)
            inside = inside + part
        else:
            inside = inside + rest[0]
            rest = rest[1:]
    if rest.startswith(")"):
        return (inside + ")", rest[1:])
    raise AssertionError("Unmatched bracket in string '" + string + "'")


def nextCategory(string):
    
    if string.startswith("("):
        return matchBrackets(string)
    return NEXTPRIM_RE.match(string).groups()


def parseApplication(app):
    
    return Direction(app[0], app[1:])


def parseSubscripts(subscr):
    
    if subscr:
        return subscr[1:-1].split(",")
    return []


def parsePrimitiveCategory(chunks, primitives, families, var):
    
    if chunks[0] == "var":
        if chunks[1] is None:
            if var is None:
                var = CCGVar()
            return (var, var)

    catstr = chunks[0]
    if catstr in families:
        (cat, cvar) = families[catstr]
        if var is None:
            var = cvar
        else:
            cat = cat.substitute([(cvar, var)])
        return (cat, var)

    if catstr in primitives:
        subscrs = parseSubscripts(chunks[1])
        return (PrimitiveCategory(catstr, subscrs), var)
    raise AssertionError(
        "String '" + catstr + "' is neither a family nor primitive category."
    )


def augParseCategory(line, primitives, families, var=None):
    
    (cat_string, rest) = nextCategory(line)

    if cat_string.startswith("("):
        (res, var) = augParseCategory(cat_string[1:-1], primitives, families, var)

    else:
        (res, var) = parsePrimitiveCategory(
            PRIM_RE.match(cat_string).groups(), primitives, families, var
        )

    while rest != "":
        app = APP_RE.match(rest).groups()
        direction = parseApplication(app[0:3])
        rest = app[3]

        (cat_string, rest) = nextCategory(rest)
        if cat_string.startswith("("):
            (arg, var) = augParseCategory(cat_string[1:-1], primitives, families, var)
        else:
            (arg, var) = parsePrimitiveCategory(
                PRIM_RE.match(cat_string).groups(), primitives, families, var
            )
        res = FunctionalCategory(res, arg, direction)

    return (res, var)


def fromstring(lex_str, include_semantics=False):
    
    CCGVar.reset_id()
    primitives = []
    families = {}
    entries = defaultdict(list)
    for line in lex_str.splitlines():
        
        line = COMMENTS_RE.match(line).groups()[0].strip()
        if line == "":
            continue

        if line.startswith(":-"):
            
            
            
            primitives = primitives + [
                prim.strip() for prim in line[2:].strip().split(",")
            ]
        else:
            
            (ident, sep, rhs) = LEX_RE.match(line).groups()
            (catstr, semantics_str) = RHS_RE.match(rhs).groups()
            (cat, var) = augParseCategory(catstr, primitives, families)

            if sep == "::":
                
                
                families[ident] = (cat, var)
            else:
                semantics = None
                if include_semantics is True:
                    if semantics_str is None:
                        raise AssertionError(
                            line
                            + " must contain semantics because include_semantics is set to True"
                        )
                    else:
                        semantics = Expression.fromstring(
                            SEMANTICS_RE.match(semantics_str).groups()[0]
                        )
                
                
                entries[ident].append(Token(ident, cat, semantics))
    return CCGLexicon(primitives[0], primitives, families, entries)


@deprecated("Use fromstring() instead.")
def parseLexicon(lex_str):
    return fromstring(lex_str)


openccg_tinytiny = fromstring(
    
)