<jupyter_start><jupyter_text>Probabilistic Viterbi parser<jupyter_code>import nltk
from nltk import  PCFG, ViterbiParser

grammar = PCFG.fromstring('''
  NP  -> NNS [0.5] | JJ NNS [0.3] | NP CC NP [0.2]
  NNS -> "cats" [0.1] | "dogs" [0.2] | "mice" [0.3] | NNS CC NNS [0.4]
  JJ  -> "big" [0.4] | "small" [0.6]
  CC  -> "and" [0.9] | "or" [0.1]
  ''')

sent = ['small', 'cats', 'and', 'mice']

parser = ViterbiParser(grammar)
parse = parser.parse(sent)
tree = next(parse)
print(tree)<jupyter_output>(NP (JJ small) (NNS (NNS cats) (CC and) (NNS mice))) (p=0.001944)<jupyter_text>Trace<jupyter_code>parser = ViterbiParser(grammar)
parser.trace(3)
parse = parser.parse(sent)
print(next(parse))<jupyter_output>Inserting tokens into the most likely constituents table...
   Insert: |=...| small
   Insert: |.=..| cats
   Insert: |..=.| and
   Insert: |...=| mice
Finding the most likely constituents spanning 1 text elements...
   Insert: |=...| JJ -> 'small' [0.6]               0.6000000000 
   Insert: |.=..| NNS -> 'cats' [0.1]               0.1000000000 
   Insert: |.=..| NP -> NNS [0.5]                   0.0500000000 
   Insert: |..=.| CC -> 'and' [0.9]                 0.9000000000 
   Insert: |...=| NNS -> 'mice' [0.3]               0.3000000000 
   Insert: |...=| NP -> NNS [0.5]                   0.1500000000 
Finding the most likely constituents spanning 2 text elements...
   Insert: |==..| NP -> JJ NNS [0.3]                0.0180000000 
Finding the most likely constituents spanning 3 text elements...
   Insert: |.===| NP -> NP CC NP [0.2]              0.0013500000 
   Insert: |.===| NNS -> NNS CC NNS [0.4]           0.0108000000 
   Insert: |.===| NP -> NNS [0.5]                   0.00540[...]<jupyter_text>Learn a treebank grammar<jupyter_code>import nltk
nltk.download('treebank')
from nltk.corpus import treebank
productions = []
S = nltk.Nonterminal('S')
for f in treebank.fileids():
    for tree in treebank.parsed_sents(f):
        productions += tree.productions()
grammar = nltk.induce_pcfg(S, productions)
grammar.productions()[10:15]<jupyter_output><empty_output><jupyter_text>Apply the learnt PCFG to Viterbi parser<jupyter_code>sent = ['it', 'is', 'a', 'small', 'group', 'of', 'workers', 'and', 'researchers']
parser = ViterbiParser(grammar)
parse = parser.parse(sent)
tree = next(parse)
print(tree)<jupyter_output>(S
  (NP-SBJ (PRP it))
  (VP
    (VBZ is)
    (NP-PRD
      (NP (DT a) (JJ small) (NN group))
      (PP
        (IN of)
        (NP (NNS workers) (CC and) (NNS researchers)))))) (p=2.64379e-21)
{'fraction_non_alphanumeric': 0.18085106382978725, 'fraction_numerical': 0.06698187549251379, 'mean_word_length': 2.7670623145400595, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '523840', 'license': 'MIT', 'n_tokens_mistral': 1109, 'n_tokens_neox': 967, 'n_words': 239, 'path': 's8/codes/s8c.ipynb', 'repo_name': 'gebakx/ihlt', 'size': '2538'}