<jupyter_start><jupyter_text>Initial list of idioms<jupyter_code>import pickle
noun_dictionary = {"act of God": ":EnvironmentalOrEcologicalEvent",
                   "armed conflict": ":War",
                   "home": ":PhysicalLocation",  
                   "breaking and entering": ":RobberyAndTheft",
                   "charges": ":LegalEvent",
                   "cold shoulder": ":RefusalAndRejection",
                   "false step": ":Mistake",
                   "false papers": ":LegalInstrument",
                   "identification papers": ":LegalInstrument",
                   "helping hand": ":ProvisionOfAid",
                   "ill will": ":HatredBitternessAndDisgust",
                   "smoke screen": ":Concealment",
                   "thirst for knowledge": ":Curiosity",
                   "legal proceedings": ":LegalEvent",
                   "lack of judgment": ":Mistake",
                   "out of business": ":DissolutionOfBusiness",
                   "personal effects": ":Resource",
                   "position": ":Employment",
                   "post": ":Employment",
                   "peace of mind": ":RelaxationAndSerenity",
                   "school": ":Organization",
                   "strong-arm tactic": ":CoercionAndIntimidation",
                   "surprise attack": ":Battle",
                   "mutual assistance": ":Cooperation",
                   "word of honor": ":PromiseAndOath",
                   "crowd": ":GroupOfAgents",
                   "mob": ":GroupOfAgents"}
verb_dictionary = {"crack": "('joke' > :Joke) | (:AttackDamageAndAssault)",
                   "end": "('marriage' > :Divorce) | (:End)", 
                   "get": ":Possession", 
                   "give": "('back' & dobj > :ReturnRecoveryAndRelease ; :has_topic dobj) | " \
                           "('birth' > :Birth) | ('thanks' > :ExpressiveAndExclamation) | " \
                           "('way' > :SurrenderAndYielding) | ('word' > :PromiseAndOath) | (:GiftAndDonation)",
                   "hold": "('captive' & dobj > :CaptureAndSeizure ; :has_affected_agent dobj) | " \
                           "(dobj > :Storage ; :has_topic dobj)",
                   "keep": "('secret' & dobj > :Concealment ; :has_topic dobj) | " \
                           "(dobj > :Storage ; :has_topic dobj)",
                   "kill": "('self' > :Suicide) | (:MurderAndHomicide)",
                   "lack": "('judgment' > :Mistake) | (dobj > :Deficiency ; :has_topic dobj)", 
                   "lay":  "('bet' > :GamblingAndWagering) | (:InProcessPlacement)", 
                   "look": "('high and low' > :Searching) | ('the other way' > :IgnoringAndOverlooking) | " \
                           "(:SensoryPerception) ",
                   "make": "('appeal' > :RequestAndAppeal) | ('decision' > :DecisionMaking) | " \
                           "('known' & dobj > :Revelation ; :has_topic dobj) | ('mistake' > :Mistake) | " \
                           "(dobj > :ProductionManufactureAndCreation ; :has_topic dobj)",
                   "marry": "(dobj > :Marriage ; :has_active_agent dobj) | " \
                            "(pobj > :Marriage ; :has_active_agent pobj)",
                   "offer": "('thanks' > :ExpressiveAndExclamation) | (dobj > :GiftAndDonation ; :has_topic dobj)", 
                   "pay": "('no attention' > :IgnoringAndOverlooking) | ('no mind' > :IgnoringAndOverlooking) | " \
                          "(:PurchaseAndSale)",
                   "place": "('bet' > :GamblingAndWagering) | (:MovementTravelAndTransportation)", 
                   "put ": "('back together' & dobj > :Repair ; :has_topic dobj) | " \
                           "(:MovementTravelAndTransportation)",
                   "raise": "(Agent dobj > :CaringForDependents ; :has_affected_agent dobj) | " \
                            "(dobj > :Increase ; :has_topic dobj)",
                   "reach": "('resolution' > :Agreement) | ('verdict' > :LegalEvent) | " \
                            "(Location dobj > :MovementTravelAndTransportation ; :has_destination dobj) | " \
                            "(:EventAndState)",
                   "set": "('fire' > :Arson) | (dobj > :MovementTravelAndTransportation ; :has_topic dobj)",
                   "show": "('lack of judgment' > :Mistake) | (:Revelation)", 
                   "slip": "('away' > :EscapeAndEvasion) | (:BodyMovement)",
                   "take": "('back' & dobj > :Possession ; :has_topic dobj) | " \
                           "('control' > :Possession) | ('hostage' > :CaptivityAndDetention) | " \
                           "('leap in the dark' > :RiskTaking) | ('offensive' > :AttackDamageAndAssault) | " \
                           "('own life' > :Suicide) | ('trip' > :MovementTravelAndTransportation) | " \
                           "('vacation' > :MovementTravelAndTransportation) | " \
                           "('life' > :MurderAndHomicide) | (dobj > :Possession ; :has_topic dobj)",
                   "tell": "('joke' > :Joke) | (:CommunicationAndSpeechAct)",
                   "turn": "('blind eye' > :IgnoringAndOverlooking) | (:BodyMovement)", 
                   "wage": "('war' > :War) | (:ProjectAndCampaign)",
                   "wed": "(dobj > :Marriage ; :has_active_agent dobj) | " \
                          "(pobj > :Marriage ; :has_active_agent pobj)"}
verb_prep_dictionary = {"care for": "prep_for (Agent pobj > :CaringForDependent ; :has_affected_agent pobj) | " \
                                    "(pobj > :AttentionAndFocus ; :has_topic pobj)",
                        "care about": "prep_about (Agent pobj > :CompassionAndEmpathy ; :has_affected_agent pobj) | " \
                                      "(pobj > :AttentionAndFocus ; :has_topic pobj)",
                        "charge with": "prep_with (pobj > :LegalEvent ; :has_topic pobj)",
                        "give into": "prep_into (pobj > :Agreement ; :has_topic pobj)",
                        "go to": "prep_to ('battle' > :Battle) | (pobj > :MovementTravelAndTransportation)",
                        "go on": "prep_on ('strike' > :ProtestAndStrike) | " \
                                 "(pobj > :MovementTravelAndTransportation ; :has_instrument pobj)",
                        "hold in": "prep_in ('esteem' & dobj > :AdmirationAndRespect ; :has_affected_agent dobj) | " \
                                   "('regard' & dobj > :AdmirationAndRespect ; :has_affected_agent dobj) | " \
                                   "('reserve' & dobj > :Storage ; :has_topic dobj) | " \
                                   "(dobj & pobj > :Storage ; :has_topic dobj ; :has_instrument pobj)",
                        "look at": "prep_at (pobj > :AttentionAndFocus ; :has_topic pobj)",
                        "look for": "prep_for (pobj > :Searching ; :has_topic pobj)",
                        "put on": "prep_on (dobj > :Grooming ; :has_topic dobj)",
                        "return to": "prep_to (Location pobj > :MovementTravelAndTransportation ; :has_destination pobj) " \
                                     "| (pobj > :ReturnRecoveryAndRelease ; :has_topic pobj)",
                        "reunite with": "prep_with (Agent pobj > :InclusionAttachmentAndUnification ; :has_active_agent pobj) " \
                                        "| (pobj > :Possession ; :has_topic pobj)",
                        "take of": "prep_of ('possession' & pobj > :Possession ; :has_topic pobj)",
                        "take for": "prep_for ('responsibility' & pobj > :Acknowledgment ; :has_topic pobj)"}
verb_prt_dictionary = {"carry out": "dobj > :Ignore, dobj verb",
                       "cut down": "(Agent dobj > :MurderAndHomicide ; :has_affected_agent dobj) | " \
                                   "(dobj > :Decrease ; :has_topic dobj)",
                       "give back": ":ReturnRecoveryAndRelease",
                       "give up": ":SurrenderAndYielding",
                       "kick off": ":StartOrBeginning",
                       "put on": ":Grooming",
                       "round up": ":CaptureAndSeizure",
                       "shoot down": ":AttackDamageAndAssault",
                       "take over": "(Agent dobj > :InvasionAndOccupation ; :has_affected_agent dobj) | " \
                                    "(dobj > :InvasionAndOccupation ; :has_topic dobj)",
                       "touch up": ":Repair"}
verb_xcomp_dictionary = {"attempt": "xcomp > :Attempt, xcomp verb",
                         "carry": "xcomp > :Continuation, xcomp verb",
                         "continue": "xcomp > :Continuation, xcomp verb",
                         "give": "xcomp > :End, xcomp verb"}
with open('noun-idioms.pickle', 'wb') as outFile:
    pickle.dump(noun_dictionary, outFile)
with open('verb-idioms.pickle', 'wb') as outFile:
    pickle.dump(verb_dictionary, outFile)
with open('verb-prep-idioms.pickle', 'wb') as outFile:
    pickle.dump(verb_prep_dictionary, outFile)
with open('verb-prt-idioms.pickle', 'wb') as outFile:
    pickle.dump(verb_prt_dictionary, outFile)
with open('verb-xcomp-idioms.pickle', 'wb') as outFile:
    pickle.dump(verb_xcomp_dictionary, outFile)

from lark import Lark, Tree

with open('lark.txt', 'r') as inFile:
    lark_details = inFile.read()
dsl = Lark(lark_details)
rules = ["('joke' > :Joke) | (:AttackDamageAndAssault)",
         "('marriage' > :Divorce) | (:End)", 
         ":Possession", 
         "('back' & dobj > :ReturnRecoveryAndRelease ; :has_topic dobj) | " \
         "('birth' > :Birth) | ('thanks' > :ExpressiveAndExclamation) | " \
         "('way' > :SurrenderAndYielding) | ('word' > :PromiseAndOath) | (:GiftAndDonation)",
         "('captive' > :CaptureAndSeizure ; :has_affected_agent dobj) | " \
         "(dobj > :Storage ; :has_topic dobj)",
         "('secret' & dobj > :Concealment ; :has_topic dobj) | " \
         "(dobj > :Storage ; :has_topic dobj)",
         "('self' > :Suicide) | (:MurderAndHomicide)",
         "('judgment' > :Mistake) | (dobj > :Deficiency ; :has_topic dobj)", 
         "('bet' > :GamblingAndWagering) | (:InProcessPlacement)", 
         "('high and low' > :Searching) | ('the other way' > :IgnoringAndOverlooking) | " \
         "(:SensoryPerception) ",
         "('appeal' > :RequestAndAppeal) | ('decision' > :DecisionMaking) | " \
         "('known' & dobj > :Revelation ; :has_topic dobj) | ('mistake' > :Mistake) | " \
         "(dobj > :ProductionManufactureAndCreation ; :has_topic dobj)",
         "(dobj > :Marriage ; :has_active_agent dobj) | " \
         "(pobj > :Marriage ; :has_active_agent pobj)",
         "('thanks' > :ExpressiveAndExclamation) | (dobj > :GiftAndDonation ; :has_topic dobj)", 
         "('no attention' > :IgnoringAndOverlooking) | ('no mind' > :IgnoringAndOverlooking) | (:PurchaseAndSale)",
         "('bet' > :GamblingAndWagering) | (:MovementTravelAndTransportation)", 
         "('back together' & dobj > :Repair ; :has_topic dobj) | (:MovementTravelAndTransportation)",
         "(Agent dobj > :CaringForDependents ; :has_affected_agent dobj) | (dobj > :Increase ; :has_topic dobj)",
         "('resolution' > :Agreement) | ('verdict' > :LegalEvent) | " \
         "(Location dobj > :MovementTravelAndTransportation ; :has_destination dobj) | (:EventAndState)",
         "('fire' > :Arson) | (dobj > :MovementTravelAndTransportation ; :has_topic dobj)",
         "('lack of judgment' > :Mistake) | (:Revelation)", 
         "('away' > :EscapeAndEvasion) | (:BodyMovement)",
         "('back' & dobj > :Possession ; :has_topic dobj) | " \
         "('control' > :Possession) | ('hostage' > :CaptivityAndDetention) | " \
         "('leap in the dark' > :RiskTaking) | ('offensive' > :AttackDamageAndAssault) | " \
         "('own life' > :Suicide) | ('trip' > :MovementTravelAndTransportation) | " \
         "('vacation' > :MovementTravelAndTransportation) | " \
         "('life' > :MurderAndHomicide) | (dobj > :Possession ; :has_topic dobj)",
         "('joke' > :Joke) | (:CommunicationAndSpeechAct)",
         "('blind eye' > :IgnoringAndOverlooking) | (:BodyMovement)", 
         "('war' > :War) | (:ProjectAndCampaign)",
         "(dobj > :Marriage ; :has_active_agent dobj) | (pobj > :Marriage ; :has_active_agent pobj)",
         "prep_for (Agent pobj > :CaringForDependent ; :has_affected_agent pobj) | " \
         "(pobj > :AttentionAndFocus ; :has_topic pobj)",
         "prep_about (Agent pobj > :CompassionAndEmpathy ; :has_affected_agent pobj) | " \
         "(pobj > :AttentionAndFocus ; :has_topic pobj)",
         "prep_with (pobj > :LegalEvent ; :has_topic pobj)",
         "prep_into (pobj > :Agreement ; :has_topic pobj)",
         "prep_to ('battle' > :Battle) | (pobj > :MovementTravelAndTransportation)",
         "prep_on ('strike' > :ProtestAndStrike) | " \
         "(pobj > :MovementTravelAndTransportation ; :has_instrument pobj)",
         "prep_in ('esteem' & dobj > :AdmirationAndRespect ; :has_affected_agent dobj) | " \
         "('regard' & dobj > :AdmirationAndRespect ; :has_affected_agent dobj) | " \
         "('reserve' & dobj > :Storage ; :has_topic dobj) | " \
         "(dobj & pobj > :Storage ; :has_topic dobj ; :has_instrument pobj)",
         "prep_at (pobj > :AttentionAndFocus ; :has_topic pobj)",
         "prep_for (pobj > :Searching ; :has_topic pobj)",
         "prep_on (dobj > :Grooming ; :has_topic dobj)",
         "prep_to (Location pobj > :MovementTravelAndTransportation ; :has_destination pobj) " \
         "| (pobj > :ReturnRecoveryAndRelease ; :has_topic pobj)",
         "prep_with (Agent pobj > :InclusionAttachmentAndUnification ; :has_active_agent pobj) " \
         "| (pobj > :Possession ; :has_topic pobj)",
         "prep_of ('possession' & pobj > :Possession ; :has_topic pobj)",
         "prep_for ('responsibility' & pobj > :Acknowledgment ; :has_topic pobj)",
         "dobj > :Ignore, dobj verb",
         "(Agent dobj > :MurderAndHomicide ; :has_affected_agent dobj) | " \
         "(dobj > :Decrease ; :has_topic dobj)",
         "(Agent dobj > :InvasionAndOccupation ; :has_affected_agent dobj) | " \
         "(dobj > :InvasionAndOccupation ; :has_topic dobj)",
         "xcomp > :Attempt, xcompattack"]
         
for rule in rules:
    print(rule)
    rule_parse = dsl.parse(rule)
    print(rule_parse)
    print(rule_parse.pretty())
    print()
rule = ":Possession"
print('class_name', rule)
rule_parse = dsl.parse(rule)
tree = rule_parse.children[0].children[0]
class_name = tree.children[0].value
print(class_name)
print()

rule = "xcomp > :Attempt, xcomp attack"
print('xcomp', rule)
rule_parse = dsl.parse(rule)
tree = rule_parse.children[0].children[0]
class_tree, verb_tree = tree.children
print(class_tree.children[0].value)
xcomp, text = verb_tree.children
print(text.children[0].value)
print()

rule = "'back' & dobj > :ReturnRecoveryAndRelease ; :has_topic dobj"
print('keyword', rule)
rule_parse = dsl.parse(rule)
tree = rule_parse.children[0].children[0]        
if len(tree.children) == 4:
    text_tree, objects_tree, class_tree, property_tree = tree.children
else:
    text_tree, objects_tree, class_tree = tree.children
    property_tree = None
looking_for = ''
for text in text_tree.children:
    looking_for += f'{text.value} '
print(looking_for[:-1])
if objects_tree:
    print(objects_tree.children[0].data)
print(class_tree.children[0].value)
if property_tree:
    print(property_tree)
    prop_names, prop_obj_type = property_tree.children
    prop_name = ':'
    for pname in prop_names.children:
        prop_name += f'{pname.value}_'
    print(prop_name[:-1])
    print(prop_obj_type.children[0].data)
print()
    
rule = "prep_for (Agent pobj > :CaringForDependent ; :has_affected_agent pobj)"
print('preposition', rule)
rule_parse = dsl.parse(rule)
tree = rule_parse.children[0].children[0] 
prep_tree, complex_rule = tree.children
print(prep_tree.value)
print(complex_rule)
print()

rule = "Agent dobj > :CaringForDependents ; :has_affected_agent dobj"
# rule = "dobj > :Ignore, dobj verb"
rule = "dobj & pobj > :Storage ; :has_topic dobj ; :has_instrument pobj"
print('object', rule)
rule_parse = dsl.parse(rule)
tree = rule_parse.children[0].children[0] 
entity_tree = tree.children[0]
obj1_tree = tree.children[1]
obj2_tree = tree.children[2]
class_tree = tree.children[3]
prop_or_verbs = []
if len(tree.children) > 4:
    for i in range(4, len(tree.children)):
        prop_or_verbs.append(tree.children[i]) 
if entity_tree:
    print(entity_tree.children[0].data)
print(obj1_tree.children[0].data)
if obj2_tree:
    print(obj2_tree.children[0].data)
print(class_tree.children[0].value)
for prop_or_verb in prop_or_verbs:
    if prop_or_verb.children[0].data == 'property_text':   
        prop_names, prop_obj_type = prop_or_verb.children[0].children
        prop_name = ':'
        for pname in prop_names.children:
            prop_name += f'{pname.value}_'
        print(prop_name[:-1])
        print(prop_obj_type.children[0].data)
    else:
        obj_type, text = prop_or_verb.children[0].children
        print(obj_type.data)
        print(text.children[0].value)<jupyter_output>class_name :Possession
Possession

xcomp xcomp > :Attempt, xcomp attack
Attempt
attack

keyword 'back' & dobj > :ReturnRecoveryAndRelease ; :has_topic dobj
back
dobj
ReturnRecoveryAndRelease
Tree(Token('RULE', 'property_text'), [Tree(Token('RULE', 'prop_name'), [Token('WORD', 'has'), Token('WORD', 'topic')]), Tree(Token('RULE', 'object'), [Tree(Token('RULE', 'dobj'), [])])])
:has_topic
dobj

preposition prep_for (Agent pobj > :CaringForDependent ; :has_affected_agent pobj)
for
Tree(Token('RULE', 'complex_rule'), [Tree(Token('RULE', 'paren_rule'), [Tree(Token('RULE', 'rule'), [Tree(Token('RULE', 'obj_rule'), [Tree(Token('RULE', 'entity_type'), [Tree(Token('RULE', 'agent'), [])]), Tree(Token('RULE', 'object'), [Tree(Token('RULE', 'pobj'), [])]), None, Tree(Token('RULE', 'class_name'), [Token('WORD', 'CaringForDependent')]), Tree(Token('RULE', 'property_or_verb'), [Tree(Token('RULE', 'property_text'), [Tree(Token('RULE', 'prop_name'), [Token('WORD', 'has'), Token('WORD', 'affected'), Toke[...]
{'fraction_non_alphanumeric': 0.15526649396461445, 'fraction_numerical': 0.0022047070495507907, 'mean_word_length': 2.506764592191728, 'pattern_counts': {'":': 118, '<': 4, '<?xml version=': 0, '>': 183, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '441547', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 6115, 'n_tokens_neox': 5674, 'n_words': 1148, 'path': 'notebooks/Backgrd_Idioms.ipynb', 'repo_name': 'ontoinsights/deep_narrative_analysis', 'size': '18143'}