<jupyter_start><jupyter_text>Full solution, receiving text blocks as array, outputting JSON<jupyter_code>import pdftotext
import os
import re

# Example of converting a PDF file into a JSON object with page text array
pdf_docs_path = os.path.join("PDF")
one_pdf_path = os.path.join(pdf_docs_path,"protect-your-home-from-wildfire.pdf")

with open(one_pdf_path, "rb") as f:
    pdf = pdftotext.PDF(f)
    
textArray = []
for page in pdf:
    docText = re.sub(r"[^a-zA-Z0-9:.,!?%$@]+", ' ', page).strip()
    textArray.append(docText)<jupyter_output><empty_output><jupyter_text>Now textArray is the object to be posted to a single call that returns a json object with a list item for each text instance, including disaster type and actions.  All NLP and other dependencies for Lambda Functions<jupyter_code>import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from collections import Counter
import string
import json
nlp = spacy.load("en_core_web_sm")<jupyter_output><empty_output><jupyter_text>Action extraction function Three rules to determine an imperative sentence1. The sentence begins with a verb Ex: Consider a sanitary wastewater backflow preventer valve to reduce the risk of sewage backup into your basement.    A caveat is if the verb in in gerund tense, there should be another verb in the sentence, so we avoid title sentences like "ADAPTING TO CLIMATE CHANGE IN COASTAL COMMUNITIES OF THE ATLANTIC PROVINCES, CANADA"2. The sentence has a structure of Always + v. / Never + v. / Please + v. / Don't + v. / not + v.Ex: Never pour kitchen grease, fats or oils into your house drains.3. The sentence has multiple clauses and one of the clause begins with a verbEx: If you live alone, develop a plan for yourself with links to neighbours and friends.Before these rules are applied, though, we need to rule out interrogative sentencesConsidering a new house?<jupyter_code># Defining the imperative sentence detector for action extraction as well as geopolitical entities (GPE)
def impSentenceExtractor(someText):
    
    doc = nlp(someText)
    impSentList=[]
    gpeList=[]
    specialWords = ("always","never","please","don't","not")
    # Just a list of words not to be confused with places
    gpeStopWords = ("Wildfire", "Flood", "Flooding", "Hurricane", "Store", "Driveway", "Building", "Uncut", "Rbon", "Complementary")
    # Extract sentences from block of text
    for sentence in doc.sents:
        
        # Check if it is not an interrogative sentence:
        if sentence[len(sentence)-1].shape_ != "?":
        
            # Rule 1
            if sentence[0].tag_=='VB' and len(sentence) > 2 and not (sentence.text in impSentList):
                impSentList.append(sentence.text.capitalize())
                
            else:
                if sentence[0].tag_=='VBG' and len(sentence) > 2 and not (sentence.text in impSentList):
                    # Caveat to rule 1: search for another verb
                    addSentence = False
                    for token in sentence:
                        if token.tag_=='VB':
                            addSentence = True
                    if addSentence:
                        impSentList.append(sentence.text.capitalize()) 

                
                # Rules 2 & 3         
                else:
                    addSentence = False
                    for token in sentence:
                        if token.lower_ in specialWords:
                            n = token.i - sentence.start #find the position of the special word in the sentence
                            if sentence[n+1].pos_=='VERB' and sentence[n+1].tag_=="VB" and len(sentence) > 2: 
                                addSentence = True
                    
                    if addSentence and not (sentence.text in impSentList):
                        impSentList.append(sentence.text.capitalize())

    # Geopolitical entity extraction
    for ent in doc.ents:
        if ent.label_ == "GPE":
            # Append it if not already there:
            if not(ent.text in gpeList) and not (ent.text in gpeStopWords):
                gpeList.append(ent.text)
    return impSentList, gpeList
#Test
someText="Burn everything thoroughly in Canada in Rbon. Burn barrels and Re pits. adapting to climate change is good. close the doors. always close the doors. Always close the doors. Acknowledging how you feel can help you manage stress. In Hurricane."
impSentenceExtractor(someText)<jupyter_output><empty_output><jupyter_text>Frequent words extractor function<jupyter_code># Defining a function for frequent word extraction, returning a simple string:
def frequentClimateWordsExtractor(text):

    # Dictionary of relevant words
    dictionary = ["snow","change","climate","heatwave","adaptation","tornado","water","icestorm","risk","impact","level","community","land","management","planning","development","http","plan","infrastructure","sea","event","action","vulnerability","flood","assessment","storm","temperature", "low","rise","resource","weather","strategy","damage","effect","precipitation","hazard","ice","protection","home","flooding","erosion","environment","emission","al","winter","heat","forest","wind","mitigation","emergency","coast","shoreline","greenhouse","elevation","carbon","wave","dike","wetland","disaster","conservation","reduction","fire","rain","drainage","ground","power","stormwater","roof","rainfall","extreme","wildfire","reference","vegetation","threat","drought","disease","coastline","sewer","nature","neutral","neutrality"]

    # Loads text with linguistic annotations from Spacy
    my_doc = nlp(text)

    filteredDoc = []
    filteredList = []
    
    # Returns a list with relevant words filtered by the dictionary
    for sentence in my_doc.sents:
        for word in sentence:
            if not(word.is_stop) and (word.pos_=='NOUN' or word.pos_=='PROPN'):
                filteredDoc.append(word.text.lower())

    nounsFreqDistribution = Counter(filteredDoc)
    
    listOfWords=""
    for word in nounsFreqDistribution.most_common(300):
        if word[0] in dictionary:
            listOfWords = listOfWords + word[0] + ", "
            filteredList.append(word[0])
    return listOfWords, filteredList<jupyter_output><empty_output><jupyter_text>Disaster Classifier (rules-based)<jupyter_code># Trying a simple rule-based approach to disaster classification
def disasterType(key_arr):
    
    # Undefined to start with
    # disaster_class = "Undefined"
    numDetected = 0
    disaster_class = []
    
    if "carbon" in key_arr and ("neutral" in key_arr or "neutrality" in key_arr):
        disaster_class.append("Carbon Neutrality")
        numDetected+=1
        
    if "adaptation" in key_arr and ("change" in key_arr or "plan" in key_arr):
        disaster_class.append("Climate Change Adaptation")
        numDetected+=1
        
    if "drought" in key_arr:
        disaster_class.append("Drought")
        numDetected+=1
        
    if "flood" in key_arr or "flooding" in key_arr or "rainfall" in key_arr or "stormwater" in key_arr or ("sea" in key_arr and "level" in key_arr and "rise" in key_arr):
        disaster_class.append("Flooding")
        numDetected+=1
        
    if ("heat" in key_arr and "extreme" in key_arr) or "heatwave" in key_arr:
        disaster_class.append("Heatwave")
        numDetected+=1
        
    if "mitigation" in key_arr:
        disaster_class.append("Mitigation")
        numDetected+=1
        
    if "wind" in key_arr or "tornado" in key_arr:
        disaster_class.append("Severe Wind")
        numDetected+=1
        
    if "snow" in key_arr or "snowstorm" in key_arr:
        disaster_class.append("Snowstorm")
        numDetected+=1

    if "temperature" in key_arr and "low" in key_arr:
        disaster_class.append("Low Temperatures")
        numDetected+=1


    if "fire" in key_arr or "wildfire" in key_arr:
        disaster_class.append("Wildfire")
        numDetected+=1
        
    if numDetected==0:
        disaster_class.append("Undefined")
        
    if "http" in key_arr or "al" in key_arr or "reference" in key_arr:
        disaster_class = ["References"]
    
    return disaster_class<jupyter_output><empty_output><jupyter_text>Core function execution, processing textArray<jupyter_code># Given textArray, loop through its text items
def mainProcessor(arrayOfText):
    returnBody=[]

    i=0
    for textBlock in arrayOfText:
        pageItem = {}
        i+=1
        impSents, places = impSentenceExtractor(textBlock)
        keywords, wordList = frequentClimateWordsExtractor(textBlock)
        disasterClass = disasterType(wordList)
        pageItem['block']=i
        pageItem['class']=disasterClass
        pageItem['places']=places
        pageItem['actions']=impSents
        if len(impSents)>0:
            returnBody.append(pageItem)
    
    return returnBody
mainProcessor(textArray)
textArray<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10432057309156033, 'fraction_numerical': 0.004253413924334005, 'mean_word_length': 3.1005048187241853, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '94577', 'license': 'MIT', 'n_tokens_mistral': 2698, 'n_tokens_neox': 2528, 'n_words': 750, 'path': 'jupyter-notebooks/Deploy - Full Solution - Classifier, Actions, Places.ipynb', 'repo_name': 'schrappe/riipen-deploy', 'size': '8934'}