<jupyter_start><jupyter_text>Import Model and Classify Text<jupyter_code>import json
import plotly
import pandas as pd
import re
import nltk
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.base import BaseEstimator, TransformerMixin
import pickle
import joblib
from spellhelper import Spellhelper
import cleaner

#from flask import Flask
#from flask import render_template, request, jsonify
#from sklearn.externals import joblib
#from sqlalchemy import create_engine
def cleanmess(text):
    '''
    import message
    clean the message
    return the cleaned message
    '''
    sents = sent_tokenize(text)
    all_sents = []
    for sent in sents:
        clean = cleaner.CleanText(sent) + '.'
        all_sents.append(clean)

    return ' '.join(all_sents)
def tokenize(text):
    '''
    input text string
    clean and lower case characters of string
    tokenize text
    lematize and remove stop words
    add bigrams
    return cleaned tokens and bigrams
    '''
    stop_words = stopwords.words("english") + ['.']
    lemmatizer = WordNetLemmatizer()
    # normalize
    s = re.sub(r"[^a-zA-Z0-9]", " ", text.lower())
    # tokenize
    tokens = word_tokenize(s)
    # stemming
    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]
    # add bi-grams
    bigrams = [a + ' ' + b for a,b in list(nltk.bigrams(tokens))]
    return tokens + bigrams
def PreProcess(text):
    '''
    input text string
    tokenize
    check if token in filtered frequencies
    return list of filtered tokens
    '''
    tokens = tokenize(text)
    return [t for t in tokens if t in dtf]
# load model
global dtf
filename = 'Logregmod18.pkl'
with open(filename,'rb') as handle:
    model, dtf = pickle.load(handle)
# initialize cleaner
fd_file = 'freq_dict_20210122203414.txt'
lu_file = 'lookup_dict_20210122203414.pkl'
cleaner.speller = Spellhelper(fd_file)
with open(lu_file, 'rb') as handle:
    cleaner.corr_dict = pickle.load(handle)
def ProcessText(text):
    '''
    import text
    clean
    tokenize
    predict
    print results along the way
    '''
    targets = ['related', 'request', 'offer', 'aid_related', 
           'medical_help', 'medical_products', 'search_and_rescue', 
           'security', 'military', 'child_alone', 'water', 'food', 
           'shelter', 'clothing', 'money', 'missing_people', 
           'refugees', 'death', 'other_aid', 'infrastructure_related', 
           'transport', 'buildings', 'electricity', 'tools', 'hospitals', 
           'shops', 'aid_centers', 'other_infrastructure', 'weather_related', 
           'floods', 'storm', 'fire', 'earthquake', 'cold', 'other_weather', 
           'direct_report']
    
    clean = cleanmess(text)
    print(clean)
    print()
    toks = tokenize(clean)
    print(toks)
    print()
    classification_labels = model.predict([clean])[0]
    classification_results = dict(zip(targets, classification_labels))
    print(classification_results)
    print()<jupyter_output><empty_output><jupyter_text>hurricane<jupyter_code>test1 = 'Please be aware that there is an emminent hurricane sandy. '+\
        'Please buckle down the hatches and seeek shelter123soooner than later. '+\
        'If at all possible, please evacuate to a safer location. '
ProcessText(test1)<jupyter_output>please be aware that there is an imminent hurricane sandy. please buckle down the hatches and seek number shelter sooner than later. if at all possible please evacuate to a safer location.

['please', 'aware', 'imminent', 'hurricane', 'sandy', 'please', 'buckle', 'hatch', 'seek', 'number', 'shelter', 'sooner', 'later', 'possible', 'please', 'evacuate', 'safer', 'location', 'please aware', 'aware imminent', 'imminent hurricane', 'hurricane sandy', 'sandy please', 'please buckle', 'buckle hatch', 'hatch seek', 'seek number', 'number shelter', 'shelter sooner', 'sooner later', 'later possible', 'possible please', 'please evacuate', 'evacuate safer', 'safer location']

{'related': 1, 'request': 0, 'offer': 0, 'aid_related': 1, 'medical_help': 0, 'medical_products': 0, 'search_and_rescue': 0, 'security': 0, 'military': 0, 'child_alone': 0, 'water': 0, 'food': 0, 'shelter': 1, 'clothing': 0, 'money': 0, 'missing_people': 0, 'refugees': 0, 'death': 0, 'other_aid': 0, 'infrastructure_related':[...]<jupyter_text>hospital<jupyter_code>test2 = 'Our hospital is running at capacity and short on medical supplies. '+\
        'Clean drinking water, prepackaged food, medicine, drugs, antibiotics, anything. '+\
        'Please charge the materials and delivery to the helpful government in charge. '
ProcessText(test2)<jupyter_output>our hospital is running at capacity and short on medical supplies. clean drinking water prepackaged food medicine drugs antibiotics anything. please charge the materials and delivery to the helpful government in charge.

['hospital', 'running', 'capacity', 'short', 'medical', 'supply', 'clean', 'drinking', 'water', 'prepackaged', 'food', 'medicine', 'drug', 'antibiotic', 'anything', 'please', 'charge', 'material', 'delivery', 'helpful', 'government', 'charge', 'hospital running', 'running capacity', 'capacity short', 'short medical', 'medical supply', 'supply clean', 'clean drinking', 'drinking water', 'water prepackaged', 'prepackaged food', 'food medicine', 'medicine drug', 'drug antibiotic', 'antibiotic anything', 'anything please', 'please charge', 'charge material', 'material delivery', 'delivery helpful', 'helpful government', 'government charge']

{'related': 1, 'request': 1, 'offer': 0, 'aid_related': 1, 'medical_help': 1, 'medical_products': 1, 'search_and_rescue': 0, 'securit[...]<jupyter_text>lost kid<jupyter_code>test3 = 'A brown hair boy alone lost his parents and cannot find his way home. '+\
        'He is nine, short and skinny with red shorts and freckles. '+\
        'Please put us in touch with his parents so they can recover their lost kid. '
ProcessText(test3)<jupyter_output>a brown hair boy alone lost his parents and can not find his way home. he is nine short and skinny with red shorts and freckles. please put us in touch with his parents so they can recover their lost kid.

['brown', 'hair', 'boy', 'alone', 'lost', 'parent', 'find', 'way', 'home', 'nine', 'short', 'skinny', 'red', 'short', 'freckle', 'please', 'put', 'u', 'touch', 'parent', 'recover', 'lost', 'kid', 'brown hair', 'hair boy', 'boy alone', 'alone lost', 'lost parent', 'parent find', 'find way', 'way home', 'home nine', 'nine short', 'short skinny', 'skinny red', 'red short', 'short freckle', 'freckle please', 'please put', 'put u', 'u touch', 'touch parent', 'parent recover', 'recover lost', 'lost kid']

{'related': 1, 'request': 1, 'offer': 0, 'aid_related': 1, 'medical_help': 0, 'medical_products': 0, 'search_and_rescue': 0, 'security': 0, 'military': 0, 'child_alone': 1, 'water': 0, 'food': 0, 'shelter': 0, 'clothing': 0, 'money': 0, 'missing_people': 0, 'refugees': 0, 'death': 0, 'oth[...]<jupyter_text>search and rescue<jupyter_code>test4 = 'A person hiking alone fell off of the side of the trail and fell down. '+\
        'Please send search and rescue to free her from the rocks down the cliff. '+\
        'It is cold outside and she is not warm enough to stay over night. '
ProcessText(test4)<jupyter_output>a person hiking alone fell off of the side of the trail and fell down. please send search and rescue to free her from the rocks down the cliff. it is cold outside and she is not warm enough to stay over night.

['person', 'hiking', 'alone', 'fell', 'side', 'trail', 'fell', 'please', 'send', 'search', 'rescue', 'free', 'rock', 'cliff', 'cold', 'outside', 'warm', 'enough', 'stay', 'night', 'person hiking', 'hiking alone', 'alone fell', 'fell side', 'side trail', 'trail fell', 'fell please', 'please send', 'send search', 'search rescue', 'rescue free', 'free rock', 'rock cliff', 'cliff cold', 'cold outside', 'outside warm', 'warm enough', 'enough stay', 'stay night']

{'related': 1, 'request': 0, 'offer': 0, 'aid_related': 1, 'medical_help': 0, 'medical_products': 0, 'search_and_rescue': 1, 'security': 0, 'military': 0, 'child_alone': 0, 'water': 0, 'food': 0, 'shelter': 0, 'clothing': 0, 'money': 0, 'missing_people': 0, 'refugees': 0, 'death': 0, 'other_aid': 0, 'infrastructure_related':[...]<jupyter_text>homeless<jupyter_code>test5 = 'Many people here have been displaced by the big bad forest fire and live in tents as homeless. '+\
        'They were sent by firemen to camp out next to the road and need help to recover from this disaster. '+\
        'Please send aid to help get them back on their feet before the winter storms set in. '
ProcessText(test5)<jupyter_output>many people here have been displaced by the big bad forest fire and live in tents as homeless. they were sent by firemen to camp out next to the road and need help to recover from this disaster. please send aid to help get them back on their feet before the winter storms set in.

['many', 'people', 'displaced', 'big', 'bad', 'forest', 'fire', 'live', 'tent', 'homeless', 'sent', 'fireman', 'camp', 'next', 'road', 'need', 'help', 'recover', 'disaster', 'please', 'send', 'aid', 'help', 'get', 'back', 'foot', 'winter', 'storm', 'set', 'many people', 'people displaced', 'displaced big', 'big bad', 'bad forest', 'forest fire', 'fire live', 'live tent', 'tent homeless', 'homeless sent', 'sent fireman', 'fireman camp', 'camp next', 'next road', 'road need', 'need help', 'help recover', 'recover disaster', 'disaster please', 'please send', 'send aid', 'aid help', 'help get', 'get back', 'back foot', 'foot winter', 'winter storm', 'storm set']

{'related': 1, 'request': 1, 'offer': 0, 'aid_relat[...]<jupyter_text>death<jupyter_code>test6 = 'The protestors have raided the capital and are violently yelling, vandalizing, and pushing doors down. '+\
        'Shots have been fired killing an American and several people were electrocuted dead by stun weapons. '+\
        'In LA they have lifted the cremation smoke ban to burn the build up of dead people lying about everywhere.'
ProcessText(test6)<jupyter_output>the protestors have raided the capital and are violently yelling vandalizing and pushing doors down. shots have been fired killing an american and several people were electrocuted dead by stun weapons. in la they have lifted the cremation smoke ban to burn the build up of dead people lying about everywhere.

['protestors', 'raided', 'capital', 'violently', 'yelling', 'vandalizing', 'pushing', 'door', 'shot', 'fired', 'killing', 'american', 'several', 'people', 'electrocuted', 'dead', 'stun', 'weapon', 'la', 'lifted', 'cremation', 'smoke', 'ban', 'burn', 'build', 'dead', 'people', 'lying', 'everywhere', 'protestors raided', 'raided capital', 'capital violently', 'violently yelling', 'yelling vandalizing', 'vandalizing pushing', 'pushing door', 'door shot', 'shot fired', 'fired killing', 'killing american', 'american several', 'several people', 'people electrocuted', 'electrocuted dead', 'dead stun', 'stun weapon', 'weapon la', 'la lifted', 'lifted cremation', 'cremation smoke', 'smoke b[...]<jupyter_text>electricity<jupyter_code>test7 = 'The high winds blew so hard that trees fell down onto power lines and now we have no electricity. '+\
        'This has been going on for several days and people are running emergency generators to get by. '+\
        'The problem with PG&E seems they have not reinvested their profits into maintaining infrastructure.'
ProcessText(test7)<jupyter_output>the high winds blew so hard that trees fell down onto power lines and now we have no electricity. this has been going on for several days and people are running emergency generators to get by. the problem with pm a seems they have not reinvested their profits into maintaining infrastructure.

['high', 'wind', 'blew', 'hard', 'tree', 'fell', 'onto', 'power', 'line', 'electricity', 'going', 'several', 'day', 'people', 'running', 'emergency', 'generator', 'get', 'problem', 'pm', 'seems', 'reinvested', 'profit', 'maintaining', 'infrastructure', 'high wind', 'wind blew', 'blew hard', 'hard tree', 'tree fell', 'fell onto', 'onto power', 'power line', 'line electricity', 'electricity going', 'going several', 'several day', 'day people', 'people running', 'running emergency', 'emergency generator', 'generator get', 'get problem', 'problem pm', 'pm seems', 'seems reinvested', 'reinvested profit', 'profit maintaining', 'maintaining infrastructure']

{'related': 1, 'request': 1, 'offer': 0, 'aid_[...]<jupyter_text>mud slide<jupyter_code>test8 = 'Mega storm has dropped two years of rain in the last 12 hours and after our fire now we have floods. '+\
        'Floods and mudslides are tearing through neighborhoods and the cleanup crew dumped the mud on the beach. '+\
        'Because there was human remains in the mud they dumped now the beach is off limits to go to for a year.'
ProcessText(test8)<jupyter_output>mega storm has dropped two years of rain in the last number hours and after our fire now we have floods. floods and mudslides are tearing through or hoods and the cleanup crew dumped the mud on the beach. because there was human remains in the mud they dumped now the beach is off limits to go to for a year.

['mega', 'storm', 'dropped', 'two', 'year', 'rain', 'last', 'number', 'hour', 'fire', 'flood', 'flood', 'mudslide', 'tearing', 'hood', 'cleanup', 'crew', 'dumped', 'mud', 'beach', 'human', 'remains', 'mud', 'dumped', 'beach', 'limit', 'go', 'year', 'mega storm', 'storm dropped', 'dropped two', 'two year', 'year rain', 'rain last', 'last number', 'number hour', 'hour fire', 'fire flood', 'flood flood', 'flood mudslide', 'mudslide tearing', 'tearing hood', 'hood cleanup', 'cleanup crew', 'crew dumped', 'dumped mud', 'mud beach', 'beach human', 'human remains', 'remains mud', 'mud dumped', 'dumped beach', 'beach limit', 'limit go', 'go year']

{'related': 1, 'request': 0, 'offer': 0, [...]<jupyter_text>not related<jupyter_code>test9 = 'Today was another day of working inside as quarantined from playing outside in the sunshine.'+\
        'Luckily there was a long overdue Udacity project that was near impossible to complete to work on.'
ProcessText(test9)<jupyter_output>today was another day of working inside as quarantined from playing outside in the sunshine luckily there was a long overdue audacity project that was near impossible to complete to work on.

['today', 'another', 'day', 'working', 'inside', 'quarantined', 'playing', 'outside', 'sunshine', 'luckily', 'long', 'overdue', 'audacity', 'project', 'near', 'impossible', 'complete', 'work', 'today another', 'another day', 'day working', 'working inside', 'inside quarantined', 'quarantined playing', 'playing outside', 'outside sunshine', 'sunshine luckily', 'luckily long', 'long overdue', 'overdue audacity', 'audacity project', 'project near', 'near impossible', 'impossible complete', 'complete work']

{'related': 0, 'request': 0, 'offer': 0, 'aid_related': 0, 'medical_help': 0, 'medical_products': 0, 'search_and_rescue': 0, 'security': 0, 'military': 0, 'child_alone': 0, 'water': 0, 'food': 0, 'shelter': 0, 'clothing': 0, 'money': 0, 'missing_people': 0, 'refugees': 0, 'death': 0, 'other_aid': [...]<jupyter_text>military<jupyter_code>test10 = 'After heavy investment in military equipment, that power hungry government is ready to battle.'+\
         'Fortunately, its just another hollywood action flick full of tanks, bombs, and guns run amok.'
ProcessText(test10)<jupyter_output>after heavy investment in military equipment that power hungry government is ready to battle fortunately its just another hollywood action flick full of tanks bombs and guns run amok.

['heavy', 'investment', 'military', 'equipment', 'power', 'hungry', 'government', 'ready', 'battle', 'fortunately', 'another', 'hollywood', 'action', 'flick', 'full', 'tank', 'bomb', 'gun', 'run', 'amok', 'heavy investment', 'investment military', 'military equipment', 'equipment power', 'power hungry', 'hungry government', 'government ready', 'ready battle', 'battle fortunately', 'fortunately another', 'another hollywood', 'hollywood action', 'action flick', 'flick full', 'full tank', 'tank bomb', 'bomb gun', 'gun run', 'run amok']

{'related': 1, 'request': 0, 'offer': 0, 'aid_related': 1, 'medical_help': 0, 'medical_products': 0, 'search_and_rescue': 0, 'security': 0, 'military': 1, 'child_alone': 0, 'water': 0, 'food': 0, 'shelter': 0, 'clothing': 0, 'money': 0, 'missing_people': 0, 'refugees': 0, 'd[...]
{'fraction_non_alphanumeric': 0.1467148884870404, 'fraction_numerical': 0.010066305003013865, 'mean_word_length': 5.090675477239354, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '288503', 'license': 'MIT', 'n_tokens_mistral': 5337, 'n_tokens_neox': 4909, 'n_words': 2010, 'path': 'models/test_frontend.ipynb', 'repo_name': 'patronical/disaster_pipeline', 'size': '16590'}