<jupyter_start><jupyter_text>Working With Data  Import all of our packages<jupyter_code>import numpy as np
import pandas as pd 
import matplotlib.pyplot as plt
import nltk 
import plotly as py
import plotly.graph_objs as go
from functools import reduce
py.offline.init_notebook_mode(connected=True)
nltk.download('wordnet')
loans = pd.read_csv('kiva_loans.csv')
mpi_regions = pd.read_csv('kiva_mpi_region_locations.csv')
MPI_subnational = pd.read_csv('MPI_subnational.csv')
MPI_national = pd.read_csv('MPI_national.csv')
uses = loans['use'].value_counts().head(2000)
data = [go.Bar(x=uses.index,
            y=uses.values)]
py.offline.offline.iplot(data)
def get_term(document):
    def normalise(word):
        """Normalises words to lowercase and stems and lemmatizes it."""
        word = word.lower()
        # word = stemmer.stem_word(word) #if we consider stemmer then results comes with stemmed word, but in this case word will not match with comment
        word = lemmatizer.lemmatize(word)
        return word
    def leaves(tree):
        """Finds NP (nounphrase) leaf nodes of a chunk tree."""
        for subtree in tree.subtrees(filter = lambda t: t.label()=='NP'):
            yield subtree.leaves()
    
    lemmatizer = nltk.WordNetLemmatizer()
    stemmer = nltk.stem.porter.PorterStemmer()
    
    sentences = nltk.sent_tokenize(document)
    sentences = [nltk.word_tokenize(sent) for sent in sentences] 
    sentences = [nltk.pos_tag(sent) for sent in sentences]
    sentences = reduce(lambda x,y: x+y,sentences)
    grammar = r"""
        NBAR:
            {<NN.*|JJ>*<NN.*>}  # Nouns and Adjectives, terminated with Nouns

        NP:
            {<NBAR>}
            {<NBAR><IN><NBAR>}  # Above, connected with in/of/etc...
    """
    cp = nltk.RegexpParser(grammar) 
    tree = cp.parse(sentences)
    for index,leaf in enumerate(leaves(tree)):
        term = ' '.join([ normalise(w) for w,t in leaf])
        if index==0:
            return(term)
        
    
uses_nouns = {}
uses_values= uses.values.tolist()
uses_names = uses.index.tolist()
print(uses_list)
for i,use in enumerate(uses_names):
    noun = get_term(use)
    if noun in uses_nouns:
        uses_nouns[noun]+=uses_values[i]
    else:
        uses_nouns[noun]=uses_values[i]
uses_nouns
uses_export = pd.DataFrame(list(uses_nouns.items()),columns=['Loans Use','Loans Count'])
uses_export = uses_export.set_index('Loans Use')
uses_export = uses_export.sort_values('Loans Count',ascending=False)
uses_export.to_csv('Loan Count by Use.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12157935887412041, 'fraction_numerical': 0.001954652071931196, 'mean_word_length': 3.5614973262032086, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '449730', 'license': 'MIT', 'n_tokens_mistral': 921, 'n_tokens_neox': 891, 'n_words': 201, 'path': 'Data_Cleaning.ipynb', 'repo_name': 'bsegal59/MSTR-Blitz', 'size': '2558'}