<jupyter_start><jupyter_text>Query NSF for Awards with Boston as awardeeCity_Committed by <NAME> (@jwacooks) 2015-12-15_<jupyter_code>import os
import pandas as pd
import numpy as np
from urllib.request import urlopen
import json
import matplotlib as plt
%matplotlib inline
import random
import seaborn as sns
sns.set(color_codes=True)
sns.set_context('talk',font_scale=1.2, rc={"lines.linewidth": 2.5})
sns.set_style("whitegrid")
from textblob import TextBlob,Word
os.chdir('/Volumes/jwa_drive1/git/nsf')
base_url = 'http://api.nsf.gov/services/v1/awards.json?rpp=25'
variables ={}
page = 1
keyword = ''
offset=1
variables['awardeeZipCode'] = ''
variables['awardeeStateCode'] = ''
variables['awardeeCity'] = 'Boston'
variables['awardeeName'] = ''
variables['perfLocation '] = ''
variables['perfCity'] = ''
variables['perfStateCode'] = ''
variables['printFields'] = 'offset,id,agency,awardeeCity,awardeeStateCode,awardeeName,date,perfCity,perfState,fundsObligatedAmt,fundProgramName,title,piFirstName,piMiddleName,piLastName,primaryProgram'
variables['offset'] = str(offset)

url = base_url

for k,v in variables.items():
    if len(v) > 0:
        url += '&' + k + '=' + v
url=url.replace(' ','')
print(url)
#http://api.nsf.gov/services/v1/awards.json?rpp=25&printFields=offset,id,agency,awardeeCity,awardeeStateCode,awardeeName,date,perfCity,perfState,fundsObligatedAmt,fundProgramName,title,piFirstName,piMiddleName,piLastName,primaryProgram&keyword=Boston

df = pd.DataFrame()
while True:
    url = base_url
    for k,v in variables.items():
        if len(v) > 0:
            url += '&' + k + '=' + v
    url=url.replace(' ','')
    response = urlopen(url).read().decode('utf-8').replace('\n','')
    response_json = json.loads(response)
    if len(response_json['response']['award']) == 0:
        break
    awards_list = response_json['response']['award']
    for award_json in awards_list:
        df = df.append(award_json, ignore_index=True)
    variables['offset'] = str(int(response_json['response']['parameters']['offset'])+25)
    
df.shape
df['year'] = pd.Series(len(df['date']),index=df.index)
int_list = list(range(0, df.shape[0]))
for x in int_list:
    df.set_value(x,'year',int(str(df.ix[x]['date'])[6:]))
    df.set_value(x,'fundsObligatedAmt',float(df.ix[x]['fundsObligatedAmt']))
                 
# Define the aggregation calculations
aggregations = {
    'fundProgramName' : {
        'total grants' : 'count'
    },
    'year' : {
        'grants per year' : 'count'
        
    },

    'primaryProgram' : {
        'Primary Program per year' : 'count'
        
    },
    'awardeeName' : {
        'Awardee': 'count'
        
    }
}
grouped = df.groupby(['awardeeName','piLastName','piFirstName','year']).agg(aggregations)
grouped<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1652922194334887, 'fraction_numerical': 0.010756543564001434, 'mean_word_length': 4.728952772073922, 'pattern_counts': {'":': 1, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '366407', 'license': 'MIT', 'n_tokens_mistral': 1020, 'n_tokens_neox': 989, 'n_words': 157, 'path': 'src/notebooks/nsf-query-boston-awards.ipynb', 'repo_name': 'bulib/playhouse', 'size': '2796'}