<jupyter_start><jupyter_text>Load the Page Ahead Book Up program information<jupyter_code>with open('./data/interim/book_up_dict.pickle', 'rb') as handle:
    bookup_dct = pickle.load(handle)
type(bookup_dct)<jupyter_output><empty_output><jupyter_text>Load the Map Assessment student information<jupyter_code>map_df = pd.read_pickle("./data/interim/map_df.pkl")
print('shape=',map_df.shape)
map_df.columns<jupyter_output>shape= (82049, 30)<jupyter_text>Load the Smarter Balance Assessment Score Information<jupyter_code>targets_df = pd.read_pickle("./data/interim/targets.pkl")
print('shape=',targets_df.shape)
targets_df.columns<jupyter_output>shape= (4438, 30)<jupyter_text>A function that returns all the data for a given student<jupyter_code>def get_student_data(studentID):
    """Gets all records for studentID from the map_df
    -----
    input: a string containing the studentID
    returns: a pandas dataframe of all columns of the map_df for studentID
    """
    df = map_df[map_df.StudentID==studentID]
    return df<jupyter_output><empty_output><jupyter_text>A get_student_data usage example for a random studentID from targets_df:* StudentID 3099442 was helpful for development of the get_treatments function.<jupyter_code>example = get_student_data(targets_df.StudentID.sample().values[0])
example = get_student_data(3099442)
example<jupyter_output><empty_output><jupyter_text>An Example usage of the bookup_dct to find schools Page Ahead was serving for a given school year and grade:<jupyter_code>bookup_dct[example.iloc[0].TestSchoolYear,example.iloc[0].TestGrade]<jupyter_output><empty_output><jupyter_text>Work to develop the get_treatments function:<jupyter_code>summary = example.groupby(['TestSchoolYear','TestSchoolID','TestSeason']).RITScore.mean()
summary
singleYear = example[example.TestSchoolYear=='2016-17']
gb=singleYear.groupby(['TestSchoolID','TestGrade','TestSeason']).RITScore.mean()
gb.index.levels
gb
summary.index.levels
for i in range(0,len(gb)):
    print(gb.index[i],summary[i])
example.TestSchoolYear.unique()<jupyter_output><empty_output><jupyter_text>Development cell for the get treatments function:<jupyter_code>Treatments = []
for yr in example.TestSchoolYear.unique():
    yr_data = example[example.TestSchoolYear==yr]
    gb=yr_data.groupby(['TestSchoolID','TestGrade','TestSeason']).RITScore.mean()
    last = 0
    last_ind = 0
    for i in range(0,len(gb)):
        position = ['Fall','Winter','Spring'].index(gb.index[i][2])
        if position > last:
            last_ind = i
            last = position
    SchoolID = gb.index[last_ind][0]
    TestGrade = gb.index[last_ind][1]
    if SchoolID in bookup_dct[yr,TestGrade]:
        Treatments.append((int(yr[:4])+1,SchoolID))
#     print(yr,SchoolID,TestGrade)
#     print('len',len(gb))
#     print(yr,gb.index[i],summary[i])
#     print(yr,gb.index[i][2])
print(Treatments,len(Treatments))<jupyter_output>[(2017, 267)] 1<jupyter_text>Functions used to build the independent variables<jupyter_code>def get_RITs(stu_data,school_year):
    """Gets the mean any available MAP RIT scores for each test season
    (Fall, Winter, Spring) of the 2015-16 school year (1st Grade).
    -----
    Inputs: A dataframe containing all the students data from the map_df
            and the school year in the form of '2015-16'
    Returns: G1_Fall_RIT, G1_Winter_RIT, G1_Spring_RIT for the given year if available.
    """
    G1_Fall_RIT, G1_Winter_RIT, G1_Spring_RIT = None, None, None
    yr_data = stu_data[stu_data.TestSchoolYear==school_year]
    RIT_by_season = yr_data.groupby(['TestSeason']).RITScore.mean()
    for season in RIT_by_season.index:
        if season=='Fall':
            G1_Fall_RIT=RIT_by_season[season]
        if season=='Spring':
            G1_Spring_RIT=RIT_by_season[season]
        if season=='Winter':
            G1_Winter_RIT=RIT_by_season[season]
    return G1_Fall_RIT, G1_Winter_RIT, G1_Spring_RIT
get_RITs(example,'2015-16')
def get_last_RIT(Fall_RIT,Winter_RIT,Spring_RIT):
    """Returns the RIT score from latest in the school year.
    ---
    input: RIT scores for Fall, Winter and Spring
    returns: the season name and the score of the score from latest in the school year.
    """
    if Spring_RIT is not None:
        return 'Spring',Spring_RIT
    elif Winter_RIT is not None:
        return 'Winter',Winter_RIT
    elif Fall_RIT is not None:
        return 'Fall', Fall_RIT
    return None, None
def get_treatments(stu_data):
    """Determines a list of the year and school students would have received 
    Book Up program treatments from Page Ahead.
    -----
    input: A dataframe containing all rows of map_df for a given student
    returns: A list of tuples of the year and school ID a student should have
             received Book Up program books in the summer.
    """
    treatments = []
    # For each year of the student's data...
    for yr in stu_data.TestSchoolYear.unique():
        yr_data = stu_data[stu_data.TestSchoolYear==yr]
        # Get a collection of the schools they tested at...
        gb=yr_data.groupby(['TestSchoolID','TestGrade','TestSeason']).RITScore.mean()
        # Determine which testing record was the last in the school year to estimate
        # which school they were attending...
        last = 0
        last_ind = 0
        for i in range(0,len(gb)):
            position = ['Fall','Winter','Spring'].index(gb.index[i][2])
            if position > last:
                last_ind = i
                last = position
        SchoolID = gb.index[last_ind][0]
        TestGrade = gb.index[last_ind][1]
        # If the school they were attending was being served by the Book Up program
        # of Page Ahead then record the year and school in the treatments list.
        if SchoolID in bookup_dct[yr,TestGrade]:
            treatments.append((int(yr[:4])+1,SchoolID))
    return treatments<jupyter_output><empty_output><jupyter_text>Example usage of the get_treatments function with the example student data:<jupyter_code>treat=get_treatments(example)
treat<jupyter_output><empty_output><jupyter_text>Building the X dataframe:<jupyter_code>X_df = pd.DataFrame(columns=['StudentID','G1_Fall_RIT','G1_Winter_RIT','G1_Spring_RIT',\
                             'Last_G1_RIT','Last_G1_RIT_Season','Treatments','nTreatments'])
for studentID in targets_df.StudentID:
    Stu_data = get_student_data(studentID)
    G1_Fall_RIT, G1_Winter_RIT, G1_Spring_RIT = get_RITs(Stu_data,'2015-16')
    Last_RIT_Season, Last_RIT = get_last_RIT(G1_Fall_RIT, G1_Winter_RIT, G1_Spring_RIT)
    treatments = get_treatments(Stu_data)
    X_df = X_df.append({'StudentID':studentID,
                        'G1_Fall_RIT':G1_Fall_RIT,
                        'G1_Winter_RIT':G1_Winter_RIT,
                        'G1_Spring_RIT':G1_Spring_RIT,
                        'Last_G1_RIT':Last_RIT,
                        'Last_G1_RIT_Season':Last_RIT_Season,
                        'Treatments':treatments,
                        'nTreatments':int(len(treatments))
                       },ignore_index=True)
X_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 4438 entries, 0 to 4437
Data columns (total 8 columns):
StudentID             4438 non-null object
G1_Fall_RIT           1619 non-null object
G1_Winter_RIT         1254 non-null object
G1_Spring_RIT         3571 non-null object
Last_G1_RIT           3625 non-null object
Last_G1_RIT_Season    3625 non-null object
Treatments            4438 non-null object
nTreatments           4438 non-null object
dtypes: object(8)
memory usage: 277.5+ KB<jupyter_text>Note that nTreatments is an ojbect.  I'll coerce it to an integer.<jupyter_code>X_df.nTreatments = pd.to_numeric(X_df.nTreatments)
X_df.info()
X_df[X_df.nTreatments>0]<jupyter_output><empty_output><jupyter_text>If Last_G1_RIT_Season varies a lot this is going to affect the variability of the predictions because this X variable will have "error".<jupyter_code>X_df.Last_G1_RIT_Season.value_counts()<jupyter_output><empty_output><jupyter_text>However, there are also 813 records with Last_G1_RIT_Season == None:<jupyter_code>X_df[X_df.Last_G1_RIT_Season.isna()].count()<jupyter_output><empty_output><jupyter_text>So, 49 of our treated students did not have and RIT Score.  So, we can't make any predictions on these students.<jupyter_code>X_df[X_df.Last_G1_RIT_Season.isna()].nTreatments.value_counts()
X_df[X_df.Last_G1_RIT_Season!='Spring'].sort_values(by='StudentID')
print(len(X_df))
X_df.nTreatments.value_counts()<jupyter_output>4438<jupyter_text>Building the Targets Dataframe<jupyter_code>Y_df = targets_df[targets_df.StudentID.isin(X_df.StudentID)][['StudentID',
                                                              'Score',
                                                              'LevelCode',
                                                              'MetStandard']]
print(len(Y_df))
Y_df.MetStandard.value_counts()<jupyter_output>4438<jupyter_text>Verify X and Y data are in the same StudentID order<jupyter_code>Y_df[list(X_df.StudentID != Y_df.reset_index().StudentID)]
Y_df<jupyter_output><empty_output><jupyter_text>Pickle X_df and Y_df<jupyter_code># X_df.to_pickle('./data/interim/X_df.pkl')
# Y_df.to_pickle('./data/interim/Y_df.pkl')<jupyter_output><empty_output><jupyter_text>MVP modeling<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
import numpy as np
X = X_df[['StudentID','Last_G1_RIT','nTreatments']].copy()
Y = Y_df[['Score','LevelCode','MetStandard']].copy()<jupyter_output><empty_output><jupyter_text>Remove all nan's.<jupyter_code>X[X.isnull().any(axis=1)].count()<jupyter_output><empty_output><jupyter_text>Need to delete the same rows in both X and Y.<jupyter_code>df = pd.concat([X, Y.reset_index(drop=True)], axis=1)
df.set_index('StudentID',inplace=True)
df['Last_G1_RIT'].replace({None:np.nan},inplace=True)
df.dropna(inplace=True)
df[df.isnull().any(axis=1)]
df
X = df[['Last_G1_RIT','nTreatments']]
Y = df[['MetStandard']]
Y.MetStandard.value_counts()
X.nTreatments.value_counts()
X.info()
X_tr, X_te, y_tr, y_te = train_test_split(X, Y, test_size=0.33, random_state=42)
lr = LogisticRegression(solver='lbfgs')
lr.fit(X_tr, y_tr.MetStandard.ravel())
f1_score(lr.predict(X_te), y_te.MetStandard.ravel(),pos_label='Y')
print(lr.coef_)
print(lr.intercept_)
print('Treatments=0',len(df[df.nTreatments==0]))
print('Treatments>0',len(df[df.nTreatments>0]))
pal = dict(enumerate(sns.color_palette("colorblind", 3)))
ax = sns.scatterplot('Last_G1_RIT', 'Score', data=df[df.nTreatments>0], hue='nTreatments',
           palette=pal, alpha=.2, edgecolor=None)
ax.set_title("Measuring English Language Arts Literacy", fontsize=20)
ax.set_xlabel('1st Grade MAP Score (RIT)',fontsize=15)
ax.set_ylabel('3rd Grade SB Score',fontsize=15)
ax.legend(loc='upper left')
plt.axvline(161, 0,2800)
plt.axvline(178, 0,2800)
ax.text(161.5, 2020, "1st Grade", horizontalalignment='left', size='medium', color='black', weight='normal')
fig = ax.get_figure()
plt.ylim(bottom=2000,top=2820)
plt.xlim(left=110,right=230)
fig.savefig('./reports/figures/Scores1-2.svg', format='svg', dpi=1200)
pal = dict(enumerate(sns.color_palette("colorblind", 3)))
pal2={'Y':pal[0],'N':pal[1]}
ax = sns.scatterplot('Last_G1_RIT', 'Score', data=df, hue='nTreatments',
           palette=pal, alpha=.2, edgecolor=None)
ax.set_title("Measuring English Language Arts Literacy", fontsize=20)
ax.set_xlabel('1st Grade MAP Score (RIT)',fontsize=15)
ax.set_ylabel('3rd Grade SB Score',fontsize=15)
plt.axvline(161, 0,2800)
plt.axvline(178, 0,2800)
ax.text(161.5, 2020, "1st Grade", horizontalalignment='left', size='medium', color='black', weight='normal')
fig = ax.get_figure()
plt.ylim(bottom=2000,top=2820)
plt.xlim(left=110,right=230)
fig.savefig('./reports/figures/Scores0-1-2.svg', format='svg', dpi=1200)
df.head()
pal = dict(enumerate(sns.color_palette("colorblind", 2)))
pal2={'Y':pal[0],'N':pal[1]}
ax = sns.scatterplot('Last_G1_RIT', 'Score', data=df, hue='MetStandard',
           palette=pal2, alpha=.2, edgecolor=None)
ax.set_title("Measuring English Language Arts Literacy", fontsize=20)
ax.set_xlabel('1st Grade MAP Score (RIT)',fontsize=15)
ax.set_ylabel('3rd Grade SB Score',fontsize=15)
#plt.axhline(2431)
plt.axvline(161, 0,2800)
plt.axvline(178, 0,2800)
ax.text(161.5, 2020, "1st Grade", horizontalalignment='left', size='medium', color='black', weight='normal')
fig = ax.get_figure()
plt.ylim(bottom=2000,top=2820)
plt.xlim(left=110,right=230)
fig.savefig('./reports/figures/ScoresMetStandard.svg', format='svg', dpi=1200)<jupyter_output><empty_output><jupyter_text>Reference for RIT score norms for each grade including first: https://www.testprep-online.com/map-scores<jupyter_code># df['Pred'] = pd.Series(lr.predict(df.Score))
# df.head()
X = df[['nTreatments']]
Y = df[['MetStandard']]
X_tr, X_te, y_tr, y_te = train_test_split(X, Y, test_size=0.33, random_state=42)
lr = LogisticRegression(solver='lbfgs')
lr.fit(X_tr, y_tr.MetStandard.ravel())
f1_score(lr.predict(X_te), y_te.MetStandard.ravel(),pos_label='Y')
print(lr.coef_)
print(lr.intercept_)
X = df[['Last_G1_RIT']]
Y = df[['MetStandard']]
X_tr, X_te, y_tr, y_te = train_test_split(X, Y, test_size=0.33, random_state=42)
lr = LogisticRegression(solver='lbfgs')
lr.fit(X_tr, y_tr.MetStandard.ravel())
f1_score(lr.predict(X_te), y_te.MetStandard.ravel(),pos_label='Y')
print(lr.coef_)
print(lr.intercept_)
Y_df.MetStandard.value_counts()
df
targets_df[targets_df.Score<500].iloc[:,:16]
df2=df.replace({'Y':1,'N':0})
df2
fig, ax_new = plt.subplots(1,3,sharey=False)
bp = df2.boxplot(by='nTreatments',ax=ax_new)
plt.ylim(bottom=2000);
Met0=pd.DataFrame(df[df.nTreatments==0].MetStandard.replace({'Y':1,'N':0}))
Met1=pd.DataFrame(df[df.nTreatments==1].MetStandard.replace({'Y':1,'N':0}))
Met2=pd.DataFrame(df[df.nTreatments==2].MetStandard.replace({'Y':1,'N':0}))
Met=pd.concat([Met0.reset_index(drop=True),\
               Met1.reset_index(drop=True),\
               Met2.reset_index(drop=True)],axis=1)
Met
fig, ax = plt.subplots()
ax.boxplot([Met0.MetStandard,Met1.MetStandard,Met2.MetStandard]);
boxplot = Met.boxplot()
fig, ax = plt.subplots()
x = [0,1,2]
y = [Met0.mean()[0],Met1.mean()[0],Met2.mean()[0]]
plt.bar(x,y);
fig, ax = plt.subplots()
ax.scatter([0,1,2],[Met0.mean(),Met1.mean(),Met2.mean()]);
Met2.MetStandard.value_counts()
132/(132+197)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13670256835128416, 'fraction_numerical': 0.03169014084507042, 'mean_word_length': 4.780127693535515, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 90, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '18426', 'license': 'MIT', 'n_tokens_mistral': 5638, 'n_tokens_neox': 5228, 'n_words': 968, 'path': 'mvp.ipynb', 'repo_name': 'spdavern/metis_project3', 'size': '14484'}