<jupyter_start><jupyter_text>Data Visualization<jupyter_code>data_all.head()
sns.countplot(x='bloom', hue='Folders', data=data_all)
# Factor Plot
sns.countplot(x='bloom', hue='Module', data=data_all[data_all['Folders']=='projects'])
subjects = pd.DataFrame()
subjects['post'] = data_all['Subject'].unique()
subjects.to_csv("subjects.csv")
# Calculate correlations
corr = data_all.corr()
 
# Heatmap
sns.heatmap(corr)<jupyter_output><empty_output><jupyter_text>Making Table for Backend<jupyter_code>module_summary = None
module_summary = pd.DataFrame()
module_summary = data_all[['Module','bloom']].groupby("Module").mean()
module_summary['question_total']=data_all[['Module','bloom']].groupby("Module").sum()
display(module_summary.head())
def value_counts_df(df, col):
    """
    Returns pd.value_counts() as a DataFrame

    Parameters
    ----------
    df : Pandas Dataframe
        Dataframe on which to run value_counts(), must have column `col`.
    col : str
        Name of column in `df` for which to generate counts

    Returns
    -------
    Pandas Dataframe
        Returned dataframe will have a single column named "count" which contains the count_values()
        for each unique value of df[col]. The index name of this dataframe is `col`.

    Example
    -------
    >>> value_counts_df(pd.DataFrame({'a':[1, 1, 2, 2, 2]}), 'a')
       count
    a
    2      3
    1      2
    """
    df = pd.DataFrame(df[col].value_counts())
    df.index.name = col
    df.columns = ['count']
    return df
# for each module
# get the rows with that module 
# count the rows with bloom == 0 / len(module)...
x = value_counts_df(data_all.groupby("Module"), 'bloom')
x = x.unstack()
x = x.fillna(0)
x.columns = x.columns.to_flat_index()
x.columns = ['count_0', 'count_1', 'count_2', 'count_3']
x = x.div(x.sum(axis=1), axis=0)
module_summary = pd.merge(module_summary, x, left_index=True, right_index=True)
module_summary.columns = ['score','question_total', 'prop_0', 'prop_1', 'prop_2', 'prop_3']
display(module_summary.head())
module_summary = module_summary.sort_values(by='question_total', ascending=False)
module_summary.to_csv("module_summary_full.csv")<jupyter_output><empty_output><jupyter_text>Student Profiles<jupyter_code>student_data = data_all.copy()
student_summary = None
student_summary = pd.DataFrame()
student_summary = student_data[['Email','bloom']].groupby("Email").mean()
student_summary['question_total']=student_data[['Email','bloom']].groupby("Email").sum()
display(student_summary.head())
# for each module
# get the rows with that module 
# count the rows with bloom == 0 / len(module)...
x = value_counts_df(student_data.groupby("Email"), 'bloom')
x = x.unstack()
x = x.fillna(0)
x.columns = x.columns.to_flat_index()
x.columns = ['count_0', 'count_1', 'count_2', 'count_3']
x = x.div(x.sum(axis=1), axis=0)
student_summary = pd.merge(student_summary, x, left_index=True, right_index=True)
student_summary.columns = ['score','question_total', 'prop_0', 'prop_1', 'prop_2', 'prop_3']
display(student_summary.head())
student_summary = student_summary.sort_values(by='question_total', ascending=False)
# plot data with seaborn
facet = sns.lmplot(data=student_summary, x='prop_1', y='prop_2', hue='question_total', 
                   fit_reg=False, legend=True, legend_out=True)
student_summary.to_csv("student_summary_full.csv")<jupyter_output><empty_output><jupyter_text>Aggregate Class Data<jupyter_code>d= {'bloom':[data_copy['bloom'].mean()], 'question_total':data_copy['bloom'].sum()}
class_summary=pd.DataFrame(data=d)
display(class_summary)
x = value_counts_df(data_copy, 'bloom')
x = x.unstack()
x = x.fillna(0)
x = pd.DataFrame(x)
display(x)
class_summary['count_0'] = x.iloc[0]
class_summary['count_1'] = x.iloc[1]
class_summary['count_2'] = x.iloc[2]
class_summary['count_3'] = x.iloc[3]
display(class_summary)
class_summary.to_csv("class_summary.csv")<jupyter_output><empty_output><jupyter_text>Training and Validation<jupyter_code>from sklearn.model_selection import train_test_split

train, val = train_test_split(labeled_all, test_size=0.2, random_state=42)
def words_in_texts(words, texts):
    '''
    Args:
        words (list-like): words to find
        texts (Series): strings to search in
    
    Returns:
        NumPy array of 0s and 1s with shape (n, p) where n is the
        number of texts and p is the number of words.
    '''
    n = len(texts)
    p = len(words)
#     print([[j] for j in texts])
    indicator_array = [[1 if words[i] in j else 0 for i in range(p)] for j in texts]
    indicator_array = np.array(indicator_array)
    return indicator_array
words_in_texts(['hello', 'bye', 'world'], 
                   pd.Series(['hello', 'hello worldhello']))
train['question']
pd.Series(['hello', 'hello worldhello'])
words_in_texts(['hello', 'bye', 'world'], 
                   train['question'])
display(train['question'].isna().sum())
import matplotlib.pyplot as plt
# test words in the dataset 
train = train.reset_index(drop=True)
test_words = ['what', 'how', 'why', 'clarify', 'explain', 'try', 'create', 'understand', 'change']

words_array = words_in_texts(test_words, pd.Series(train['question']))
words = pd.DataFrame(words_array, columns=test_words)

train_new = pd.concat([train, words], axis=1)
train_new = train_new.filter(items=test_words + ['bloom'])

def bloom_category(x):
    if x.bloom == 1:
        return 'recall'
    elif x.bloom == 2:
        return 'analyze'
    elif x.bloom == 3:
        return 'create'
    else:
        return 'other'
    
train_new['type'] = train_new.apply(lambda row: bloom_category(row), axis=1) 
train_new = train_new.melt(id_vars=['type'], value_vars=test_words)
train_new
plt.figure()
sns.set(style="whitegrid")
sns.barplot(x='variable', y='value', hue='type', data=train_new)
plt.title('Frequency of words in questions')
plt.xlabel('Words')
plt.ylabel('Frequency of word')
train_copy = train.copy()
train_copy['length'] = train_copy.apply(lambda x: len(x.question), axis=1)
train_copy['type'] = train_copy.apply(lambda row: bloom_category(row), axis=1) 
train_copy
recall_length = train_copy.loc[train_copy['type']=='recall']['length']
analyze_length = train_copy.loc[train_copy['type']=='analyze']['length']
create_length = train_copy.loc[train_copy['type']=='create']['length']
other_length = train_copy.loc[train_copy['type']=='other']['length']
sns.distplot(recall_length, label='recall')
sns.distplot(analyze_length, label='analyze')
sns.distplot(create_length, label='create')
sns.distplot(other_length, label='other')
plt.xlim(0, 1200)
plt.legend()
plt.title('Distribution of email lengths in questions')
plt.xlabel('Length of question body')
plt.ylabel('Distribution')
some_words = ['what', 'how', 'why', 'clarify', 'explain', 'try', 'create', 'understand', 'change']


X_train = words_in_texts(some_words, train['question'])
Y_train = train['bloom']

X_train[:5], Y_train[:5]
from sklearn.linear_model import LogisticRegression

model = LogisticRegression()
model.fit(X_train, Y_train)

training_accuracy = model.score(X_train, Y_train)
print("Training Accuracy: ", training_accuracy)<jupyter_output>Training Accuracy:  0.5564516129032258
{'fraction_non_alphanumeric': 0.16027339935834845, 'fraction_numerical': 0.011577625889245362, 'mean_word_length': 5.524112829845314, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '467421', 'license': 'MIT', 'n_tokens_mistral': 2569, 'n_tokens_neox': 2486, 'n_words': 510, 'path': 'datapiazza.ipynb', 'repo_name': 'graceyraspberry/Infocamp2020data', 'size': '7169'}