<jupyter_start><jupyter_text>Generate a synthetic 1PL/2PL IRT model and sample an interaction history from it<jupyter_code>num_students = 2000
num_assessments = 3000
num_ixns_per_student = 1000

USING_2PL = False # False => using 1PL
proficiencies = np.random.normal(0, 1, num_students)
difficulties = np.random.normal(0, 1, num_assessments)

if USING_2PL:
    discriminabilities = np.random.normal(0, 1, num_assessments)
else:
    discriminabilities = np.ones(num_assessments)

student_ids = ['S'+str(x) for x in xrange(num_students)]
assessment_ids = ['A'+str(x) for x in xrange(num_assessments)]
ixns = [None] * (num_students * num_ixns_per_student)
assessment_idxes = range(num_assessments)
for student_idx, student_id in enumerate(student_ids):
    for t in xrange(num_ixns_per_student):
        module_idx = random.choice(assessment_idxes)
        pass_likelihood = 1 / (1 + math.exp(-(discriminabilities[module_idx]*proficiencies[student_idx] + difficulties[module_idx])))
        ixns[student_idx * num_ixns_per_student + t] = {
            'student_id' : student_id, 
            'module_id' : assessment_ids[module_idx], 
            'module_type' : datatools.AssessmentInteraction.MODULETYPE,
            'outcome' : np.random.random() < pass_likelihood, 
            'timestep' : t+1
        }
history = datatools.InteractionHistory(pd.DataFrame(ixns))
history.idx_of_student_id = lambda x: int(x[1:])
history.idx_of_assessment_id = lambda x: int(x[1:])
mirt_model = models.MIRTModel(history, dims=1, using_assessment_factors=USING_2PL)
estimator = est.MIRTMAPEstimator(
    regularization_constant=1e-3,
    ftol=1e-5,
    debug_mode_on=True)
mirt_model.fit(estimator)
onepl_model = models.OneParameterLogisticModel(
    history.data, select_regularization_constant=True)
onepl_model.fit()
twopl_model = models.TwoParameterLogisticModel(
    history.data, select_regularization_constant=True)
twopl_model.fit()
student_idxes = [int(k[1:]) for k in history.data['student_id'].unique()]
assessment_idxes = [int(k[1:]) for k in history.data['module_id'].unique()]<jupyter_output><empty_output><jupyter_text>Verify that `models.OneParameterLogisticModel` can recover parameters. We would only expect this to be possible when `USING_2PL = False`.<jupyter_code>plt.xlabel('True difficulties')
plt.ylabel('Estimated difficulties')
plt.scatter(difficulties[assessment_idxes], onepl_model.model.coef_[0, num_students:])
plt.show()

plt.xlabel('Estimated difficulty - true difficulty')
plt.ylabel('Frequency (number of assessments)')
plt.hist(onepl_model.model.coef_[0, num_students:] - difficulties[assessment_idxes], bins=20)
plt.show()
plt.xlabel('True proficiencies')
plt.ylabel('Estimated proficiencies')
plt.scatter(proficiencies[student_idxes], onepl_model.model.coef_[0, :num_students])
plt.show()

plt.xlabel('Estimated proficiency - true proficiency')
plt.ylabel('Frequency (number of students)')
plt.hist(onepl_model.model.coef_[0, :num_students] - proficiencies[student_idxes], bins=20)
plt.show()<jupyter_output><empty_output><jupyter_text>Verify that `models.TwoParameterLogisticModel` can recover parameters. We would only expect this to be possible when `USING_2PL = True`.<jupyter_code>plt.xlabel('True difficulties')
plt.ylabel('Estimated difficulties')
plt.scatter(difficulties[assessment_idxes], twopl_model.model.coef_[0, (num_students*num_assessments):])
plt.show()

plt.xlabel('Estimated difficulty - true difficulty')
plt.ylabel('Frequency (number of assessments)')
plt.hist(twopl_model.model.coef_[0, (num_students*num_assessments):] - difficulties[assessment_idxes], bins=20)
plt.show()
est_params = twopl_model.model.coef_[0, :(num_students*num_assessments)]
true_params = discriminabilities[:, None].dot(proficiencies[:, None].T).ravel()

plt.xlabel('True proficiency*discriminability')
plt.ylabel('Estimated proficiency*discriminability')
plt.scatter(true_params, est_params)
plt.show()

plt.xlabel('Estimated proficiency*discriminability - true proficiency*discriminability')
plt.ylabel('Frequency (number of student-assessment pairs)')
plt.hist(est_params - true_params, bins=20)
plt.show()<jupyter_output><empty_output><jupyter_text>Verify that `models.MIRTModel` can recover parameters<jupyter_code>plt.xlabel('True difficulties')
plt.ylabel('Estimated difficulties')
plt.scatter(difficulties, mirt_model.assessment_offsets)
plt.show()

plt.xlabel('Estimated difficulty - true difficulty')
plt.ylabel('Frequency (number of assessments)')
plt.hist(mirt_model.assessment_offsets - difficulties, bins=20)
plt.show()
plt.xlabel('True proficiencies')
plt.ylabel('Estimated proficiencies')
plt.scatter(proficiencies, mirt_model.student_factors[:, 0])
plt.show()

plt.xlabel('Estimated proficiency - true proficiency')
plt.ylabel('Frequency (number of students)')
plt.hist(mirt_model.student_factors[:, 0] - proficiencies, bins=20)
plt.show()
plt.xlabel('True discriminabilities')
plt.ylabel('Estimated discriminabilities')
plt.scatter(discriminabilities, mirt_model.assessment_factors[:, 0])
plt.show()

plt.xlabel('Estimated discriminability - true discriminability')
plt.ylabel('Frequency (number of assessments)')
plt.hist(mirt_model.assessment_factors[:, 0] - discriminabilities, bins=20)
plt.show()<jupyter_output><empty_output><jupyter_text>Verify that all models achieve similar training AUCs<jupyter_code># models.OneParameterLogisticModel
evaluate.training_auc(onepl_model, history, plot_roc_curve=True)
# models.TwoParameterLogisticModel
evaluate.training_auc(twopl_model, history, plot_roc_curve=True)
# models.MIRTModel
evaluate.training_auc(mirt_model, history, plot_roc_curve=True)
# true model
true_model = copy.deepcopy(mirt_model)
true_model.student_factors[:, 0] = proficiencies
true_model.assessment_factors[:, 0] = discriminabilities
true_model.assessment_offsets = difficulties
evaluate.training_auc(true_model, history, plot_roc_curve=True)<jupyter_output><empty_output><jupyter_text>Construct a synthetic embedding<jupyter_code>num_students = 10000
num_assessment_interactions_per_step = 100
grid_size = 5
embedding_dimension = 2

num_assessments = grid_size ** 2
num_lessons = 2 * grid_size * (grid_size - 1)
num_lesson_interactions_per_student = 2 * (grid_size - 1) + 2
S = np.zeros((num_students, embedding_dimension, num_lesson_interactions_per_student))
A = np.zeros((num_assessments, embedding_dimension))
L = np.zeros((num_lessons, embedding_dimension))
Q = np.zeros((num_lessons, embedding_dimension))

lesson_idx_of_loc = {}
assessment_idx_of_loc = {}

cell_size = 10 / (grid_size - 1)
lesson_count = 0
for i in xrange(grid_size):
    for j in xrange(grid_size):
        A[grid_size * i + j, :] = [i, j]
        assessment_idx_of_loc[(i, j)] = grid_size * i + j
        
        if j < grid_size - 1:
            Q[lesson_count, :] = [i, j]
            L[lesson_count, :] = [0, 1]
            lesson_idx_of_loc[(i, j, 0, 1)] = lesson_count
            lesson_count += 1
        
        if i < grid_size - 1:
            Q[lesson_count, :] = [i, j]
            L[lesson_count, :] = [1, 0]
            lesson_idx_of_loc[(i, j, 1, 0)] = lesson_count
            lesson_count += 1
            
A *= cell_size
Q *= cell_size
L *= cell_size

A = np.maximum(1e-3, A)
Q = np.maximum(1e-3, Q)<jupyter_output><empty_output><jupyter_text>Sample interactions from the synthetic embedding<jupyter_code>id_of_loc = lambda x: '-'.join(str(z) for z in x)
data = []
for student_idx in xrange(num_students):
    student_id = 'S' + str(student_idx)
    steps = ([(0, 1)] * (grid_size - 1)) +  ([(1, 0)] * (grid_size - 1))
    random.shuffle(steps)
    
    x, y = 0, 0
    t = 1
    assessment_idx = assessment_idx_of_loc[(0, 0)]
    assessment_id = id_of_loc(assessment_loc_of_idx[assessment_idx])
    pass_likelihood = 1 / (1 + math.exp(-(np.dot(S[student_idx, :, t], A[assessment_idx, :]) / np.linalg.norm(A[assessment_idx, :]) - np.linalg.norm(A[assessment_idx, :]))))
    outcome = random.random() < pass_likelihood
    data.append({
        'student_id' : student_id, 
        'module_id' : assessment_id,
        'module_type' : datatools.AssessmentInteraction.MODULETYPE,
        'timestep' : t,
        'outcome' : outcome})
    
    for i, j in steps:
        lesson_idx = lesson_idx_of_loc[(x, y, i, j)]
        lesson_id = id_of_loc(lesson_loc_of_idx[lesson_idx])
        data.append({
            'student_id' : student_id,
            'module_id' : lesson_id,
            'module_type' : datatools.LessonInteraction.MODULETYPE,
            'timestep' : t, 
            'outcome' : None})
        
        x += i
        y += j
        # DEBUG
        S[student_idx, :, t+1] = S[student_idx, :, t] + L[lesson_idx, :]# / (1 + math.exp(-(np.dot(S[student_idx, :, t], Q[lesson_idx, :]) / np.linalg.norm(Q[lesson_idx, :]) - np.linalg.norm(Q[lesson_idx, :]))))
        
        t += 1
        for _ in xrange(num_assessment_interactions_per_step):
            assessment_idx = random.randint(0, num_assessments - 1)
            assessment_id = id_of_loc(assessment_loc_of_idx[assessment_idx])
            pass_likelihood = 1 / (1 + math.exp(-(np.dot(S[student_idx, :, t], A[assessment_idx, :]) / np.linalg.norm(A[assessment_idx, :]) - np.linalg.norm(A[assessment_idx, :]))))
            outcome = random.random() < pass_likelihood
            # BEGIN DEBUG
            if assessment_idx_of_loc[(0, 0)] == assessment_idx:
                outcome = random.random() < 0.1
            # END DEBUG
            data.append({
                'student_id' : student_id,
                'module_id' : assessment_id,
                'module_type' : datatools.AssessmentInteraction.MODULETYPE,
                'timestep' : t,
                'outcome' : outcome})
history = datatools.InteractionHistory(pd.DataFrame(data))

assessment_idx_map = {id_of_loc(loc): idx for idx, loc in assessment_loc_of_idx.iteritems()}
lesson_idx_map = {id_of_loc(loc): idx for idx, loc in lesson_loc_of_idx.iteritems()}
history.compute_idx_maps(assessment_idx=assessment_idx_map, lesson_idx=lesson_idx_map)
len(history.data)
history_path = os.path.join('data', 'lse_synthetic_history.pkl')
with open(history_path, 'wb') as f:
    pickle.dump(history, f, pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>Estimate an embedding from the sampled interactions<jupyter_code>model = models.EmbeddingModel(
    history, embedding_dimension=2, 
    using_lessons=True, using_prereqs=False, using_bias=True, 
    learning_update_variance_constant=0.5)

estimator = est.EmbeddingMAPEstimator(
    regularization_constant=1e-3, using_scipy=True, 
    debug_mode_on=True, ftol=1e-4)

model.fit(estimator)
model = models.OneParameterLogisticModel(history.data, select_regularization_constant=True)
model.fit()
evaluate.training_auc(model, history, plot_roc_curve=True)<jupyter_output><empty_output><jupyter_text>Visualize the estimated embedding vs. the true embedding<jupyter_code>plt.scatter(A[:, 0], A[:, 1])
for assessment_idx in xrange(num_assessments):
    plt.annotate(id_of_assessment_idx(assessment_idx), (A[assessment_idx, 0], A[assessment_idx, 1]))
"""
for i in xrange(grid_size):
    for j in xrange(grid_size):
        if j < grid_size - 1:
            assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i, j + 1)]]
            plt.plot(A[assessment_idxes, 0], A[assessment_idxes, 1], c='black')
            
        if i < grid_size - 1:
            assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i + 1, j)]]
            plt.plot(A[assessment_idxes, 0], A[assessment_idxes, 1], c='black')
"""
plt.show()
plt.scatter(model.assessment_embeddings[:, 0], model.assessment_embeddings[:, 1])
for assessment_idx in xrange(num_assessments):
    plt.annotate(id_of_assessment_idx(assessment_idx), (model.assessment_embeddings[assessment_idx, 0], model.assessment_embeddings[assessment_idx, 1]))
"""
for i in xrange(grid_size):
    for j in xrange(grid_size):
        if j < grid_size - 1:
            assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i, j + 1)]]
            plt.plot(model.assessment_embeddings[assessment_idxes, 0], model.assessment_embeddings[assessment_idxes, 1], c='black')
            
        if i < grid_size - 1:
            assessment_idxes = [assessment_idx_of_loc[(i, j)], assessment_idx_of_loc[(i + 1, j)]]
            plt.plot(model.assessment_embeddings[assessment_idxes, 0], model.assessment_embeddings[assessment_idxes, 1], c='black')
"""
plt.show()
plt.quiver(Q[:, 0], Q[:, 1], L[:, 0], L[:, 1], pivot='tail', color='black')
"""
for i in xrange(grid_size):
    for j in xrange(grid_size):
        if j < grid_size - 1:
            lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i, j + 1)]]
            plt.plot(Q[lesson_idxes, 0], Q[lesson_idxes, 1], c='black')
            
        if i < grid_size - 1:
            lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i + 1, j)]]
            plt.plot(Q[lesson_idxes, 0], Q[lesson_idxes, 1], c='black')
"""
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
plt.quiver(model.prereq_embeddings[:, 0], model.prereq_embeddings[:, 1], model.lesson_embeddings[:, 0], model.lesson_embeddings[:, 1], pivot='tail', color='black')
"""
for i in xrange(grid_size):
    for j in xrange(grid_size):
        if j < grid_size - 1:
            lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i, j + 1)]]
            plt.plot(model.prereq_embeddings[lesson_idxes, 0], model.prereq_embeddings[lesson_idxes, 1], c='black')
            
        if i < grid_size - 1:
            lesson_idxes = [lesson_idx_of_loc[(i, j)], lesson_idx_of_loc[(i + 1, j)]]
            plt.plot(model.prereq_embeddings[lesson_idxes, 0], model.prereq_embeddings[lesson_idxes, 1], c='black')
"""
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
right_lesson_idxes = [lesson_idx_of_loc[(i, j, 1, 0)] for i in xrange(grid_size) for j in xrange(grid_size) if (i, j, 1, 0) in lesson_idx_of_loc]
up_lesson_idxes = [lesson_idx_of_loc[(i, j, 0, 1)] for i in xrange(grid_size) for j in xrange(grid_size) if (i, j, 0, 1) in lesson_idx_of_loc]
plt.quiver(0, 0, L[right_lesson_idxes, 0], L[right_lesson_idxes, 1], pivot='tail', color='red', alpha=0.25)
plt.quiver(0, 0, L[up_lesson_idxes, 0], L[up_lesson_idxes, 1], pivot='tail', color='blue', alpha=0.25)

plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
plt.quiver(0, 0, model.lesson_embeddings[right_lesson_idxes, 0], model.lesson_embeddings[right_lesson_idxes, 1], pivot='tail', color='red', alpha=0.25)
plt.quiver(0, 0, model.lesson_embeddings[up_lesson_idxes, 0], model.lesson_embeddings[up_lesson_idxes, 1], pivot='tail', color='blue', alpha=0.25)

plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.xlim([-1, 11])
plt.ylim([-1, 11])
plt.show()
plt.scatter(L[right_lesson_idxes, 0], L[right_lesson_idxes, 1], color='red', label='1-0')
plt.scatter(L[up_lesson_idxes, 0], L[up_lesson_idxes, 1], color='blue', label='0-1')
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.legend(loc='best')
plt.show()
plt.scatter(model.lesson_embeddings[right_lesson_idxes, 0], model.lesson_embeddings[right_lesson_idxes, 1], color='red', label='1-0')
plt.scatter(model.lesson_embeddings[up_lesson_idxes, 0], model.lesson_embeddings[up_lesson_idxes, 1], color='blue', label='0-1')
plt.xlabel('Skill 1')
plt.ylabel('Skill 2')
plt.legend(loc='best')
plt.show()
student_idxes = random.sample(range(num_students), 10)
for student_idx in student_idxes:
    plt.scatter(S[student_idx, 0, :], S[student_idx, 1, :], c='black')
    for i in xrange(num_lesson_interactions_per_student):
        plt.plot(S[student_idx, 0, i:(i+2)], S[student_idx, 1, i:(i+2)], c='black')
    plt.xlabel('Skill 1')
    plt.ylabel('Skill 2')
    plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
    plt.show()
for student_idx in student_idxes:
    plt.scatter(model.student_embeddings[student_idx, 0, :], model.student_embeddings[student_idx, 1, :], c='black')
    for i in xrange(num_lesson_interactions_per_student):
        plt.plot(model.student_embeddings[student_idx, 0, i:(i+2)], model.student_embeddings[student_idx, 1, i:(i+2)], c='black')
    plt.xlabel('Skill 1')
    plt.ylabel('Skill 2')
    plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
    plt.show()
for student_idx in student_idxes:
    for i in xrange(embedding_dimension):
        plt.plot(S[student_idx, i, :], '-s', label='Skill 1')
    plt.xlabel('Timestep')
    plt.ylabel('Skill')
    plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
    plt.legend(loc='best')
    plt.show()
for student_idx in student_idxes:
    for i in xrange(embedding_dimension):
        plt.plot(model.student_embeddings[student_idx, i, :], '-s', label='Skill 1')
    plt.xlabel('Timestep')
    plt.ylabel('Skill')
    plt.title('student_id = %s' % history.id_of_student_idx(student_idx))
    plt.legend(loc='best')
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14177007726527746, 'fraction_numerical': 0.01685787871692812, 'mean_word_length': 4.725536193029491, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '78376', 'license': 'Apache-2.0', 'n_tokens_mistral': 6624, 'n_tokens_neox': 6262, 'n_words': 1115, 'path': 'nb/synthetic_experiments.ipynb', 'repo_name': 'Knewton/lentil', 'size': '17084'}