<jupyter_start><jupyter_text>Evaluation of results<jupyter_code>import matplotlib.pyplot as plt
# sets the default fonts on matplotlib to type2 postcript
plt.rcParams['pdf.fonttype'] = 42
plt.rcParams['ps.fonttype'] = 42
#file = "res_points100-750.txt"
#file = "res_points20-200.txt"
file = "res.txt"
#file = "res.txt"

with open(file, "rb") as fp:   # Unpickling
    res = pickle.load(fp)
results = {}
results['grad'] = np.array(res['grad'])
results['gn'] = np.array(res['gn'])
results['lm-e'] = np.array(res['lm-e'])
results['lm-s'] = np.array(res['lm-s'])
results['grad-ini'] = np.array(res['grad-ini'])
results['gn-ini'] = np.array(res['gn-ini'])
results['lm-e-ini'] = np.array(res['lm-e-ini'])
results['lm-s-ini'] = np.array(res['lm-s-ini'])
results['land'] = np.array(res['land'])
# analize results
# Nplanes[0], Nposes[1], Npoints[2], iters[3],  process-time[4],
    #         ini_error[5] error[6], eigenvalues[7-12]
    #         erro_ini_rot[13] erro_ini_tran[14] erro_rot[15] erro_tran[16]
def print_statistics(index, results):
    print('Gradient    : ', np.percentile(results['grad'][:,index],25),
          np.median(results['grad'][:,index]), np.percentile(results['grad'][:,index],75))
    print('Gradient-ini: ', np.percentile(results['grad-ini'][:,index],25),
      np.median(results['grad-ini'][:,index]), np.percentile(results['grad-ini'][:,index],75))
    x = results['gn'][:,index][np.logical_not(np.isnan(results['gn'][:,index]))]
    print('Gauss Newton: ', np.percentile(x,25),
          np.median(x), np.percentile(x,75))
    x = results['gn-ini'][:,index][np.logical_not(np.isnan(results['gn-ini'][:,index]))]
    print('Gauss New ini: ', np.percentile(x,25),
          np.median(x), np.percentile(x,75))
    print('LM Ellispe   : ', np.percentile(results['lm-e'][:,index],25),
          np.median(results['lm-e'][:,index]), np.percentile(results['lm-e'][:,index],75))
    print('LM Ellisp INI: ', np.percentile(results['lm-e-ini'][:,index],25),
          np.median(results['lm-e-ini'][:,index]), np.percentile(results['lm-e-ini'][:,index],75))
    print('LM Sphere    : ', np.percentile(results['lm-s'][:,index],25),
          np.median(results['lm-s'][:,index]), np.percentile(results['lm-s'][:,index],75))
    print('LM Sphere INI: ', np.percentile(results['lm-s-ini'][:,index],25),
          np.median(results['lm-s-ini'][:,index]), np.percentile(results['lm-s-ini'][:,index],75))
    print('FGraph Land  : ', np.percentile(results['land'][:,index],25),
          np.median(results['land'][:,index]), np.percentile(results['land'][:,index],75))

def print_mean(index,results):
    print('Gradient    : ', np.mean(results['grad'][:,index]), np.std(results['grad'][:,index]))
    print('Gradient-ini: ', np.mean(results['grad-ini'][:,index]), np.std(results['grad-ini'][:,index]))
    x = results['gn'][:,index][np.logical_not(np.isnan(results['gn'][:,index]))]
    print('Gauss Newton: ', np.mean(x), np.std(x))
    x = results['gn-ini'][:,index][np.logical_not(np.isnan(results['gn-ini'][:,index]))]
    print('Gauss New ini: ', np.mean(x), np.std(x))
    print('LM Ellispe   : ', np.mean(results['lm-e'][:,index]), np.std(results['lm-e'][:,index]))
    print('LM Ellisp INI: ', np.mean(results['lm-e-ini'][:,index]), np.std(results['lm-e-ini'][:,index]))
    print('FGraph landm: ', np.mean(results['land'][:,index]), np.std(results['land'][:,index]))
print('Mean Iterations')
#print_statistics(3,results)
print_mean(3,results)
print('Mean Processing time')
#print_statistics(4,results)
print_mean(4,results)
print('Rotation error')
print_statistics(15,results)
print('Translation error')
print_statistics(16,results)
print('Success rate')
print_mean(17,results)
i=2
plt.hist(results['lm-e-ini'][:,i],100)
plt.figure()
plt.hist(results['land'][:,i],100)
plt.show()
def statistics_errors(res):
    xrot = res[:,15][np.logical_not(np.isnan(res[:,15]))]
    xtra = res[:,16][np.logical_not(np.isnan(res[:,16]))]
    return [[np.median(xrot), np.median(xtra)],
            [np.percentile(xrot,25), np.percentile(xtra,25)],
            [np.percentile(xrot,75), np.percentile(xtra,75)]]

# plot bar for static overall conditions

labels = ['Rotation error [rad]', 'Translation error [m]']
methods = ['Grad-EF', 'Newton-EF', 'LM-EF','Land']

x = np.arange(len(labels))  # the label locations
width = 0.45  # the width of the bars

res0 = statistics_errors(results['grad-ini'])
res1 = statistics_errors(results['gn-ini'])
res2 = statistics_errors(results['lm-e-ini'])
res3 = statistics_errors(results['land'])


fig, ax = plt.subplots(figsize=(8,4))
rects1 = ax.bar(x - width*0.75, res0[0], width/2, yerr=[res0[1],res0[2]], label=methods[0])
rects2 = ax.bar(x - width*0.25, res1[0], width/2, yerr=[res1[1],res1[2]], label=methods[1])
rects3 = ax.bar(x + width*0.25, res2[0], width/2, yerr=[res2[1],res2[2]], label=methods[2])
rects4 = ax.bar(x + width*0.75, res3[0], width/2, yerr=[res3[1],res3[2]], label=methods[3])

# Add some text for labels, title and custom x-axis tick labels, etc.
ax.set_ylim([0,0.1])
ax.set_ylabel('Pose Error')
ax.set_title('Error to Ground Truth Last Pose')
ax.set_xticks(x)
ax.set_xticklabels(labels)
ax.legend(loc='upper left')
plt.savefig('error-gt-bar.pdf', bbox_inches='tight')
# plot function of poses
# preprocess results
def create_index_points(N_ref):
    cont = 0
    ind = []
    for n in N_points_frame:
        for pi in N_planes:
            for p in N_poses:
                for s in point_noise:
                    for samples in range(100):
                        if n == N_ref:
                            ind.append(cont)
                        cont = cont+1
    return ind

# create statistics for each number of points
x = N_points_frame
res_ef = []
res_land = []
for n in N_points_frame:
    ind = create_index_points(n)
    tmp = statistics_errors(results['lm-e-ini'][ind,:])
    res_ef.append(np.array(tmp).reshape(1,6))
    tmp = statistics_errors(results['land'][ind,:])
    res_land.append(np.array(tmp).reshape(1,6))
plt.figure(figsize=(3,3))
res_ef = np.asarray(res_ef)[:,0,:]
plt.plot(x, res_ef[:,0], 'g', label='LM-EF')
#plt.fill_between(x,res_ef[:,2],res_ef[:,4], color='green', alpha = 0.2 )
res_land = np.asarray(res_land)[:,0,:]
plt.plot(x, res_land[:,0], 'r--', label='Landmark')
#plt.fill_between(x,res_land[:,2],res_land[:,4], color='red', alpha = 0.2 )
plt.title('Rotation Error [rad]')
plt.xlabel('number of points per pose')
plt.xticks((x[0],x[3],x[4],x[5]))
plt.legend()
plt.savefig('rotation-number-points.pdf', bbox_inches='tight')

# Translation error
plt.figure(figsize=(3,3))
plt.plot(x, res_ef[:,1], 'g', label='LM-EF')
#plt.fill_between(x,res_ef[:,3],res_ef[:,5], color='green', alpha = 0.2 )
plt.plot(x, res_land[:,1], 'r--', label='Landmark')
#plt.fill_between(x,res_land[:,3],res_land[:,5], color='red', alpha = 0.2 )
plt.title('Translation Error [m]')
plt.xlabel('number of points per pose')
plt.xticks((x[0],x[3],x[4],x[5]))
plt.legend()
plt.savefig('trans-number-points.pdf', bbox_inches='tight')
# create problem and evaluate
def solve_new_problem_lm(result,N_points,N_planes,N_poses, point_noise,bias):
    synthetic_points = registration.CreatePoints(N_points,N_planes,N_poses, point_noise,bias)#add bias noise param
    problem = registration.PlaneRegistration() #empty creator
    synthetic_points.create_plane_registration(problem)
    T_gt = synthetic_points.get_ground_truth_last_pose()
    
    # 3.1) Solve LM Ellipse optimization INI
    r = solve_particular_problem(problem, registration.LM_ELLIP,T_gt,True)
    result.append(r)
# create an example with a fixed number of points per plane (1 for example) and augment with poses
# rerun cell with defintoin of results as lists
res=[]
for n in [3]:
    for pi in [3]:
        for p in np.arange(2,20):
            for s in [0.005]: # point_noise:
                for bias in [0.0]: #N_bias:
                    print('Iteration = ',n,pi,p,s)
                    for samples in range(100):
                        solve_new_problem_lm(res,n,pi,p,s,bias)
x = np.arange(2,20)
res_ef = []
res = np.asarray(res)
for n in x:
    ind = np.where(res[:,1]== n)
    tmp = statistics_errors(res[ind,:])
    res_ef.append(np.array(tmp).reshape(1,6))
plt.figure(figsize=(8,6))
res_ef = np.asarray(res_ef)[:,0,:]
plt.plot(x, res_ef[:,0], 'g', label='LM-EF')
#plt.fill_between(x,res_ef[:,2],res_ef[:,4], color='green', alpha = 0.2 )
plt.title('Rotation Error [rad]')
plt.xlabel('number of points per pose')
plt.legend()
# the problem becomes illposes and diverges quickly...
#plt.savefig('rotation-1point-poses.pdf', bbox_inches='tight')
np.median(res[:,5])
# function to plot any of the historgrams results:
def plot_hist(index, results):
    plt.figure(figsize=(18,8))
    plt.subplot(2,4,1)
    plt.hist(results['grad'][:,index],  label='grad')
    plt.title('Gradient')
    
    plt.subplot(2,4,2)
    plt.hist(results['gn'][:,index],  label='gn')
    plt.title('Gauss Newton')
    
    plt.subplot(2,4,3)
    plt.hist(results['lm-e'][:,index],  label='lm-e')
    plt.title('LM Ellipse')
    
    plt.subplot(2,4,4)
    plt.hist(results['lm-s'][:,index],  label='lm-s')
    plt.title('LM spehere')
    
    plt.subplot(2,4,5)
    plt.hist(results['grad-ini'][:,index],  label='grad-ini')
    plt.title('Gradient-INI')
    
    plt.subplot(2,4,6)
    plt.hist(results['gn-ini'][:,index],  label='gn')
    plt.title('Gauss Newton INI')
    
    plt.subplot(2,4,7)
    plt.hist(results['lm-e-ini'][:,index],  label='lm-e')
    plt.title('LM Ellipse INI')
    
    plt.subplot(2,4,8)
    plt.hist(results['lm-s-ini'][:,index],  label='lm-s')
    plt.title('LM spehere INI')
# function to plot any of the historgrams results:
def plot_all_hist(x):
    plt.figure(figsize=(18,9))
    t =1
    for index in [3,4,5,6,13,14,15,16]:
        plt.subplot(2,4,t)
        plt.hist(x[:,index])
        plt.title(index)
        t = t+1
#function to plot any of the results
def plot_results(index,results):
    plt.figure(figsize=(8,5))
    plt.plot(results['grad'],  label='grad')
    #TODO
# analize results
# Nplanes[0], Nposes[1], Npoints[2], iters[3],  process-time[4],
    #         ini_error[5] error[6], eigenvalues[7-12]
    #         erro_ini_rot[13] erro_ini_tran[14] erro_rot[15] erro_tran[16]
plot_hist(16,results)
plot_all_hist(results['lm-e-ini'])<jupyter_output><empty_output><jupyter_text>Plane Landmark<jupyter_code>#moved up to do a single 
# create problem and evaluate
def solve_new_problem_landmark(result,N_points,N_planes,N_poses, point_noise):
    synthetic_points = registration.CreatePoints(N_points,N_planes,N_poses, point_noise,0.1)
    problem = registration.PlaneRegistration()
    synthetic_points.create_plane_registration(problem)
    T_gt = synthetic_points.get_ground_truth_last_pose()
    

    # 3.1) Solve LM Ellipse optimization INI
    r = solve_particular_problem(problem, registration.LM_ELLIP,T_gt,True)
    result['ef'].append(r)
    
    r = solve_landmark_particular_problem(problem,synthetic_points,N_planes,N_poses)
    result['land'].append(r)
results_land = {}
results_land['ef']=[]
results_land['land']=[]
solve_new_problem_landmark(results_land,50,3,3,0.05)
# analize results
# Nplanes[0], Nposes[1], Npoints[2], iters[3],  process-time[4],
    #         ini_error[5] error[6], eigenvalues[7-12]
    #         erro_ini_rot[13] erro_ini_tran[14] erro_rot[15] erro_tran[16]
i=16
print(results_land['ef'][0][i])
print(results_land['land'][0][i])<jupyter_output>0.10468393020794706
3.717437078070865
{'fraction_non_alphanumeric': 0.2011314186248912, 'fraction_numerical': 0.032550043516100954, 'mean_word_length': 4.641138929798724, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '215116', 'license': 'MIT', 'n_tokens_mistral': 4545, 'n_tokens_neox': 4385, 'n_words': 703, 'path': 'notebooks/synthetic-planes-benchmark.ipynb', 'repo_name': 'DmiitriyJarosh/PlaneDetector', 'size': '11490'}