<jupyter_start><jupyter_text>Generate the data<jupyter_code>m = np.random.rand()
n = np.random.rand()
num_of_points = 100
x = np.random.random(num_of_points)
y = x*m + n + 0.15*np.random.random(num_of_points)
fig = go.Figure(data=[go.Scatter(x=x, y=y, mode='markers', name='all points')],
                   layout=go.Layout(
        xaxis=dict(range=[np.min(x), np.max(x)], autorange=False),
        yaxis=dict(range=[np.min(y), np.max(y)], autorange=False)
        )
               )
fig.show()
print("m=" + str(m) + " n=" + str(n) )

# fmin
def stright_line_fmin(x,y):
    dist_func = lambda p: (((y-x*p[0]-p[1])**2).mean())
    p_opt = optimize.fmin(dist_func, np.array([0,0]))
    return p_opt
stright_line_fmin(x,y)
# PCA
def straight_line_pca(x,y):
    
    X = np.append(x-x.mean(),y-y.mean(), axis=1)
    # Data matrix X, assumes 0-centered
    
    n, m = X.shape
    # Compute covariance matrix
    C = np.dot(X.T, X) / (n-1)
    # Eigen decomposition
    eigen_vals, eigen_vecs = np.linalg.eig(C)
    # Project X onto PC space
    X_pca_inv = np.dot(np.array([[1,0],[-1,0]]), np.linalg.inv(eigen_vecs))
    X_pca = np.dot(X, eigen_vecs)
    x_min = (x-x.mean()).min()
    x_max = (x-x.mean()).max()
    fig = go.Figure(data=[
        go.Scatter(x=x.ravel(), y=y.ravel(), mode='markers', name='all points'),
        go.Scatter(x=X_pca_inv[:, 0]+x.mean(), y=X_pca_inv[:,1]+y.mean(), mode='lines', name='pca estimation')])
    fig.show()
    return X_pca_inv[1, 1]/X_pca_inv[1, 0], y.mean() - x.mean()*X_pca_inv[1, 1]/X_pca_inv[1, 0]
c = straight_line_pca(x[:, np.newaxis],y[:, np.newaxis])
c
#leaset squares
def least_square_fit(x, y):
    # model: y_i = h*x_i
    # cost: (Y-h*X)^T * (Y-h*X)
    # solution: h = (X^t *X)^-1 * X^t * Y
    return np.dot(np.linalg.inv(np.dot(x.transpose(), x)), np.dot(x.transpose() , y))
least_square_fit(np.append(x[:, np.newaxis], np.ones_like(x[:, np.newaxis]), axis=1), y)
# SVd
def svd_fit(x, y):
    # model: y_i = h*x_i
    # minimize: [x_0, 1, -y_0; x1, 1, -y_1; ...]*[h, 1] = Xh = 0
    # do so by: eigenvector coresponds to smallest eigenvalue of X
    X = np.append(x, -y, axis=1)
    u, s, vh = np.linalg.svd(X)
    return vh[-1, :2]/vh[-1,-1]
m_, n_ = svd_fit(np.append(x[:, np.newaxis], np.ones_like(x[:, np.newaxis]), axis=1), y[:, np.newaxis])
print(m_, n_)
#Ransac
def ransac(src_pnts, distance_func, model_func, num_of_points_to_determine_model, 
           dist_th, inliers_ratio=0.7, p=0.95):
    """Summary or Description of the Function

    Parameters:
    src_pnt : data points used by Ransac to find the model
    distance_func : a function pointer to a distance function. 
    The distance function takes a model and a point and calculate the cost
    p : success probabilaty

    Returns:
    int:Returning value

   """

    min_x = src_pnts[:, 0].min()
    max_x = src_pnts[:, 0].max()
    print(min_x, max_x)
    num_of_points = src_pnts.shape[0]
    num_of_iter = int(np.ceil(np.log(1-p)/np.log(1-inliers_ratio**num_of_points_to_determine_model)))
    proposed_line = []
    max_num_of_inliers = 0
    for i in range(num_of_iter):
        indx = np.random.permutation(num_of_points)[:num_of_points_to_determine_model]
        curr_model = model_func(src_pnts[indx, :])
        x=np.array([min_x, max_x])
        y=curr_model(x)
        print(y)
        d = distance_func(curr_model, src_pnts)
        num_of_inliers = np.sum(d<dist_th)
        proposed_line.append((curr_model, x, y, indx, d, num_of_inliers))
        if num_of_inliers > max_num_of_inliers:
            max_num_of_inliers = num_of_inliers
            best_model = curr_model
    return best_model, proposed_line
        
        
def stright_line_from_two_points(pnts):
    m = (pnts[1, 1]-pnts[0,1])/(pnts[1,0]-pnts[0,0])
    n = (pnts[1,0]*pnts[0,1]-pnts[0,0]*pnts[1,1])/(pnts[1,0]-pnts[0,0])
    mod_func = lambda x : x*m + n
    return mod_func
src_pnts = np.array([x, y]).transpose()
distance_func = lambda model, pnts : (model(pnts[:, 0]) - pnts[:, 1])**2
model_func = stright_line_from_two_points
num_of_points_to_determine_model = 2
dist_th = 0.2
best_model, ransac_run = ransac(src_pnts, distance_func, model_func, num_of_points_to_determine_model, dist_th)
print(x.min())
print(x.max())
x_ransac = np.array([x.min(), x.max()])
y_ransac = best_model(x_ransac)
print(y_ransac)
scatter_xy = go.Scatter(x=x, y=y, mode='markers', name="all points")
frames=[go.Frame(
    data=[scatter_xy, 
          go.Scatter(x=x[item[3]], y=y[item[3]], mode='markers', line=dict(width=2, color="red"), name="selected points"), 
          go.Scatter(x=item[1], y=item[2], mode='lines', name='current line')]) for item in ransac_run]
fig = go.Figure(
    data=[go.Scatter(x=x, y=y, mode='markers', name='all points'), 
          go.Scatter(x=x, y=y, mode='markers', name="selected points"), 
          go.Scatter(x=x, y=y, mode='markers', name="current line"),
          go.Scatter(x=x_ransac, y=y_ransac, mode='lines', name="best selection")],
    layout=go.Layout(
        xaxis=dict(range=[np.min(x), np.max(x)], autorange=False),
        yaxis=dict(range=[np.min(y), np.max(y)], autorange=False),
        title="Ransac guesses",
        updatemenus=[dict(
            type="buttons",
            buttons=[dict(label="Play",
                          method="animate",
                          args=[None])])]
    ),
    frames=frames
)

fig.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1786770140428677, 'fraction_numerical': 0.015890613451589063, 'mean_word_length': 3.3972380178716493, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '9525', 'license': 'MIT', 'n_tokens_mistral': 2299, 'n_tokens_neox': 2220, 'n_words': 400, 'path': 'fit_a_stright_line.ipynb', 'repo_name': 'moshebou/50-ways-to-fit-a-straight-line', 'size': '5412'}