<jupyter_start><jupyter_text>remove x y points based on threshold ( of rmse distance from previous frame )<jupyter_code>direc = r"../"
visitnum = 'FirstVisit/'

path = os.path.join(direc, visitnum)
trackslist = glob.glob(path + '*.csv')
name = 'c-10_m11'
f = [file for file in trackslist if name in file]
f
axes = pd.read_csv(f[0]).loc[:,['x', 'y', 'likelihood']].plot(subplots=True, figsize=(15,4))
circ_parameters_path = glob.glob('../CircleParameters/' + '*.csv')
circ_parameters = pd.read_csv(circ_parameters_path[0])
data = f[0]
# name = os.path.basename(data)[:-4]
file = pd.read_csv(data)
x = file.x.values
y = file.y.values

x_notinView = x <=5
y_notinView = y <=5

x[x_notinView & y_notinView]=np.nan
y[x_notinView & y_notinView]=np.nan

if x.size == 0 or y.size == 0:
    print(name + 'has emtpy x y tracks')
plt.plot(x,y, 'o-.', alpha = 0.2, markersize=1.0)
plt.xlim(0, 648)
plt.ylim(0,488)
plt.title('camera view')
name = [n for n in circ_parameters.name if n + '_' in f[0]][0]
circ_x = circ_parameters.loc[circ_parameters.name == name, 'circ_x'].values
circ_y = circ_parameters.loc[circ_parameters.name == name, 'circ_y'].values
circ_radii = circ_parameters.loc[circ_parameters.name == name, 'circ_radii'].values
cent_x = x - circ_x
cent_y = x - circ_y

r = np.linalg.norm([cent_x, cent_y], axis = 0)
r = r/circ_radii
_,ax = plt.subplots(3,1, figsize = (20,6))
ax[0].plot(x)
ax[1].plot(y)
ax[2].plot(r)
ax[2].set_ylabel('Radial Distance')
ax[2].set_xlabel('Frames')
plt.suptitle ('Raw data')
trajectory = pd.DataFrame([x, y, r]).T
trajectory.columns = ['x', 'y', 'r']
axes = trajectory.plot(subplots=True, figsize=(15,4))
# Apply filters
ROLLING_WINDOW = [11, 21, 31][1]
trajectory2 = trajectory.copy()
print(trajectory.shape, trajectory2.shape)
for colname in trajectory.columns:
#     trajectory.loc[:, colname] = signal.medfilt(trajectory.loc[:, colname], kernel_size=11)
#     trajectory.loc[:,colname] = trajectory.loc[:,colname].interpolate(method = 'polynomial', order = 3)
    trajectory.loc[:, colname] = trajectory.loc[:, colname].rolling(ROLLING_WINDOW).median(center=True).interpolate(method='pad') 
    trajectory.loc[:, colname] = sp.signal.savgol_filter( trajectory.loc[:, colname], window_length=7, polyorder=2, axis=0) 


axes = pd.concat([trajectory, trajectory2], axis=1).plot(subplots=True, figsize=(15,8))
print(trajectory.shape, trajectory2.shape)
# axes = plt.plot(trajectory2['x'], trajectory2['y'])
plt.plot(trajectory2['x'], trajectory2['y'], 'o-.', alpha = 0.2, markersize=1.0)
plt.xlim(0, 648)
plt.ylim(0,488)
plt.title('camera view')
# Remove noisy part at end
# axes = plt.plot(trajectory2['x'], trajectory2['y'])
plt.plot(trajectory2.loc[:1400, 'x'], trajectory2.loc[:1400, 'y'], 'o-.', alpha = 0.2, markersize=1.0)
plt.xlim(0, 648)
plt.ylim(0,488)
plt.title('camera view')
# Stop

x = np.array(trajectory2['x'])
y = np.array(trajectory2['y'])
r = np.array(trajectory2['r'])
# get rmse values for subsequent frames
rmse = GetRMSE(x[1:], y[1:], x[:-1], y[:-1])

filtered_x = np.copy(x[1:])
filtered_y = np.copy(y[1:])

filtered_x[(rmse > cutoff) | (rmse == np.nan)] = np.nan
filtered_y[(rmse > cutoff) | (rmse == np.nan)] = np.nan

filtered_r = np.linalg.norm([filtered_x - circ_x, filtered_y - circ_y], axis = 0)
filtered_r = filtered_r/circ_radii
_,ax = plt.subplots(3,1, figsize = (20,6))
ax[0].plot(filtered_x)
ax[1].plot(filtered_y)
ax[2].plot(filtered_r)
ax[2].set_xlabel('Frames')
plt.suptitle('filtered after RMSE threshold')
x = pd.Series(filtered_x)
y = pd.Series(filtered_y)

x_interpl = x.interpolate(method='polynomial', order=interpol_order)
y_interpl = y.interpolate(method='polynomial', order=interpol_order)

x_interpl = x_interpl[~np.isnan(x_interpl)]
y_interpl= y_interpl[~np.isnan(y_interpl)]

r_interpl = np.linalg.norm([x_interpl - circ_x, y_interpl - circ_y], axis = 0)
r_interpl = r_interpl/circ_radii
_,ax = plt.subplots(3,1, figsize = (20,6))
ax[0].plot(x_interpl)
ax[1].plot(y_interpl)
ax[2].plot(r_interpl)
ax[2].set_xlabel('Frames')
plt.suptitle('interpolated Data')
# savitzky-golay method for smoothening
x_savgol = signal.savgol_filter(x_interpl, savgol_win, savgol_polyorder)
y_savgol = signal.savgol_filter(y_interpl, savgol_win, savgol_polyorder)
r_savgol = np.linalg.norm([x_savgol- circ_x, y_savgol- circ_y], axis = 0)
r_savgol = r_savgol/circ_radii
_,ax = plt.subplots(3,1, figsize = (20,6))
ax[0].plot(x_savgol)
ax[1].plot(y_savgol)
ax[2].plot(r_savgol)
ax[2].set_xlabel('Frames')
plt.suptitle('smoothened Data')<jupyter_output><empty_output><jupyter_text>I don't believe the last bit of the data, look at DLC confidence in that interval<jupyter_code>plt.scatter(np.arange(len(r_savgol)), r_savgol, c = file.likelihood[2:-1][:len(r_savgol)], cmap = plt.cm.cool )
plt.colorbar()<jupyter_output><empty_output><jupyter_text>Possible solution - use DLC likehood and remove points less than 0.4<jupyter_code>x = pd.Series(filtered_x)
y = pd.Series(filtered_y)

x_interpl = x.interpolate(method='polynomial', order=interpol_order)
y_interpl = y.interpolate(method='polynomial', order=interpol_order)

x_interpl = x_interpl[~np.isnan(x_interpl)]
y_interpl= y_interpl[~np.isnan(y_interpl)]
plt.plot(x_interpl, y_interpl, 'o-')<jupyter_output><empty_output><jupyter_text>OHHH FUCK!  Should be using 2D interpolation, shouldn't I?<jupyter_code># lets first look at DLC likelihood

# plt.scatter(np.arange(len(filtered_x)),filtered_x, c = file.likelihood[1:], cmap = plt.cm.cool)
# plt.show()

plt.scatter(filtered_x, filtered_y, c = file.likelihood[1:], cmap = plt.cm.cool)
plt.colorbar()<jupyter_output><empty_output><jupyter_text>get angle and magnitude<jupyter_code>def Unitvector(x_gauss, y_gauss):
    from sklearn import preprocessing
    
    # get the slope of the tangent
    trajectory = np.asarray([x_gauss, y_gauss])
    m = np.gradient(trajectory, axis = 1)
    m_atx = m[1]/m[0]
    
    # get the tangent vector at x = x0 + 1
    tangent_x = x_gauss+1
    tangent_y = m_atx + y_gauss
    
    # get the unit tangent vector
    u_x = []
    u_y = []

    for x,y,x0,y0 in zip(tangent_x, tangent_y, x_gauss, y_gauss):
        if np.any(np.isnan([x, y])) or np.any(np.isinf([x, y])):
            unit_x = np.nan
            unit_y = np.nan
        else:
            vector = np.asarray([x-x0, y-y0]).reshape(1,-1)
            [unit_x, unit_y] = preprocessing.normalize(vector, norm = 'l2')[0]
        u_x.append(unit_x)
        u_y.append(unit_y)
    u_x = np.asarray(u_x)
    u_y = np.asarray(u_y)
    return(u_x, u_y)

def getAngle(loc, tangent):
    cross = np.cross(tangent, loc)
    dot = np.dot(tangent, loc)
    angle = np.arctan2(cross, dot)*180/np.pi
    return(angle)

def wrapAngle(angle):
    angle = np.absolute(angle)
    for i,a in enumerate(angle):
        if a > 90:
            a = 180 - a
            angle[i] = a
    return(angle)
r = np.linalg.norm([x_interpl, y_interpl], axis = 0)
r = r/circ_radii

# savitzky-golay method
x_savgol = signal.savgol_filter(x_interpl, savgol_win, savgol_polyorder)
y_savgol = signal.savgol_filter(y_interpl, savgol_win, savgol_polyorder)
r_savgol = np.linalg.norm([x_savgol, y_savgol], axis = 0)
r_savgol_norm = r_savgol/circ_radii

# save all usable variables as series

df1 = pd.Series(data = x_savgol, name = 'x_savgol')
df2 = pd.Series(data = y_savgol, name = 'y_savgol')
df3 = pd.Series(data = r_savgol_norm, name = 'radial distance savgol')


#calculate the unit tangent vectors - savitzky-golay vector
u_x, u_y = Unitvector(x_savgol, y_savgol)

angle_savgol = []
for x0, y0, x, y in zip(x_savgol, y_savgol, u_x, u_y):
    loc = [x0, y0]
    tangent = [x, y]
    a = getAngle(loc, tangent)
    angle_savgol.append(a)

angle_savgol = wrapAngle(angle_savgol)

df4 = pd.Series(data = angle_savgol, name = 'angle_savgol')
       
#         new_file = pd.concat([file, df1, df2, df3, df4], axis = 1)
#         new_file.to_csv(newpath + name + 'RadiusAndAngle.csv', index_label = False)

df4.rolling(11).median().plot(figsize=(15,1))
# fig = plt.figure()
# ax = fig.add_subplot(111, projection='polar')
# c = ax.scatter(df4, np.arange(len(df4)), alpha=0.75, s=1.0)
# c = ax.plot(df4, np.arange(len(df4)), alpha=0.5, markersize=1.0)
# https://stackoverflow.com/questions/28465850/rolling-window-polynomial-fit-in-pandas
coefs = df4.rolling(21).apply(lambda x: np.polyfit(range(len(x)), x, 3)[0])
coefs.plot(figsize=(15,1))<jupyter_output>/home/satsingh/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: FutureWarning: Currently, 'apply' passes the values as ndarrays to the applied function. In the future, this will change to passing it as Series objects. You need to specify 'raw=True' to keep the current behaviour, and you can pass 'raw=False' to silence this warning
{'fraction_non_alphanumeric': 0.1622520353170508, 'fraction_numerical': 0.026373122348354545, 'mean_word_length': 4.9172320217096335, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '641433', 'license': 'MIT', 'n_tokens_mistral': 3766, 'n_tokens_neox': 3558, 'n_words': 690, 'path': 'PaperPinelineCodes/CodesOnLongerUsed/Proboscis_Sat.ipynb', 'repo_name': 'TanviDeora/ProboscisTracking', 'size': '8721'}