<jupyter_start><jupyter_text>This script is designed to get indexes of invisible beacon from the beacon file by comparing intervals if less than 60 seconds between sanity check is then to compare to metadata if for some session it fits.  next step is to be able to index into the position file and get behavioral state or outcome - comparing to straightness trajectories.<jupyter_code>root3 = '//10.153.170.3/storage2/fabian/data/raw/FS03/Event_files_FS03/'
root4 = '//10.153.170.3/storage2/fabian/data/raw/FS04/Event_files_FS04'
figures = 'C:/Users/Fabian/Desktop/Analysis/Round3_FS03_FS06/Figures/'
animal = '//10.153.170.3/storage2/fabian/data/raw/FS03/Event_files_FS03/'<jupyter_output><empty_output><jupyter_text>Individual session example.<jupyter_code>FS03_position= pd.read_csv(root3+'position_20210312-172823.txt',sep=" ", header=None)
#FS04_position = pd.read_csv(root4+'position 20200128-151826.txt',sep=" ", header=None)

FS03_beacon = pd.read_csv(root+'beacons_20210312-172823.txt',sep=" ", header=None)
#FS04_beacon = pd.read_csv(root+'beacons 20200128-151826.txt',sep=" ", header=None)

FS03_metadata = pd.read_csv(root+'metadata_20210312-172823.txt',sep=" ", header=None)
#FS04_metadata = pd.read_csv(root+'beacons 20200128-151826.txt',sep=" ", header=None)
FS03_beacon.head()
pd.options.display.float_format = '{:.5f}'.format
FS03_beacon.head()<jupyter_output><empty_output><jupyter_text>diffs = np.diff(trajectory, axis=0)dt = diffs[:, 0]dx = np.sqrt(np.square(diffs[:, 1]) + np.square(diffs[:, 2]))<jupyter_code>df = FS03_beacon[0].diff().to_frame()
df.head()
invis=[]
#for ind in FS03_beacon.index:
df = FS03_beacon[0].diff().to_frame()
for ind in df.index:
    if df[0][ind]<60 and not (ind % 2 == 0):
        invis.append(ind)
        print(FS03_beacon[0][ind])

        
invis
pos_index=[]
invis=[]
#for ind in FS03_beacon.index:
df = FS03_beacon[0].diff().to_frame()
for ind in df.index:
    if df[0][ind]<60 and not (ind % 2 == 0):
        invis.append(ind)
        print(FS03_beacon[0][ind])
        k = FS03_position.iloc[(FS03_position[0] - FS03_beacon[0][ind]).abs().argsort()[:1]]
        #print(list(k.index))
        print(FS03_position[0][list(k.index)])
        pos_index.append(int(''.join(map(str,list(k.index)))))
invis
pos_index<jupyter_output>1615566694.1
11402   1615566694.10000
Name: 0, dtype: float64
1615566894.46
23433   1615566894.46000
Name: 0, dtype: float64
1615567446.89
56610   1615567446.90000
Name: 0, dtype: float64
1615567680.72
70657   1615567680.72000
Name: 0, dtype: float64
1615568260.29
105457   1615568260.30000
Name: 0, dtype: float64<jupyter_text>This is almost correct - so now I can try to match position of animal with the time.  Sanity check<jupyter_code>result=pd.DataFrame()

for dirpath, dirnames, files in os.walk(root3, topdown=True):
    fullstring = dirpath
    for metadata in files:
        if fnmatch.fnmatch(metadata, 'metadata_*'):
            k=(animal+'/'+metadata)
            day = pd.read_csv(k,sep=" : ", header=None,engine='python')
            df=day.T
            df= df.rename(columns=df.iloc[0])
            df=df.drop(df.index[0])
            if int(df['Pellets'].values[0])>1:
                result = result.append(df, ignore_index=True,sort=False)
sorted_data = result.sort_values('Computer time was',)
sorted_data
invis =[]
for ind in sorted_data.index:
    l=sorted_data['invisible_list'].values[ind]
    print(l)
    if not l:
        for num in l: 
            print(num)
            invis.append(l[num])
invis<jupyter_output>[]
[7, 15, 19]
[29, 33, 35]
[]
[1, 5, 7, 13, 15]
[13, 23, 39, 43]
[1, 7, 17, 35, 39]
[15]
[15, 25]
[3, 7, 9, 33, 35, 37, 51, 53, 55, 57, 59]
[13, 15, 17, 19]
[21, 23, 25, 27, 47, 51]
[]
[]
[3, 7]
[15]
[]
[3]
[29]
[5, 29]
[29, 39, 45, 55, 61, 69]
[5, 11, 33, 47]
[19, 21, 33]
[9, 13, 21, 31, 39]
[33]
[9]
[]
[3, 5, 7, 9, 11, 27, 49]
[41]
[13, 25, 37]
[13, 17, 19]
[9, 17, 19, 27, 29, 37, 45, 49]
[31, 35, 37, 39, 49]
[33, 37, 39]
[]
[]
[23, 25, 27, 29, 43]
[15, 17]
[25, 29]
[9, 25, 27, 41]
[31, 33, 35, 37, 39, 41, 47, 49, 53]
[3, 13, 23, 29, 33, 37, 39, 47, 49, 55]
[1, 3, 7, 11, 13, 15, 17, 19]
[5, 9, 13, 17, 19, 21]
[17, 19, 31, 33, 35, 37, 41, 43, 45, 49]<jupyter_text>imports<jupyter_code>import math
import pandas as pd 
import matplotlib.pyplot as plt 
import numpy as np
import seaborn as sns
import matplotlib.lines as mlines
import matplotlib.patches as mpatches
from numpy import median
from scipy.stats import ranksums

import numpy as np
import scipy.stats

import os
import fnmatch<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1610989010989011, 'fraction_numerical': 0.14615384615384616, 'mean_word_length': 4.171590909090909, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '409851', 'license': 'Apache-2.0', 'n_tokens_mistral': 2323, 'n_tokens_neox': 1848, 'n_words': 455, 'path': 'Code/.ipynb_checkpoints/20210525_FS_Identify_invisible_beacon-checkpoint.ipynb', 'repo_name': 'Fabalinks/Rearing_analysis', 'size': '4550'}