<jupyter_start><jupyter_text>Set data paths<jupyter_code>data_path = 'Sample data'
spectroscopy_path = os.path.join(data_path, 'Spectroscopy')
loading_path = os.path.join(data_path, 'Loading Time')
temperature_path = os.path.join(data_path, 'Release and Recapture')<jupyter_output><empty_output><jupyter_text>Functions to parse the data<jupyter_code>def get_txt_csv(path):
    files = os.listdir(path)
    file_names = [ file for file in files if file.endswith( ('.txt','.TXT','.CSV','.csv') ) ]
    file_paths = [os.path.join(path, file) for file in file_names]
    return file_names, file_paths

def make_oscilloscope_df(path):
    file_names, file_paths = get_txt_csv(path)
    dfs = []
    for file_path, file_name in zip(file_paths, file_names):
        df = pd.read_csv(file_path, header=None)
        df.drop([0,1,2,5], axis=1, inplace=True)
#         df = df.astype(np.float64)
#         df.index = df.index.astype(np.float64)
        df.set_index(3, inplace=True)
        df.index.name = 'time [s]'
        df.columns = [file_name[:-4]]
        dfs.append(df)
    df = pd.concat(dfs, axis=1)
    return dfs

def make_spectroscopy_df(path):
    pass

def plot_dfs(dfs):
    plot_columns = ceil(len(dfs)/4)
    fig, axes = plt.subplots(plot_columns,4, figsize=(15, plot_columns*2), facecolor='w', edgecolor='k')
    axes = axes.ravel()
    for i, df in enumerate(dfs):
        axes[i].plot(df.index,df.values, '.')
    return fig, axes<jupyter_output><empty_output><jupyter_text>Analysing loading curves  1 - Starting point<jupyter_code>dfs = make_oscilloscope_df(loading_path)
fig, axes = plot_dfs(dfs)
plt.show(fig)<jupyter_output><empty_output><jupyter_text>2 - Filtering values that are equal to the minimum value<jupyter_code>def filter_loading(dfs, rolling=1):
    for i, df in enumerate(dfs):
        dfs[i] = df[df != df.min()].rolling(window=rolling, center=True).mean().dropna()
    return dfs
dfs = filter_loading(dfs, rolling=10)
fig, axes = plot_dfs(dfs)
plt.show(fig)<jupyter_output><empty_output><jupyter_text>3 - Fitting loading curves<jupyter_code>from lmfit import Parameters, fit_report, minimize

def loading_residual(pars, x , data=None):
    vals = pars.valuesdict()
    v1 = vals['v1']
    v2 = vals['v2']
    offset = vals['offset']
    t0 = vals['t0']
    dt = vals['dt']
    tau = vals['tau']
    
    model = v2+(v1-v2)*(1-np.exp(-np.abs((x-t0-dt)/tau)))+offset
    model[x <= t0+dt] = v2
    model[x <= t0] = v1
    if data is None:
        return model
    return model - data


def fit_loading_dfs(dfs, offset_on=False):
    fit_data = {
        'file': [df.columns[0] for df in dfs],
        'amp': [],
        'offset': [],
#         'amp_std': [],
        'tau': [],
#         'tau_std': [],
        'redchi': [],
    }
    for i, df in enumerate(dfs):
        x = df.index.values
        y = df.iloc[:,0].values
        
        # predicting parameters
        vmin = np.min(y)
        vmax = np.max(y)
        dv = vmax - vmin
        tmin = np.min(x)
        tmax = np.max(x)
        tscan = tmax - tmin
        
        p = Parameters()
        p.add('v1', value=vmax, min=vmax-dv/10, max=vmax)
        p.add('v2', value=vmin, min=vmin, max=np.min(y)+0.03)
        p.add('offset', value=0, vary=offset_on, min=0, max=dv, brute_step=dv/20)
        p.add('t0', value=tscan/5, min=tmin, max=tmax, brute_step=tscan/20)
        p.add('dt', value=0, min=0, max=tscan, brute_step=tscan/5)
        p.add('tau', value=1, min=0, max=60, brute_step=10)
        
        mi = minimize(loading_residual, p, args=(x,), kws={'data': y}, method='powell')
#         mi = minimize(loading_residual, mi.params, args=(x,), kws={'data': y}, method='leastsq')
        dfs[i]['best fit'] = loading_residual(mi.params, x)
#         dfs[i]['init fit'] = loading_residual(mi.init_values, x)
        
        # storing fit results
        fit_data['amp'].append(mi.params['v1'].value-mi.params['v2'].value)
        fit_data['tau'].append(mi.params['tau'].value)
        fit_data['offset'].append(mi.params['offset'].value)
        fit_data['redchi'].append(mi.redchi)
        
    fit_df = pd.DataFrame(data=fit_data)
    fit_df = fit_df.set_index('file', drop=True).sort_index(level=0)
    return dfs, fit_df

def plot_dfs_with_fits(dfs):
    plot_columns = ceil(len(dfs)/4)
    fig, axes = plt.subplots(plot_columns,4, figsize=(15, plot_columns*2), facecolor='w', edgecolor='k')
    axes = axes.ravel()
    for i, df in enumerate(dfs):
        axes[i].plot(df.index,df.iloc[:,0].values, '.')
#         axes[i].plot(df.index,df['init fit'].values, 'k--')
        axes[i].plot(df.index,df['best fit'].values, 'r-')
    return fig, axes
dfs_fit, fit_df = fit_loading_dfs(dfs, offset_on=False)
fig, axes = plot_dfs_with_fits(dfs_fit)
fit_df<jupyter_output><empty_output><jupyter_text>Release and Recapture<jupyter_code>dfs_rr = make_oscilloscope_df(temperature_path)
fig, axes = plot_dfs(dfs_rr)
dfs_fit_rr, fit_df_rr = fit_loading_dfs(dfs_rr, offset_on=True)
fig, axes = plot_dfs_with_fits(dfs_fit_rr)
fit_df_rr<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1605695076132094, 'fraction_numerical': 0.014435436029266363, 'mean_word_length': 3.444639718804921, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '154250', 'license': 'MIT', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1958, 'n_words': 344, 'path': 'FP 20 notebook.ipynb', 'repo_name': 'RFerraciniAlves/FP-20-notebooks', 'size': '5057'}