<jupyter_start><jupyter_text>Notebook A: Data ProcesssingThis notebook takes experimental data and converts it to a form that is usable for machine learning algorithm training.This involves smoothing the data using the Savitzky–Golay filter, and adding slope values by comparing consecutive concentration values, and dividing by time elapsed.   Setup Imports<jupyter_code>import pandas as pd
import numpy as np
import math
from scipy.signal import savgol_filter<jupyter_output><empty_output><jupyter_text>Import experimental data<jupyter_code>exp_df = pd.read_csv('../data/experimental_data.csv')
exp_df.head(10)<jupyter_output><empty_output><jupyter_text>Define function to smooth a single trial of experimental data using a second order Savitzky-Golay filter<jupyter_code># set up savgol filter to have a window length of 15, polynomial order of 2, 
# and to output the smoothed values.
savgol = lambda x: savgol_filter(x, 15, 2, deriv=0)

def smooth_single_trial(trial_df, delta):
    # get list of time values
    times = list(trial_df.time)

    # get smoothed time values
    max_time = times[-1]
    new_times = np.arange(0,max_time, delta)  

    # temporarily set time as the index of the dataframe
    trial_df.set_index('time', inplace=True, drop=True)

    # set the index of the df to the new times
    trial_df = trial_df.reindex(trial_df.index.union(new_times))

    # find the linear interpolation for all the new data points
    trial_df = trial_df.interpolate()

    # remove experimental time points, unless they are an exact multiple of delta
    times_to_remove = set(times) - (set(times) & set(new_times))
    trial_df = trial_df.loc[~trial_df.index.isin(times_to_remove)]

    # convert linear interpolation to smoothed polynomial fit
    trial_df = trial_df.apply(savgol)
    
    # round negative smoothed values to 0
    trial_df.clip(lower=0, inplace=True)
    
    # Prevent floating point errors later
    trial_df.composition = round(trial_df.composition, 0)
    trial_df.trial = round(trial_df.trial, 0)

    # remove time's status as index column
    trial_df.insert(2, 'time',trial_df.index)
    trial_df.reset_index(inplace=True, drop=True)
    
    return trial_df<jupyter_output><empty_output><jupyter_text>Apply smoothing function to each trial of experimental data<jupyter_code># This parameter tunes the distance between
delta = 0.1

smoothed_df = pd.DataFrame()

# Loop through compositions
for composition in set(exp_df.composition):
    
    # For each composition, loop over trials
    composition_df = exp_df[exp_df.composition == composition]
    for trial in set(composition_df.trial):
        # isolate the data for a single trial
        trial_df = exp_df[(exp_df.composition == composition) & (exp_df.trial == trial)]
        
        # get the smooth data for that trial
        smooth_trial_df = smooth_single_trial(trial_df, delta)
        
        # add the trial to smoothed_df
        smoothed_df = pd.concat([smoothed_df, smooth_trial_df])

smoothed_df.head(10)<jupyter_output><empty_output><jupyter_text>Define function to add production rates to a single trial<jupyter_code>def add_rates_to_trial(trial_df, delta):
    
    trial_rates_df = trial_df.copy()

    biomass_rates = []
    ethanol_rates = []
    acetate_rates = []
    butanol_rates = []
    butyrate_rates = []
    
    
    # loop over rows of dataframe
    for index, row in trial_df.iterrows():
        # if it is the first row, don't calculate slope
        if row.time == 0:
            biomass_rates.append(0)
            ethanol_rates.append(0)
            acetate_rates.append(0)
            butanol_rates.append(0)
            butyrate_rates.append(0)
        # find diffence between current and previous value divide by the delta value to get production rate
        else:
            previous_row = trial_df[trial_df.index == index-1]
            biomass_rates.append(float((row['biomass (g/L)'] - previous_row['biomass (g/L)'])/delta))
            ethanol_rates.append(float((row['ethanol (mM)'] - previous_row['ethanol (mM)'])/delta))
            acetate_rates.append(float((row['acetate (mM)'] - previous_row['acetate (mM)'])/delta))
            butanol_rates.append(float((row['butanol (mM)'] - previous_row['butanol (mM)'])/delta))
            butyrate_rates.append(float((row['butyrate (mM)'] - previous_row['butyrate (mM)'])/delta))

    # add production rates to the data frame
    trial_rates_df['biomass rate'] = biomass_rates
    trial_rates_df['ethanol rate'] = ethanol_rates
    trial_rates_df['acetate rate'] = acetate_rates
    trial_rates_df['butanol rate'] = butanol_rates
    trial_rates_df['butyrate rate'] = butyrate_rates

    return trial_rates_df<jupyter_output><empty_output><jupyter_text>Apply function to every trial<jupyter_code>rates_df = pd.DataFrame()

# Loop through compositions
for composition in set(smoothed_df.composition):
    
    # For each composition, loop over trials
    composition_df = smoothed_df[smoothed_df.composition == composition]
    for trial in set(composition_df.trial):
        
        # isolate the data for a single trial
        trial_df = smoothed_df[(smoothed_df.composition == composition) & (smoothed_df.trial == trial)]
        
        # get the rates data for that trial
        trial_rates_df = add_rates_to_trial(trial_df, delta)
        
        # add the trial to rates_df
        rates_df = pd.concat([rates_df, trial_rates_df])

rates_df.head(10)<jupyter_output><empty_output><jupyter_text>Remove data from the first 24 hrs of fermentation (glucose was present)<jupyter_code># remove all time points before one day has passed
rates_df = rates_df[rates_df['time'] >= 1]

# subtract one day from all times
rates_df['time'] = rates_df['time'] - 1

rates_df.head(10)<jupyter_output><empty_output><jupyter_text>Save rates data as csv<jupyter_code>rates_df.to_csv(f'../data/rates_data.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09202764200235969, 'fraction_numerical': 0.005899207820664082, 'mean_word_length': 3.8839506172839506, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '556662', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2026, 'n_tokens_neox': 1815, 'n_words': 491, 'path': 'notebooks/A_data_processing.ipynb', 'repo_name': 'NKWBTB/SyngasMachineLearning', 'size': '5933'}