<jupyter_start><jupyter_text>1. Specifiy IRIS version<jupyter_code>#Valid choices are : IRIS_1, IRIS_2, or IRIS_3
IRIS_VERSION = 'IRIS_1'<jupyter_output><empty_output><jupyter_text>2. Specify IRIS raw file<jupyter_code>filepath = '../IRIS_FILE_RETRIEVAL/20220322.TXT'
# Create file date object and read file as pandas dataframe
date_string = filename.split('/')[-1].split('.')[0]

raw_file = pd.read_csv(filename, sep=',', names=['timestamp', 'voltage'])
raw_file['timestamp'] = pd.to_datetime(date_string + ' ' + raw_file['timestamp'])<jupyter_output><empty_output><jupyter_text>3. Parse raw IRIS files into lists based on time interval between measurements<jupyter_code>def compute_time_difference(row):
    #Handle first row exception
    if row.name == 0:
        return timedelta(seconds=0)
    else :
        time_difference = row['timestamp'] - raw_file['timestamp'][int(row.name) - 1]
        return time_difference
raw_file['delta_t'] = raw_file.apply(lambda x : compute_time_difference(x), axis=1)

timestamp = raw_file['timestamp'].to_list()
voltage = raw_file['voltage'].to_list()
delta_t = raw_file['delta_t'].to_list()
c_list_volt = []
c_list_time = []
groups_time = []
groups_volt = []

for idx, interval in enumerate(delta_t):
    if interval < pd.to_timedelta('10s'):
        c_list_volt.append(voltage[idx])
        c_list_time.append(timestamp[idx])
    else :
        groups_volt.append(c_list_volt)
        groups_time.append(c_list_time)
        c_list_volt = [voltage[idx]]
        c_list_time = [timestamp[idx]]<jupyter_output><empty_output><jupyter_text>4. Read measure labels (calib + height) in excel sheet<jupyter_code>in_shape_df = pd.read_excel("C:/Users/paulb/Documents/Doctorat/ROGERS_PASS_2022/2022-03-22/Fidelity_20220322_playground.xlsx",
             sheet_name="IRIS_labels")
# Average voltage measurements and generate calibration and measures dataframe
average_volt = [np.mean(measures) for measures in groups_volt]
in_shape_df['voltage'] = average_volt
calib_df  = in_shape_df[in_shape_df['labels'].str.contains('Spectralon', na=False)].copy()
measures_df = in_shape_df[~in_shape_df['labels'].str.contains('Spectralon', na=False)].copy()<jupyter_output><empty_output><jupyter_text>5. Process IRIS data  Compute reflectance voltage averages for spectralon and measures<jupyter_code>calib_df.rename({'labels': 'spectralon'}, inplace=True, axis=1)
calib_df['spectralon'] = calib_df['spectralon'].apply(lambda x : int(x.split(' ')[-1][:-1]))
calib_df = calib_df.groupby(['spectralon']).mean()<jupyter_output><empty_output><jupyter_text>Create averaged voltages per heights<jupyter_code>measures_df.rename({'labels': 'height'}, inplace=True, axis=1)<jupyter_output><empty_output><jupyter_text>Compute IRIS parameters<jupyter_code># Voltage to reflectance polynomial fit
polynom = iris.calibration_polynom_fit(calib_df.index.values, calib_df['voltage'].values)
# Compute reflectance
measures_df['reflectance'] = measures_df['voltage'].apply(lambda x : iris.voltage_to_reflectance(x, polynom))
# Compute SSA from reflectance
measures_df['ssa'] = measures_df['reflectance'].apply(lambda x : iris.reflectance_to_ssa(x, IRIS_VERSION))
# Compute optical radius from SSA
measures_df['optical radius'] = measures_df['ssa'].apply(lambda x : iris.ssa_to_optical_radius(x))<jupyter_output><empty_output><jupyter_text>Processed data preview<jupyter_code>measures_df.sort_index(ascending=False, inplace=True)
measures_df<jupyter_output><empty_output><jupyter_text>5. Write dataframe to a .csv file<jupyter_code>measures_df.to_csv(iris_file.split('.')[0] + '_processed.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13517241379310344, 'fraction_numerical': 0.01406896551724138, 'mean_word_length': 7.1851015801354405, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '633511', 'license': 'MIT', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1271, 'n_words': 206, 'path': 'IRIS_date_parsing.ipynb', 'repo_name': 'grimp-lab/IRIS_processing', 'size': '3625'}