<jupyter_start><jupyter_text>- set up environment<jupyter_code># set up environment

# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow import keras

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import FastICA

# for Set working folder, etc.
import os
import os.path

# for sampling 
import random

# set working folder
# fw slashes for windows
cwd_desired = "C:/Users/Bonita/Documents/work/velexi/neural networks" 
os.chdir(cwd_desired) 

# test code
# print(os.getcwd())<jupyter_output><empty_output><jupyter_text>- ETL spectrum files<jupyter_code># etl spectrum files

# turn files into lists and cut off first line
# put into database "spectra"
# turn deleted points (flagged as -1.23e+34) into 0's


# init spectra and class databases
spectra_wd = str(cwd_desired + "/spectra2")
num_samples = len([name for name in os.listdir(spectra_wd) if \
                   os.path.isfile(os.path.join(spectra_wd,name))])
# test code 
print(num_samples)

spectrum_len = 480                                 # 480 >= the BECK spectrometer spectrum length
spectra = np.zeros((num_samples,spectrum_len))             
y = np.zeros((num_samples, 1))

# fill databases
i = 0

# find spectra folder
os.chdir(spectra_wd)

# etl the data
for filename in os.listdir(spectra_wd):
    file_object  = open(filename, 'r').readlines()
    # strip off header, add to matrix 'spectra'
    spectra[i,:] = file_object[1:]                  
    
    # label spectrum class, based on header
    # actinolite: 0, alunite: 1, calcite: 2
    material_name = file_object[0]
    
    # testcode
    # print(material_name)
    
    if material_name.find('Actinolite',)!= -1: #if material name contains actinolite
        y[i,0] = 0
    elif material_name.find('Alun',)!= -1:
        y[i,0] = 1
    else:                   #calcite
        y[i,0] = 2         
               
    # turn deleted points into 0
    for j in range(spectrum_len):  
        if spectra[i,j] <0:
            spectra[i,j]=0                          
    i+=1

# test code
# print(y[:])
# print(spectra[0:2,0])<jupyter_output>32<jupyter_text>- Load data<jupyter_code># divide up data randomly

# 80% training data 20% test for this pilot
# eventually, 60% training set, 20% validation set, 20% test set
sample_indices = list(range(0, num_samples))
train_set_size = 4*(num_samples//5)
test_set_size= num_samples-train_set_size
train_set_indices = random.sample(sample_indices,train_set_size)
test_set_indices = np.setdiff1d(sample_indices,train_set_indices)     #fixed bug: take remaining samples after making train set

# test code
# print(train_set_indices)
# print(test_set_indices)

# make train and test sets

train_set = spectra[train_set_indices, :]
train_labels = y[train_set_indices, :]
test_set = spectra[test_set_indices, :]
test_labels = y[test_set_indices, :]<jupyter_output><empty_output><jupyter_text>- spectrum plots<jupyter_code>num0 = 0 #number of samples of class 0
num1 = 0
num2 = 0

# count the number of each class to make spectra0, spectra1, spectra2 databases
for i in range(num_samples):
    if y[i,0]== 0:
        num0 += 1
    elif y[i,0]== 1:
        num1 += 1
    elif y[i,0]== 2:
        num2 += 1

# make class-specific databases spectra0, ...1, ...2
spectra0 = np.zeros((num0,spectrum_len)) 
spectra1 = np.zeros((num1,spectrum_len)) 
spectra2 = np.zeros((num2,spectrum_len)) 

# make counters for each database to place spectra
i0 = 0
i1 = 0
i2 = 0

# populate class-specific databases spectra0, ...1, ...2
for i in range(num_samples):
    if y[i,0]== 0:
        spectra0[i0,:] = spectra[i,:]
        i0 +=1
    elif y[i,0]== 1:
        spectra1[i1,:] = spectra[i,:]
        i1 +=1
    else:
        spectra2[i2,:] = spectra[i,:]
        i2 +=1

# plot each class-specific database separately
for i in range(i0):
    plt.plot(spectra0[i,:])
plt.figure()

for i in range(i1):
    plt.plot(spectra1[i,:])
plt.figure()

for i in range(i2):
    plt.plot(spectra2[i,:])<jupyter_output><empty_output><jupyter_text>- ICA plots<jupyter_code># --- Generate ICA model

# ICA Parameters
num_components = 3

# Create FastICA object
ica = FastICA(n_components=num_components)

# Fit ICA model
X = spectra.T
S_ = ica.fit_transform(X)

print(X.shape)
print(S_.shape)

print("Number of components generated:", num_components)
print("Number of iterations:", ica.n_iter_)

# Display independent components
for i in range(num_components):
    plt.title('Component {}'.format(i))
    plt.plot(S_[:,i])
    plt.figure()<jupyter_output>(480, 32)
(480, 3)
Number of components generated: 3
Number of iterations: 6<jupyter_text>- spectrum plots, individualized<jupyter_code># display real spectra
for i in range(num_components):
    plt.title('Spectra {}'.format(i))
    plt.plot(spectra[i])
    plt.figure()<jupyter_output><empty_output><jupyter_text>- reflectance spectra plots, reconstructed from functional group spectra<jupyter_code>A_ = ica.mixing_
print(A_.shape)
print(S_.shape)
X_ = np.dot(S_, A_.T) + ica.mean_
print(X_.shape)
for i in range(num_components):
    plt.title('Reconstructed Reflectance Spectra {}'.format(i))
    plt.plot(X_[:,i])
    plt.figure()<jupyter_output>(32, 3)
(480, 3)
(480, 32)<jupyter_text>- functional group spectra plots, deconstructed from reflectance spectra<jupyter_code>Ainv = ica.components_
print(Ainv.shape)
print(X.shape)
S_2 = np.dot(X, Ainv.T)
print(S_2.shape)
for i in range(num_components):
    plt.title('Reconstructed Functional Group Spectra {}'.format(i))
    plt.plot(S_2[:,i])
    plt.figure()<jupyter_output>(3, 32)
(480, 32)
(480, 3)
{'fraction_non_alphanumeric': 0.12381801962533452, 'fraction_numerical': 0.02711864406779661, 'mean_word_length': 3.7670068027210886, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '259311', 'license': 'Apache-2.0', 'n_tokens_mistral': 2120, 'n_tokens_neox': 1990, 'n_words': 512, 'path': 'lab-notebook/bsong/2019-07-09 - bootstrap rev 1 - implement/backups and checkpoints/2019-01-29-BPS-spectra reconstruction and deconstruction.ipynb', 'repo_name': 'velexi-corporation/spectra-ml', 'size': '5605'}