<jupyter_start><jupyter_text>Estimating a continuous variable from a near-infrared spectraIn this example data set we aim to predict a continous variable. This is the case for instance in the food industry, where one wants to predict nutrient content.  Data exploration<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
df=pd.read_csv("./data/nir.csv", header=None, index_col=0, sep=";")
df.head(10)
df.columns = ['target']+[i for i in range(1,681)]
from ipywidgets import interact
ymin=df.iloc[:,1:].min().min()
ymax=df.iloc[:,1:].max().max()
def choose_spectra(val):
    tol = 0.5
    ix=np.where((df.iloc[:,0]>=val-tol) & (df.iloc[:,0]<=val+tol) )[0][0]
    plt.figure(figsize=(5,5))
    plt.plot(df.iloc[ix,1:])
    plt.ylim(ymin,ymax)
    plt.show()
minval=df.iloc[:,0].min()
maxval=df.iloc[:,0].max()
interact(choose_spectra, val=(minval,maxval));<jupyter_output><empty_output><jupyter_text>Baseline modelWe will use PLS as a baseline model<jupyter_code>X_train=df.iloc[:,1:].values
y_train=df.iloc[:,0].values
from sklearn.cross_decomposition import PLSRegression
from sklearn.model_selection import GridSearchCV

pls = PLSRegression()
gs=GridSearchCV(pls, {'n_components':[5,10,15,20,25]})
gs.fit(X_train,y_train)
gs.best_estimator_
gs.best_score_
import seaborn as sns
X_train_r = gs.transform(X_train)
plt.figure(figsize=(10,7))
sns.scatterplot(X_train_r[:,0], X_train_r[:,1], hue=y_train)
from sklearn.decomposition import PCA
pca = PCA()
X_pca= pca.fit_transform(X_train)
plt.figure(figsize=(10,7))
sns.scatterplot(X_pca[:,0], X_pca[:,1], hue=y_train)
y_train_clf = np.where(y_train>50, 1, 0)
y_train_clf
X_train_clf, _ = pls.fit_transform(X_train, y_train_clf)
plt.figure(figsize=(10,7))
sns.scatterplot(X_pca[:,0], X_pca[:,1], hue=y_train_clf)
plt.figure(figsize=(10,7))
sns.scatterplot(X_train_clf[:,0], X_train_clf[:,1], hue=y_train_clf)
X_train_clf.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16223132036847493, 'fraction_numerical': 0.028147389969293755, 'mean_word_length': 7.319148936170213, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '256643', 'license': 'MIT', 'n_tokens_mistral': 839, 'n_tokens_neox': 818, 'n_words': 146, 'path': '03 Regression of NIR spectra.ipynb', 'repo_name': 'jpmaldonado/ml-for-iot', 'size': '1954'}