<jupyter_start><jupyter_text>Pumpkin PricingLoad up required libraries and dataset. Convert the data to a dataframe containing a subset of the data: - Only get pumpkins priced by the bushel- Convert the date to a month- Calculate the price to be an average of high and low prices- Convert the price to reflect the pricing by bushel quantity<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report 
from sklearn.metrics import roc_curve, roc_auc_score
from sklearn.linear_model import LogisticRegression
import sklearn
import numpy as np
import calendar
import seaborn as sns

pumpkins = pd.read_csv('../data/US-pumpkins.csv')

pumpkins.head()


pumpkins = pumpkins[pumpkins['Package'].str.contains('bushel', case=True, regex=True)]

new_columns = ['Package', 'Variety', 'City Name', 'Month', 'Low Price', 'High Price', 'Date', 'City Num', 'Variety Num']


pumpkins = pumpkins.drop([c for c in pumpkins.columns if c not in new_columns], axis=1)

price = (pumpkins['Low Price'] + pumpkins['High Price']) / 2

month = pd.DatetimeIndex(pumpkins['Date']).month


new_pumpkins = pd.DataFrame({'Month': month, 'Variety': pumpkins['Variety'], 'City': pumpkins['City Name'], 'Package': pumpkins['Package'], 'Low Price': pumpkins['Low Price'],'High Price': pumpkins['High Price'], 'Price': price})

new_pumpkins.loc[new_pumpkins['Package'].str.contains('1 1/9'), 'Price'] = price/1.1

new_pumpkins.loc[new_pumpkins['Package'].str.contains('1/2'), 'Price'] = price*2

new_pumpkins.head()<jupyter_output><empty_output><jupyter_text>A basic scatterplot reminds us that we only have month data from August through December. We probably need more data to be able to draw conclusions in a linear fashion.<jupyter_code>new_pumpkins["Month_str"] = new_pumpkins['Month'].apply(lambda x: calendar.month_abbr[x])
plt.scatter('Month_str', 'Price', data=new_pumpkins)
plt.scatter("City", "Price", data=new_pumpkins)
new_pumpkins.iloc[:, 0:-1] = new_pumpkins.iloc[:, 0:-1].apply(LabelEncoder().fit_transform)
new_pumpkins.head(10)
print(new_pumpkins["City"].corr(new_pumpkins["Price"]))
print(new_pumpkins["Package"].corr(new_pumpkins["Price"]))
new_pumpkins.dropna(inplace=True)

new_columns = ["Package", "Price"]

lil_pumpkins = new_pumpkins.drop([c for c in new_pumpkins.columns if c not in new_columns], axis="columns")
X = lil_pumpkins.values[:, :1]
y = lil_pumpkins.values[:, 1:2]
X
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

lil_reg = LinearRegression()
lil_reg.fit(X_train, y_train)

pred = lil_reg.predict(X_test)

accuracy_score = lil_reg.score(X_train, y_train)
print(f"Model accuracy: {accuracy_score}")
plt.scatter(X_test, y_test, color="black")
plt.plot(X_test, pred, color="blue", linewidth=3)

plt.xlabel("Package")
plt.ylabel("Price")
lil_reg.predict([[2.75]])
new_columns = ['Variety', 'Package', 'City', 'Month', 'Price']
poly_pumpkins = new_pumpkins.drop([c for c in new_pumpkins.columns if c not in new_columns], axis="columns")
corr = poly_pumpkins.corr()
corr.style.background_gradient(cmap="coolwarm")
X = poly_pumpkins.iloc[:, 3:4].values
y = poly_pumpkins.iloc[:, 4:5].values
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline

pipeline = make_pipeline(PolynomialFeatures(4), LinearRegression())

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

pipeline.fit(X_train, y_train)

y_pred = pipeline.predict(X_test)
df = pd.DataFrame({"x": X_test[:,0], "y":y_pred[:,0]})
df.sort_values(by="x", inplace=True)

# chiamo pd.DataFrame per creare un nuovo df
points = pd.DataFrame(df).to_numpy()
plt.plot(points[:, 0], points[:, 1], color="blue", linewidth=3)
plt.xlabel("Package")
plt.ylabel("Price")
plt.scatter(X, y, color="black")
accuracy_score = pipeline.score(X_train, y_train)
accuracy_score
pipeline.predict([[2.75]])<jupyter_output><empty_output><jupyter_text>Lecture 1-4 - Binary classification<jupyter_code>pumpkins = pd.read_csv('../data/US-pumpkins.csv')

new_columns = ['Color','Origin','Item Size','Variety','City Name','Package']

new_pumpkins = pumpkins[new_columns]
new_pumpkins.dropna(inplace=True)
new_pumpkins = new_pumpkins.apply(LabelEncoder().fit_transform)
new_pumpkins
g = sns.PairGrid(new_pumpkins)
g.map(sns.scatterplot)
sns.swarmplot(x="Color", y="Item Size", data=new_pumpkins)
sns.catplot(x="Color", y="Item Size", kind="violin", data=new_pumpkins)
Selected_features = ['Origin','Item Size','Variety','City Name','Package']

X = new_pumpkins[Selected_features]
y = new_pumpkins["Color"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
model = LogisticRegression()
model.fit(X_train, y_train)
predictions = model.predict(X_test)

print(classification_report(y_test, predictions))
# print(f"Predicted labels: {predictions}")
print(f"Accuracy: {sklearn.metrics.accuracy_score(y_test, predictions)}")
from sklearn.metrics import confusion_matrix

confusion_matrix(y_test, predictions)
y_scores = model.predict_proba(X_test)

fpr, tpr, thresholds = roc_curve(y_test, y_scores[:, 1])
sns.lineplot([0, 1], [0, 1])
sns.lineplot(fpr, tpr)
auc = roc_auc_score(y_test, y_scores[:, 1])
print(auc)<jupyter_output>0.6976998904709748
{'fraction_non_alphanumeric': 0.14242424242424243, 'fraction_numerical': 0.01265597147950089, 'mean_word_length': 7.619047619047619, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '1803', 'license': 'MIT', 'n_tokens_mistral': 2069, 'n_tokens_neox': 2025, 'n_words': 442, 'path': '2-Regression/3-Linear/notebook.ipynb', 'repo_name': 'GDaglio/ML-For-Beginners', 'size': '5610'}