<jupyter_start><jupyter_text>Identify:Please fill-in your ID in the following cell, **only id numbers, no names!**<jupyter_code>id1 = "204762124"
id2 = "324573732"
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn as sk
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error as mse
df = pd.DataFrame([id1])<jupyter_output><empty_output><jupyter_text>Uncomment in case two students are doing the assignment together:<jupyter_code>df = pd.DataFrame([id1,id2])
df.to_clipboard(index=False,header=False)<jupyter_output><empty_output><jupyter_text>--- ----<jupyter_code>url = 'https://raw.githubusercontent.com/nlihin/data-analytics/main/datasets/avocado.csv'
data = pd.read_csv(url)
data.head()
data.dtypes<jupyter_output><empty_output><jupyter_text>Q1<jupyter_code>tavla = data.groupby(['region','year'])[['total_sold']].sum()
tavla<jupyter_output><empty_output><jupyter_text>Q2 We chose Atlanta<jupyter_code>anuliks= data.loc[(data['region']=='Atlanta') & (data['year']==2016)]
anuliks.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 104 entries, 2860 to 12036
Data columns (total 14 columns):
 #   Column        Non-Null Count  Dtype  
---  ------        --------------  -----  
 0   Unnamed: 0    104 non-null    int64  
 1   Date          104 non-null    object 
 2   AveragePrice  104 non-null    float64
 3   total_sold    104 non-null    float64
 4   small_sold    104 non-null    float64
 5   large_sold    104 non-null    float64
 6   sma           104 non-null    float64
 7   Total Bags    104 non-null    float64
 8   Small Bags    104 non-null    float64
 9   Large Bags    104 non-null    float64
 10  XLarge Bags   104 non-null    float64
 11  type          104 non-null    object 
 12  year          104 non-null    int64  
 13  region        104 non-null    object 
dtypes: float64(9), int64(2), object(3)
memory usage: 12.2+ KB<jupyter_text>there are 104 rows for our new data  Q3<jupyter_code>anuliks['total_sold'].mean()<jupyter_output><empty_output><jupyter_text>Q4<jupyter_code>anuliks['Date']=pd.to_datetime(anuliks.Date)
anuliks['month'] = pd.DatetimeIndex(anuliks.Date).month
september = anuliks.loc[(anuliks['month']==9)]
september['Total Bags'].sum()<jupyter_output><ipython-input-12-dd9727666118>:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  anuliks['Date']=pd.to_datetime(anuliks.Date)
<ipython-input-12-dd9727666118>:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  anuliks['month'] = pd.DatetimeIndex(anuliks.Date).month<jupyter_text>Q5<jupyter_code>sns.barplot(x = 'type', y = 'total_sold',data= data)<jupyter_output><empty_output><jupyter_text>Q6<jupyter_code>features = ['total_sold', 'Total Bags', 'AveragePrice', 'type_organic']

# continue here
data = pd.get_dummies(data, columns=['type'])

corrs = data[features].corr(method = 'kendall')
plt.figure(figsize = (15,10))
sns.heatmap(corrs,cmap ='bone',center = 0,annot = True)<jupyter_output><empty_output><jupyter_text>Q7<jupyter_code>sns.scatterplot(x='total_sold', y='Total Bags',data = data, hue='year')<jupyter_output><empty_output><jupyter_text>--- GOOD LUCK !!!!---<jupyter_code>data.isnull().sum()
data.fillna(data.mean(), inplace = True)
data.head()

sns.histplot(data, y= 'total_sold',log_scale = True)
plt.hist(data['total_sold'],log = True, bins = 50)

data.pivot_table('AveragePrice', index =['region','year'], columns = 'type_organic')
data['region']=data['region'].astype('category').cat.codes
features = ['region', 'type_organic', 'AveragePrice', 'Total Bags']
X = data[features]
y = data['total_sold']
X_train, X_test, y_train, y_test = sk.model_selection.train_test_split(X, y, test_size=0.3, random_state=2)
dt = sk.tree.DecisionTreeRegressor(random_state=0)
dt.fit(X_train, y_train)


def mse(a,b): #function to RMSE
    return np.sqrt((np.square(a-b)).mean())
print("The predictions are:")
test_pred = dt.predict(X_test)
print(test_pred)
print("RMSE: {:.3f}".format(mse(y_test.values,test_pred)))
print('the mse is very high so the model is not predecting well')<jupyter_output>The predictions are:
[ 118455.23 6374842.9   540540.5  ...   97959.55  268415.49 6927745.15]
RMSE: 503013.292
the mse is very high so the model is not predecting well
{'fraction_non_alphanumeric': 0.13739669421487602, 'fraction_numerical': 0.04979338842975207, 'mean_word_length': 5.043695380774032, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '215108', 'license': 'MIT', 'n_tokens_mistral': 1981, 'n_tokens_neox': 1776, 'n_words': 375, 'path': 'hw5_solution_template.ipynb', 'repo_name': 'NirNagary/DAProject', 'size': '4840'}