<jupyter_start><jupyter_text>- mad2306: <NAME>- pds2136: <NAME><jupyter_code>import pandas as pd
import numpy as np

from sklearn.preprocessing import StandardScaler,Normalizer
from category_encoders import TargetEncoder
from sklearn.pipeline import Pipeline,make_pipeline
from sklearn.model_selection import cross_val_score, GridSearchCV, train_test_split
from sklearn.compose import make_column_transformer
from sklearn.linear_model import Ridge,LinearRegression

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer<jupyter_output><empty_output><jupyter_text>1.1<jupyter_code>data = pd.read_csv('winemag-data-130k-v2.csv')
data = data.drop(columns = ['Unnamed: 0'])
data = data.loc[data['country'] == 'US']
# number of missing values in each column
data.isnull().sum()
# below columns have a lot of missing values and the value of country for all is 'US' this dropping the respective columns
data = data.drop(columns = ['taster_name', 'taster_twitter_handle', 'designation', 'country']) 

# removing text data for the first part
data1 = data.drop(columns = ['description', 'title'])

# dropping the rows with missing values
data1 = data1.dropna() 
y1 = data1[['points']]
X1 = data1.drop(columns = 'points')
preprocess = make_column_transformer((StandardScaler(), ['price']), remainder = TargetEncoder())

X_train, X_test, y_train, y_test = train_test_split(X1, y1, test_size=0.2, random_state = 42)
# baseline using Linear Regression
pipe = make_pipeline(preprocess, LinearRegression())
score = cross_val_score(pipe, X_train, y_train, cv = 5)
pipe.fit(X_train,y_train)

print("The cross-validation score for Linear Regression is: "+ str(np.mean(score)))
print("The score on test set is: " + str(pipe.score(X_test,y_test)))
# Ridge
pipe = make_pipeline(preprocess, Ridge())
score = cross_val_score(pipe, X_train, y_train, cv = 5)
pipe.fit(X_train,y_train)

print("The cross-validation score for Ridge Regression is: "+ str(np.mean(score)))
print("The score on test set is: " + str(pipe.score(X_test,y_test)))<jupyter_output>The cross-validation score for Ridge Regression is: 0.3947542271071985
The score on test set is: 0.4215590685281599<jupyter_text>1.2<jupyter_code>text_data = data[['description', 'points']]
y2 = text_data[['points']]
X2 = text_data.drop(columns = 'points')

X_train, X_test, y_train, y_test = train_test_split(X2, y2, test_size=0.2, random_state = 42)
X_train = X_train['description'].tolist()
X_test = X_test['description'].tolist()
vect = CountVectorizer()
X_train = vect.fit_transform(X_train)
X_test = vect.transform(X_test)
lr = LinearRegression().fit(X_train, y_train)
s1 = lr.score(X_test, y_test)
print("Score from Linear Regression on test set is: " +str(s1))
r = Ridge().fit(X_train, y_train)
s1 = r.score(X_test, y_test)
print("Score from Ridge Regression on test set is: " +str(s1))<jupyter_output>Score from Ridge Regression on test set is: 0.675389761886473<jupyter_text>We see that with using text features, the R2 score is increasing from 0.42 to 0.67. Next, we noticed there are a lot of duplicate values in the 'description' feature, so we dropped them and again run our model.<jupyter_code>text_data = text_data.drop_duplicates('description', keep=False)
y2 = text_data[['points']]
X2 = text_data.drop(columns = 'points')

X_train, X_test, y_train, y_test = train_test_split(X2, y2, test_size=0.2, random_state = 42)
X_train = X_train['description'].tolist()
X_test = X_test['description'].tolist()
vect = CountVectorizer()
X_train = vect.fit_transform(X_train)
X_test = vect.transform(X_test)
lr = LinearRegression().fit(X_train, y_train)
s1 = lr.score(X_test, y_test)
print("Score from Linear Regression on test set is: " +str(s1))
r = Ridge().fit(X_train, y_train)
s1 = r.score(X_test, y_test)
print("Score from Ridge Regression on test set is: " +str(s1))<jupyter_output>Score from Ridge Regression on test set is: 0.6761203670215854<jupyter_text>We see that after dropping duplicates, our r2 increases by 0.01.So moving forward in 1.3 we use the data after removing the duplicates.  1.3<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X2, y2, test_size=0.2, random_state = 42)
X_train = X_train['description'].tolist()
X_test = X_test['description'].tolist()
# adding n-grams with minimum frequency = 4
vect = CountVectorizer(ngram_range=(1, 3), min_df=4)
X_train_ngram = vect.fit_transform(X_train)
X_test_ngram = vect.transform(X_test)

r = Ridge().fit(X_train_ngram, y_train)
s1 = r.score(X_test_ngram, y_test)
print("Score from Ridge Regression with default parameter setting on test set is: " +str(s1))

param_grid = [{'alpha':  np.logspace(-2,2,10)}]
grid = GridSearchCV(Ridge(), param_grid, cv = 5)
grid.fit(X_train_ngram, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("Score from Ridge Regression with default parameter setting on test set is: " 
      +str(grid.score(X_test_ngram, y_test)))
# adding tf-idf rescaling
tfidf = make_pipeline(CountVectorizer(ngram_range=(1, 3), min_df=4), TfidfTransformer())
X_train_tfidf = tfidf.fit_transform(X_train)
X_test_tfidf = tfidf.transform(X_test)

r = Ridge().fit(X_train_tfidf, y_train)
s1 = r.score(X_test_tfidf, y_test)
print("Score from Ridge Regression with default parameter setting on test set is: " +str(s1))

param_grid = [{'alpha': np.logspace(-2,2,10)}]
grid = GridSearchCV(Ridge(), param_grid, cv = 5)
grid.fit(X_train_tfidf, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("Score from Ridge Regression with default parameter setting on test set is: " 
      +str(grid.score(X_test_tfidf, y_test)))
# adding characters
char = make_pipeline(CountVectorizer(ngram_range=(1, 3), min_df=4, analyzer="char_wb"), TfidfTransformer())
X_train_char = char.fit_transform(X_train)
X_test_char = char.transform(X_test)

r = Ridge().fit(X_train_char, y_train)
s1 = r.score(X_test_char, y_test)
print("Score from Ridge Regression with default parameter setting on test set is: " +str(s1))

param_grid = [{'alpha': np.logspace(-2,2,10)}]
grid = GridSearchCV(Ridge(), param_grid, cv = 5)
grid.fit(X_train_char, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("Score from Ridge Regression with default parameter setting on test set is: " 
      +str(grid.score(X_test_char, y_test)))
# removing stop words and trying the previous model
char1 = make_pipeline(CountVectorizer(ngram_range=(1, 3), min_df=4, stop_words="english", analyzer="char_wb"), TfidfTransformer())
X_train_char1 = char1.fit_transform(X_train)
X_test_char1 = char1.transform(X_test)

r = Ridge().fit(X_train_char1, y_train)
s1 = r.score(X_test_char1, y_test)
print("Score from Ridge Regression with default parameter setting on test set is: " +str(s1))

param_grid = [{'alpha': np.logspace(-2,2,10)}]
grid = GridSearchCV(Ridge(), param_grid, cv = 5)
grid.fit(X_train_char1, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("Score from Ridge Regression with default parameter setting on test set is: " 
      +str(grid.score(X_test_char1, y_test)))
# removing charater analysis from the aboove model 
char1 = make_pipeline(CountVectorizer(ngram_range=(1, 3), min_df=4, stop_words="english"), TfidfTransformer())
X_train_char1 = char1.fit_transform(X_train)
X_test_char1 = char1.transform(X_test)

r = Ridge().fit(X_train_char1, y_train)
s1 = r.score(X_test_char1, y_test)
print("Score from Ridge Regression with default parameter setting on test set is: " +str(s1))

param_grid = [{'alpha':np.logspace(-2,2,10)}]
grid = GridSearchCV(Ridge(), param_grid, cv = 5)
grid.fit(X_train_char1, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("Score from Ridge Regression with default parameter setting on test set is: " 
      +str(grid.score(X_test_char1, y_test)))
# adding Normalizer to the aboove model 
char2 = make_pipeline(CountVectorizer(ngram_range=(1, 3), min_df=4, stop_words="english"),Normalizer(), TfidfTransformer())
X_train_char2 = char2.fit_transform(X_train)
X_test_char2 = char2.transform(X_test)

r = Ridge().fit(X_train_char2, y_train)
s1 = r.score(X_test_char2, y_test)
print("Score from Ridge Regression with default parameter setting on test set is: " +str(s1))

param_grid = [{'alpha': np.logspace(-2,2,10)}]
grid = GridSearchCV(Ridge(), param_grid, cv = 5)
grid.fit(X_train_char2, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("Score from Ridge Regression with default parameter setting on test set is: " 
      +str(grid.score(X_test_char2, y_test)))<jupyter_output>Score from Ridge Regression with default parameter setting on test set is: 0.7004720431097023
The best parameters after grid search: {'alpha': 0.5994842503189409}
Score from Ridge Regression with default parameter setting on test set is: 0.6959062801460703<jupyter_text>After looking at all the models, we see that the best R2 is coming when we used the following - 1. n-gram ranging from 1-32. Minimum frequency = 43. Removed stop words4. Normalizing the bag of words vector5. Tranforming the tokens using tf-idf transformerWe can also see that the regularization parameter changes based on the setting we used for tuning the bag to words model.  1.4<jupyter_code># below columns have a lot of missing values and the value of country for all is 'US' this dropping the respective columns
data2 = data.drop(columns = ['title']) 

# dropping the rows with missing values
data2 = data2.dropna()
data2 = (data2.drop_duplicates(keep = 'first'))
# adding non-text features
y3 = data2[['points']]
X3 = data2.drop(columns = 'points')

# splitting the data into training, validation and testing
X_train, X_test, y_train, y_test = train_test_split(X3, y3, test_size=0.2, random_state = 42)
text_pipeline = make_pipeline(CountVectorizer(ngram_range=(1, 3), min_df = 4),Normalizer(),TfidfTransformer())
preprocess = make_column_transformer((StandardScaler(), ['price']),
                                     (text_pipeline, 'description'),
                                     remainder = TargetEncoder())

pipe = make_pipeline(preprocess, Ridge())
pipe.fit(X_train, y_train)
print("The score on test set with default parameters is: " + str(pipe.score(X_test,y_test)))
pipeline_new = Pipeline([('processor', preprocess),('regressor', Ridge())])
param_grid = [{'regressor__alpha': np.logspace(-2,2,10)}]
grid = GridSearchCV(pipeline_new, param_grid, cv = 5)
grid.fit(X_train, y_train)
print("The best parameters after grid search: " + str(grid.best_params_))
print("The score on test set after grid search is " + str(grid.score(X_test, y_test)))<jupyter_output>The best parameters after grid search: {'regressor__alpha': 0.5994842503189409}
The score on test set after grid search is 0.7658084506602063
{'fraction_non_alphanumeric': 0.11910009182736456, 'fraction_numerical': 0.032231404958677684, 'mean_word_length': 6.217362491716369, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '555288', 'license': 'MIT', 'n_tokens_mistral': 3901, 'n_tokens_neox': 3732, 'n_words': 1021, 'path': 'Predicting Wine-Quality from review texts and other properties/task1.ipynb', 'repo_name': 'Prasham8897/Applied_Machine_Learning', 'size': '10918'}