<jupyter_start><jupyter_text>Latest Access to the dataset: 12/03/2020<jupyter_code>import pandas as pd
import numpy as np
file_name = pd.read_csv('https://query.data.world/s/gib6aa6n3tmtuvrqpbdlhcbgdg7tk4')

# Best Practice: Make a copy of the raw data to work on
reviews = file_name.copy()

# Sanity check
reviews.head()<jupyter_output><empty_output><jupyter_text>Creating New Features<jupyter_code>reviews = reviews.sort_values('review_overall', ascending=False)

reviews = reviews.drop_duplicates(subset= ['review_profilename', 'beer_name'], keep='first')
reviews = reviews[(reviews['review_overall'] >= 1)|\
       (reviews['review_appearance'] >= 1)]
# Convert 'object' to 'category' 
#reviews[reviews.select_dtypes(['object']).columns] = reviews.select_dtypes(['object']).\
                                                         #apply(lambda x: x.astype('category'))
reviews = reviews.dropna()
reviews.info()
reviews['Score'] = ((reviews['review_appearance'] + reviews['review_aroma'] + reviews['review_palate'] + reviews['review_taste'])*5)
# create a list of our conditions
conditions = [
    (reviews['beer_abv'] >= 0.0)&(reviews['beer_abv'] < 0.5),
    (reviews['beer_abv'] >= 0.5) & (reviews['beer_abv'] < 3.2),
    (reviews['beer_abv'] >= 3.2) & (reviews['beer_abv'] < 6.0),
    (reviews['beer_abv'] >= 6.0)
    ]

# create a list of the values we want to assign for each condition
values = ['NA', 'Low', 'Reg', 'High']

# create a new column and use np.select to assign values to it using our lists as arguments
reviews['beer_strength'] = np.select(conditions, values)

# display updated DataFrame
reviews.head()
#reviews = reviews.drop(['brewery_id', 'review_time', 'review_overall', 'review_aroma', 'review_appearance', 'review_profilename', 'review_palate', 'review_taste', 'beer_beerid'], axis=1)
import matplotlib.pyplot as plt
%matplotlib inline 
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(reviews, test_size=0.2, random_state=42)
## separate labels and data

reviews = train_set.drop("review_overall", axis=1)
reviews_labels = train_set["review_overall"].copy()
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy= 'mean')
reviews_num = reviews.loc[: ,reviews.dtypes !=object]
imputer.fit(reviews_num)
temp_num_data = imputer.transform(reviews_num)

reviews_tr = pd.DataFrame(temp_num_data, columns=reviews_num.columns)
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer()
beer_style_1hot = encoder.fit_transform(reviews['beer_style']) 
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
num_pipeline = Pipeline([
        ('imputer', SimpleImputer(strategy="mean")),
        ('std_scaler', StandardScaler()),
    ])
reviews_num_tr = num_pipeline.fit_transform(reviews_num)

try:
    from sklearn.compose import ColumnTransformer
except ImportError:
    from future_encoders import ColumnTransformer # Scikit-Learn < 0.20
from sklearn.preprocessing import OneHotEncoder
num_attribs = list(reviews_num)
cat_attribs = ["beer_name"]

full_pipeline = ColumnTransformer([
        ("num", num_pipeline, num_attribs),
        ("cat", OneHotEncoder(), cat_attribs),
    ])

reviews_prepared = full_pipeline.fit_transform(reviews)
from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(reviews_num, reviews_labels)
from sklearn.tree import DecisionTreeRegressor

tree_reg = DecisionTreeRegressor(random_state=42)
tree_reg.fit(reviews_prepared, reviews_labels)
reviews_predictions = tree_reg.predict(reviews_prepared)
tree_mse = mean_squared_error(reviews_labels, reviews_predictions)
tree_rmse = np.sqrt(tree_mse)
tree_rmse
from sklearn.model_selection import cross_val_score

scores = cross_val_score(tree_reg, reviews_prepared, reviews_labels,
                         scoring="neg_mean_squared_error", cv=10)
tree_rmse_scores = np.sqrt(-scores)



def display_scores(scores):
    print("Scores:", scores)
    print("Mean:", scores.mean())
    print("Standard deviation:", scores.std())

display_scores(tree_rmse_scores)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12415295256534366, 'fraction_numerical': 0.010648596321393998, 'mean_word_length': 5.934563758389261, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '635240', 'license': 'MIT', 'n_tokens_mistral': 1435, 'n_tokens_neox': 1384, 'n_words': 278, 'path': 'Technical Notebook/DecisionTreeRegressor.ipynb', 'repo_name': 'tvanwer1/Final-Project', 'size': '4132'}