<jupyter_start><jupyter_text>Использовать dataset [MovieLens](https://grouplens.org/datasets/movielens/latest/)- Построить рекомендации (регрессия, предсказываем оценку) на фичах:    - TF-IDF на тегах и жанрах    - Средние оценки (+ median, variance, etc.) пользователя и фильма- Оценить RMSE на тестовой выборке<jupyter_code># import os
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# import seaborn as sns

from datetime import datetime
from math import sqrt
from tqdm import tqdm_notebook
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer, CountVectorizer
from sklearn.metrics import accuracy_score, roc_auc_score, precision_score, recall_score, mean_squared_error
from sklearn.neighbors import NearestNeighbors
from sklearn.preprocessing import StandardScaler

# from pylab import rcParams
# rcParams['figure.figsize'] = (14,7)

import warnings
warnings.simplefilter('ignore')

# import matplotlib
# matplotlib.style.use('ggplot')

# sns.set()
%matplotlib inline
# %config InlineBackend.figure_format = 'retina'

# !ls *.csv
movies = pd.read_csv('movies.csv')
ratings = pd.read_csv('ratings.csv')
tags = pd.read_csv('tags.csv')
# links = pd.read_csv('links.csv')
display(movies.head(3))
display(ratings.head(3))
display(tags.head(3))
def change_string(s):
    return ' '.join(s.replace(' ', '').replace('-', '').split('|')) # Убираем лишние символы (применим для 'Жанры')


def extract_year(row):
    match = re.search('\(\D*\d{4}\D*\)', row)
    if match:
        return int(re.search('\d{4}', match[0])[0]) # Получить новую фичу 'Год'
    return 0


def movies_preprocessing(movies):
    movies['genres_'] = [change_string(i) for i in movies['genres'].values] # Обработка 'Жанры'
    movies['year'] = movies['title'].apply(extract_year) # Добавляем фичу 'Год'
    movies.drop('genres', axis='columns', inplace=True) # Удаляем не обработанный столбец 'Жанры'
    movies.columns = ['movieId', 'title', 'genres', 'year'] # Переименовываем колонки
    movies = movies[movies['year']!=0] # Через срез оставим только корректные данные, по фиче 'Год' нули не учитываем 
    movies.drop('title', axis='columns', inplace=True) # Удалил 'Название фильмов' они не нужны для дальнейшей обработки
    return movies


def tags_preprocessing(tags):
    tags['tag'] = tags['tag'].str.lower() # Приведем к нижнему регистру
    groupby_tags = tags.groupby('movieId')[['tag']].agg(' '.join) # Группируем 'Тэги' через объединение через пробел
    groupby_tags.columns = ['tags'] # Убираем lvl после группировки (путем преопределения названия колонки)
    tags = pd.merge(tags, groupby_tags, left_on='movieId', right_index=True) # Полученный результат группировки объединем с данными файла 'Тэги'
    tags = tags.drop(['tag', 'timestamp', 'userId'], axis=1) # Удаляем лишние колонки
    tags = tags.drop_duplicates(subset=['movieId'], keep='first') # Удаляем дубликаты
    return tags<jupyter_output><empty_output><jupyter_text>TF-IDF на 'genres'<jupyter_code>movies = movies_preprocessing(movies)
display(movies.head())
def tfidf_genres(movies):
    
    count_vect = CountVectorizer()
    X_train_counts = count_vect.fit_transform(movies['genres'])

    tfidf_transformer = TfidfTransformer()
    X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts) # Получаем векторное представление
    
    for key, value in count_vect.vocabulary_.items():
        movies[key+'_genres'] = X_train_tfidf.toarray().T[value] # Записываем полученный результат
    
    movies.drop('genres', axis='columns', inplace=True) # Удаляем 'Жанры' т.к. теперь у нас есть векторное представление
    movies = pd.get_dummies(movies, columns=['year']) # get_dummies для 'Года' 
    
    return movies
movies = tfidf_genres(movies)
movies.head()<jupyter_output><empty_output><jupyter_text>TF-IDF на 'tags'<jupyter_code>tags = tags_preprocessing(tags)
display(tags.head())
def tfidf_tags(tags):
    
    count_vect = CountVectorizer()
    X_train_counts = count_vect.fit_transform(tags['tags'])

    tfidf_transformer = TfidfTransformer()
    X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts) # Получаем векторное представление
    
    for key, value in count_vect.vocabulary_.items():
        tags[key+'_tags'] = X_train_tfidf.toarray().T[value] # Записываем полученный результат
    
    tags.drop('tags', axis='columns', inplace=True) # Удаляем 'Тэги' т.к. теперь у нас есть векторное представление
    
    return tags
tags = tfidf_tags(tags)
display(tags.head())<jupyter_output><empty_output><jupyter_text>Получим дополнительные фичи для target (по условию задачи)<jupyter_code>rating_count = ratings.groupby('movieId', sort=False)['rating'].count()
rating_mean = ratings.groupby('movieId', sort=False)[['rating']].mean()<jupyter_output><empty_output><jupyter_text>Объеденим предобработанные данные в один dataframe<jupyter_code>def merge_data(tags, movies, rating_count, rating_mean):
    data = tags.merge(movies, on='movieId')
    data = pd.merge(data, pd.DataFrame(rating_count).rename(columns={'rating':'rating_count'}), left_on='movieId', right_index=True)
    data = pd.merge(data, rating_mean.rename(columns={'rating':'mean_rating'}), left_on='movieId', right_index=True)
    return data
data = merge_data(tags, movies, rating_count, rating_mean)
data.head()<jupyter_output><empty_output><jupyter_text>Разбиваем данные:    1. X и y    2. train и test<jupyter_code>X = data.drop(columns=['mean_rating'])
y = data['mean_rating'] # Предсказываем среднее значение
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)<jupyter_output><empty_output><jupyter_text>Стандартизируем данные<jupyter_code>scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>Построим несколько Регрессионных моделей<jupyter_code>from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold, KFold, cross_val_score
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, BaggingRegressor, AdaBoostRegressor
from sklearn.linear_model import LinearRegression, SGDRegressor
from sklearn.metrics import mean_squared_error, r2_score, explained_variance_score
from mlxtend.regressor import StackingCVRegressor, StackingRegressor
# model_LR = LinearRegression()
model_ABR = AdaBoostRegressor()
model_BR = BaggingRegressor()
model_GBR = GradientBoostingRegressor()
# model_RFR = RandomForestRegressor(n_estimators=27, max_depth=8, min_samples_leaf=5, max_features=0.5, n_jobs=-1)
# model_SGDR = SGDRegressor(alpha = 0.01, max_iter = 1000)

model_meta_RFR = RandomForestRegressor(n_estimators=27, max_depth=8, min_samples_leaf=5, max_features=0.5, n_jobs=-1)
stack = StackingCVRegressor(regressors=(
#     model_LR, 
    model_ABR, 
    model_BR, 
    model_GBR, 
#     model_RFR, 
#     model_SGDR
),meta_regressor=model_meta_RFR,use_features_in_secondary=True)
stack.fit(X_train, y_train)
def rmse(y_train, y_pred):
     return np.sqrt(mean_squared_error(y_train, y_pred))

def cv_rmse(model, X=X_train, y=y_train):    
    return np.sqrt(-cross_val_score(model, X, y, scoring='neg_mean_squared_error', cv=kf))

kf = KFold(n_splits=10, shuffle=True, random_state=555)
models = {
#     'LinearRegression': model_LR,
    'AdaBoostRegressor': model_ABR,
    'BaggingRegressor': model_BR,
    'GradientBoostingRegressor': model_GBR, 
#     'RandomForestRegressor': model_RFR,
#     'SGDRegressor': model_SGDR, 
    'StackingCVRegressor_meta_RandomForestRegressor': model_meta_RFR
}
predictions = {}
predictions_s = {}

predictions_t = {}
predictions_ss = {}

scores = {}
scores_s = {}

scores_t = {}
scores_ss = {}

# model.fit(X_train, y_train)
# y_pred = model.predict(X_test)
# print(accuracy_score(y_test, y_pred))

for name, model in models.items():
    start = datetime.now()
    print('[{}] Running {}'.format(start, name))
    
    model.fit(X_train, y_train)
    
    predictions[name] = np.expm1(model.predict(X_train))
    predictions_t[name] = np.expm1(model.predict(X_test))
    
    predictions_s[name] = model.predict(X_train)
    predictions_ss[name] = model.predict(X_test)
    
    score = cv_rmse(model, X=X_train, y=y_train)
    score_t = cv_rmse(model, X=X_test, y=y_test)
    
    scores[name] = (score.mean(), score.std())
    scores_t[name] = (score_t.mean(), score_t.std())
    
    scores_s[name] = (r2_score(y_train, model.predict(X_train)), explained_variance_score(y_train, model.predict(X_train)))
    scores_ss[name] = (r2_score(y_test, model.predict(X_test)), explained_variance_score(y_test, model.predict(X_test)))    
    
    end = datetime.now()
    print('[{}] Finished Running {} in {:.2f}s'.format(end, name, (end - start).total_seconds()))
    print('[{}] {} _score:\ntrain --- RMSE: {:.6f} / Std: {:.6f} / r2_score: {:.6f} / explained_variance_score: {:.6f}'.format(
        datetime.now(), name, scores[name][0], scores[name][1], scores_s[name][0], scores_s[name][1]))
    print('test --- RMSE: {:.6f} / Std: {:.6f} / r2_score: {:.6f} / explained_variance_score: {:.6f}\n'.format(
        scores_t[name][0], scores_t[name][1], scores_ss[name][0],scores_ss[name][1]))

print('\n[END]')
pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13346207488466902, 'fraction_numerical': 0.007188069949576226, 'mean_word_length': 5.711303095752339, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '459961', 'license': 'MIT', 'n_tokens_mistral': 3570, 'n_tokens_neox': 3637, 'n_words': 627, 'path': 'Homework notebooks/(HW notebooks) netology Machine learning/28. Content Based Recommendations/hw2_rs.ipynb', 'repo_name': 'Alex110117/data_analysis', 'size': '9321'}