<jupyter_start><jupyter_text><jupyter_code>import os
import datetime as dt
import numpy as np
import pandas as pd
import pandas as pd
from sklearn import tree
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.feature_selection import mutual_info_classif
% matplotlib inline
np.random.seed(0)
# print(os.getcwd())
'''
students_performance = pd.read_csv('StudentsPerformance.csv')
print(students_performance.columns)
# print(students_performance.head)
standart_lunch = students_performance[(students_performance['lunch'] == 'standard')]
free_lunch = students_performance[(students_performance['lunch'] == 'free/reduced')]
print(standart_lunch['math score'].mean())
print(standart_lunch['reading score'].mean())
print(standart_lunch['writing score'].mean())
print(free_lunch['math score'].mean())
print(free_lunch['reading score'].mean())
print(free_lunch['writing score'].mean())
print(standart_lunch['math score'].std())
print(standart_lunch['reading score'].std())
print(standart_lunch['writing score'].std())
print(free_lunch['math score'].std())
print(free_lunch['reading score'].std())
print(free_lunch['writing score'].std())
# standart_lunch.filter(like='score', axis = 1)
# standart_lunch.select_dtypes(include='number')

# students_performance.groupby('gender').mean()
# students_performance.groupby('gender').aggregate({'math score': 'mean'})
students_performance.groupby('gender', as_index=False) \
    .aggregate({'math score': 'mean'}).rename(columns = {'math score': 'mean math score'})
students_performance.groupby(['gender', 'lunch'], as_index=False) \
    .aggregate({'math score': 'mean'}).rename(columns = {'math score': 'mean math score'})

mean_scores = students_performance.groupby(['gender', 'race/ethnicity']) \
    .aggregate({'math score': 'mean'}).rename(columns = {'math score': 'mean math score'})
# mean_scores
mean_scores.loc[('female', 'group A')]
students_performance.sort_values(['gender', 'math score'], ascending=False).groupby('gender').head()
students_performance.columns = [x.replace(" ", "_") for x in students_performance.columns]
students_performance['total_score'] = students_performance.math_score \
    + students_performance.reading_score + students_performance.writing_score
students_performance = students_performance.assign(total_score_log = np.log(students_performance.total_score))
students_performance.head()
dota_df = pd.read_csv('dota_hero_stats.csv')
print(dota_df.head())
print(dota_df.columns)
print(dota_df['legs'].value_counts(sort=True, dropna=False))

acc_df = pd.read_csv('accountancy.csv')
print(acc_df.head())
print(acc_df.columns)
print(acc_df.groupby(['Type', 'Executor']).aggregate({'Salary': 'mean'}))

dota_df = pd.read_csv('dota_hero_stats.csv')
# print(dota_df.head())
# print(dota_df.columns)
print(dota_df.groupby(['attack_type', 'primary_attr']).count())
# print(dota_df.groupby(['attack_type', 'primary_attr']).head())
# print(dota_df.groupby(['attack_type', 'primary_attr']).describe())
algae_df = pd.read_csv('algae.csv')
print(algae_df.head())
print(algae_df.columns)
# print(algae_df.groupby(['genus']).mean())
# print(algae_df.groupby(['genus']).filter(like=['Fucus'], axis=1).head())
# print(algae_df.groupby(['genus']).agg(np.mean).round(decimals=2))
# print(algae_df.groupby(['genus', 'alanin']).agg(np.min).round(decimals=2))
print(algae_df.groupby(['group']).agg('max')['sucrose']-algae_df.groupby(['group']).agg('min')['sucrose'])
df = pd.read_csv("dataset_209770_6.txt", sep=" ")
# df = DataFrame({'income': np.random.rand(100), 'time': np.arange(0,100)})
print(df.head())
print(df.columns)
# df['income'].plot()
sns.scatterplot(x=df[df.columns[0]], y=df[df.columns[1]])
# df.income.plot()
# sns.lineplot(data=df)
# df.plot()
# plt.plot(df.time, df.income)
# df.plot(kind='line')


df = pd.read_csv("genome_matrix.csv")
print(df.head)
print(df.columns)
# df.drop()
print(df[df.columns[1:]])
#print(df.values)
g = sns.heatmap(df[df.columns[1:]], cmap='viridis')
g.xaxis.set_ticks_position('top')
g.xaxis.set_tick_params(rotation=90)

df = pd.read_json("heroes.json")
print(df.columns)
df.role_count = df.roles.map(lambda x: len(eval('x')))
print(df.role_count)
df.role_count.hist()


df = pd.read_csv("iris.csv")
print(df.columns)
print(df.head())

for column in df:
    if column != df.columns[0]:
        g1 = sns.kdeplot(df[column])
for column in df:
    if column != df.columns[0]:
        g2 = sns.distplot(df[column])
sns.violinplot(x=None, y=df['petal length'])

sns.pairplot(df, hue="species")
my_data = pd.DataFrame({'type': ['A', 'A', 'B', 'B'], 'value': [10, 14, 12, 23]})
print(my_data)

my_stat = pd.read_csv("my_stat.csv")
print(df.columns)
print(df.head())

subset_1 = df[['V1', 'V2']].iloc[:10]
subset_2 = df[['V2', 'V4']].drop([0, 4], axis=0)
print(subset_1)
print(subset_2)


subset_1 = df.query('V1 > 0 and V3 == "A"')
subset_2 = df.query('V2 != 10 or V4 >= 1')
print(subset_1)
print(subset_2)

df['V5'] = df['V1'] + df['V4']
print(df.head())
df['V6'] = np.log(df['V1'])
print(df.head())


my_stat = my_stat.rename(mapper={'V1': 'session_value', 'V2':'group', 'V3': 'time', 'V4': 'n_users'}, axis=1)

my_stat['session_value'][0, 1] = np.NaN
my_stat['n_users'][0, 1, 2, 3, 4, 5, 6, 7] = -100
print(my_stat['session_value'].value_counts(dropna=False)) 
my_stat['session_value'].fillna(0, inplace=True)
median = my_stat.query('n_users >= 0')['n_users'].median()
# median = my_stat.query('n_users >= 0')['n_users']
subset =  my_stat.query('n_users < 0')['n_users'].values
my_stat['n_users'].replace(subset, median, inplace=True)
print(subset)
print(median)
print(my_stat.head())

print(my_stat.groupby('group', as_index=False).agg({'session_value': np.mean}).rename(index=str, columns={'session_value': 'mean_session_value'}))

event_df = pd.read_csv("event_data_train.zip", compression='zip')
subs_df = pd.read_csv("submissions_data_train.zip", compression='zip')

print(event_df.columns)
print(event_df.head())
print(subs_df.columns)
print(subs_df.head())
print(event_df[0:500].groupby(['user_id', 'action']).agg('count'))
print(subs_df[0:500].groupby(['user_id', 'submission_status']).agg('count'))
'''

'''
piv_t = pd.pivot_table(data=event_df[0:1000], index='user_id', columns='action', values='step_id', aggfunc='count').reset_index()
print(piv_t)
de_df = event_df.groupby(['user_id']).agg(list)
print(type(de_df))
de_df['range'] = de_df['timestamp'].apply(max) - de_df['timestamp'].apply(min)
print(de_df['range'].sort_values())
# print(de_df.loc[(slice(None), 'passed'), ])
# print(subs_df.groupby(['user_id', 'submission_status']).agg('count'))
subs_df[subs_df.submission_status == 'correct'].groupby('user_id').agg({'submission_status': 'count'}).sort_values(by=['submission_status'], ascending=False).head(20)
'''
'''
event_df = pd.read_csv("cats.csv")
print(event_df)
x = 4/10
y = 6/10
entr = -x*np.log2(x)-y*np.log2(y)
print(entr)
print(0.97-5/10*0.72)
'''

'''
train_df = pd.read_csv("train_iris.csv", index_col=0)
print(train_df.columns)
print(train_df.head())
test_df = pd.read_csv("test_iris.csv", index_col=0)
print(test_df.columns)
print(test_df.head())
np.random.seed(0)
x_train = train_df.drop('species', axis=1)
y_train = train_df['species']
x_test = test_df.drop('species', axis=1)
y_test = test_df['species']
train_scores = []
test_scores = []
for tree_length in range(1, 100):
    clf = DecisionTreeClassifier(max_depth=tree_length)
    clf.fit(x_train, y_train)
    train_scores.append(clf.score(x_train, y_train))
    test_scores.append(clf.score(x_test, y_test))
result_df = DataFrame(data={'train_scores': train_scores, 'test_scores': test_scores})
print(result_df.head())
sns.lineplot(data=result_df)
'''
'''
train_df = pd.read_csv("dogs_n_cats.csv")
print(len(train_df))
print(train_df.columns)
print(train_df.head())
x = train_df.drop('Вид', axis=1)
y = pd.get_dummies(train_df['Вид'])['котик']
x_train, x_test, y_train, y_test = train_test_split(x, y, shuffle=True)
train_scores = []
test_scores = []
mean_cross_val_scores = []
# print(y_train)
for tree_length in range(1, 5):
    clf = DecisionTreeClassifier(max_depth=tree_length, criterion='entropy')
    mean_cross_val_score = cross_val_score(clf, x_train, y_train, cv=5).mean()
    clf.fit(x_train, y_train)
    train_scores.append(clf.score(x_train, y_train))
    test_scores.append(clf.score(x_test, y_test))
    mean_cross_val_scores.append(mean_cross_val_score)
    tree.plot_tree(clf, feature_names=x_train.columns)
scores = {'train_scores': train_scores, 'test_scores': test_scores, 'cv_srore':  mean_cross_val_scores}
result_df = DataFrame(data=scores)
print(result_df.head())
sns.lineplot(data=result_df)
print(train_df.query('Высота > 11.8').groupby('Вид').agg('count'))
best_clf = DecisionTreeClassifier(max_depth=1, criterion='entropy')
best_clf.fit(x_train, y_train)
test_df = pd.read_json("dataset_209691_15.txt")
print(len(test_df))
print(test_df.columns)
print(test_df.head())
neworder = train_df.columns
df_test = test_df.reindex(columns=neworder)
out = clf.predict(test_df)
print(out)
print(pd.Series(out).value_counts())
'''
'''
df = pd.read_csv('train_data_tree.csv')
print(df.columns)
print(df.head())
x = df.drop('num', axis=1)
y = df['num']
clf = DecisionTreeClassifier(criterion='entropy')
clf.fit(x, y)
tree.plot_tree(clf, feature_names=x.columns)
l_node = clf.tree_.children_left[0]
print(clf.tree_.children_left[0])
print(clf.tree_.impurity)
print(clf.tree_.n_node_samples)
'''
# 0.996-((157/238)*0.903 + (81/238)*0.826)<jupyter_output><empty_output><jupyter_text><jupyter_code># print(list(range(1,10)))
'''
subs_df = pd.read_csv("submissions_data_train.zip", compression='zip')
print(subs_df.columns)
print(subs_df.head())
print(subs_df[0:500].groupby(['user_id', 'submission_status']).agg('count'))
subset = subs_df.query('submission_status == "wrong"')
print(subset.head())
df1 = subset.groupby('step_id', as_index=False).agg({'submission_status': 'count'})
print(df1.head())
print(df1.sort_values(by=['submission_status'], ascending=False))
'''
'''
pd.set_option('display.max_columns', 500)
subs_df = pd.read_csv("heart-disease-uci.zip", compression='zip')
print(subs_df.columns)
print(subs_df.head())
np.random.seed(0)
rf = RandomForestClassifier(10, max_depth=5)
x_train = subs_df.drop('target', axis=1)
y_train = subs_df['target']
rf.fit(x_train, y_train)
imp = pd.DataFrame(rf.feature_importances_, index=x_train.columns, columns=['importance'])
imp.sort_values('importance').plot(kind='barh', figsize=(12, 8))
'''
'''
rom sklearn.model_selection import GridSearchCV
df = pd.read_csv("training_mush.csv")
print(subs_df.columns)
print(subs_df.head())
clf = RandomForestClassifier(random_state=0)
params = {'n_estimators': range(10, 50, 10), 'max_depth': range(1, 12, 2), \
          'min_samples_leaf': range(1, 7, 1), 'min_samples_split': range(2, 9, 2)}
x_train = df.drop('class', axis=1)
y_train = df['class']
search = GridSearchCV(clf, param_grid = params, n_jobs=-1, cv=3)
search.fit(x_train, y_train)
'''
'''
print(search.best_params_) 
imp = pd.DataFrame(search.best_estimator_.feature_importances_, index=x_train.columns, columns=['importance'])
imp.sort_values('importance').plot(kind='barh', figsize=(12, 8))
'''
'''
test_df = pd.read_csv("testing_mush.csv")
print(test_df.columns)
print(test_df.head())
predictions = DataFrame({'pred': search.best_estimator_.predict(test_df)})
print(predictions['pred'].value_counts())
'''
'''
# ! unzip -P Cool!Move_forward! testing_y_mush.csv.zip
from sklearn.metrics import confusion_matrix
anot_df = pd.read_csv("testing_y_mush.csv")
print(anot_df.columns)
print(anot_df.head())
y_true = anot_df['class']
y_pred = predictions['pred']
sns.heatmap(confusion_matrix(y_true, y_pred), annot=True, cmap="Blues")
'''
'''
train_df = pd.read_csv("invasion.csv")
print(train_df.columns)
print(train_df.head())
train_x = train_df.drop('class', axis=1)
le = LabelEncoder()
le.fit(train_df['class'])
print(le.classes_)
train_y = le.transform(train_df['class'])
clf = RandomForestClassifier(random_state=0)
params = {'n_estimators': range(10, 50, 10), 'max_depth': range(1, 12, 2), \
          'min_samples_leaf': range(1, 7, 1), 'min_samples_split': range(2, 9, 2)}
search = GridSearchCV(clf, param_grid = params, n_jobs=-1, cv=5)
search.fit(train_x, train_y)
'''
'''
test_df = pd.read_csv("operative_information.csv")
print(test_df.columns)
print(test_df.head())
predictions = search.best_estimator_.predict(test_df)
out = pd.DataFrame({'ship_type': le.inverse_transform(predictions)})
print(out['ship_type'].value_counts())
print(search.best_estimator_.feature_importances_)
'''
'''
train_df = pd.read_csv("space_can_be_a_dangerous_place.csv")
print(train_df.columns)
print(train_df.head())
print(train_df.corr())
sns.heatmap(train_df.corr())
'''
df = pd.read_csv("iris.csv")
from time import time
before = time()
df.apply(np.mean)
after = time()
print(after - before)
before = time()
df.mean(axis=0)
after = time()
print(after - before)
before = time()
df.describe().loc['mean']
after = time()
print(after - before)
before = time()
df.apply('mean')
after = time()
print(after - before)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1908729265180515, 'fraction_numerical': 0.01778878630938978, 'mean_word_length': 7.9482874412357285, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '565545', 'license': 'Apache-2.0', 'n_tokens_mistral': 5345, 'n_tokens_neox': 5107, 'n_words': 741, 'path': 'Stepik_Ml.ipynb', 'repo_name': 'bvm84/open_colab_ml', 'size': '13323'}