<jupyter_start><jupyter_text>数据分析实战  04 Python科学计算：用NumPy快速处理数据<jupyter_code>x=2
x*=2
print(x)
import numpy as np
a=np.array([1,2,3])
b=np.array([[1,2,3],[4,5,6],[7,8,9]])
print(a.shape)
print(b.shape)
print(a.dtype)
print(b)
import numpy as np

# 自定义类型
person_type = np.dtype({
    'names': ['name', 'age', 'chinese', 'math', 'english'],
    'formats': ['S32', 'i', 'i', 'i', 'f']})

peoples = np.array([
    ("ZhangFei", 32, 75, 100, 90),
    ("GuanYu", 24, 85, 96, 88.5),
    ("ZhaoYun", 28, 85, 92, 96.5),
    ("HuangZhong", 29, 65, 85, 100)], dtype=person_type)

ages = peoples[:]['age']
chinese = peoples[:]['chinese']
math = peoples[:]['math']
english = peoples[:]['english']

# 计算平均值
print(np.mean(ages))
print(np.mean(chinese))
print(np.mean(math))
print(np.mean(english))
import numpy as np

# 初始值、终值和步长
x1=np.arange(1,11,2)

#  初始值、终值和元素个数
x2=np.linspace(1,9,5)

print(x1)
print(x2)
# 算术运算
print(np.add(x1,x2))
print(np.subtract(x1,x2))
print(np.multiply(x1,x2))
print(np.divide(x1,x2))
print(np.power(x1,x2))
print(np.remainder(x1,x2))
# 统计运算
import numpy as np
a = np.array([[1,2,3],[4,5,6],[7,8,9]])

# 统计矩阵最小值
print(np.amin(a))
print(np.amin(a,0))
print(np.amin(a,1))

# 统计矩阵最大值
print(np.amax(a))
print(np.amax(a,0))
print(np.amax(a,1))
# 统计最大值和最小值之差
print(np.ptp(a))
print(np.ptp(a,0))
print(np.ptp(a,1))
# 统计数组的百分位数 percentile()
print(np.percentile(a,50))
print(np.percentile(a,50,axis=0))
print(np.percentile(a,50,axis=1))
# 统计数组中的中位数 median()
print(np.median(a))
print(np.median(a,axis=0))
print(np.median(a,axis=1))
# 统计数组中的平均数 mean()
print(np.mean(a))
print(np.mean(a,axis=0))
print(np.mean(a,axis=1))
# 加权平均值 average()
source=np.array([1,2,3,4])
wts=np.array([1,2,3,4])
print(np.average(a))
# 加权
print(np.average(source,weights=wts))
# 统计数组中的标准差 std() 、方差 var()
source = np.array([1, 2, 3, 4])
print(np.std(source))
print(np.var(source))
# numpy 排序
source=np.array([[4,2,3,1],[2,4,1,7],[3,2,6,0]])
print(np.sort(source))
print(np.sort(source,axis=None))
print(np.sort(source,axis=0))
print(np.sort(source,axis=0))
print(np.sort(source,axis=1))
# 实例
import numpy as np

def show(name,data):
    print('%s，平均成绩：%d,最小成绩：%d,最大成绩：%d,方差：%d,标准差：%d' % (name,np.average(data),np.amin(data),np.amax(data),np.var(data),np.std(data)))

score_type=np.dtype({
    'names':['name','chinese','english','math'],
    'formats':['S32','i','i','i']
})

students=np.array([
    ('zhangfei',66,65,30),
    ('guanyu',95,85,98),
    ('zhaoyun',93,92,96),
    ('huangzhong',90,88,77),
    ('dianwei',80,90,90)
],dtype=score_type)

show('chinese',students[:]['chinese'])
show('english',students[:]['english'])
show('math',students[:]['math'])

rank=sorted(students,key=lambda x:x[1]+x[2]+x[3],reverse=True)
print(rank)<jupyter_output><empty_output><jupyter_text>05 Python科学计算：Pandas<jupyter_code>import pandas as pd
from pandas import Series,DataFrame

x1=Series([1,2,3,4,5,6,7,8,9])
x2=Series(data=[1,2,3,4,5],index=['a','b','c','d','e'])

data={'a':1,'b':2,'c':3,'d':4,'e':5}
x3=Series(data)

print(x1)
print(x2)
print(x3)
import pandas as pd
from pandas import Series, DataFrame
import numpy as np

data = {'Chinese': [66, 95, 93, 90,80],'English': [65, 85, 92, 88, 90],'Math': [30, 98, 96, 77, 90]}
df1= DataFrame(data)
df2 = DataFrame(data, index=['ZhangFei', 'GuanYu', 'ZhaoYun', 'HuangZhong', 'DianWei'], columns=['English', 'Math', 'Chinese'])

print(df1)
print(df2)<jupyter_output><empty_output><jupyter_text>清洗数据<jupyter_code># 删除 DataFrame 中的不必要的列和行
del_columns=df2.drop(columns=['Chinese'])
print(del_columns)

del_line=df2.drop(index=['ZhangFei'])
print(del_line)
# 重命名列名 columns
df2.rename(columns={'Chinese':'YunWen','English':'YinYu'},inplace=True)
print(df2)
# 去除重复行
remove_duplicate=df2.drop_duplicates()
df2['YunWen'].astype(np.int64)
df2['YunWen']= df2['YunWen'].astype(str)
# 删除左右两边空格
df2['YunWen']=df2['YunWen'].map(str.strip)
# 删除左边空格
df2['YunWen']=df2['YunWen'].map(str.lstrip)
# 删除右边空格
df2['YunWen']=df2['YunWen'].map(str.rstrip)

# 去除特殊字符
df2['YunWen']=df2['YunWen'].str.strip('$')

# 全部大写
df2.columns=df2.columns.str.upper()
# 全部小写
df2.columns=df2.columns.str.lower()
# 首字母大写
df2.columns=df2.columns.str.title()
# 统计函数合集
df3 = DataFrame({'name':['ZhangFei', 'GuanYu', 'a', 'b', 'c'], 'data1':range(5)})
print(df3.describe())
# 数据表合并
df4 = DataFrame({'name':['ZhangFei', 'GuanYu', 'a', 'b', 'c'], 'data1':range(5)})
df5 = DataFrame({'name':['ZhangFei', 'GuanYu', 'A', 'B', 'C'], 'data2':range(5)})

# 内连接（交集）
df6 = pd.merge(df4,df5,how='inner')
# 左连接
df6 = pd.merge(df4,df5,how='left')
# 右连接
df6 = pd.merge(df4,df5,how='right')
# 外连接（并集）
df6 = pd.merge(df4,df5,how='outer')
# 使用 SQL
import pandas as pd
from pandas import DataFrame
from pandasql import sqldf, load_meat, load_births
df7 = DataFrame({'name':['ZhangFei', 'GuanYu', 'a', 'b', 'c'], 'data1':range(5)})
pysqldf = lambda sql: sqldf(sql, globals())
sql = "select * from df7 where name ='ZhangFei'"
print(pysqldf(sql))
# 例子
import pandas as pd
from pandas import Series, DataFrame
import numpy as np

data = {'Chinese': [66, 95, 95, 90,80,80],'English': [65, 85, 92, 88, 90,90],'Math': [None, 98, 96, 77, 90,90]}
df = DataFrame(data, index=['ZhangFei', 'GuanYu', 'ZhaoYun', 'HuangZhong', 'DianWei','DianWei'], columns=['Chinese','English', 'Math' ])

print(df)

df=df.drop_duplicates()
df['Total']=df.sum(axis=1)
print(df)<jupyter_output><empty_output><jupyter_text>13 数据变换<jupyter_code># 1. Min-Max 规范化
from sklearn import preprocessing
import numpy as np

x = np.array([
    [0.,-3.,1.],
    [3.,1.,2.],
    [0.,1.,-1.]
])

min_max_scaler=preprocessing.MinMaxScaler()
minmax_x=min_max_scaler.fit_transform(x)

print(minmax_x)
# 2. Z-Score 规范化
from sklearn import preprocessing
import numpy as np

x = np.array([
    [0.,-3.,1.],
    [3.,1.,2.],
    [0.,1.,-1.]
])

scaled_x=preprocessing.scale(x)
print(scaled_x)
# 3. 小数定标规范化
from sklearn import preprocessing
import numpy as np

x = np.array([
    [0.,-3.,1.],
    [3.,1.,2.],
    [0.,1.,-1.]
])

j=np.ceil(np.log10(np.max(abs(x))))
scaled_x=x/(10**j)
print(scaled_x)
# 习题
from sklearn import preprocessing
import numpy as np

x = np.array([
    [5000.],
    [16000.],
    [58000.]
])

min_max_scaler=preprocessing.MinMaxScaler()
minmax_x=min_max_scaler.fit_transform(x)

print(minmax_x)<jupyter_output><empty_output><jupyter_text>15 数据可视化<jupyter_code># 散点图
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

N=1000
x=np.random.rand(N)
y=np.random.rand(N)

plt.scatter(x,y,marker='x')
plt.show()
df = pd.DataFrame({'x':x,'y':y})
sns.jointplot(x='x',y='y',data=df,kind='scatter')
plt.show()
# 折线图
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

x = [2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019]
y = [5, 3, 6, 20, 17, 16, 19, 30, 32, 35]

plt.plot(x,y)
plt.show()
df=pd.DataFrame({'x':x,'y':y})
sns.lineplot(x='x',y='y',data=df)
plt.show()
# 直方图
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

a=np.random.rand(100)
s=pd.Series(a)

plt.hist(s)
plt.show()
sns.distplot(s,kde=False)
plt.show()
sns.distplot(s,kde=True)
plt.show()
# 条形图
import matplotlib.pyplot as plt
import seaborn as sns

x=['Cat1','Cat2','Cat3','Cat4','Cat5']
y=[5,4,8,12,7]

plt.bar(x,y)
plt.show()
sns.barplot(x,y)
plt.show()
# 箱线图
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

data=np.random.normal(size=(10,4))
labels=['A','B','C','D']

plt.boxplot(data,labels=labels)
plt.show()
df=pd.DataFrame(data,columns=labels)
sns.boxplot(data=df)
plt.show()
# 饼图
import matplotlib.pyplot as plt
# 数据准备
nums = [25, 37, 33, 37, 6]
labels = ['High-school','Bachelor','Master','Ph.d', 'Others']
# 用Matplotlib画饼图
plt.pie(x = nums, labels=labels)
plt.show()
# 热力图
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from pandas import DataFrame
# 数据准备
flights = DataFrame(pd.read_csv('~/Project/seaborn-data/flights.csv'))
data=flights.pivot('year','month','passengers')
# 用Seaborn画热力图
sns.heatmap(data)
plt.show()
# 蜘蛛图
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.font_manager import FontProperties

labels=np.array([u"推进","KDA",u"生存",u"团战",u"发育",u"输出"])
stats=[83, 61, 95, 67, 76, 88]

angles=np.linspace(0,2*np.pi,len(labels),endpoint=False)
stats=np.concatenate((stats,[stats[0]]))
angles=np.concatenate((angles,[angles[0]]))

fig=plt.figure()
ax=fig.add_subplot(111,polar=True)
ax.plot(angles, stats, 'o-', linewidth=2)
ax.fill(angles, stats, alpha=0.25)
# 设置中文字体
font = FontProperties(fname=r"C:\Windows\Fonts\simhei.ttf", size=14)  
ax.set_thetagrids(angles * 180/np.pi, labels, FontProperties=font)
plt.show()
# 二元变量分布
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from pandas import DataFrame
# 数据准备
tips = DataFrame(pd.read_csv('seaborn-data/tips.csv'))
# print(tips.head(10))
# 用Seaborn画二元变量分布图（散点图，核密度图，Hexbin图）
sns.jointplot(x="total_bill", y="tip", data=tips, kind='scatter')
sns.jointplot(x="total_bill", y="tip", data=tips, kind='kde')
sns.jointplot(x="total_bill", y="tip", data=tips, kind='hex')
plt.show()
# 成对关系
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
from pandas import DataFrame
# 数据准备
iris = DataFrame(pd.read_csv('~/Project/seaborn-data/iris.csv'))
# 用Seaborn画成对关系
sns.pairplot(iris)
plt.show()<jupyter_output><empty_output><jupyter_text>决策树（上）：要不要去打篮球？决策树来告诉你<jupyter_code>import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz
from sklearn.datasets import load_iris
from pandas import DataFrame
import graphviz

# 准备数据集
iris = load_iris()
# 获取特征集和分类标识
features = iris.data
labels = iris.target

# 随机抽取 33% 的数据作为测试集，其余为训练集
train_features,test_features,train_labels,test_labels = train_test_split(features,labels,test_size=0.33,random_state=0)

# 创建 CART 分类树
clf = DecisionTreeClassifier(criterion='gini')
# 拟合构造 CART 分类树
clf = clf.fit(train_features,train_labels)
# 用 CART 分类树做预测
test_predict = clf.predict(test_features)
score = accuracy_score(test_labels,test_predict)
print("CART 分类树准确率 %.4lf" % score)
# 决策树
# dot_data = export_graphviz(clf,out_file=None)
# graph = graphviz.Source(dot_data)
# graph.render('iris')<jupyter_output><empty_output><jupyter_text>决策树（中）：CART，一棵是回归树，另一棵是分类树<jupyter_code>from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_boston
from sklearn.metrics import r2_score,mean_absolute_error,mean_squared_error
from sklearn.tree import DecisionTreeRegressor

# 准备数据集
boston = load_boston()
# 探索数据
print(boston.feature_names)
# 获取特征集和房价
features = boston.data
prices = boston.target
# 随机抽取33%的数据作为测试集，其余为训练集
train_features,test_features,train_price,test_price = train_test_split(features,prices,test_size=0.33)
# 创建CART回归树
dtr = DecisionTreeRegressor()
# 拟合构造CART回归树
dtr.fit(train_features,train_price)
# 预测测试集中的房价
predict_price = dtr.predict(test_features)
# 测试集的结果评价
print('回归树二乘偏差均值',mean_squared_error(test_price,predict_price))
print('回归树绝对值偏差均值',mean_absolute_error(test_price,predict_price))<jupyter_output><empty_output><jupyter_text>决策树（下）：泰坦尼克乘客生存预测<jupyter_code>import pandas as pd
import numpy as np
from sklearn.feature_extraction import DictVectorizer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import cross_val_score

# 数据加载
train_data = pd.read_csv('~/Project/Titanic_Data/train.csv')
test_data = pd.read_csv('~/Project/Titanic_Data/test.csv')
# print(train_data.info())
# print(train_data.describe())

# 使用平均年龄来填充年龄中的 nan 值
train_data['Age'].fillna(train_data['Age'].mean(),inplace=True)
test_data['Age'].fillna(test_data['Age'].mean(),inplace=True)
# 使用票价的均值填充票价中的 nan 值
train_data['Fare'].fillna(train_data['Fare'].mean(),inplace=True)
test_data['Fare'].fillna(test_data['Fare'].mean(),inplace=True)

# print(train_data['Embarked'].value_counts())

# 使用登陆最多的港口来填充登陆港口的 nan 值
train_data['Embarked'].fillna('S',inplace=True)
test_data['Embarked'].fillna('S',inplace=True)

# 特征选择
features = ['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']
train_features = train_data[features]
train_labels = train_data['Survived']
test_features = test_data[features]

dvec = DictVectorizer(sparse=False)
train_features = dvec.fit_transform(train_features.to_dict(orient='record'))
print(dvec.feature_names_)

# 决策树模型
# 构造 ID3 决策树
clf = DecisionTreeClassifier(criterion='entropy')
# 决策树训练
clf.fit(train_features,train_labels)

test_features = dvec.transform(test_features.to_dict(orient='record'))
# 决策树预测
pred_labels=clf.predict(test_features)

# 得到决策树的准确率
acc_decision_tree = round(clf.score(train_features,train_labels),6)
print(u'score 准确率 %.4lf' % acc_decision_tree)

# 使用 k 折交叉验证 统计决策树准确率
print(u'cross_val_score 准确率 %.4lf' % np.mean(cross_val_score(clf,train_features,train_labels,cv=10)))<jupyter_output><empty_output><jupyter_text>朴素贝叶斯分类（下）：如何对文档进行分类？<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vec = TfidfVectorizer()

documents = [
    'this is the bayes document',
    'this is the second second document',
    'and the third one',
    'is this the document'
]
tfidf_matrix = tfidf_vec.fit_transform(documents)

print('不重复的词:',tfidf_vec.get_feature_names())
print('每个单词的ID:',tfidf_vec.vocabulary_)
print('每个单词的 tfidf 值:',tfidf_matrix.toarray())
import jieba
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
from sklearn.naive_bayes import MultinomialNB

#  加载停用词列表
def load_stop_wrods():
    stop_word=''
    with open('text_classification/text classification/stop/stopword.txt','r', encoding='utf-8') as f:
        stop_word=f.read();
        stop_word= stop_word.encode('utf-8').decode('utf-8-sig')
        stop_word=stop_word.split('\n')
    return stop_word

# 加载文件
def load_document(file_dir,label):
    file_list = os.listdir(file_dir)
    words_list = []
    label_list = []
    for file in file_list:
        file_path=os.path.join(file_dir,file)
        words_list.append(cut_text(file_path))
        label_list.append(label)
    return words_list,label_list

# 分词
def cut_text(file_path):
    word_spilit=''
    result = ''
    with open(file_path,'r', encoding='gb18030') as f:
        text=f.read()
        cut_text = jieba.cut(text)
        result=word_spilit.join(cut_text)
    return result


# 训练数据
train_words_list1, train_labels1 = load_document('text_classification/text classification/train/女性', '女性')
train_words_list2, train_labels2 = load_document('text_classification/text classification/train/体育', '体育')
train_words_list3, train_labels3 = load_document('text_classification/text classification/train/文学', '文学')
train_words_list4, train_labels4 = load_document('text_classification/text classification/train/校园', '校园')

train_words_list = train_words_list1 + train_words_list2 + train_words_list3 + train_words_list4
train_labels = train_labels1 + train_labels2 + train_labels3 + train_labels4

# 测试数据
test_words_list1, test_labels1 = load_document('text_classification/text classification/test/女性', '女性')
test_words_list2, test_labels2 = load_document('text_classification/text classification/test/体育', '体育')
test_words_list3, test_labels3 = load_document('text_classification/text classification/test/文学', '文学')
test_words_list4, test_labels4 = load_document('text_classification/text classification/test/校园', '校园')

test_words_list = test_words_list1 + test_words_list2 + test_words_list3 + test_words_list4
test_labels = test_labels1 + test_labels2 + test_labels3 + test_labels4

stop_words=load_stop_wrods()

# 计算权重
train_tf=TfidfVectorizer(stop_words=stop_words,max_df=0.5)
train_features=train_tf.fit_transform(train_words_list)

# 多项式贝叶斯分类器
clf=MultinomialNB(alpha=0.01).fit(train_features,train_labels)

# 使用生成分类器做预测
test_tf=TfidfVectorizer(stop_words=stop_words,max_df=0.5,vocabulary=tf.vocabulary_)
test_features=test_tf.fit_transform(test_words_list)

predicted_labels=clf.predict(test_features)

# 计算准确率
print('准确率为：', metrics.accuracy_score(test_labels, predicted_labels))<jupyter_output><empty_output><jupyter_text>SVM（下）：如何进行乳腺癌检测？<jupyter_code>from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import metrics
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# 加载数据
data=pd.read_csv('breast_cancer_data/data.csv')
pd.set_option('display.max_columns',None)
# print(data.columns)
# print(data.head(5))
# print(data.describe())

# 将特征字段分为3组
features_mean=list(data.columns[2:12])
features_se=list(data.columns[12:22])
features_worst=list(data.columns[22:32])

# 删除没有用的列
data.drop('id',axis=1,inplace=True)
# 将B良性替换为0，M恶性替换为1
data['diagnosis']=data['diagnosis'].map({'M':1,'B':0})
# 将肿瘤诊断结果可视化
sns.countplot(data['diagnosis'],label='Count')
plt.show()

# 用热力图呈现 features_mean 字段之间的相关性
corr=data[features_mean].corr()
plt.figure(figsize=(15,15))
sns.heatmap(corr,annot=True)
plt.show()

# 颜色越浅代表相关性越大

# 特征选择
features_remain = ['radius_mean','texture_mean', 'smoothness_mean','compactness_mean','symmetry_mean', 'fractal_dimension_mean'] 
# 抽取 30% 的数据作为测试集，其余作为训练集
train,test=train_test_split(data,test_size=0.3)
train_x=train[features_remain]
train_y=train['diagnosis']
test_x=test[features_remain]
test_y=test['diagnosis']

ss=StandardScaler()
train_x=ss.fit_transform(train_x)
test_x=ss.transform(test_x)

# 创建 SVM 分类器
# model=svm.SVC()
model=svm.LinearSVC()
# 用训练集做训练
model.fit(train_x,train_y)
# 用测试集做预测
prediction=model.predict(test_x)
print('准确率：',metrics.accuracy_score(prediction,test_y))<jupyter_output><empty_output><jupyter_text>KNN（下）：如何对手写数字进行识别？<jupyter_code># KNN 分类器
from sklearn.neighbors import KNeighborsClassifier
# KNN 回归
from sklearn.neighbors import KNeighborsRegressor

from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.metrics import accuracy_score
from sklearn.datasets import load_digits
from sklearn.svm import SVC
from sklearn.naive_bayes import MultinomialNB
from sklearn.tree import DecisionTreeClassifier
import matplotlib.pyplot as plt

digits = load_digits()
data = digits.data

print(data.shape)
print(digits.images[0])
print(digits.target[0])

plt.gray()
plt.imshow(digits.images[0])
plt.show()
train_x,test_x,train_y,test_y = train_test_split(data,digits.target,test_size=0.25,random_state=33)
# 采用 Z-Score 规范
ss=preprocessing.StandardScaler()
train_ss_x = ss.fit_transform(train_x)
test_ss_x = ss.transform(test_x)
# 创建 KNN 分类器
knn = KNeighborsClassifier()
knn.fit(train_ss_x, train_y)
predict_y = knn.predict(test_ss_x)
print("KNN 准确率: %.4lf"%accuracy_score(test_y,predict_y))
# 创建 SVM 分类器
svm = SVC()
svm.fit(train_ss_x,train_y)
predict_y=svm.predict(test_ss_x)
print('SVM 准确率: %0.4lf'%accuracy_score(test_y,predict_y))
# 采用Min-Max规范化
mm = preprocessing.MinMaxScaler()
train_mm_x = mm.fit_transform(train_x)
test_mm_x = mm.transform(test_x)
# 创建Naive Bayes分类器
mnb = MultinomialNB()
mnb.fit(train_mm_x, train_y)
predict_y = mnb.predict(test_mm_x)
print("多项式朴素贝叶斯准确率: %.4lf" % accuracy_score(test_y, predict_y))
# 创建CART决策树分类器
dtc = DecisionTreeClassifier()
dtc.fit(train_mm_x, train_y)
predict_y = dtc.predict(test_mm_x)
print("CART决策树准确率: %.4lf" % accuracy_score(test_y, predict_y))<jupyter_output>多项式朴素贝叶斯准确率: 0.8844
CART决策树准确率: 0.8644<jupyter_text>K-Means（上）：如何给20支亚洲球队做聚类？<jupyter_code>from sklearn.cluster import KMeans
from sklearn import preprocessing
import pandas as pd
import numpy as np

# 数据源 https://github.com/cystanford/kmeans
data = pd.read_csv('data.csv', encoding='gbk')
train_x = data[['2019年国际排名', '2018世界杯', '2015亚洲杯']]
kmeans = KMeans(n_clusters=3)

# 规范化到[0,1]空间
min_max_scaler = preprocessing.MinMaxScaler()
train_x = min_max_scaler.fit_transform(train_x)

# kmeans 算法
kmeans.fit(train_x)
predict_y = kmeans.predict(train_x)

# 合并具类结果，插入到原数据中
result = pd.concat((data, pd.DataFrame(predict_y)), axis=1)
result.rename({0: u'具类'}, axis=1, inplace=True)
print(result)
from sklearn.cluster import KMeans
from sklearn import preprocessing
import PIL.Image as image
import pandas as pd
import numpy as np


# 加载图像，并对数据进行规范化
def load_image(image_path):
    data = []
    with open(image_path, 'rb') as f:
        image_pixel=image.open(f)
        width,height=image_pixel.size
        for x in range(width):
            for y in range(height):
                c1,c2,c3=image_pixel.getpixel((x,y))
                data.append([c1,c2,c3])
    mm = preprocessing.MinMaxScaler()
    data=mm.fit_transform(data)
    return np.mat(data),width,height

image_mat,width,height=load_image('weixin.jpg')
kmeans=KMeans(n_clusters=2)
label= kmeans.fit_predict(image_mat)
label= label.reshape([width,height])
pic_mark=image.new("L",(width,height))
for x in range(width):
    for y in range(height):
        pic_mark.putpixel((x,y),int(256/(label[x][y]+1))-1)
pic_mark.save('weixin_mark.jpg','JPEG')
weixin_mark=image.open('weixin_mark.jpg')
weixin_mark.show()<jupyter_output><empty_output><jupyter_text>![](weixin.jpg)![](weixin_mark.jpg)<jupyter_code>from skimage import color

label_color=(color.label2rgb(label)*255).astype(np.uint8)
label_color=label_color.transpose(1,0,2)
images=image.fromarray(label_color)
images.save('weixin_mark_color.jpg')<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: FutureWarning: The new recommended value for bg_label is 0. Until version 0.19, the default bg_label value is -1. From version 0.19, the bg_label default value will be 0. To avoid this warning, please explicitly set bg_label value.
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>![](weixin_mark_color.jpg)<jupyter_code>from sklearn.cluster import KMeans
from sklearn import preprocessing
import PIL.Image as image
import pandas as pd
import numpy as np
from skimage import color

# 加载图像，并对数据进行规范化
def load_image(image_path):
    data = []
    with open(image_path, 'rb') as f:
        image_pixel=image.open(f)
        width,height=image_pixel.size
        for x in range(width):
            for y in range(height):
                c1,c2,c3=image_pixel.getpixel((x,y))
                data.append([c1,c2,c3])
    mm = preprocessing.MinMaxScaler()
    data=mm.fit_transform(data)
    return np.mat(data),width,height

image_mat,width,height=load_image('baby.jpg')
kmeans=KMeans(n_clusters=2)
label=kmeans.fit_predict(image_mat)
label=label.reshape([width,height])
label_color=(color.label2rgb(label)*255).astype(np.uint8)
label_color=label_color.transpose(1,0,2)
images=image.fromarray(label_color)
images.save('baby_mark_color.jpg')<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:26: FutureWarning: The new recommended value for bg_label is 0. Until version 0.19, the default bg_label value is -1. From version 0.19, the bg_label default value will be 0. To avoid this warning, please explicitly set bg_label value.<jupyter_text>EM聚类（下）：用EM算法对王者荣耀英雄进行划分<jupyter_code>import pandas as pd
import csv
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler

data_ori=pd.read_csv('heros.csv',encoding='gb18030')
features=[u'最大生命',u'生命成长',u'初始生命',u'最大法力', u'法力成长',u'初始法力',u'最高物攻',u'物攻成长',u'初始物攻',u'最大物防',u'物防成长',u'初始物防', u'最大每5秒回血', u'每5秒回血成长', u'初始每5秒回血', u'最大每5秒回蓝', u'每5秒回蓝成长', u'初始每5秒回蓝', u'最大攻速', u'攻击范围']
data=data_ori[features]

plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus']=False

corr=data[features].corr()
plt.figure(figsize=(14,14))
plt.show()

features_remain=[u'最大生命', u'初始生命', u'最大法力', u'最高物攻', u'初始物攻', u'最大物防', u'初始物防', u'最大每5秒回血', u'最大每5秒回蓝', u'初始每5秒回蓝', u'最大攻速', u'攻击范围']
data=data_ori[features_remain]
data[u'最大攻速']=data[u'最大攻速'].apply(lambda x:float(x.strip('%'))/100)
data[u'攻击范围']=data[u'攻击范围'].map({'远程':1,'近战':0})

ss=StandardScaler()
data=ss.fit_transform(data)

gmm=GaussianMixture(n_components=30,covariance_type='full')
gmm.fit(data)

prediction=gmm.predict(data)
print(prediction)

data_ori.insert(0,'分组',prediction)
data_ori.to_csv('hero_out.csv',index=False,sep=',')<jupyter_output><empty_output><jupyter_text>关联规则挖掘<jupyter_code>from efficient_apriori import apriori

data=[('牛奶','面包','尿布'),           
    ('可乐','面包', '尿布', '啤酒'),           
    ('牛奶','尿布', '啤酒', '鸡蛋'),           
    ('面包', '牛奶', '尿布', '啤酒'),           
    ('面包', '牛奶', '尿布', '可乐')]

itemsets,rules=apriori(data,min_support=0.5,min_confidence=1)
print(itemsets)
print(rules)<jupyter_output>{1: {('面包',): 4, ('牛奶',): 4, ('尿布',): 5, ('啤酒',): 3}, 2: {('尿布', '牛奶'): 4, ('尿布', '面包'): 4, ('牛奶', '面包'): 3, ('啤酒', '尿布'): 3}, 3: {('尿布', '牛奶', '面包'): 3}}
[{牛奶} -> {尿布}, {面包} -> {尿布}, {啤酒} -> {尿布}, {牛奶, 面包} -> {尿布}]
{'fraction_non_alphanumeric': 0.16490367212323534, 'fraction_numerical': 0.028838032417648714, 'mean_word_length': 6.959026888604353, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 65, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '207946', 'license': 'Apache-2.0', 'n_tokens_mistral': 11468, 'n_tokens_neox': 11487, 'n_words': 1679, 'path': 'geektime_practice.ipynb', 'repo_name': 'ab300819/jupyter_notebook', 'size': '24863'}