<jupyter_start><jupyter_text>This contains lots of methods including cluster, dimension reduction and regression for test.<jupyter_code>data = pd.read_csv(folder + '/match_total.csv')
data = data[data.columns[~data.columns.str.contains('gk')]]<jupyter_output>/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning:

Columns (933,934,976,977) have mixed types.Specify dtype option on import or set low_memory=False.<jupyter_text>**获取label**<jupyter_code>y = data['home_team_goal'] - data['away_team_goal']<jupyter_output><empty_output><jupyter_text>获取某些data<jupyter_code>'''
feature_list = ['overall_rating', # 总体评分
        'potential', # 总体评分上升潜力

        'ball_control', # 对方传来的球能不能接住
        'dribbling', # 带球
        'agility', # 变换能力
        'sprint_speed', # 冲刺

        'short_passing', # 短传
        'long_passing', # 长传
        'crossing', # 横传
        'vision', # 对周围环境的理解力，这对提升长传能力有效

        'volleys', # 凌空传球
        'curve', # 香蕉球
        'finishing', # 进球

        'sliding_tackle', # 铲球
        'standing_tackle', # 拦截
        'interceptions', # 抢断
        'marking', # 盯人

        'stamina', # 体力
        'reactions', # 对身边环境的反应能力
        'positioning', # 站位
]
'''
feature_list = ['long_passing', 'curve', 'short_passing', 'crossing',
'overall_rating', 'ball_control', 'potential', 'reactions',
'vision', 'finishing', 'volleys', 'positioning',
'marking', 'sliding_tackle', 'standing_tackle', 'interceptions',
'dribbling', 'agility', 'sprint_speed', 'stamina']
def get_data(feat, try_data=None):
  if try_data is None:
    home = data[data.columns[data.columns.str.contains(feat) & data.columns.str.contains('home')]]
    away = data[data.columns[data.columns.str.contains(feat) & data.columns.str.contains('away')]]
    return home, away
  else:
    home = try_data[data.columns[try_data.columns.str.contains(feat) & try_data.columns.str.contains('home')]]
    away = try_data[data.columns[try_data.columns.str.contains(feat) & try_data.columns.str.contains('away')]]
    return home, away
home_dict = {}
away_dict = {}
home_mean = pd.DataFrame()
away_mean = pd.DataFrame()
for feat in feature_list:
  a, b = get_data(feat)
  home_dict[feat] = a
  away_dict[feat] = b
  home_mean[feat] = a.mean(axis=1)
  away_mean[feat] = b.mean(axis=1)
mean_df = pd.concat([home_mean, away_mean], axis=0)
corr_mat = mean_df.corr()
corr_mat
sns.heatmap(mean_df.corr())
sns.heatmap(mean_df[np.random.permutation(mean_df.columns)].corr())
sns.heatmap(mean_df.corr('spearman'))<jupyter_output><empty_output><jupyter_text>Flat Clustering<jupyter_code>dissimilarity = 1 - np.abs(mean_df.corr())
hierarchy = linkage(squareform(dissimilarity), method='average')
labels = fcluster(hierarchy, 0.3, criterion='distance')
labels
mean_df.columns
c = Counter(labels)
c
temp = rearrange(mean_df.copy(), dict(zip(mean_df.columns, labels)))
sns.heatmap(temp)
temp.columns
hierarchy<jupyter_output><empty_output><jupyter_text>DBSCAN<jupyter_code>from sklearn.cluster import DBSCAN
dist = mean_df.corr()
X = 1 - dist.values
clustering = DBSCAN(eps=0.5, min_samples=3).fit(X)
clustering.labels_
temp = rearrange(mean_df.copy(), dict(zip(mean_df.columns, clustering.labels_.tolist())))
sns.heatmap(temp)
Counter(clustering.labels_)
temp.columns<jupyter_output><empty_output><jupyter_text>Block Modeling Clustering (old version)<jupyter_code>n_variables = 20
n_clusters = 5
cluster_size = n_variables // n_clusters

C = mean_df.corr().values
belongs_to_cluster = np.repeat(range(n_clusters), cluster_size)
def score(C):
    '''
    Function to assign a score to an ordered covariance matrix.
    High correlations within a cluster improve the score.
    High correlations between clusters decease the score.
    '''
    score = 0
    for cluster in range(n_clusters):
        inside_cluster = np.arange(cluster_size) + cluster * cluster_size
        outside_cluster = np.setdiff1d(range(n_variables), inside_cluster)

        # Belonging to the same cluster
        score += np.sum(C[inside_cluster, :][:, inside_cluster])

        # Belonging to different clusters
        score -= np.sum(C[inside_cluster, :][:, outside_cluster])
        score -= np.sum(C[outside_cluster, :][:, inside_cluster])

    return score


initial_C = C
initial_score = score(C)
initial_ordering = np.arange(n_variables)

plt.figure()
plt.imshow(C, interpolation='nearest')
plt.title('Initial C')
print('Initial ordering:', initial_ordering)
print('Initial covariance matrix score:', initial_score)
# Pretty dumb greedy optimization algorithm that continuously
# swaps rows to improve the score
def swap_rows(C, var1, var2):
    '''
    Function to swap two rows in a covariance matrix,
    updating the appropriate columns as well.
    '''
    D = C.copy()
    D[var2, :] = C[var1, :]
    D[var1, :] = C[var2, :]

    E = D.copy()
    E[:, var2] = D[:, var1]
    E[:, var1] = D[:, var2]

    return E

current_C = C
current_ordering = initial_ordering
current_score = initial_score

max_iter = 1000
for i in range(max_iter):
    # Find the best row swap to make
    best_C = current_C
    best_ordering = current_ordering
    best_score = current_score
    for row1 in range(n_variables):
        for row2 in range(n_variables):
            if row1 == row2:
                continue
            option_ordering = best_ordering.copy()
            option_ordering[row1] = best_ordering[row2]
            option_ordering[row2] = best_ordering[row1]
            option_C = swap_rows(best_C, row1, row2)
            option_score = score(option_C)

            if option_score > best_score:
                best_C = option_C
                best_ordering = option_ordering
                best_score = option_score

    if best_score > current_score:
        # Perform the best row swap
        current_C = best_C
        current_ordering = best_ordering
        current_score = best_score
    else:
        # No row swap found that improves the solution, we're done
        break


# Output the result
plt.figure()
plt.imshow(current_C, interpolation='nearest')
plt.title('Best C')
print('Best ordering:', current_ordering)
print('Best score:', current_score)
print()
print('Cluster     [variables assigned to this cluster]')
print('------------------------------------------------')
for cluster in range(n_clusters):
  meaning_list = [mean_df.columns[i] for i in current_ordering[cluster*cluster_size:(cluster+1)*cluster_size]]
  print('Cluster %02d  %s' % (cluster + 1, meaning_list))<jupyter_output>Best ordering: [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19]
Best score: -345.768936391987

Cluster     [variables assigned to this cluster]
------------------------------------------------
Cluster 01  ['long_passing', 'curve', 'short_passing', 'crossing']
Cluster 02  ['overall_rating', 'ball_control', 'potential', 'reactions']
Cluster 03  ['vision', 'finishing', 'volleys', 'positioning']
Cluster 04  ['marking', 'sliding_tackle', 'standing_tackle', 'interceptions']
Cluster 05  ['dribbling', 'agility', 'sprint_speed', 'stamina']<jupyter_text>Block Modeling Clustering (new version)<jupyter_code>n_variables = 20
n_clusters = 5
cluster_size = n_variables // n_clusters
columns_num_dict = dict(zip(list(mean_df.columns), np.repeat(range(n_clusters), cluster_size)))
class_count = Counter(columns_num_dict.values())
reg = sum([i ** 2 for i in class_count.values()])
df = abs(mean_df.corr())

lamda = 2
def score(df, columns_num_dict, verbose=False):
  '''
  Function to assign a score to an ordered covariance matrix.
  High correlations within a cluster improve the score.
  High correlations between clusters decease the score.
  '''
  m, n = df.shape
  C = df.values
  assert m == n
  score = 0
  for cluster in range(n_clusters):
    is_inside = np.array([columns_num_dict[i] == cluster for i in df.columns])
    inside_cluster = np.arange(m)[is_inside]
    outside_cluster = np.setdiff1d(range(m), inside_cluster)

    # Belonging to the same cluster
    score += np.sum(C[inside_cluster, :][:, inside_cluster])

    # Belonging to different clusters
    score -= np.sum(C[inside_cluster, :][:, outside_cluster])
    score -= np.sum(C[outside_cluster, :][:, inside_cluster])
  
  if verbose:
    return score, - lamda * reg, score - lamda * reg
  return score - lamda * reg
def recombinant(var1, var2, columns_num_dict):
  '''
  Function to swap two rows in a covariance matrix,
  updating the appropriate columns as well.
  '''
  p1 = columns_num_dict[var1]
  p2 = columns_num_dict[var2]
  columns_num_dict[var1] = p2
  columns_num_dict[var2] = p1
  return columns_num_dict
def metamorphosis(var, j, columns_num_dict):
  global reg
  columns_num_dict[var] = j
  class_count = Counter(columns_num_dict.values())
  reg = sum([i ** 2 for i in class_count.values()])
  if len(class_count.values()) != 5:
    return False, columns_num_dict
  return True, columns_num_dict
  
def rearrange(mean_df, column_dict):
  col_list = []
  for i in column_dict.keys():
    col_list.append((column_dict[i], i))
  col_list.sort()
  cols = [i for _, i in col_list]
  return mean_df[cols].corr()

def print_cluster(column_dict):
  for i in range(n_clusters):
    print('Cluster', i, end='')
    print(':', end=' ')
    for j in column_dict.keys():
      if column_dict[j] == i:
        print(j, end='\t')
    print()
initial_df = df
initial_score = score(df, columns_num_dict)
intial_dict = columns_num_dict

plt.figure()
sns.heatmap(mean_df.corr())
plt.title('Initial df')
print_cluster(intial_dict)
import copy

current_df = df
current_dict = copy.deepcopy(intial_dict)
current_score = initial_score

max_iter = 1000
for p in range(max_iter // 30):
  # if p == 0:
  #   var = np.random.choice(df.columns)
  #   j = np.random.randint(0, 5)
  #   res, d = metamorphosis(var, j, copy.deepcopy(current_dict))
  #   if not res:
  #     continue
  #   current_dict = d
  for i in range(max_iter):
    # Find the best row swap to make
    best_dict = copy.deepcopy(current_dict)
    best_score = current_score
    for row1 in range(n_variables):
      for row2 in range(n_variables):
        if row1 == row2:
          continue
        option_dict = recombinant(df.columns[row1], df.columns[row2], copy.deepcopy(best_dict))
        option_score = score(df, option_dict)

        if option_score > best_score:
          best_dict = copy.deepcopy(option_dict)
          best_score = option_score

    if best_score > current_score:
      # Perform the best row swap
      current_C = best_C
      current_dict = copy.deepcopy(best_dict)
      current_score = best_score
      print_cluster(best_dict)
      print(score(df, best_dict, True))
    else:
      # No row swap found that improves the solution, we're done
      break
current_score
df = rearrange(mean_df, current_dict)
current_dict
sns.heatmap(df)
print_cluster(best_dict)<jupyter_output><empty_output><jupyter_text>LDA  Data Preprocessing<jupyter_code>'''
From <NAME>
'''
# def avg_max_data(data, k):
#   result_control = []
#   for i in range(len(data)):
#     curr_arr = data.iloc[i].values
#     curr_arr = curr_arr[~np.isnan(curr_arr)]
#     result_control.append(np.nanmean(sorted(curr_arr, reverse=True)[:k]))
  
#   return np.array(result_control)

# for feat in tqdm(feature_list, total=len(feature_list)):
#   a, b = get_data(feat)
#   home_mean[feat] = avg_max_data(a, 6)
#   away_mean[feat] = avg_max_data(b, 6)
# home_mean.to_csv(folder + '/home_mean.csv')
# away_mean.to_csv(folder + '/away_mean.csv')
home_mean = pd.read_csv(folder + '/home_mean.csv', index_col=0)
away_mean = pd.read_csv(folder + '/away_mean.csv', index_col=0)
y = data['home_team_goal'] - data['away_team_goal']
y.name = 'label'
lda_data = pd.concat([home_mean - away_mean, y], axis=1)
lda_data = lda_data.loc[lda_data.label != 0]
lda_data['label'] = lda_data['label'] > 0
lda_data['label']
feature_cluster_list = [df.columns[i * 4:i * 4 + 4] for i in range(5)]
ans = pd.DataFrame()
trans_list = []
minmax_list = []
for i in range(5):
  val = lda_data[df.columns[i * 4:i * 4 + 4]].values
  y = lda_data['label']
  minmax = MinMaxScaler()
  minmax.fit(val)
  val = minmax.transform(val)
  lda = LinearDiscriminantAnalysis(n_components=1)
  lda.fit(val, y)
  trans_list.append(lda)
  minmax_list.append(minmax)
  ans['feat' + str(i)] = lda.transform(val)[:, 0]
ans.feat0.plot(kind='hist')
trans_list[0].coef_
for i in range(5):
  for j in range(4):
    print(round(trans_list[i].coef_[0][j], 3), '*', df.columns[i * 4 + j], sep=' ', end=' + ')
  print()
for i in range(5):
  print(trans_list[i].explained_variance_ratio_)<jupyter_output>[1.]
[1.]
[1.]
[1.]
[1.]<jupyter_text>Build Evaluation System  For Classification<jupyter_code>y = lda_data['label'].astype(int)
team_features = ['buildUpPlaySpeed', 'buildUpPlayPassing', 'chanceCreationPassing', 'chanceCreationCrossing',
'chanceCreationShooting', 'defencePressure', 'defenceAggression', 'defenceTeamWidth']
mean_df
system_df = lda_data[df.columns]
system_df
sys_lda_df = pd.DataFrame()
name_list = ['passing', 'overall', 'shot', 'tackle', 'physical']
for i in range(5):
  mmtrans = minmax_list[i]
  ldatrans = trans_list[i]
  val = system_df.iloc[:, i * 4:i * 4 + 4].values
  val = ldatrans.transform(mmtrans.transform(val))
  sys_lda_df[name_list[i]] = np.squeeze(val)
sys_lda_df.passing.plot(kind='hist')
X = np.array(sys_lda_df.iloc[:, [0, 2, 3, 4]]).reshape(-1, 4)
reg = sm.Logit(y, sm.add_constant(X)).fit()
reg.summary()
X = np.array(sys_lda_df.iloc[:]).reshape(-1, 5)
reg = sm.Logit(y, sm.add_constant(X)).fit()
reg.summary()
def get_team_data(feat):
  home = data[data.columns[data.columns.str.endswith(feat) & data.columns.str.contains('home')]]
  away = data[data.columns[data.columns.str.endswith(feat) & data.columns.str.contains('away')]]
  return home, away

team_df = pd.DataFrame()
for feat in team_features:
  home, away = get_team_data(feat)
  team_df[feat] = (home.values - away.values)[:, 0]
  print((home.values - away.values).shape)
team_df
team_df.dropna(how='all')<jupyter_output><empty_output><jupyter_text>For Regression<jupyter_code>y = data['home_team_goal'] - data['away_team_goal']
system_df = home_mean[df.columns] - away_mean[df.columns]
system_df
sys_lda_df = pd.DataFrame()
name_list = ['passing', 'general', 'shot', 'tackle', 'quality']
for i in range(5):
  mmtrans = minmax_list[i]
  ldatrans = trans_list[i]
  val = system_df.iloc[:, i * 4:i * 4 + 4].values
  val = ldatrans.transform(mmtrans.transform(val))
  sys_lda_df[name_list[i]] = np.squeeze(val)
X = np.array(sys_lda_df.iloc[:, [0, 2, 3, 4]]).reshape(-1, 4)
reg = sm.OLS(y, sm.add_constant(X)).fit()
reg.summary()
X = np.array(sys_lda_df.iloc[:]).reshape(-1, 5)
reg = sm.OLS(y, sm.add_constant(X)).fit()
reg.summary()
sys_lda_df
sys_lda_df.quality.plot(kind='hist')
home_mean_df = home_mean[df.columns]

home_sys_lda_df = pd.DataFrame()
name_list = ['passing', 'general', 'shot', 'tackle', 'quality']
for i in range(5):
  mmtrans = minmax_list[i]
  ldatrans = trans_list[i]
  val = home_mean_df.iloc[:, i * 4:i * 4 + 4].values
  val = ldatrans.transform(mmtrans.transform(val))
  home_sys_lda_df[name_list[i]] = np.squeeze(val)
home_score_df = home_sys_lda_df.copy()
home_score_df['passing'] = home_score_df['passing'].apply(lambda x: np.clip(7 * x - 13, 60, 100))
home_score_df['general'] = home_score_df['general'].apply(lambda x: np.clip(7 * x - 27, 60, 100))
home_score_df['shot'] = home_score_df['shot'].apply(lambda x: np.clip(7 * x + 1, 60, 100))
home_score_df['tackle'] = home_score_df['tackle'].apply(lambda x: np.clip(7 * x + 1, 60, 100))
home_score_df['quality'] = home_score_df['quality'].apply(lambda x: np.clip(7 * x - 27, 60, 100))
away_mean_df = away_mean[df.columns]

away_sys_lda_df = pd.DataFrame()
name_list = ['passing', 'general', 'shot', 'tackle', 'quality']
for i in range(5):
  mmtrans = minmax_list[i]
  ldatrans = trans_list[i]
  val = away_mean_df.iloc[:, i * 4:i * 4 + 4].values
  val = ldatrans.transform(mmtrans.transform(val))
  away_sys_lda_df[name_list[i]] = np.squeeze(val)
away_score_df = away_sys_lda_df.copy()
away_score_df['passing'] = away_score_df['passing'].apply(lambda x: np.clip(7 * x - 13, 60, 100))
away_score_df['general'] = away_score_df['general'].apply(lambda x: np.clip(7 * x - 27, 60, 100))
away_score_df['shot'] = away_score_df['shot'].apply(lambda x: np.clip(7 * x + 1, 60, 100))
away_score_df['tackle'] = away_score_df['tackle'].apply(lambda x: np.clip(7 * x + 1, 60, 100))
away_score_df['quality'] = away_score_df['quality'].apply(lambda x: np.clip(7 * x - 27, 60, 100))
away_score_df.to_csv(folder + '/away_score.csv')
home_score_df.iloc[:, 3].plot(kind='hist')
away_mean_df = away_mean[df.columns]
home_score_df = home_sys_lda_df.applymap(lambda x: np.clip(80 + 5 * x, 60, 100))
home_sys_lda_df.quality.plot(kind='hist')
home_score_df
score_df.to_csv(folder + '/score.csv')<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>pca_data = MinMaxScaler().fit(mean_df).transform(mean_df)
pca_data = pd.DataFrame(pca_data, columns=mean_df.columns)
pca = PCA(n_components=10)
pca = pca.fit(pca_data)
var_exp_list = pca.explained_variance_ratio_
var_exp_list
plt.plot((1 - var_exp_list.cumsum()))
pca.components_.shape
wgt = abs(pd.DataFrame(pca.components_, columns=mean_df.columns))
sns.heatmap(wgt)<jupyter_output><empty_output><jupyter_text>**potential**<jupyter_code>home_potential, away_potential = get_data('potential')
home_potential<jupyter_output><empty_output><jupyter_text>取平均分<jupyter_code>mean_home = home_potential.mean(axis=1)
mean_away = away_potential.mean(axis=1)
mean_diff = mean_home - mean_away
X = np.array(mean_diff).reshape(-1, 1)
reg = sm.OLS(y, sm.add_constant(X)).fit()
reg.summary()
y_pred = reg.predict(sm.add_constant(X))
plt.scatter(X, y)
plt.plot(X, y_pred, color='r')
plt.xlabel('mean_potential_difference')
plt.ylabel('score_difference')
plt.show()<jupyter_output><empty_output><jupyter_text>平方后取平均<jupyter_code>mean_home = home_potential.apply(lambda x: x ** 2).mean(axis=1)
mean_away = away_potential.apply(lambda x: x ** 2).mean(axis=1)
mean_diff = mean_home - mean_away
X = np.array(mean_diff).reshape(-1, 1)
reg = sm.OLS(y, sm.add_constant(X)).fit()
reg.summary()
y_pred = reg.predict(sm.add_constant(X))
plt.scatter(X, y)
plt.plot(X, y_pred, color='r')
plt.xlabel('mean_potential_difference')
plt.ylabel('score_difference')
plt.show()<jupyter_output><empty_output><jupyter_text>立方后取平均<jupyter_code>mean_home = home_potential.apply(lambda x: x ** 3).mean(axis=1)
mean_away = away_potential.apply(lambda x: x ** 3).mean(axis=1)
mean_diff = mean_home - mean_away
X = np.array(mean_diff).reshape(-1, 1)
reg = sm.OLS(y, sm.add_constant(X)).fit()
reg.summary()
y_pred = reg.predict(sm.add_constant(X))
plt.scatter(X, y)
plt.plot(X, y_pred, color='r')
plt.xlabel('mean_potential_difference')
plt.ylabel('score_difference')
plt.show()<jupyter_output><empty_output><jupyter_text>指数后取平均<jupyter_code>mean_home = home_potential.apply(lambda x: np.exp(x - 100)).mean(axis=1)
mean_away = away_potential.apply(lambda x: np.exp(x - 100)).mean(axis=1)
mean_diff = mean_home - mean_away
X = np.array(mean_diff).reshape(-1, 1)
reg = sm.OLS(y, sm.add_constant(X)).fit()
reg.summary()
y_pred = reg.predict(sm.add_constant(X))
plt.scatter(X, y)
plt.plot(X, y_pred, color='r')
plt.xlabel('mean_potential_difference')
plt.ylabel('score_difference')
plt.show()<jupyter_output><empty_output><jupyter_text><jupyter_code>import matplotlib    
from matplotlib.font_manager import * 
import numpy as np
import matplotlib.pyplot as plt
matplotlib.rcParams['axes.unicode_minus']=False

#=======自己设置开始============
#标签
labels = np.array(['Passing', 'Overall', 'Shot', 'Tackle', 'Physical'])
#数据个数
dataLenth = 5
#数据
data = away_score_df.iloc[471]# data = np.array([60,70,80,90,100])
labels = [labels[i] + '(' + str(round(data[i])) + ')' for i in range(5)]
#========自己设置结束============

angles = np.linspace(0, 2*np.pi, dataLenth, endpoint=False)
data = np.concatenate((data, [data[0]])) # 闭合 # #将数据结合起来
angles = np.concatenate((angles, [angles[0]])) # 闭合

fig = plt.figure()
ax = fig.add_subplot(121, polar=True)# polar参数！！121代表总行数总列数位置
ax.plot(angles, data, 'bo-', linewidth=1)# 画线四个参数为x,y,标记和颜色，闲的宽度
ax.fill(angles, data, facecolor='r', alpha=0.5)# 填充颜色和透明度
ax.set_thetagrids(angles * 180/np.pi , labels)
ax.set_rlim(0,100)
ax.grid(True)
home_score_df.iloc[471]
away_score_df.iloc[471]
data.iloc[471].home_team_id
data.iloc[471].away_team_id
data.iloc[471].home_team_goal
data.iloc[471].away_team_goal<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.154296875, 'fraction_numerical': 0.018603515625, 'mean_word_length': 4.762802476083286, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '311161', 'license': 'MIT', 'n_tokens_mistral': 8183, 'n_tokens_neox': 7803, 'n_words': 1490, 'path': 'analysis.ipynb', 'repo_name': 'sandrawing/APAC_Citadel_Datathon', 'size': '20482'}