<jupyter_start><jupyter_text>Import Data - FIFA 2018 Player Dataset<jupyter_code>import pandas as pd
data = pd.read_csv('CompleteDataset.csv', low_memory=False)
data.head()<jupyter_output><empty_output><jupyter_text>Cleaning Data Feature Noise<jupyter_code>columns_needed = data.drop(['Unnamed: 0','Age','Photo', 'Flag', 'Club Logo', 'CAM','CB','CDM','CF','CM','ID','LAM','LB','Preferred Positions','RAM','LCB','LCM','LDM','LF','LM','LS','LW','LWB', 'RB', 'RCB', 'RCM', 'RDM', 'RF', 'RM', 'RS', 'RW', 'RWB', 'ST'], axis=1)

def handle_calculations(digit_str):
    cell = digit_str.split('+')
    if len(cell) == 1:
        cell = digit_str.split('-')
        cell = list(map(int, cell))
        return cell[0] - cell[1]
    else:
        cell = list(map(int, cell))
        return sum(cell)

# Handling attribute noise e.g. 71+9 for a certain skill

pd.options.mode.chained_assignment = None
none_essential_features = ['Name','Overall','Nationality', 'Potential','Club', 'Value', 'Wage', 'Special']
features = [feature for feature in columns_needed.columns.tolist() if feature not in none_essential_features]

for feature in features:
    columns_needed[feature] = columns_needed[feature].apply(lambda x: handle_calculations(x) if len(x) > 2 else x)<jupyter_output><empty_output><jupyter_text>Duplicates<jupyter_code>columns_needed[columns_needed.duplicated()]
columns_needed_no_duplicates = columns_needed.drop_duplicates() # Removing exact duplicates, default keeping first entry
columns_needed_no_duplicates.groupby(['Name', 'Nationality', 'Club']).size()<jupyter_output><empty_output><jupyter_text>Duplicates Partial<jupyter_code>columns_needed_no_duplicates = columns_needed_no_duplicates.drop_duplicates(subset=['Name', 'Nationality', 'Club']) # Removing partial duplicates
columns_needed_no_duplicates.groupby(['Name', 'Nationality', 'Club']).size()<jupyter_output><empty_output><jupyter_text>Change Data type<jupyter_code>columns_needed_no_duplicates.dtypes
columns_needed_no_duplicates[features] = columns_needed_no_duplicates[features].apply(pd.to_numeric, errors='coerce',downcast='integer',axis=1)
columns_needed_no_duplicates.dtypes<jupyter_output><empty_output><jupyter_text>Label Noise<jupyter_code>def convert_currency(currency_str):
    currency_str = currency_str.replace("€","").replace("K","000").replace("M","000000")
    if "." not in currency_str:
        return currency_str
    else:
        currency_str = currency_str.replace(".","")
        currency_str = currency_str[:-1]
        return currency_str

columns_needed_no_duplicates['Value'] = columns_needed_no_duplicates['Value'].apply(convert_currency).apply(pd.to_numeric, errors='coerce',downcast='integer')
columns_needed_no_duplicates['Wage'] = columns_needed_no_duplicates['Wage'].apply(convert_currency).apply(pd.to_numeric, errors='coerce',downcast='integer')
columns_needed_no_duplicates
columns_needed_no_duplicates.apply(lambda x: (x != 0).all()) # Label noise in Value and Wage!
# Domain knowledge: If a player doesn't belong to a club the player has no Value or Wage
# Hence 0 in Wage and Value must be removed
columns_needed_no_duplicates[pd.isnull(columns_needed_no_duplicates).any(axis=1)]
columns_needed_no_duplicates_final = columns_needed_no_duplicates.drop(['Name', "Nationality", "Club"], axis=1)
columns_needed_no_duplicates_wage = columns_needed_no_duplicates_final.drop(['Value'], axis=1)
columns_needed_no_duplicates_value = columns_needed_no_duplicates_final.drop(['Wage'], axis=1)
print(columns_needed_no_duplicates_wage[columns_needed_no_duplicates_wage.Wage == 0].shape)
print(columns_needed_no_duplicates_wage.shape)
print(columns_needed_no_duplicates_value[columns_needed_no_duplicates_value.Value == 0].shape)
print(columns_needed_no_duplicates_value.shape)
columns_needed_no_duplicates_wage_clean = columns_needed_no_duplicates_wage[columns_needed_no_duplicates_wage.Wage != 0]
columns_needed_no_duplicates_value_clean = columns_needed_no_duplicates_value[columns_needed_no_duplicates_value.Value != 0]
print(columns_needed_no_duplicates_wage_clean.shape)
print(columns_needed_no_duplicates_value_clean.shape)
columns_needed_no_duplicates_wage_clean
columns_needed_no_duplicates_value_clean<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
def plot(x, y):
    axis = plt.scatter(x, y)
    zed = np.polyfit(x, y, 1)
    poly = np.poly1d(zed)
    plt.plot(x, poly(x), "#ff7f0e")
    plt.show()
    
plot(x=columns_needed_no_duplicates_value_clean['Overall'], y=columns_needed_no_duplicates_value_clean['Value'])
plot(x=columns_needed_no_duplicates_wage_clean['Overall'], y=columns_needed_no_duplicates_wage_clean['Wage'])
columns_needed_no_duplicates_wage_clean['LoggedWage'] = np.log(columns_needed_no_duplicates_wage_clean['Wage'])
columns_needed_no_duplicates_value_clean['LoggedValue'] = np.log(columns_needed_no_duplicates_value_clean['Value'])
plot(x=columns_needed_no_duplicates_value_clean['Overall'], y=columns_needed_no_duplicates_value_clean['LoggedValue'])
plot(x=columns_needed_no_duplicates_wage_clean['Overall'], y=columns_needed_no_duplicates_wage_clean['LoggedWage'])
columns_needed_no_duplicates_wage_clean.to_csv("player_set_wage.csv")
columns_needed_no_duplicates_value_clean.to_csv("player_set_value.csv")
import seaborn as sns

corr_value = columns_needed_no_duplicates_value_clean.corr()

fig, ax = plt.subplots(figsize=(10,10))
ax.set_title("Skill Correlation - Value")
sns.heatmap(corr_value, 
            xticklabels=corr_value.columns.values,
            yticklabels=corr_value.columns.values, ax=ax)

corr_wage = columns_needed_no_duplicates_wage_clean.corr()
fig, ax = plt.subplots(figsize=(10,10))
ax.set_title("Skill Correlation - Wage")
sns.heatmap(corr_wage, 
            xticklabels=corr_wage.columns.values,
            yticklabels=corr_wage.columns.values, ax=ax)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1250845165652468, 'fraction_numerical': 0.00777552400270453, 'mean_word_length': 8.87813021702838, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '38090', 'license': 'MIT', 'n_tokens_mistral': 2215, 'n_tokens_neox': 2077, 'n_words': 278, 'path': 'FIFA 2018 Player Dataset Handling.ipynb', 'repo_name': 'bragner/FIFA18-player-predictions', 'size': '5916'}