<jupyter_start><jupyter_text>Scrape data<jupyter_code>club_power_index = get('https://projects.fivethirtyeight.com/global-club-soccer-rankings/', 'lxml')


''' Grabbing the country names, scores and ranks '''

club_power_index = BeautifulSoup(club_power_index.text, 'html.parser')


team_list=[p.text for p in club_power_index.findAll('div', attrs={'class':'name'})] 
score_list=[p.text for p in club_power_index.findAll('td', attrs={'class':'num'}) if len(p.text)>3] 
league_list=[p.text.strip() for p in club_power_index.findAll('td', attrs={'class':'league drop-5'}) ] 
country_list=[p.text.strip() for p in club_power_index.findAll('td', attrs={'class':'country drop-1'}) ] 

team_list=team_list[:600]
score_list=score_list[:600]
league_list=league_list[:600]
country_list=country_list[:600]

club_score_df=pd.DataFrame(list(zip(score_list, league_list, country_list)),
             index=team_list, columns=["score", "league", "country"])
club_score_df.head(10)<jupyter_output><empty_output><jupyter_text>Clean raw data<jupyter_code>club_score_df.reset_index(level=0, inplace=True)
club_score_df.rename(columns={'index':'club'}, inplace=True)
pickle.dump(club_score_df[['club','score']], open('./temporary_pkl/club_score_df.pkl', 'wb'))
# keep only teams in 7 major european leagues 
# we find that Austrian top soccer league is also named Bundasliga and Russian's league named 
# Premier League. To avoid ambiguity, we use country as a selection criteria


mask2=club_score_df['country'].isin(['England', 'Germany', 'France', 'Spain', 'Italy', 'Netherlands',
       'Portugal'])

club_score_df=club_score_df[mask2]

club_score_df['score']=pd.to_numeric(club_score_df['score'])


club_score_df
# load transfer data , for future merge use
pkl_file = open('./temporary_pkl/transfer_span_reduced.pkl','rb')
transfer_span_reduced = pickle.load(pkl_file)
pkl_file.close()
club_std_name=list(transfer_span_reduced['club_name'].unique())

#club_std_name looks like this:
# ['Arsenal', 'Stoke City', 'Wolverhampton Wanderers', 'Chelsea',
#        'Newcastle United', 'Fulham', 'West Bromwich Albion',
#        'Manchester United', 'Wigan Athletic', 'Aston Villa',
#        'West Ham United', 'Everton', 'Liverpool', 'Tottenham Hotspur',
#        'Blackburn Rovers', 'Manchester City', 'Sunderland AFC',
#        'Birmingham City', 'Bolton Wanderers', 'Blackpool', 'AC Milan',
#        'AS Roma', 'Brescia Calcio', 'Internazionale', 'Parma', 'GenoaC',
#        'AS Bari', 'SS Lazio', 'Udinese Calcio', 'AFiorentina',
#        'US Palermo', 'Juventus', 'Chievo Verona', 'US Lecce', 'Bologna ',
#        'UC Sampdoria', 'Cagliari Calcio', 'AC Cesena', 'Calcio Catania',
#        'SSC Napoli', 'Girondins Bordeaux', 'AS Monaco',
#        'Olympique Marseille', '<NAME>', '<NAME>', 'Toulouse',
#        'OGC Nice', 'Paris Saint-Germain', 'AS Saint-Étienne',
#        'Sochaux-Montbéliard', 'RC Lens', 'HSC Montpellier',
#        'Olympique Lyon', 'LOSC Lille', 'Lorient', 'AS Nancy-Lorraine',
#        'SM Caen', 'Valenciennes', 'Stade Brest ',
#        'Athlétic Club Arlésien', 'SL Benfica', 'Sporting CP', 'Porto',
#        'CD Nacional', 'SC Braga', 'Vitória Setúbal', 'CS Marítimo',
#        'SC Beira-Mar', 'Vitória Guimarães SC', 'Rio Ave',
#        'União de Leiria', 'Académica Coimbra', 'Paços de Ferreira',
#        'SC Olhanense', 'Naval º de Maio', 'Portimonense SC',
#        '<NAME>', 'Bayern Munich', 'Schalke ',
#        '<NAME>', '<NAME>', 'VfB Stuttgart',
#        'Hannover ', 'VfL Wolfsburg', 'SV Werder Bremen', '.Nuremberg',
#        'Eintracht Frankfurt', 'TSG  Hoffenheim', 'SC Freiburg',
#        '.FSV Mainz ', 'Hamburger SV', '. Köln', '.Kaiserslautern',
#        'St. Pauli', 'Atlético Madrid', 'Barcelona', 'Real Zaragoza',
#        'RCD Mallorca', 'CA Osasuna', 'Sevilla', 'Real Madrid',
#        'Racing Santander', 'Real Sociedad', 'RCD Espanyol Barcelona',
#        'Deportivo de La Coruña', 'Valencia', 'Villarreal', 'Málaga',
#        'Sporting Gijón', 'UD Almería', 'Levante UD', 'Getafe', 'Hércules',
#        'Athletic Bilbao', 'NAC Breda', 'Roda JC Kerkrade', 'Utrecht',
#        'Groningen', 'Feyenoord Rotterdam', 'SC Heerenveen',
#        'Twente Enschede', 'PSV Eindhoven', 'Wil<NAME> Tilburg',
#        'NEC Nijmegen', 'Vitesse Arnhem', 'Ajax Amsterdam',
#        'De <NAME>', 'SBV Excelsior Rotterdam',
#        'AZ Alkmaar', 'ADO Den Haag', '<NAME>', 'VVV-Venlo',
#        'Queens Park Rangers', 'Norwich City', 'Swansea City',
#        'Atalanta BC', 'AC Siena', '<NAME> ', 'AC Ajaccio',
#        'FCO Dijon', '<NAME>', '<NAME>', 'CD Feirense',
#        'Augsburg', 'Hertha BSC', '<NAME>', '<NAME>',
#        'Granada', 'RKC Waalwijk', 'Southampton', 'Reading',
#        'Del<NAME> ', 'Torino', 'SC Bastia', 'ES Troyes AC',
#        'Stade Reims', 'GD Estoril', 'Moreirense', 'SpVgg Greuther Fürth',
#        '<NAME>', 'Real Valladolid', 'Celta de Vigo',
#        'PEC Zwolle', 'Cardiff City', 'Hull City', 'Crystal Palace',
#        'Hellas Verona', 'AS Livorno', 'US Sassuolo', 'EA Guingamp',
#        'Nantes', 'Belenenses SAD', 'Arouca', 'Eintracht Braunschweig',
#        'Elche', 'SC Cambuur-Leeuwarden', 'Go Ahead Eagles Deventer',
#        'Burnley', 'Leicester City', 'Empoli', 'Inter Milan', 'Metz',
#        'Penafiel', 'Boavista Porto', 'SC Paderborn ', 'SD Eibar',
#        'Córdoba', 'Dordrecht', 'ABournemouth', 'Watford', 'Carpi ',
#        '<NAME>', 'SCO Angers', 'GAjaccio', 'CD Tondela',
#        'União Madeira', 'Ingolstadt ', 'SV Darmstadt ', 'UD Las Palmas',
#        'Middlesbrough', 'Crotone', 'GD Chaves', 'RB Leipzig',
#        'Deportivo Alavés', 'CD Leganés', 'Sparta Rotterdam',
#        'Brighton & Hove Albion', 'Huddersfield Town', '<NAME>',
#        'SPAL ', 'SC Amiens', 'RC Strasbourg Alsace', 'Desportivo Aves',
#        'Girona', '<NAME> ', 'SPAL', 'Nîmes Olympique',
#        'CD Santa Clara', 'SD Huesca', 'Emmen', 'Fortuna Sittard',
#        'Sheffield United', 'Famalicão', '.Union Berlin']

from fuzzywuzzy import process
def rename(club,choices):
    if club=='PSG':
        return 'Paris Saint-Germain'
    else:
        return process.extractOne(club, choices)[0]

club_score_cleaned_df=club_score_df.copy()
club_score_cleaned_df['club']=club_score_df['club'].apply(lambda x: x if x in club_std_name else rename(x,club_std_name))



club_score_cleaned_df=club_score_cleaned_df.drop_duplicates('club')
# save the dataframe for future use
pickle.dump(club_score_cleaned_df[['club','score']], open('./temporary_pkl/club_score_df.pkl', 'wb'))
# AS there are clubs not figuring in the clubs score list, we will need to fill up their score by a reasonable value 
# Noting that they are typically small clubs not reputed, we decide to fill up their value by the min value of 
# their league minus 10
avrg_sc_by_league_dict = club_score_cleaned_df.groupby(
    by='league')['score'].min().apply(lambda x:x-10).round(3).to_dict()
avrg_sc_by_league_dict
pickle.dump(avrg_sc_by_league_dict, open('./temporary_pkl/avrg_sc_by_league_dict.pkl', 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17070750035445909, 'fraction_numerical': 0.0038281582305401958, 'mean_word_length': 4.5109375, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '590721', 'license': 'MIT', 'n_tokens_mistral': 2663, 'n_tokens_neox': 2610, 'n_words': 613, 'path': 'scrape+clean_data.ipynb', 'repo_name': 'zx-joe/Network-Analysis-of-European-Football-Transfer-Market', 'size': '7220'}