<jupyter_start><jupyter_text>Data PreparationThis notebook loads the 2018 World Cup dataset provided by StatsBomb and converts it to the [SPADL format](https://github.com/ML-KULeuven/socceraction). **Disclaimer**: this notebook is compatible with the following package versions:- tqdm 4.42.1- pandas 1.0- socceraction 0.1.1<jupyter_code>import os; import sys
from tqdm.notebook import tqdm

import math
import pandas as pd

import socceraction.spadl as spadl
import socceraction.spadl.statsbomb as statsbomb<jupyter_output><empty_output><jupyter_text>Configure leagues and seasons to download and convertThe two dictionaries below map my internal season and league IDs to Statsbomb's IDs. Using an internal ID makes it easier to work with data from multiple providers.<jupyter_code>seasons = {
    3: '2018',
}
leagues = {
    'FIFA World Cup': 'WC',
}<jupyter_output><empty_output><jupyter_text>Configure folder names and download URLsThe two cells below define the URLs from where the data are downloaded and were data is stored.<jupyter_code>free_open_data_remote = "https://raw.githubusercontent.com/statsbomb/open-data/master/data/"
spadl_datafolder = "../data/statsbomb_opensource"
raw_datafolder = f"../data/statsbomb_opensource/raw"

# Create data folder if it doesn't exist
for d in [raw_datafolder, spadl_datafolder]:
    if not os.path.exists(d):
        os.makedirs(d, exist_ok=True)
        print(f"Directory {d} created ")<jupyter_output>Directory ../data/statsbomb_opensource/raw created<jupyter_text>Set up the statsbombloader<jupyter_code>SBL = statsbomb.StatsBombLoader(root=free_open_data_remote, getter="remote")<jupyter_output><empty_output><jupyter_text>Select competitions to load and convert<jupyter_code># View all available competitions
df_competitions = SBL.competitions()
set(df_competitions.competition_name)
df_selected_competitions = df_competitions[df_competitions.competition_name.isin(
    leagues.keys()
)]

df_selected_competitions<jupyter_output><empty_output><jupyter_text>Convert to the SPADL format<jupyter_code>for competition in df_selected_competitions.itertuples():
    # Get matches from all selected competition
    matches = SBL.matches(competition.competition_id, competition.season_id)

    matches_verbose = tqdm(list(matches.itertuples()), desc="Loading match data")
    teams, players, player_games = [], [], []
    
    competition_id = leagues[competition.competition_name]
    season_id = seasons[competition.season_id]
    spadl_h5 = os.path.join(spadl_datafolder, f"spadl-statsbomb_opensource-{competition_id}-{season_id}.h5")
    with pd.HDFStore(spadl_h5) as spadlstore:
        
        spadlstore["actiontypes"] = spadl.actiontypes_df()
        spadlstore["results"] = spadl.results_df()
        spadlstore["bodyparts"] = spadl.bodyparts_df()
        
        for match in matches_verbose:
            # load data
            teams.append(SBL.teams(match.match_id))
            players.append(SBL.players(match.match_id))
            events = SBL.events(match.match_id)

            # convert data
            player_games.append(statsbomb.extract_player_games(events))
            spadlstore[f"actions/game_{match.match_id}"] = statsbomb.convert_to_actions(events,match.home_team_id)

        games = matches.rename(columns={"match_id": "game_id", "match_date": "game_date"})
        games.season_id = season_id
        games.competition_id = competition_id
        spadlstore["games"] = games
        spadlstore["teams"] = pd.concat(teams).drop_duplicates("team_id").reset_index(drop=True)
        spadlstore["players"] = pd.concat(players).drop_duplicates("player_id").reset_index(drop=True)
        spadlstore["player_games"] = pd.concat(player_games).reset_index(drop=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10188378880339613, 'fraction_numerical': 0.005571769700185725, 'mean_word_length': 4.872274143302181, 'pattern_counts': {'":': 2, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '495635', 'license': 'Apache-2.0', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1225, 'n_words': 242, 'path': 'notebooks/1-load-and-convert-statsbomb-data.ipynb', 'repo_name': 'ML-KULeuven/soccer_xg', 'size': '3769'}