<jupyter_start><jupyter_text>Scraping live betting odds from [flashscore.pl/tenis](https://www.flashscore.pl/tenis/)<jupyter_code>import numpy as np
import pandas as pd
from bs4 import BeautifulSoup as BS
import re
import requests
import json
from datetime import datetime
from datetime import timedelta
pd.set_option('display.max_rows', 200)
pd.set_option('display.max_columns', 40)
# Prepare a data frame
data = pd.DataFrame(columns = ['event', 'match_time', 'player_1', 'player_2', 
                               'player_1_score_sets', 'player_1_score_games', 'player_1_score_points',
                               'player_2_score_sets', 'player_2_score_games', 'player_2_score_points', 
                               'serving',
                               'player_1_nationality', 'player_2_nationality', 
                               'player_1_rank', 'player_2_rank',
                               'player_1_link', 'player_2_link',
                               'player_1_eFortuna', 'player_2_eFortuna',
                               'player_1_STS', 'player_2_STS',
                               'player_1_Betclic', 'player_2_Betclic',
                               'player_1_Betfan', 'player_2_Betfan',
                               'player_1_Pzbuk', 'player_2_Pzbuk',
                               'player_1_Lvbet', 'player_2_Lvbet',
                               'player_1_Totolotek', 'player_2_Totolotek'
                              ])

# Retrieve data file for the main page for today's matches, including current scores
headers = {"Accept":  "*/*",
           "Accept-Encoding": "gzip, deflate, br",
           "Accept-Language": "pl,en-US;q=0.7,en;q=0.3",
           "Connection": "keep-alive",
           "Host": "d.flashscore.pl",
           "Referer": "https://d.flashscore.pl/x/feed/proxy-fetch",
           "TE": "Trailers",
           "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0",
           "x-fsign": "SW9D1eZo"
          }

s = requests.Session()

main_page = s.get('https://d.flashscore.pl/x/feed/f_2_0_2_pl_1', 
                        headers = headers)

# Split the data for the matches (Note: the 1st element of `matches` will not be a match, but we do not worry, 
# because this element will be filtered out in the next step.)
matches = re.split('¬~AA÷', main_page.text)

# It seems that that the ended/cancelled/interrupted matches contain the tag '¬AB÷3', the current ones - '¬AB÷2', 
# and the future ones - '¬AB÷1'. We will keep only those matches, for which there are active/future bets.
matches = [match for match in matches if '¬AB÷1' in match or '¬AB÷2' in match]

for match in matches:
    
    new_row = {}
    
    # Retrieve match ID that will be used for retriving detailed information for the match (betting odds)
    match_id = re.search('.*?¬AD÷', match).group()[:-4]
    url = 'https://www.flashscore.pl/mecz/' + match_id + '/#zestawienie-kursow/home-away/koniec-meczu'
    
    # Retrieve match time
    # Add 2 hours (time zone difference).
    match_time = re.search('¬AD÷.*?¬ADE÷', match).group()[4:-5]
    match_time = datetime.utcfromtimestamp(int(match_time))
    match_time = match_time + timedelta(hours = 2)
    new_row['match_time'] = match_time.strftime('%d-%m-%Y %H:%M:%S')
    
    # Retrieve current match score from the file (if match not started, NoType object is found)
    player_1_score = re.search('¬AG÷.*?¬OA÷', match)
    player_2_score = re.search('¬AH÷.*?¬OB÷', match)
    
    # Strip the score from unnecesary symbols and split it into sets, games and points
    if player_1_score:
        player_1_score = player_1_score.group()[4:-4]
        # (Possibly remove the part of the score accounting for tie-break points.)
        player_1_score = re.sub('¬D.÷\d+', '', player_1_score)
        player_1_score = re.split('¬.*?÷', player_1_score)
        new_row['player_1_score_sets'] = player_1_score[0]
        new_row['player_1_score_games'] = ' '.join(map(str, player_1_score[1:-1]))
        new_row['player_1_score_points'] = player_1_score[-1]

    else:
        new_row['player_1_score_sets'] = 0
        new_row['player_1_score_games'] = ''
        new_row['player_1_score_points'] = ''

    if player_2_score:
        player_2_score = player_2_score.group()[4:-4]
        player_2_score = re.sub('¬D.÷\d+', '', player_2_score)
        player_2_score = re.split('¬.*?÷', player_2_score)
        new_row['player_2_score_sets'] = player_2_score[0]
        new_row['player_2_score_games'] = ' '.join(map(str, player_2_score[1:-1]))
        new_row['player_2_score_points'] = player_2_score[-1]

    else:
        new_row['player_2_score_sets'] = 0
        new_row['player_2_score_games'] = ''
        new_row['player_2_score_points'] = ''
        
    # Retrieve information on who is serving right now. It seems that '¬WC÷1' signifies that it is the 1st player,
    # and '¬WC÷2' - the 2nd player.
    if '¬WC÷1' in match:
        new_row['serving'] = 'player_1'
    elif '¬WC÷2' in match:
        new_row['serving'] = 'player_2'
    else:
        new_row['serving'] = np.nan
        
    try:
        html = s.get(url)
        match_page = BS(html.content, 'html.parser')
        
        # Retrieve the event title from the the match's site
        new_row['event'] = match_page.head.find_all('meta')[5]['content']
        
        # Retrieve players' names
        players = match_page.head.find_all('meta')[4]['content']
        players = re.split(' - ', players)
        new_row['player_1'] = players[0]
        new_row['player_2'] = players[1]
        
        # Retrieve a JSON part containing some details on the players
        script = match_page.find_all('script')[1].string
        script = re.search('\{.*\}', script).group()
        details = json.loads(script)
        
        details_player_1 = details['participantsData']['home']
        details_player_2 = details['participantsData']['away']
        
        # Retrieve players' nationality
        # In the case of doubles, seprate the information by ' / '.
        new_row['player_1_nationality'] = ' / '.join([player['country'] for player in details_player_1])
        new_row['player_2_nationality'] = ' / '.join([player['country'] for player in details_player_2])

        # Retrieve players' rankings
        # In the case of doubles, seprate the information by ' / '.
        new_row['player_1_rank'] = ' / '.join([player['rank'][1] if player['rank']!=[] else '' for player \
                                               in details_player_1])
        new_row['player_2_rank'] = ' / '.join([player['rank'][1] if player['rank']!=[] else '' for player \
                                               in details_player_2])
        
        # Retrieve links to players' pages on flashscore.pl
        # In the case of doubles, seprate the information by ' / '.
        new_row['player_1_link'] = ' / '.join(['https://www.flashscore.pl' + player['detail_link'] for player \
                                               in details_player_1]) 
        new_row['player_2_link'] = ' / '.join(['https://www.flashscore.pl' + player['detail_link'] for player \
                                               in details_player_2])

        
    except:
        new_row['event'] = np.nan
        # (For players' names and nationalities we can use another method of retrieving this information, 
        # not using the match_page, but the main_page).
        details_player_1 = re.search('¬AE÷.*?¬JA÷', match).group()[4:-4]
        details_player_2 = re.search('¬AF÷.*?¬JB÷', match).group()[4:-4]
        details_player_1 = re.split(' \(', details_player_1)
        details_player_2 = re.split(' \(', details_player_2)
        
        new_row['player_1'] = details_player_1[0]
        new_row['player_2'] = details_player_2[0]
        # (We need to distinguish singles - players' nationalities is given on the main page - and doubles
        # - nationalities not given).
        if len(details_player_1)==2:
            new_row['player_1_nationality'] = details_player_1[1][:-1]
        else:
            new_row['player_1_nationality'] = np.nan
        if len(details_player_2)==2:
            new_row['player_2_nationality'] = details_player_2[1][:-1]
        else:
            new_row['player_2_nationality'] = np.nan
        
        new_row['player_1_rank'] = np.nan
        new_row['player_2_rank'] = np.nan
        new_row['player_1_link'] = np.nan
        new_row['player_2_link'] = np.nan
        
        
    try:
        # Retrieve a file including betting odds from the match's site
        match_odds = s.get('https://d.flashscore.pl/x/feed/df_od_1_' + match_id, headers = headers)

        # Retrieve the part of data including end of match betting odds
        match_odds = re.search('(home-away).*?(Set 1)', match_odds.text).group()
        
        # It seems that invalid (crossed-out) odds are those that end up with '¬OG÷0', and the valid ones
        # end up with '¬OG÷1'.
        match_odds = re.findall('¬OD÷.*?¬OG÷.', match_odds)
        valid_odds = ''.join([o for o in match_odds if o[-1]=='1'])

        # Retrieve the lists of the bookmakers' names and the odds
        bookmakers = re.findall('¬OD.*?¬OPI', valid_odds)
        player_1_odds = re.findall('¬XB÷.*?¬XC', valid_odds) 
        player_2_odds = re.findall('¬XC÷.*?¬OG', valid_odds)
        
        # Strip the bookmakers names from unnecessary symbols
        bookmakers = [i[4:-4] for i in bookmakers]
        
        # Strip the odds data from unnecessary symbols, split on the last change of the odds, 
        # and take the last value
        player_1_odds = [re.split('\[.\]', i[4:-3])[-1] for i in player_1_odds]
        player_2_odds = [re.split('\[.\]', i[4:-3])[-1] for i in player_2_odds]
        
        # Assign betting odds for 7 possible bookmakers, for both players.
        # (Regex is used to produce an appropriate form of a variable (column name).)
        for bookmaker in ['eFortuna.pl', 'STS.pl', 'Betclic.pl', 'Betfan.pl', 
                          'Lvbet.pl', 'Pzbukpl', 'Totolotek.pl']:
            
            try:
                new_row['player_1_' + re.search('\w*(?=[\.p])', bookmaker).group()] = \
                player_1_odds[bookmakers.index(bookmaker)]
            except:
                new_row['player_1_' + re.search('\w*(?=[\.p])', bookmaker).group()] = np.nan

            try:
                new_row['player_2_' + re.search('\w*(?=[\.p])', bookmaker).group()] = \
                player_2_odds[bookmakers.index(bookmaker)]
            except:
                new_row['player_2_' + re.search('\w*(?=[\.p])', bookmaker).group()] = np.nan
            
    except:
        
        for bookmaker in ['eFortuna.pl', 'STS.pl', 'Betclic.pl', 'Betfan.pl', 
                          'Lvbet.pl', 'Pzbukpl', 'Totolotek.pl']:
            
            new_row['player_1_' + re.search('\w*(?=[\.p])', bookmaker).group()] = np.nan
            new_row['player_2_' + re.search('\w*(?=[\.p])', bookmaker).group()] = np.nan
            
    data = data.append(new_row, ignore_index = True)

scraping_time = datetime.now()
data
data.dtypes
# Hand-pick non-numeric columns
character_columns = ['event', 'match_time', 'player_1', 'player_2', 'player_1_score_games', 'player_2_score_games',
                    'player_1_score_points', 'player_2_score_points', 'serving', 'player_1_nationality',
                    'player_2_nationality', 'player_1_rank', 'player_2_rank', 'player_1_link', 'player_2_link']

# Note that `player_1_score_games` (and `player_2_score_games`) are not integers, because it contains games won
# in the subsequent sets, separated by a space, and `player_1_score_points` (and `player_2_score_points`) is not
# an integer column either, because it may contain 'A' (advantage).
# `player_1_rank` and `player_2_rank` are also not integers, since for doubles they contain an expression 'x / y'.

# Convert numberic (float/integer) columns to their proper types
for col in data.drop(character_columns, axis = 1).columns:
    data[col] = pd.to_numeric(data[col], downcast = 'integer')
data.dtypes
# Export the data, naming the file after the time scraping was finished
scraping_time = scraping_time.strftime("%d%m%y_%H%M%S")
data.to_csv('../data/data_' + scraping_time + '.csv', index = False)
data.to_csv('../data/data_current.csv', index = False)
# Note: the retrieved match result would not be correct for the matches that are finished. However, since we are 
# interested in the matches that one can bet, we filter out the ended matches.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13633481293595434, 'fraction_numerical': 0.01894419784400761, 'mean_word_length': 2.481512141280353, 'pattern_counts': {'":': 9, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 7, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '284724', 'license': 'MIT', 'n_tokens_mistral': 4152, 'n_tokens_neox': 4049, 'n_words': 950, 'path': 'src/Scraping_flashscore_tennis.ipynb', 'repo_name': 'MichalSalach/RR_Project', 'size': '12616'}