<jupyter_start><jupyter_text>Scraping the Nintendo Switch game list from Wikipedia---<jupyter_code>import requests
from bs4 import BeautifulSoup
from tqdm import tqdm

# link urls and corresponding table classes
urls = [('https://en.wikipedia.org/wiki/List_of_Nintendo_Switch_games_(A–L)',
         'wikitable sortable plainrowheaders'),
        ('https://en.wikipedia.org/wiki/List_of_Nintendo_Switch_games_(M–Z)',
         'wikitable plainrowheaders sortable')]

elements = []
for url, url_class in urls:
    
    #a access the web
    website_url = requests.get(url).text

    soup = BeautifulSoup(website_url,'lxml')
    #print(soup.prettify())

    # find table
    table = soup.find('table', class_=url_class)

    for row in tqdm(table.findAll('tr')):
        tqdm._instances.clear()

        # only find game names with links to their own pages
        try:
            game = row.findAll('th')
            game_name = game[0].find(text=True)
            game_link = game[0].find(href=True)['href']
        
        except: pass

        # find rest of cells
        else:
            cells = row.findAll('td')
            atts = []
            for i in range(len(cells)-1):
                att = cells[i].find(text=True)
                atts.append(att)
            
            # append full row to list    
            elements.append([game_name, game_link] + atts)
import pandas as pd

cols = ['Title', 'Link', 'Genre', 'Developer', 'Publisher', 'Release_JP', 
        'Release_NA', 'Release_Pal']

df_games = pd.DataFrame(elements, columns=cols)
df_games = df_games.astype(str)

# clean links 
df_games = df_games.applymap(lambda x: x.replace('\n', ''))
df_games = df_games.applymap(lambda x: x.replace(':', ''))


print('Shape of dataframe: ', df_games.shape, '\n')
df_games.head()
# Visit game own pages and extract plots

wiki_url = 'https://en.wikipedia.org'

plots = []
for idx, row in tqdm(df_games.iterrows(), total=df_games.shape[0]):
    tqdm._instances.clear()
    
    url = wiki_url + row['Link']
    
    website_url = requests.get(url).text

    soup = BeautifulSoup(website_url,'lxml')
    #print(soup.prettify())

    text = ''
    
    for section in soup.find_all('h2'):
        
        if section.text.startswith('Game') or section.text.startswith('Plot'):

            text += section.text + '\n\n'

            for element in section.next_siblings:
                if element.name and element.name.startswith('h'):
                    break

                elif element.name == 'p':
                    text += element.text + '\n'

        else: pass
    
    if not text:
        plots.append(None)
    else:
        plots.append(text)
len(plots)
# Clean texts

import re

plots_clean = []
for text in plots:
    if text is not None:
        text = re.sub(r'\[.*?\]+', '', text)
        text = text.replace('\n', ' ')
        text = text.replace('Gameplay ', '')
        text = text.replace('Game-play ', '')
        text = text.replace('Plot ', '')
        plots_clean.append(text)
    else:
        plots_clean.append(None)

df_games['Plots'] = plots_clean

df_games.head()
# Drop 'Untitled' games
idx_todrop = df_games[df_games.Title=='Untitled '].index.tolist()
df_games.drop(index=idx_todrop, inplace=True)

# Rename cols
rename = {'Release_JP': 'Released in: Japan', 'Release_NA': 'North America', 
 'Release_Pal': 'Rest of countries'}
df_games.rename(columns=rename, inplace=True)
df_games.info()
df_games.dropna().to_csv('datasets/Games_dataset.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13929078014184398, 'fraction_numerical': 0.001702127659574468, 'mean_word_length': 2.6652806652806653, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '157240', 'license': 'MIT', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1150, 'n_words': 251, 'path': 'Switch_games.ipynb', 'repo_name': 'hectoramirez/RecommendationSystem', 'size': '3525'}