<jupyter_start><jupyter_text>Hey there! I'm writing this file because I believe Yahoo Finance recently changed their website's backend infrastructure, because of which old ways of web-scraping data such as the details of multiple stocks in an industry (Which my stock simulator makes use of) were made redundant. Thus, I had to effectively re-code all the initial scraping parts of my simulator While there isn't too much of a difference, some scraping parameters need to be updated if you want to continue to get stock data from Yahoo Finance via web-scraping, and I hope this notebook provides an avenue for y'all to do so<jupyter_code>#Importing the required libraries
import pandas as pd
from bs4 import BeautifulSoup
import requests<jupyter_output><empty_output><jupyter_text>Here's the inital code I used to scrape - circa Jan '21<jupyter_code>#Defining lists to hold the required data

prices = []
tickers = []
names = []
changes = []
percentChanges = []
marketCaps = []
totalVolumes = []
#Scraping the data and appending it to the lists

url = "https://sg.finance.yahoo.com/industries/energy"
r = requests.get(url)
data = r.text
soup = BeautifulSoup(data, features = "lxml")
 
counter = 40
for i in range(40, 404, 14):
    for row in soup.find_all('tbody'):
        for srow in row.find_all('tr'):
            for ticker in srow.find_all('td', attrs={'class':'data-col0'}):
                tickers.append(ticker.text)
            for name in srow.find_all('td', attrs={'class':'data-col1'}):
                names.append(name.text)
            for price in srow.find_all('td', attrs={'class':'data-col2'}):
                prices.append(price.text)
            for change in srow.find_all('td', attrs={'class':'data-col4'}):
                changes.append(change.text)
            for percentChange in srow.find_all('td', attrs={'class':'data-col5'}):
                percentChanges.append(percentChange.text)
            for volume in srow.find_all('td', attrs={'class':'data-col6'}):
                totalVolumes.append(volume.text)
            for marketcap in srow.find_all('td', attrs={'class':'data-col8'}):
                marketCaps.append(marketcap.text)
#Consolidating the output

output_original = pd.DataFrame({"Ticker": tickers, "Name": names, "Price": prices, "Market Cap": marketCaps, "Intra-Day Change": changes, "Percentage Change": percentChanges, "Trading Volume": totalVolumes})
output_original #Gives an empty dataframe now with the updated backend infrastructure<jupyter_output><empty_output><jupyter_text>Here's the new code I'm using - Accurate as on 4th Oct, 21<jupyter_code>#Defining lists to hold the required data

prices_n = []
tickers_n = []
names_n = []
changes_n = []
percentChanges_n = []
marketCaps_n = []
totalVolumes_n = []
#Scraping the data and appending it to the lists

url = "https://sg.finance.yahoo.com/industries/energy"
r = requests.get(url)
data = r.text
soup = BeautifulSoup(data, features = "lxml")

counter = 40
for i in range(40, 404, 14):
    for listing in soup.find_all('tr'):
        for ticker in listing.find_all('td', attrs = {'aria-label': 'Symbol'}):
            tickers_n.append(ticker.text)
        for name in listing.find_all('td', attrs = {'aria-label': 'Name'}):
            names_n.append(name.text)
        for price in listing.find_all('td', attrs = {'aria-label': 'Last price'}):
            prices_n.append(price.text)
        for change in listing.find_all('td', attrs = {'aria-label': 'Change'}):
            changes_n.append(change.text)
        for pchange in listing.find_all('td', attrs = {'aria-label': '% change'}):
            percentChanges_n.append(pchange.text)
        for cap in listing.find_all('td', attrs = {'aria-label': 'Market cap'}):
            marketCaps_n.append(cap.text)
        for volume in listing.find_all('td', attrs = {'aria-label': 'Volume'}):
            totalVolumes_n.append(volume.text)
#Consolidating the output

output_new = pd.DataFrame({"Ticker": tickers_n, "Name": names_n, "Price": prices_n, "Market Cap": marketCaps_n, "Intra-Day Change": changes_n, "Percentage Change": percentChanges_n, "Trading Volume": totalVolumes_n})
output_new #Gives a datframe complete with the desired scraped output<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12391048292108363, 'fraction_numerical': 0.007302709069493522, 'mean_word_length': 3.7600896860986546, 'pattern_counts': {'":': 14, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '345609', 'license': 'MIT', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1302, 'n_words': 392, 'path': 'How To Scrape Yahoo Finance.ipynb', 'repo_name': 'VAyush11/Python-Stock-Simulator', 'size': '4245'}