<jupyter_start><jupyter_text>Mission Objective > * establish a connection with the target url> * extract the content of the target url (extract the html of that page we made a request to)> * parse that html content using the BeautifulSoup class that we imported<jupyter_code>def scrape_yahoo(url: str):
    resp = requests.get(url)
    bsoup_object = BeautifulSoup(resp.text, "lxml")
    
    pre_frame = {
        "ticker": [][:30], # make sure to shrink the column to having exactly 30 pieces of data so that no size/shape error occurs
        "name": [][:30],
        "last_price": [][:30],
        "market_time": [][:30],
        "change": [][:30],
        "change_percent": [][:30],
        "volume": [][:30],
        "market_cap": [][:30]
    }
    
    for row in bsoup_object.select('tr'):
        try:
            pre_frame["ticker"].append(row.select('td')[0].text)
            pre_frame["name"].append(row.select('td')[1].text)
            pre_frame["last_price"].append(row.select('td')[2].text)
            pre_frame["market_time"].append(row.select('td')[3].text)
            pre_frame["change"].append(row.select('td')[4].text)
            pre_frame["change_percent"].append(row.select('td')[5].text)
            pre_frame["volume"].append(row.select('td')[6].text)
            pre_frame["market_cap"].append(row.select('td')[7].text)
            
        except:
            continue 
            
    df = pd.DataFrame(pre_frame) # index=False we don't want an additional useless column made which has index numbers
    df.to_csv("finance.csv", index=False)
    
    return f"Exported {len(df)} rows of data"
# if __name__ == "__main__":
#     while True:
#         scrape_yahoo("https://finance.yahoo.com/trending-tickers")
#         time.sleep(10)
scrape_yahoo("https://finance.yahoo.com/trending-tickers")<jupyter_output><empty_output><jupyter_text>Data Cleansing<jupyter_code>import pandas as pd
data_frame = pd.read_csv("finance.csv")
data_frame.head()
data_frame["change"] = data_frame["change"].str.replace("+", "")
data_frame["change"] = data_frame["change"].str.replace(",", "")
data_frame["change"] = data_frame["change"].str.replace("-", "")
data_frame.info()
data_frame["change"] = data_frame["change"].astype(float)
data_frame["last_price"] = data_frame["last_price"].str.replace(",", "").astype(float)
data_frame.info()
data_frame.drop("change_percent", axis=1, inplace=True)
data_frame.head()
data_frame["volume"] = data_frame["volume"].str.replace("M", "")
data_frame["volume"] = data_frame["volume"].str.replace("B", "")
data_frame["volume"] = data_frame["volume"].str.replace(",", "").astype(float)
data_frame.head()
data_frame["market_cap"] = data_frame["market_cap"].str.replace("M", "")
data_frame["market_cap"] = data_frame["market_cap"].str.replace("B", "")
data_frame["market_cap"] = data_frame["market_cap"].str.replace(",", "").astype(float)
data_frame.head()
data_frame.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 30 entries, 0 to 29
Data columns (total 7 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   ticker       30 non-null     object 
 1   name         30 non-null     object 
 2   last_price   30 non-null     float64
 3   market_time  30 non-null     object 
 4   change       30 non-null     float64
 5   volume       30 non-null     float64
 6   market_cap   30 non-null     float64
dtypes: float64(4), object(3)
memory usage: 1.8+ KB
{'fraction_non_alphanumeric': 0.17334871646957023, 'fraction_numerical': 0.020190366310931643, 'mean_word_length': 3.486416558861578, 'pattern_counts': {'":': 9, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '387356', 'license': 'MIT', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1132, 'n_words': 226, 'path': 'yahoo_scrape.ipynb', 'repo_name': 'wareaglee/yahoo-finance-scrape', 'size': '3467'}