<jupyter_start><jupyter_text>Table of contents:* [Part 1: Scrape ticker list from Wikipedia](part-1)* [Part 2: Exploring data from Wharton Data Research Services](part-2)  Part 1: scrape ticker list from Wikipedia<jupyter_code># <NAME>
# July 1 2020
# side project: predict stock price gains from company financials

# part 1) get list of companies in S and P 500
import requests
from urllib.request import urlopen
from bs4 import BeautifulSoup

# URL from which to scrape
url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'

# get HTML data from URL via requests
#stock_page = requests.get(url)

# get HTML data from URL vis urllib
stock_page = urlopen(url)

# store HTML in soup object (from reqeusts package)
#stock_page_soup = BeautifulSoup(page.content, 'html.parser')

# store HTML in soup object (from urllib package)
stock_page_soup = BeautifulSoup(stock_page, 'html.parser')

# quick save HTML to file for external viewing
#printToFile('stockPageHTML.txt',stock_page_view)

# find the table tag
stock_page_soup_table = stock_page_soup.find_all('table')
# get descriptors of features from wharton research database

url_features = 'https://wrds-web.wharton.upenn.edu/wrds/ds/compd/funda/index.cfm?navId=83'
features_page = urlopen(url_features)
features_page_soup = BeautifulSoup(features_page, 'html.parser')
# featuresPage_soup
# to look at data
import sys

def printToFile(filename,var_to_print):
    """
    quick function for printing variable to file
    printToFile(filename,var_to_print)
    """
    with open(filename,'w') as f:
        print(var_to_print, file=f)
    #...
#...
# print(printToFile.__doc__)
# print(stock_page)
# print(stock_page_soup_table)
# printToFile('stockPageTable.txt', stock_page_soup_table)
# list for ticker symbols
tickers = []

# NOTE: following tutorial here: https://towardsdatascience.com/step-by-step-tutorial-web-scraping-wikipedia-with-beautifulsoup-48d7f2dfa52d

# print(len(stock_Page_soup_table))
# print(stock_page_soup_table[1])

# loop through tables identified
counter = -1
for table in stock_page_soup_table:
    counter += 1
    
    # only work on first table; second table is for changes to s and p index
    if counter == 0 :
        rows = table.find_all('tr')

        for row in rows:
            cells = row.find_all('td')

            if len(cells) > 1:
                ticker = cells[0]
                tickers.append(ticker.text.strip())

            #...
        #...
    #...
#...

# printToFile('tickerSymbolsForSNP500.txt',tickers)
# for ticker in tickers:
#     print(ticker)
# print(counter)
# print(tickers)
# len(tickers)<jupyter_output><empty_output><jupyter_text>Part 2: exploring data from Wharton Data Research Services<jupyter_code># part 2) explore data queried from wharton data research services
import pandas as pd

# load csv file from wharton data research services for all company financials
company_financials = pd.read_csv('companyFinancials.csv')

# DtypeWarning: Columns (26,30,958,969,970) have mixed types.

# get unique gvkeys to ID each company (for accessing financial ratios for firms on WRD)
gvkeys = company_financials.gvkey.unique()
# print(len(gvkeys))
# for gvkey in gvkeys:
#     print(gvkey)
#...
# print('Variable names of features in company financials data frame')
# company_financials.columns
# for col in company_financials.columns:
#     print(col)
# load csv files from wharton data research services for financial ratios for firms
company_ratios = pd.read_csv('companyRatios.csv')

# load csv file from wharton data research services for monthly stock prices
company_prices = pd.read_csv('companyStockPrices.csv')
# company_prices_2 = pd.read_csv('companyStockPrices_2.csv')
print(company_prices.shape)
# print(company_prices_2.shape)

# remove rows with nan values
company_prices = company_prices[company_prices['COMNAM'].notna()]


# DtypeWarning: Columns (5,9) have mixed types
# company_financials.columns
print('column names for ratios df')
company_ratios.columns
print('column names for prices df')
company_prices.columns
print('company name, stock price close annual, Debt due in one year, fiscal year')
company_financials[['conm', 'prcc_c', 'dd1', 'fyear']] #,companyFinancials.conm,companyFinancials.fyear]
# check if company names are same in financials and stock prices data frames
stock_names_prices_df = company_prices.TICKER.unique()
# print(stock_names_prices_df)
# print(' ')
stock_names_financials_df = company_financials.conm.unique()
# print(sorted(stock_names_financials_df))

stock_gvkeys_ratios_df = company_ratios.gvkey.unique()
print('n companies in prices df:', len(stock_names_prices_df))
print('n companies in financials df:', len(stock_names_financials_df))
print('n companies in ratios df:', len(stock_gvkeys_ratios_df))

# test = [stock_names_prices_df == stock_names_financials_df]

company_prices[company_prices['COMNAM'].notna()].shape
company_prices.shape
company_ratios['cash_ratio']<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1010243020686885, 'fraction_numerical': 0.011648925487045592, 'mean_word_length': 5.148148148148148, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '188835', 'license': 'MIT', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1625, 'n_words': 433, 'path': 'script_SS_predictingStockPriceGains.ipynb', 'repo_name': 'sagarsetru/s_and_p500_companyFinancials', 'size': '4984'}