<jupyter_start><jupyter_text>Grabbing pricing information en masse for a larger list of companies.  Python list of the S&P 500 companies.<jupyter_code># BeautifulSoup basically turns source code into a BeautifulSoup object that suddenly can be treated 
# much more like a typical Python object.
import bs4 as bs
import pickle
import requests

def save_sp500_tickers():
    resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, 'lxml')
    table = soup.find('table', {'class': 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text
        tickers.append(ticker)
        
    with open("sp500tickers.pickle","wb") as f:
        pickle.dump(tickers,f)
        
    return tickers

save_sp500_tickers()<jupyter_output><empty_output><jupyter_text>Pulling stock pricing data of all companies.<jupyter_code>import bs4 as bs
import datetime as dt
import os
import pandas_datareader.data as web
import pickle
import requests
# save_sp500_tickers()
def get_data_from_yahoo(reload_sp500=False):
    if reload_sp500:
        tickers = save_sp500_tickers()
    else:
        with open("sp500tickers.pickle", "rb") as f:
            tickers = pickle.load(f)
  if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')
 start = dt.datetime(2010, 1, 1)
 end = dt.datetime.now()
 for ticker in tickers:
        # just in case your connection breaks, we'd like to save our progress!
        if not os.path.exists('stock_dfs/{}.csv'.format(ticker)):
            df = web.DataReader(ticker, 'morningstar', start, end)
            df.reset_index(inplace=True)
            df.set_index("Date", inplace=True)
            df = df.drop("Symbol", axis=1)
            df.to_csv('stock_dfs/{}.csv'.format(ticker))
        else:
            print('Already have {}'.format(ticker))
            
get_data_from_yahoo()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11371749107598164, 'fraction_numerical': 0.021927587965323816, 'mean_word_length': 3.3697104677060135, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '256796', 'license': 'MIT', 'n_tokens_mistral': 684, 'n_tokens_neox': 641, 'n_words': 159, 'path': 'Python_in_finance4.ipynb', 'repo_name': 'jairajc/Python-in-Finance', 'size': '1961'}