<jupyter_start><jupyter_text>Getting and analysing data from the S&P 500 Index<jupyter_code>import bs4 as bs
import datetime as dt
import os
import pandas as pd
import pandas_datareader as web
import pickle
import requests<jupyter_output><empty_output><jupyter_text>Scraping the wikipedia page for the tickers<jupyter_code>def save_sp500_tickers():
    resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, 'lxml')
    table = soup.find('table', {'class': 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text
        tickers.append(ticker)
        
    with open("sp500tickers.pickle","wb") as f:
        pickle.dump(tickers,f)
        
    return tickers
save_sp500_tickers()<jupyter_output><empty_output><jupyter_text>Getting all the data from the individual companies (!)<jupyter_code>def data_from_yahoo(reload_sp500=True):
    if reload_sp500:
        tickers = save_sp500_tickers()
    else:
        with open("sp500tickers.pickle","rb") as f:
            tickers = pickle.load(f)
    
    if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')    
        
    start = dt.datetime(2000,1,1)
    finish = dt.datetime(2018, 9, 17)
    
    for ticker in tickers:
        if ticker.find('.'):
            ticker = ticker.replace('.','-')
        print(ticker)
        if not os.path.exists('stock_dfs/{}.csv'.format(ticker)):
            df = web.DataReader(ticker, 'yahoo', start, finish)
            df.to_csv('stock_dfs/{}.csv'.format(ticker))
        else:
            print('Already have {}'.format(ticker))
            
data_from_yahoo()
def compile_data():
    with open("sp500tickers.pickle", "rb") as f:
        tickers = pickle.load(f)

    main_df = pd.DataFrame()

    for count,ticker in enumerate(tickers):
        
        if ticker.find('.'):
            ticker = ticker.replace('.','-')
            
        df = pd.read_csv('stock_dfs/{}.csv'.format(ticker))
        df.set_index('Date', inplace=True)

        df.rename(columns={'Adj Close': ticker}, inplace=True)
        df.drop(['Open', 'High', 'Low', 'Close', 'Volume'], 1, inplace=True)

        if main_df.empty:
            main_df = df
        else:
            main_df = main_df.join(df, how='outer')

        if count % 10 == 0:
            print(count)
            
    print(main_df.head())
    main_df.to_csv('sp500_joined_closes.csv')<jupyter_output><empty_output><jupyter_text>So, this below should be the way to continue this code, but it always crashes because of memory error. Will try o fix later.<jupyter_code>compile_data()
main_df = pd.read_csv('sp500_joined_closes.csv')
main_df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13405797101449277, 'fraction_numerical': 0.021014492753623187, 'mean_word_length': 2.933048433048433, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '666705', 'license': 'MIT', 'n_tokens_mistral': 1000, 'n_tokens_neox': 940, 'n_words': 191, 'path': 'sp500.ipynb', 'repo_name': 'joaopppobr/FinancialPython', 'size': '2760'}