<jupyter_start><jupyter_text>Algo Trading Notebook  Initiation<jupyter_code>!pip freeze
import datetime as dt
import os
import matplotlib.pyplot as plt
from matplotlib import style
from matplotlib.finance import candlestick_ohlc
import matplotlib.dates as mdates
import pandas as pd
import pandas_datareader.data as web 
style.use('ggplot')
start = dt.datetime(2000,1,1)
end = dt.datetime(2019,1,1)
df = pd.read_csv('tsla.csv', parse_dates=True, index_col=0)<jupyter_output><empty_output><jupyter_text>Data manipulation - 100 ma<jupyter_code># 100 moving average
df['100ma'] = df['Adj Close'].rolling(window=100).mean()
df.dropna(inplace=True)
df[['100ma', 'Adj Close']].plot()
plt.show()

# 2 sub graphs one displaying 2 lines on 1 graph with vol below it
ax1 = plt.subplot2grid((6, 1), (0, 0), rowspan=5, colspan=1)
ax2 = plt.subplot2grid((6, 1), (5, 0), rowspan=1, colspan=1, sharex = ax1)
ax1.plot(df.index, df['Adj Close'])
ax1.plot(df.index, df['100ma'])
ax2.bar(df.index, df['Volume'])

plt.show()<jupyter_output><empty_output><jupyter_text>Data manipulation - resampling<jupyter_code># open high low close
df_ohlc = df['Adj Close'].resample('10D').ohlc()
df_volume = df['Volume'].resample('10D').sum()

print(df_ohlc.head())
# creating a candlestick ohlc
df_ohlc.reset_index(inplace=True)
# converting dates to mdates i.e. really big numbers
df_ohlc['Date'] = df_ohlc['Date'].map(mdates.date2num)
ax1 = plt.subplot2grid((6, 1), (0, 0), rowspan=5, colspan=1)
ax2 = plt.subplot2grid((6, 1), (5, 0), rowspan=1, colspan=1, sharex = ax1)

# create the candlestick graph
candlestick_ohlc(ax1, df_ohlc.values, width=2, colorup='g')

# fill from 0 to the y for volumes
ax2.fill_between(df_volume.index.map(mdates.date2num), df_volume.values, 0, color='b')

plt.show()<jupyter_output><empty_output><jupyter_text>Notice the dates on the bottom are in mdate format as opposed to normal  Web scraping using beautifulsoup<jupyter_code>import bs4 as bs
# pickle serialises any objects so saves the list
import pickle
import requests
def save_sp500_tickers():
    resp = requests.get('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, 'lxml')
    table = soup.find('table', {'class' : 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        # get the value of the first item in the row (ticker)
        ticker = row.findAll('td')[0].text
        tickers.append(ticker)
    
    with open("sp500tickers.pickle", "wb") as f:
        pickle.dump(tickers, f)
        
    print(tickers)
    
    return tickers
    
save_sp500_tickers()
def get_data_from_yahoo(reload_sp500=False):
    if reload_sp500:
        tickers = save_sp500_tickers()
    else:
        with open ("sp500tickers.pickle", "rb") as f:
            tickers = pickle.load(f)
        
    if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')
    
    for ticker in tickers:
        print(ticker)
        if not os.path.exists('stock_dfs/{}.csv'.format(ticker)):
            df = web.DataReader(ticker, 'yahoo', start, end)
            df.to_csv('stock_dfs/{}.csv'.format(ticker))
        else:
            print('Already have {}'.format(ticker))
        
get_data_from_yahoo()<jupyter_output>MMM
{'fraction_non_alphanumeric': 0.13823891625615764, 'fraction_numerical': 0.03417487684729064, 'mean_word_length': 4.3088235294117645, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '325086', 'license': 'MIT', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1192, 'n_words': 272, 'path': 'algoTrading/algoNotebook.ipynb', 'repo_name': 'umerhasan17/mlds', 'size': '3248'}