<commit_before>import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import pandas.io.data as web

plt.ion()

instruments = pd.read_csv('instruments2.csv')

startdate = '2005-01-28'
enddate = '2014-11-10'

# Download the asset data if the data isn't there,
# or it doesn't match the instruments in the csv file.
try:
    baseetfs = pd.read_csv('data/base_price_data.csv')
    baseetfs = baseetfs.set_index(
        baseetfs.columns[0]
    ).convert_objects(convert_numeric=True)
    baseetfs.index = baseetfs.index.to_datetime()
except IOError:
    baseetfs = web.DataReader(instruments.values.flatten().tolist(), 'google', startdate, enddate)
    baseetfs = baseetfs.Close.convert_objects(convert_numeric=True)
    baseetfs.to_csv('data/base_price_data.csv')

# Weekly ETF prices.
wetfs = baseetfs.resample('W-WED', how='first', fill_method='pad')

#logreturns =<commit_msg>dlogreturns<commit_after>import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import pandas.io.data as web

plt.ion()

instruments = pd.read_csv('instruments2.csv')

startdate = '2005-01-28'
enddate = '2014-11-10'

# Download the asset data if the data isn't there,
# or it doesn't match the instruments in the csv file.
try:
    baseetfs = pd.read_csv('data/base_price_data.csv')
    baseetfs = baseetfs.set_index(
        baseetfs.columns[0]
    ).convert_objects(convert_numeric=True)
    baseetfs.index = baseetfs.index.to_datetime()
except IOError:
    baseetfs = web.DataReader(instruments.values.flatten().tolist(), 'google', startdate, enddate)
    baseetfs = baseetfs.Close.convert_objects(convert_numeric=True)
    baseetfs.to_csv('data/base_price_data.csv')

# Filter out ETFs with a low number of observations
baseetfs = baseetfs.loc[:, baseetfs.count() > 2400]

print "Using {} ETFs with more than 2400 price entries".format(len(baseetfs.columns))

# Weekly ETF prices. Missing data is filled forward.
wetfs = baseetfs.resample('W-WED', how='first', fill_method='pad')

dlogreturns = np.log(wetfs).diff()
{'fraction_non_alphanumeric': 0.11889491859891466, 'fraction_numerical': 0.021706956092747903, 'mean_word_length': 4.761363636363637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4351017', 'n_tokens_mistral': 740, 'n_tokens_neox': 699, 'n_words': 166}