<jupyter_start><jupyter_text>Load the historical price data from CoinDesk<jupyter_code>df = pd.read_csv("/Users/kyledunn/Desktop/crypto/coindesk-bpi-USD-close_data-2010-07-17_2018-06-12.csv").iloc[:-2]
df.tail()<jupyter_output><empty_output><jupyter_text>Split out the month-day and year elements seperately (for YoY trend comparisons)<jupyter_code>def getMonthAndDay(v):
    return "{}-{}".format(v.Date[5:7], v.Date[8:10])

df['monthday'] = df.apply(getMonthAndDay, axis=1)

def getYear(v):
    return "{}".format(v.Date[0:4])

df['year'] = df.apply(getYear, axis=1)
df['ts'] = df.Date.map(lambda s: pd.to_datetime(s, format="%Y-%m-%d %H:%M:%S") if "-" in s else None)
df.tail()<jupyter_output><empty_output><jupyter_text>Create year-over-year time series<jupyter_code>yearOverYearSeries =\
df\
    .sort_values("ts", ascending=True)\
    .groupby(["year", "monthday"])['Close Price']\
    .agg(lambda df: df.reset_index(drop=True))\
    .unstack()\
    .T[:-1]

yearOverYearSeries.tail()
%matplotlib inline

#df_p = yearOverYearSeries.drop(["This", "http"], axis=1) #.head(90)

df_norm = pd.DataFrame()
for c in yearOverYearSeries.columns:
    
    if c not in ["2013", "2014", "2017", "2018"]:
        continue
    
    maxVal = yearOverYearSeries[c].max()
    df_norm[c] = yearOverYearSeries[c].map(lambda v: v/maxVal)<jupyter_output><empty_output><jupyter_text>Look at different years over time<jupyter_code>ax = df_norm.plot(figsize=(16, 9), title="Comparing across years")
ax.set_ylabel("% max annual price", size=14)<jupyter_output><empty_output><jupyter_text>Benford testing BTC & ETH for erroneous price data<jupyter_code>import benford as bf
from IPython.core.display import display, HTML

# Using log-returns
def runBenford(data, columnName, confidence=[99.99999, 99.99, 99, 99, 99], force=False):
    
    tmp = data.copy()

    # Calculate simple returns
    tmp['p_r'] = tmp[columnName]/tmp[columnName].shift()-1  #simple returns

    # Calculate log-simple returns (this is what get used)
    tmp['l_r'] = np.log(tmp[columnName]/tmp[columnName].shift())  #log returns

    if tmp['l_r'].mean() < tmp['l_r'].median():
        print("Mean is less than median, try passing the force flag")
        if not force: return
    
    display(HTML('<h3>First digit test - CI: {}</h3>'.format(confidence[0])))
    f1d = bf.first_digits(tmp.l_r, digs=1, decimals=8, confidence=confidence[0])

    display(HTML('<h3>First <i>two</i> digits test - CI: {}</h3>'.format(confidence[1])))
    f2d = bf.first_digits(tmp.l_r, digs=2, decimals=8, confidence=confidence[1])

    display(HTML('<h3>First <i>three</i> digits test - CI: {}</h3>'.format(confidence[2])))
    f3d = bf.first_digits(tmp.l_r, digs=3, decimals=8, confidence=confidence[2])

    display(HTML('<h3>Second digit test - CI: {}</h3>'.format(confidence[3])))
    sd = bf.second_digit(tmp.l_r, decimals=8, confidence=confidence[3])

    display(HTML('<h3>Last two digits test - CI: {}</h3>'.format(confidence[4])))
    l2d = bf.last_two_digits(tmp.l_r, decimals=8, confidence=confidence[4])
    
# Using the pricing data directly
def runBenfordDirect(data, columnName, confidence=[99.99999, 99.99, 99, 99, 99], force=False):
    
    if data[columnName].mean() < data[columnName].median():
        print("Mean is less than median, try log-returns")
        if not force: return

    tmp = data.copy()

    display(HTML('<h3>First digit test - CI: {}</h3>'.format(confidence[0])))
    f1d = bf.first_digits(tmp[columnName], digs=1, decimals=2, confidence=confidence[0])

    display(HTML('<h3>First <i>two</i> digits test - CI: {}</h3>'.format(confidence[1])))
    f2d = bf.first_digits(tmp[columnName], digs=2, decimals=2, confidence=confidence[1])

    display(HTML('<h3>First <i>three</i> digits test - CI: {}</h3>'.format(confidence[2])))
    f3d = bf.first_digits(tmp[columnName], digs=3, decimals=2, confidence=confidence[2])

    display(HTML('<h3>Second digit test - CI: {}</h3>'.format(confidence[3])))
    sd = bf.second_digit(tmp[columnName], decimals=2, confidence=confidence[3])

    display(HTML('<h3>Last two digits test - CI: {}</h3>'.format(confidence[4])))
    l2d = bf.last_two_digits(tmp[columnName], decimals=2, confidence=confidence[4])<jupyter_output><empty_output><jupyter_text>Run Benford test for BTC, picking the _highest_ CI values which show anomolies<jupyter_code>runBenfordDirect(df, "Close Price", confidence=[99.99999, 99.99999, 99.999, 99.99, 99.9])
dfe = pd.read_csv("/Users/kyledunn/Desktop/crypto/coindesk-ETH-close_data-2015-08-31_2018-06-12.csv")<jupyter_output><empty_output><jupyter_text>Run Benford test for ETH, picking the _highest_ CI values which show anomolies<jupyter_code>runBenfordDirect(dfe, "Close Price", confidence=[99, 99.99, 99.999, 99.9, 95])<jupyter_output><empty_output><jupyter_text>Compare BTC price and Google search interest<jupyter_code>from pytrends.request import TrendReq

pytrends = TrendReq(hl='en-US', tz=360)

kw_list = ["Bitcoin"]
pytrends.build_payload(kw_list, cat=0, timeframe='today 5-y', geo='', gprop='')
df_interest = pytrends.interest_over_time()
import matplotlib.pyplot as plt

f = plt.figure(figsize=(16, 9))
ax = f.subplots(1, 1)

ax.plot(df_interest.index[-90:], df_interest.Bitcoin[-90:], 'green', label="Interest")

ax2 = ax.twinx()

ax2.plot(df['ts'][-665:], df['Close Price'][-665:], 'black', label="Price")
ax2.set_ylabel("Price [USD]")

ax.set_title("Bitcoin Price (USD) and Google search interest over time", size=16)
ax.legend(["Interest"])
ax2.legend(["Price - USD"], loc=[0.89, 0.9])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1755834829443447, 'fraction_numerical': 0.04290843806104129, 'mean_word_length': 5.359589041095891, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '406172', 'license': 'MIT', 'n_tokens_mistral': 2279, 'n_tokens_neox': 2068, 'n_words': 400, 'path': 'btc-eth-benford.ipynb', 'repo_name': 'kdunn926/crypto-notebooks', 'size': '5570'}