<jupyter_start><jupyter_text>EDA - Time Series Impact on Predictions- Should I use Time Series CV?- What is the time period used?- Is there a public source that aligns to this time series?<jupyter_code>import pandas as pd
import numpy as np
import datetime as dt
import matplotlib.pylab as plt
%matplotlib inline
train_id = pd.read_csv('../../inputs/train_identity.csv')
test_id = pd.read_csv('../../inputs/test_identity.csv')
train_tr = pd.read_csv('../../inputs/train_transaction.csv')
test_tr = pd.read_csv('../../inputs/test_transaction.csv')
ss = pd.read_csv('../../inputs/sample_submission.csv')

train_id = train_id.set_index('TransactionID')
train_tr = train_tr.set_index('TransactionID')
test_id = test_id.set_index('TransactionID')
test_tr = test_tr.set_index('TransactionID')
train_tr.shape
train_id.shape
train = train_tr.merge(train_id, left_index=True, right_index=True, how='outer')
test = test_tr.merge(test_id, left_index=True, right_index=True, how='outer')
train.shape
test.shape
# All data combined
data = pd.concat([train, test], sort=False)
data.to_parquet('../../data/raw_combined.parquet')
data['isTrain'] = True
data.loc[data['isFraud'].isna(), 'isTrain'] = False<jupyter_output><empty_output><jupyter_text>Create Day Feature<jupyter_code>offset = 0.58
data['day'] = np.floor((data['TransactionDT'] / (3600 * 24))-1+offset)
data.groupby('day').count()['TransactionDT'].plot(figsize=(15, 5))<jupyter_output><empty_output><jupyter_text>Look at Spikes. These are days in high spending....<jupyter_code>data.groupby('day')['TransactionDT'].count().sort_values(ascending=False).head()
389 - 354
17 + 21
data.query('day >= 17 and day <= 38')[['day','TransactionDT']] \
    .groupby('day') \
    .count().plot(figsize=(15, 5))
381 + 21
data.query('day >= 381 and day <= 402')[['day','TransactionDT']] \
    .groupby('day') \
    .count().plot(figsize=(15, 5))
382 - 18
data['day_year_lag'] = data['day'] - 364
data.loc[data['day_year_lag'] < 0, 'day_year_lag'] = np.nan
ax = data.groupby('day')['TransactionDT'].count().plot(figsize=(15, 5))
data.groupby('day_year_lag')['TransactionDT'].count().plot(figsize=(15, 5), ax=ax)
ax = data.groupby('day')['TransactionDT'].count().plot(figsize=(15, 5), xlim=(0, 32), ylim=(0, 8000), style='.-')
data.groupby('day_year_lag')['TransactionDT'].count().plot(figsize=(15, 5), ax=ax, style='.-')<jupyter_output><empty_output><jupyter_text>Expected High Spending Days    2015	Fri	Nov 27	Black Friday	Observance    2016	Fri	Nov 25	Black Friday	Observance    2017	Fri	Nov 24	Black Friday	Observance    2018	Fri	Nov 23	Black Friday	Observance    2019	Fri	Nov 29	Black Friday	Observance  Is day 1 black friday?<jupyter_code>print(pd.to_datetime('Nov 25 2016') + dt.timedelta(days=25))
print(pd.to_datetime('Nov 24 2017') + dt.timedelta(days=25))
print(pd.to_datetime('Nov 23 2018') + dt.timedelta(days=25))
print(pd.to_datetime('Nov 24 2017') + dt.timedelta(days=29))
print(pd.to_datetime('Nov 23 2018') + dt.timedelta(days=29))
pd.to_datetime('Nov 25 2016') - pd.to_datetime('Nov 27 2015')<jupyter_output><empty_output><jupyter_text>Compare to Google Trends data for "Shopping<jupyter_code>gt = pd.read_csv('../../data/google_trends_shopping.csv', header=1)#['shopping: (United States)'].plot(figsize=(15, 5))
gt.columns = gt.columns.str.replace(': \(United States\)', '')
gt['Week'] = pd.to_datetime(gt['Week'])
gt['shopping'] = (gt['shopping'] * 40) # Scale
gt['present'] = (gt['present'] * 50) # Scale

gt.head()
data['date_guess'] = pd.to_datetime('1-NOV-2017') + data['day'].apply(dt.timedelta)
ax = data.groupby('date_guess')['TransactionDT'].count().plot(figsize=(15, 5), style='.-')
gt.set_index('Week')[['shopping','present']].plot(figsize=(15, 5), xlim=('1-Oct-2017','1-Jan-2019'), ax=ax)
plt.show()
data.groupby('date_guess')[['TransactionDT']].count().sort_values('TransactionDT', ascending=False).head()
# fig, ax = plt.subplots()
ax = data.set_index('TransactionDT')['isFraud'].rolling(5000) \
        .mean() \
        .plot(figsize=(15, 5))
ax = data.groupby('day')['isFraud'] \
        .mean() \
        .plot(figsize=(15, 5))

(data.groupby('day')['TransactionDT'] \
        .count() / 70000) \
        .plot(figsize=(15, 5), ax=ax)
test.shape
506691 * 0.2
# https://www.kaggle.com/kevinbonnes/transactiondt-starting-at-2017-12-01
START_DATE = '2017-12-01'
startdate = dt.datetime.strptime(START_DATE, '%Y-%m-%d')
data['date'] = data['TransactionDT'].apply(lambda x: (startdate + dt.timedelta(seconds = x)))
data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19636524822695037, 'fraction_numerical': 0.05031028368794326, 'mean_word_length': 5.735820895522388, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '402230', 'license': 'Apache-2.0', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1639, 'n_words': 300, 'path': 'notebooks/EDA/EDA-001-Time_Series_Impacts.ipynb', 'repo_name': 'RobMulla/kaggle-ieee-fraud-detection', 'size': '4512'}