<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy  as np

from sklearn.ensemble      import GradientBoostingRegressor
from sklearn.linear_model  import LinearRegression
from sklearn.preprocessing import StandardScaler<jupyter_output><empty_output><jupyter_text>**Configuration**Tune the regressor's parameter here. We set our file names also here.<jupyter_code># GB Parameters
LOSS = 'huber'
CRITERION = 'friedman_mse'
MAX_DEPTH = 32

# Configure Data Options
ENABLE_TWEET = True

# File Names
SUB_FILE_NAME          = 'SUBMISSION_GB_HUBER_FMSE_32.csv'
TRAIN_TIMESERIES       = 'train_timeseries2.csv'
CLEAN_TWEET_TRAIN_FILE = 'clean_tw_data_train_2.csv'
ACTUAL_DATA_FILE       = 'sample_submission_2_actual_values.csv'
TEST_TIMESERIES        = 'test_timeseries2.csv'
CLEAN_TWEET_DATA_TEST  = 'clean_tw_data_test.csv'<jupyter_output><empty_output><jupyter_text>**Gradient Boost**Loss = huber ; Criteria = FMSE, Max Depth = 32 ; Columns = Selected=> 0.9346<jupyter_code>columns = ['close',          
           'cost_per_transaction',
           'est_transaction_volume_USD',
           'est_transaction_volume',
           'cost_per_transaction_percent' ,
           'transaction_fees',
           'n_transactions_per_block',
           'utxo_count_max' ,
           'utxo_count_min' ,
           'mempool_size_max'
            ]

columns = list(set(columns))  #This is just to ensure we have unique column names and no duplicates entered by mistake
# Read Time Series Data
data = pd.read_csv(TRAIN_TIMESERIES, index_col=0)
data = data[columns]
data.head()
# Read Tweet Data (from clean file)
data_tweets = pd.read_csv(CLEAN_TWEET_TRAIN_FILE)
data_tweets.drop(data_tweets.columns[0], axis=1, inplace=True)

# Remove Account and Favorite Count columns
data_tweets.drop(['account'], axis=1, inplace=True)
data_tweets.drop(['favorite_count'], axis=1, inplace=True)


data_tweets.set_index(['date'], inplace=True)
data_tweets.head()
data_close = data[columns]
data_w_tweets = data_tweets.merge(data_close, left_index=True, right_index=True)
data_w_tweets.head()
if ENABLE_TWEET:
  data = data_w_tweets
  
data.head()
len(data)
data[data.isnull().any(axis=1)]
data = data.dropna(how='any')
data.head()
data_train = data.sample(frac=0.8, random_state=0)
data_test = data.drop(data_train.index)
train_stat = data_train.describe()
train_stat.pop('close')
train_stat = train_stat.transpose()
y_train = data_train.pop("close")
y_test  = data_test.pop("close")<jupyter_output><empty_output><jupyter_text>**Normalize the data**<jupyter_code>sc = StandardScaler()
normed_X_train = sc.fit_transform(data_train)
normed_X_test = sc.transform(data_test)

# normed_X_train = data_train
# normed_X_test  = data_test
reg = GradientBoostingRegressor(loss=LOSS, criterion = CRITERION, max_depth=MAX_DEPTH)
  
reg.fit(normed_X_train, y_train)
y_pred = reg.predict(normed_X_test)
y_pred = y_pred.round(decimals=2)
score = reg.score(normed_X_test, y_test)
score
y_pred_df = pd.DataFrame(y_pred, columns=['close'], index=y_test.index)
y_pred_df.head()<jupyter_output><empty_output><jupyter_text>**Prepare Submission File**<jupyter_code>columns.remove('close')
data_sub = pd.read_csv(TEST_TIMESERIES, index_col=0)
data_sub = data_sub[columns]

data_tweets_sub = pd.read_csv(CLEAN_TWEET_DATA_TEST)
data_tweets_sub.drop(data_tweets_sub.columns[0], axis=1, inplace=True)
data_tweets_sub.drop(['account'], axis=1, inplace=True)
data_tweets_sub.drop(['favorite_count'], axis=1, inplace=True)


data_tweets_sub.set_index(['date'], inplace=True)

data_close_sub = data_sub[columns]
data_w_tweets_sub = data_tweets_sub.merge(data_close_sub, left_index=True, right_index=True)

#Enable Tweet Data
if ENABLE_TWEET:
  data_sub = data_w_tweets_sub  

data_sub = data_sub.dropna(how='any')

normalized_sub = sc.transform(data_sub)
# normalized_sub = data_sub

y_sub = reg.predict(normalized_sub)
y_sub = y_sub.round(decimals=2)
y_sub_df = pd.DataFrame(y_sub, columns=['close'], index=data_sub.index)
y_sub_df.head()
y_sub_df = y_sub_df.groupby('date').agg('mean')<jupyter_output><empty_output><jupyter_text>**Validate Output file**<jupyter_code>data_val = pd.read_csv(ACTUAL_DATA_FILE, index_col=0)
data_val.head()
y_sub_df.columns[0]
y_sub_df.head()
val_data = data_val.merge(y_sub_df, left_index=True, right_index=True)
val_data.head()
val_data['diff'] = abs(val_data.close_x - val_data.close_y) / val_data.close_x
val_data.head()
stats = val_data.mean()
score = 1-stats['diff']
score<jupyter_output><empty_output><jupyter_text>**Download Submission File**Only if the score is at least 90%<jupyter_code>if score > 0.90:
  y_sub_df.to_csv(SUB_FILE_NAME)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11798715203426124, 'fraction_numerical': 0.00835117773019272, 'mean_word_length': 6.497592295345104, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '515469', 'license': 'MIT', 'n_tokens_mistral': 1838, 'n_tokens_neox': 1809, 'n_words': 278, 'path': 'Deloitte/competetion_01/Bitcoin_clean.ipynb', 'repo_name': 'achirde/prctc_machine_learning', 'size': '4670'}