<jupyter_start><jupyter_text>Taking Last 8 Quarters for testing<jupyter_code>def split(X,Y):
    X_train = X[:-8]
    X_test = X[-8:]
    y_train = Y[:-8]
    y_test = Y[-8:]
    
    return X_train, X_test, y_train, y_test
from sklearn.model_selection import train_test_split

def random_forest(df):
    df = df.drop(columns=['Quarter end'])
    df = df.replace([np.inf, -np.inf], np.nan)
    df = df.dropna()
    X = df
    Y = X['Price']
    Y = Y[1:]
    
    #Getting Next Quarter %change of price 
    #This will be our target variable

    X = X[:-1]
    X = X.values
    Y = Y.values
    
    
    X_train, X_test, y_train, y_test = split( X, Y)
    X_train_means = np.mean(X_train, axis = 0)
    X_train_std = np.std(X_train, axis = 0)
    X_train_scaled = (X_train - X_train_means)/X_train_std
    
    #print(np.shape(X_train_scaled)[1])

    X_test_scaled = (X_test - X_train_means)/X_train_std
    # Import the model we are using
    from sklearn.ensemble import RandomForestRegressor
    # Instantiate model with 1000 decision trees
    rf = RandomForestRegressor(n_estimators = 1000,max_depth = 30 ,max_features = 'sqrt',min_samples_leaf=4, random_state = 42)
    # Train the model on training data
    rf.fit(X_train_scaled, y_train)
    # Use the forest's predict method on the test data
    predictions = rf.predict(X_test_scaled)
    score = rf.score(X_train_scaled,y_train)
    
    test_score = rf.score(X_test_scaled,y_test)
    
    importances = rf.feature_importances_
    std = np.std([tree.feature_importances_ for tree in rf.estimators_],
             axis=0)
    indices = np.argsort(importances)[::-1]
    
    return predictions,y_test,score,test_score,importances
df = pd.read_csv("./clean_data/AXP_quarterly_financial_data.csv", delimiter = ',')
df.head()
df.columns<jupyter_output><empty_output><jupyter_text>Running Random Forest For all stocks<jupyter_code>filenames = open('filenames.txt','r').read()
filenames = filenames.split('\n')[:-1]

all_preds = {}
all_labels = {}

i = 0

scores = []
test_scores =[]
importances = []

#for i in range(15):
for i in range(len(filenames)):
    try:
        df = pd.read_csv("./new_clean_data/"+str(filenames[i]), delimiter = ',')
        preds,labels,score,test_score,importance = random_forest(df)
        
        scores.append(score)
        test_scores.append(test_score)
        importances.append(importance)
        
        all_preds[str(filenames[i])] = preds
        all_labels[str(filenames[i])] = labels
    except:
        s=2
all_preds
np.mean(scores),np.std(scores)
threshold_for_cutoff = np.mean(scores) - np.std(scores)
threshold_for_cutoff
new_all_preds = {}
count = 0
for i in all_preds:
    if scores[count] >= threshold_for_cutoff:
        new_all_preds[i]=all_preds[i]
    else:
        print('removes')
        
    count+=1
New_Predictions = pd.DataFrame.from_dict(new_all_preds, orient = 'index')
New_Predictions.to_csv('./Labels_and_Preds/Predictions_Filtered_RF.csv')
Predictions = pd.DataFrame.from_dict(all_preds, orient='index')
labels_df = pd.DataFrame.from_dict(all_labels, orient='index')
New_Predictions
labels_df
# Predictions.to_csv('./Labels_and_Preds/Predictions_RF.csv')
# labels_df.to_csv('./Labels_and_Preds/Labels.csv')<jupyter_output><empty_output><jupyter_text>Getting Results for Quarter<jupyter_code>def give_quarter_performance(Predictions,Labels,quarter):
    Q = Predictions[quarter]
    Q_labels = Labels[quarter]
    Q = Q.sort_values(ascending=False)
    Q_top_30 = Q[:30]
    
    top_30_labels = []

    for index in Q_top_30.index:
        row = Q_labels.loc[str(index)]
        top_30_labels.append(row)
    
    labels_series = pd.Series(top_30_labels)
    final_df = pd.DataFrame({'Preds': Q_top_30.values,'Labels': labels_series.values},index=Q_top_30.index)
    
    return final_df
Q_1_result = give_quarter_performance(Predictions,labels_df,1)
def quarter_performance(Quarter_result):    
    portfolio_start = 30
    portfolio_end = 0
    for value in Quarter_result['Labels'].values:
        portfolio_end = portfolio_end + 1 + value
    
    perfomance = ((portfolio_end-portfolio_start)/portfolio_start)*100
    return perfomance
        
Q_1_per = quarter_performance(Q_1_result)
Q_1_per
import matplotlib.pyplot as plt

def total_performance(num_quarters,Predictions,labels_df):
    total_returns = 100
    q_turnouts = [100]
    q_pers = []
    for quarter in range(num_quarters):
        Q_result = give_quarter_performance(Predictions,labels_df,quarter)
        Q_per = quarter_performance(Q_result)
        
        total_returns = total_returns * (1+(Q_per/100))
        print('Quarter :'+str(quarter+1)+'\t Result :' +str(Q_per)+' %')
        q_turnouts.append(total_returns)
        q_pers.append(Q_per)
    
    compounded_result = total_returns-100
    q_avg_return = pow(total_returns/100, 1/num_quarters) - 1
    CAGR = pow(total_returns/100, 1/(num_quarters/4)) - 1
    
    print('Compounded Result :'+str(compounded_result)+str(' %'))
    print('Quarterly Compounded Growth Rate :', q_avg_return*100, '%')
    print('Annual Compounded Growth Rate :', CAGR*100, '%')
    print('Sharpe Ratio :', pow(q_avg_return*100/np.std(q_pers),1/4))
    
    plt.plot(q_turnouts)
    plt.ylabel('Performance')
    plt.xlabel('Quarters')
    plt.show()

total_performance(8)
Q_1_result
np.mean(scores),np.mean(test_scores)
importances<jupyter_output><empty_output><jupyter_text>Running Random Forest on New cleaned Data<jupyter_code>filenames = open('filenames.txt','r').read()
filenames = filenames.split('\n')[:-1]

all_preds_v2 = {}
all_labels_v2 = {}

i = 0

scores = []
test_scores =[]
importances = []

#for i in range(15):
for i in range(len(filenames)):
    try:
        df = pd.read_csv("./new_clean_data_v2/"+str(filenames[i]), delimiter = ',')
        preds,labels,score,test_score,importance = random_forest(df)
        
        scores.append(score)
        test_scores.append(test_score)
        importances.append(importance)
        
        all_preds_v2[str(filenames[i])] = preds
        all_labels_v2[str(filenames[i])] = labels
    except:
        s=2
Predictions_v2 = pd.DataFrame.from_dict(all_preds_v2, orient='index')
labels_df_v2 = pd.DataFrame.from_dict(all_labels_v2, orient='index')
total_performance(8,Predictions_v2,labels_df_v2)
np.mean(scores),np.mean(test_scores)
dict1 = {}
for array in importances:
    if np.shape(array)[0] in dict1:
        dict1[np.shape(array)[0]]+=1
    else:
        dict1[np.shape(array)[0]] = 1
dict1
plt.hist(scores,bins=20)
importance_df = pd.DataFrame(importances)
importance_df.mean(axis=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12586779354059766, 'fraction_numerical': 0.01931783881678237, 'mean_word_length': 3.8620689655172415, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '281707', 'license': 'MIT', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2424, 'n_words': 397, 'path': '003_random_forest.ipynb', 'repo_name': 'dxcv/Portfolio-Management', 'size': '6626'}