<jupyter_start><jupyter_text>**Just Change these attributes for re-running the model.**<jupyter_code>#Don't change source path
source_path = '/mnt/nfs/scratch1/dpariyani/data/structured' #Read files from this directory

#Just mention a directory with company wise seperate csv files in it.

destination_path = '/mnt/nfs/scratch1/dpariyani/xml_data/final_data_scaled/NewXMLData/CompanyWiseData_revenue2/Scaled'
# destination_path = '/mnt/nfs/scratch1/dpariyani/data/structured/structured_data_scaled_revenue' #Write files to this directory
# destination_path = '/mnt/nfs/scratch1/dpariyani/xml_data/input_links/CompanyWiseData_payment'

#Don't change these values
window = 2 #Window size for regression
train_period = [2011,2016] #Training data size for each company
test_period = [2017,2018] #Test data size for each company


#Mention the variable in your data here.
var  = 'Revenue'
# var  = 'NetIncomeLoss' #Variable under consideration. Need to train 5 models for 5 vars.
# var = 'PaymentsToAcquireProperty'
# var = 'value_usd'

#Mention the time column name and indicator column name here in your data.
time_column = 'year'
ind_column = 'Company'


prep = DataPreProcessing(var,source_path,destination_path,window,train_period,test_period,time_column,ind_column)

#Loading the processed data, ready for modelling.
train,test = prep.load_data()

#Removing the null values left after preprocessing, as some companies don't have even a single time point available.
train_data = train[~pd.to_numeric(train[var], errors='coerce').isnull()]
test_data = test[~pd.to_numeric(test[var], errors='coerce').isnull()]

train_data = train_data[~train_data.isin([np.nan, np.inf, -np.inf]).any(1)]
test_data = test_data[~test_data.isin([np.nan, np.inf, -np.inf]).any(1)]

train_data = train_data[train_data[var].between(train_data[var].quantile(.05), train_data[var].quantile(.95))]
test_data = test_data[test_data[var].between(test_data[var].quantile(.05), test_data[var].quantile(.95))]

#Calling the regression module.
reg = RegressionBaseline(window,train_data,test_data,var,time_column,ind_column)

model = reg.fit()
final_train = reg.predict(model,'train')
final_test = reg.predict(model,'test')
final_train.to_csv(destination_path+'/preds_train_'+var+'.csv')
final_test.to_csv(destination_path+'/preds_test_'+var+'.csv')

# Typecasting, just in case.
final_train[var]=final_train[var].astype(float)
final_train['y_pred']=final_train['y_pred'].astype(float)
final_test[var]=final_test[var].astype(float)
final_test['y_pred']=final_test['y_pred'].astype(float)

# Evaluation Metrics Test
final_test['smape'] = 1*(final_test['y_pred'] - final_test[var]).abs()/(final_test['y_pred'].abs() + final_test[var].abs())
final_test['rmse'] = ((final_test.y_pred - final_test[var]) ** 2)
print('Average Test SMAPE:',str(final_test[final_test[var]!=0].smape.mean()*100) + ' %')
print('Average Test RMSE:',"{:2e}".format(final_test.rmse.astype(float).mean()**.5))
print('R2 score Test :',"{:2e}".format(r2_score(final_test[var], final_test['y_pred'])))
final_test['residuals'] = final_test[var] - final_test['y_pred']

#Evaluation Metrics Train
final_train['smape'] = 1*(final_train['y_pred'] - final_train[var]).abs()/(final_train['y_pred'].abs() + final_train[var].abs())
final_train['rmse'] = ((final_train.y_pred - final_train[var]) ** 2)
print('Average Train SMAPE:',str(final_train[final_train[var]!=0].smape.mean()*100) + ' %')
print('Average Train RMSE:',"{:2e}".format(final_train.rmse.astype(float).mean()**.5))
print('R2 score Train :',"{:2e}".format(r2_score(final_train[var], final_train['y_pred'])))
final_train['residuals'] = final_train[var] - final_train['y_pred']
train_data.head(42).tail(20)
import pandas as pd
import numpy as np
from sklearn import datasets, linear_model
from sklearn.linear_model import LinearRegression
import statsmodels.api as sm
from scipy import stats

diabetes = datasets.load_diabetes()
X = diabetes.data
y = diabetes.target

X2 = sm.add_constant(X)
est = sm.OLS(y, X2)
est2 = est.fit()
print(est2.summary())
final_test.head()
final_test.sort_values(['smape'],ascending=True).head(100).tail(10)
df = final_train.sort_values(['year'],ascending=True)
df2 = final_test.sort_values(['year'],ascending=True)
stock='VOYA'
plt.plot(df[df['Company']==stock]['year'],df[df['Company']==stock]['NetIncomeLoss'])
plt.plot(df2[df2['Company']==stock]['year'],df2[df2['Company']==stock]['NetIncomeLoss'])
plt.plot(df2[df2['Company']==stock]['year'],df2[df2['Company']==stock]['y_pred'])
final_train.Revenue.hist(bins=100)
plt.title('Residual plots for Revenue')
plt.ylabel('')
final_train.y_pred.hist(bins=100)
plt.title('Residual plots for Revenue')
plt.ylabel('')
final_test.residuals.hist(bins=100)
plt.title('Residual plots for Revenue')
plt.ylabel('')
final_test.NetIncomeLoss.hist(bins=100)
plt.title('Histogram of Actual Revenue')
final_test.y_pred.hist(bins=100)
plt.title('Histogram of Actual Revenue')
test[test.Company=='MORN']
train.Company.unique()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16666666666666666, 'fraction_numerical': 0.01858513189448441, 'mean_word_length': 8.149908592321754, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '589704', 'license': 'MIT', 'n_tokens_mistral': 1896, 'n_tokens_neox': 1787, 'n_words': 311, 'path': 'CS696/Modelling/regression_baseline.ipynb', 'repo_name': 'tiwarikajal/Stock_fundamental_prediction', 'size': '5004'}