<jupyter_start><jupyter_text>Analysis Of Misiing Values<jupyter_code>Target = DataKarpos[['PM10']].copy(deep=True)
Target.shape
null_data_PM10 = Target[Target.isnull().any(axis=1)] # extract data sets with null values
null_data_PM10.to_csv("PM10_NUll.txt")# save to file
Target.to_csv("R_target.csv")# save to file
null_data_PM10.head()
def getPctMissing(series):
    '''
    Returns percentage of Missing data in a data set.
    Input : Pandas series
    Output: Percentage of missing data in data set
    '''
    num = series.isnull().sum()
    Total_data = len(series)
    return 100*(num/Total_data)
print("Number of null values in PM10 : ");Target.isnull().sum()
print("Number of non null values in PM10 : ");Target.count()
print("Percentage of Missing Value: "); getPctMissing(Target)<jupyter_output>Percentage of Missing Value:<jupyter_text>Fill Missing Values With Nan<jupyter_code>pwd
Target.fillna(method='ffill',inplace=True)
Target.fillna(Target.median,inplace=True) # Fill Missing value with 0. Default model
Target.describe()
Target_list = Target.values.tolist() # create a list of values for use in data model<jupyter_output><empty_output><jupyter_text>Prepare Data<jupyter_code>counter=0
def dataGetter(datalist):
    """
    Returns a single PM10 value.
    Input: PM10 Data list named Target_list above
    Output: single PM10 value
    """
    global counter;
    assert type(datalist) == list;
    
    
    value = datalist[counter]
    #print(counter)


    counter = counter +1
             
    return np.array([value])
# Define window size or learning
sliding_window = []
lag=25 # 24 hours time step from 0 to 24 plus the window_step_move (25) which is to be predicted
window_move_step = 1
for i in range(lag - window_move_step):
    sliding_window.append(dataGetter(Target_list))
len(sliding_window)
def get_pair(Target):
    """
    Returns an (current, target) 
    Target is lag steps 24 ahead of current
    """
 
    global sliding_window
    sliding_window.append(dataGetter(Target))
    #print(len(sliding_window))
    
    input_value = sliding_window[0:24]
    
    input_value=np.array(input_value)
    #print(input_value.shape)
    input_value=np.reshape(input_value,(1,24))
    ##print(input_value)
    output_value = sliding_window[-1]
    ##print(output_value)
    sliding_window = sliding_window[1:]
    #print(len(sliding_window))
    return input_value, output_value
len(sliding_window)<jupyter_output><empty_output><jupyter_text>Input Paramsinput_dim = 1 To maintain statelast_value = np.array([0 for i in range(input_dim)])last_derivative = np.array([0 for i in range(input_dim)]) def get_total_input_output(Target):    """    Get Total feed vector - input and output     """    global last_value, last_derivative    raw_i, raw_o = get_pair(Target)    raw_i = raw_i[0]    l1 = list(raw_i)    derivative = raw_i - last_value    l2 = list(derivative)    last_value = raw_i    l3 = list(derivative - last_derivative)    last_derivative = derivative    return np.array([l1 + l2 + l3]), raw_o  Build Model<jupyter_code>#Imports
import tensorflow as tf
#from tensorflow.nn.rnn import *
from tensorflow.python.ops  import *

#Input Params
with tf.name_scope("input_target_placeholders"):    
    input_dim = 1
    ##The Input Layer as a Placeholder
    #Since we will provide data sequentially, the 'batch size'
    #is 1.
    input_layer = tf.placeholder(tf.float32, [1, input_dim*24],name="input_data")
    correct_output = tf.placeholder(tf.float32, [1, input_dim],name="target_data")
###inistate = tf.Variable(lstm_cell_with_dropout.zero_state(BATCH_SIZE, tf.float32), trainable=False)
with tf.name_scope("lstmLayer"):    
    lstm_layer1 = rnn_cell.BasicLSTMCell(input_dim*24,state_is_tuple=False)
    #The LSTM state as a Variable initialized to zeroes
    lstm_state1 = tf.Variable(tf.zeros([1, lstm_layer1.state_size]),trainable=False,name="initial_state")
    #lstm_state1 = tf.Variable(lstm_layer1.zero_state(1,lstm_layer1.state_size[-1] ), trainable=False)
    #Connect the input layer and initial LSTM state to the LSTM cell
    lstm_output1, lstm_state_output1 = lstm_layer1(input_layer, lstm_state1)
    #The LSTM state will get updated
    lstm_update_op1 = lstm_state1.assign(lstm_state_output1)
lstm_state_output1.get_shape()# verify shape
with tf.name_scope("weight_Bias_learning_rate"):
    global_step = tf.Variable(0, trainable=False,name="global_step")
    starter_learning_rate = 0.09
    learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step,
                                           1000, 0.8, staircase=False,name ="Exponential_decay")
    ##The Regression-Output Layer
    #The Weights and Biases matrices first
    output_W1 = tf.Variable(tf.truncated_normal([input_dim*48, input_dim]),name="weight")
    output_b1 = tf.Variable(tf.zeros([input_dim]),name="bias")<jupyter_output><empty_output><jupyter_text>lambda_l2_reg=0.5l2 = lambda_l2_reg * sum(    tf.nn.l2_loss(tf_var)        for tf_var in tf.trainable_variables()        if not ("noreg" in tf_var.name or "bias" in tf_var.name))loss += l2<jupyter_code>with tf.name_scope("prediction"):
    #Compute the output
    final_output = tf.matmul(lstm_state_output1, output_W1) + output_b1
final_output.get_shape() # verify output shape
output_W1 # verify weight shape
with tf.name_scope("RMS_error"):
    ##Calculate the Sum-of-Squares Error
    error = tf.pow(tf.sub(final_output, correct_output), 2)+l2
#error = tf.reshape(error,(1,))
with tf.name_scope("optimizer"):
    ##The Optimizer
    #Adam works best
    train_step = tf.train.AdamOptimizer(learning_rate).minimize(error)
# Create a summary to monitor MSE
mse=tf.summary.tensor_summary("errors_Summary",error)
# Create a summary to monitor  predictions
prediction=tf.summary.tensor_summary("predictions_Summmary", final_output)
# Create a summary to monitor bias
bias_vec=tf.summary.tensor_summary("bias", output_b1)
# create sumary
#rate_vec=tf.summary.scalar("rate", learning_rate)


#histogram plot

error_stats=tf.histogram_summary("errors_Histogram",error)
weight_stats=tf.histogram_summary("weights_Histogram",output_W1)
bias_stats=tf.histogram_summary("biases_Histogram",output_b1)
#learning_stats=tf.histogram_summary("biases_Histogram",learning_rate)


#merged_summary_op =  tf.merge_all_summaries()
merged_summary_op =   tf.merge_summary([mse,prediction,bias_vec,error_stats,weight_stats,bias_stats])
##Session
sess = tf.Session()
#Initialize all Variables
sess.run(tf.initialize_all_variables())

len(Target_list)<jupyter_output><empty_output><jupyter_text>Evaluation Data set<jupyter_code>logs_path = '/Users/Seychelles/Desktop/GoogleTensorflow/finals/tensorboardData/sliding_window/'
##Training Parameters
n_iter = 80000 
inner_iter = 8000
actual_output1 = []
network_output1 = []
validation_prediction=[];
validation_target =[];
lower_bound=0;
upper_bound=800;
prediction_window=800
import copy
#new_list = copy.deepcopy(old_list)
Test_eval=copy.deepcopy(Target_list[80001:])
len(Test_eval)
len(Target_list)-80000
# We have 800000 iterations
# Between itertions and every 10000 steps we make all initial_state = 0 and perform cross-validate
# We have 80000 validation set
# We test with the remainder 
Remainder = len(Test_eval) - (n_iter/inner_iter*prediction_window)
test_part =48;
Remainder= Remainder-test_part
Remainder = int(Remainder)
Remainder<jupyter_output><empty_output><jupyter_text>error_=[]with tf.Session() as sess:        writer = tf.train.SummaryWriter(logs_path, graph= tf.get_default_graph())     counter =0     assert counter==0    for i in range(n_iter):        input_v, output_v = get_pair(Target_list)        _, _, network_output,errors,summary = sess.run([lstm_update_op1,                                     train_step,                                     final_output,error,merged_summary_op],                                    feed_dict = {                                        input_layer: input_v,                                        correct_output: output_v})        writer.add_summary(summary)        error_.append(errors)        if i%inner_iter==0 & i!=0:            assert i!=0;            sess.run(lstm_state1.assign(tf.zeros([1, lstm_layer1.state_size])))                    for j in range(len(Test_eval[lower_bound:upper_bound])):                input_val, output_val = get_pair(Test2)                _, network_output = sess.run([lstm_update_op1,                                      final_output],                                     feed_dict = {                                     input_layer: input_val,                                     correct_output: output_val})                lower_bound= lower_bound+prediction_window;                upper_bound = upper_bound + prediction_window;                validation_target.append(output_val)                validation_prediction.append(network_output1)        sess.run(lstm_state1.assign(tf.zeros([1, lstm_layer1.state_size])))        actual_output1.append(output_v)        actual_output2.append(output_v[0][1])        network_output1.append(network_output)        network_output2.append(network_output[0][1])        x_axis.append(i)<jupyter_code>error_=[]

    


writer = tf.train.SummaryWriter(logs_path, graph= tf.get_default_graph()) 

counter =0 
assert counter==0

for i in range(n_iter):
        input_v, output_v = get_pair(Target_list)
        _, _, network_output,errors,summary = sess.run([lstm_update_op1,
                                     train_step,
                                     final_output,error,merged_summary_op],
                                    feed_dict = {
                                        input_layer: input_v,
                                        correct_output: output_v})
        writer.add_summary(summary)
        error_.append(errors)
        if i%inner_iter==0 & i!=0:
            assert i!=0;
            sess.run(lstm_state1.assign(tf.zeros([1, lstm_layer1.state_size])))
        
            for j in range(len(Test_eval[lower_bound:upper_bound])):
                input_val, output_val = get_pair(Test2)
                _, network_output = sess.run([lstm_update_op1,
                                      final_output],
                                     feed_dict = {
                                     input_layer: input_val,
                                     correct_output: output_val})
                lower_bound= lower_bound+prediction_window;
                upper_bound = upper_bound + prediction_window;
                validation_target.append(output_val)
                validation_prediction.append(network_output1)
        #sess.run(lstm_state1.assign(tf.zeros([1, lstm_layer1.state_size])))

        actual_output1.append(output_v)
        #actual_output2.append(output_v[0][1])
        network_output1.append(network_output)
        #network_output2.append(network_output[0][1])
        #x_axis.append(i)<jupyter_output><empty_output><jupyter_text>Errors Plot<jupyter_code>errorplot = np.array(error_) # make errors into arrays
errorplot = errorplot.reshape(80000,1)
import matplotlib.mlab as mlab

fig, ax = plt.subplots()
n, bins, patches=plt.hist(errorplot,100,normed=1,facecolor='deepskyblue',alpha=0.9,label='Histogram')
mu= np.mean(errorplot);sigma=np.std(errorplot);
y = mlab.normpdf( bins,mu,sigma)
#y = mlab.normpdf( bins, mu, sigma)
plt.plot(bins, y, 'r--', linewidth=5,label="pdf plot Over Histogram")
#ax.set_yscale('log')
#ax.set_xscale('log')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.title("Histogram plot for error",fontweight='bold')
plt.ylabel("Error",fontweight='bold')
plt.xlabel("Training example",fontweight='bold')

plt.show()
import matplotlib.mlab as mlab

fig, ax = plt.subplots()
plt.hist(errorplot,bins=200,normed=1,facecolor='deepskyblue',label='Error Histogram plot (log scale)')

ax.set_yscale('log')
#ax.set_xscale('log')
plt.title("Error plot",fontweight='bold')
plt.ylabel("Error",fontweight='bold')
plt.xlabel("Training  Samples",fontweight='bold')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()<jupyter_output><empty_output><jupyter_text>Extract Date in the series<jupyter_code>from matplotlib.dates  import  date2num
Data = pd.read_csv('ItemListWithDummy.csv')
pd.to_datetime(Data.date);
x_axis = Data['date']
x_axis = pd.to_datetime(x_axis)
#80000-len(network_output1)
network = np.array(network_output1)
actual= np.array(actual_output1)
actual.shape
import  matplotlib
plt.rcParams["figure.figsize"] = (8,5)
fig = plt.figure(figsize=(100, 100))

fig, ax = plt.subplots()
ax.plot(x_axis[0:48], network.reshape(80000,1)[0:48], 'r-',color="deepskyblue")
ax.plot_date(x_axis[0:48], actual.reshape(80000,1)[0:48], 'b-',color='goldenrod')
#ax.xaxis.set_minor_locator(dates.MonthLocator())
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M:%S'))
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%d\n%a'))
#ax.xaxis.grid(True, which="minor")
#ax.yaxis.grid()
#ax.xaxis.set_major_locator(dates.DayLocator())
#ax.xaxis.set_major_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
ax.xaxis.set_major_formatter( matplotlib.dates.DateFormatter('%Y-%b-%a %H:%M:%S'))

ax.xaxis.grid(True, which="minor")

#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))


plt.xticks( rotation=25 )
plt.tight_layout()
plt.ylabel("PM10 Values",fontweight='bold')
plt.title("PM10 Prediction for Small Time interval During Training",fontweight='bold')
plt.xlabel('Date',fontweight='bold')
plt.show()<jupyter_output><empty_output><jupyter_text>Plots predictions during Training<jupyter_code>#fig = plt.figure(figsize=(30, 2))
plt.rcParams["figure.figsize"] = (8,5)
fig, ax = plt.subplots()
ax.plot(x_axis[0:80000], network.reshape(80000,1)[0:80000], 'r-',color = 'deepskyblue',label="Traninning Prediction")
ax.plot_date(x_axis[0:80000], actual.reshape(80000,1)[0:80000], 'b-',color='goldenrod',label='Actual Prediction')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.,fontsize='x-large')

#ax.xaxis.set_minor_locator(dates.MonthLocator())
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M:%S'))
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%d\n%a'))
#ax.xaxis.grid(True, which="minor")
#ax.yaxis.grid()
#ax.xaxis.set_major_locator(dates.DayLocator())
#ax.xaxis.set_major_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
ax.xaxis.set_major_formatter( matplotlib.dates.DateFormatter('%Y-%b-%a %H:%M:%S'))
ax.xaxis.grid(True, which="minor")

#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))


plt.xticks( rotation=25 )
plt.tight_layout()
plt.ylabel("PM10 values",fontweight='bold')
plt.xlabel("Date",fontweight='bold')
plt.title("PM10 Training prediction and Actual Value Plots",fontweight='bold')
plt.show()<jupyter_output><empty_output><jupyter_text>Model Testing<jupyter_code># tf.get_default_session()
validation_set = Test_eval[-Remainder:-48]
#Reset counter
counter = 0
##Testing
 

#Flush LSTM state
sess.run(lstm_state1.assign(tf.zeros([1, lstm_layer1.state_size])))
actual_output_test = []
network_output_test = []

 
for i in range(len(validation_set)):
    input_v, output_v = get_pair(validation_set)
    _, network_output = sess.run([lstm_update_op1,
                                  final_output],
                                 feed_dict = {
                                     input_layer: input_v,
                                     correct_output: output_v})
        
 
    actual_output_test.append(output_v)
   
    network_output_test.append(network_output)
 

actual_output_test[-48:];
import matplotlib.pyplot 


fig = plt.figure(figsize=(30, 2))

fig, ax = plt.subplots()
ax.plot( np.array(network_output_test[-48:]).reshape(48,1), 'r-',color = 'deepskyblue',label='Predicted Values for 48 hrs Time Frame')
ax.plot( np.array(actual_output_test[-48:]).reshape(48,1), 'b-',color = 'goldenrod',label='Actual Values  48 hours Time Frame')
#ax.xaxis.set_minor_locator(dates.MonthLocator())
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.,fontsize='x-large')
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M:%S'))
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%d\n%a'))
#ax.xaxis.grid(True, which="minor")
#ax.yaxis.grid()
#ax.xaxis.set_major_locator(dates.DayLocator())
#ax.xaxis.set_major_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
#ax.xaxis.set_major_formatter(dates.DateFormatter('%Y-%b-%a %H:%M:%S'))
#ax.xaxis.grid(True, which="minor")


plt.title("PM10 Validation prediction and Actual Value Plots",fontweight='bold')
#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
plt.ylabel("PM10 values",fontweight='bold')
plt.xlabel("Time/hr",fontweight='bold')

#plt.xticks( rotation=25 )
#plt.tight_layout()

plt.show()
import matplotlib.pyplot 


fig = plt.figure(figsize=(30, 2))

fig, ax = plt.subplots()
ax.plot( np.array(network_output_test).reshape(len(network_output_test),1), 'r-',color = 'deepskyblue',label="Validation Prediction")
ax.plot( np.array(actual_output_test).reshape(len(actual_output_test),1), 'b-',color = 'goldenrod',label='Actual Values  ')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.,fontsize='x-large')
#ax.xaxis.set_minor_locator(dates.MonthLocator())
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M:%S'))
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%d\n%a'))
#ax.xaxis.grid(True, which="minor")
#ax.yaxis.grid()
#ax.xaxis.set_major_locator(dates.DayLocator())
#ax.xaxis.set_major_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
#ax.xaxis.set_major_formatter(dates.DateFormatter('%Y-%b-%a %H:%M:%S'))
#ax.xaxis.grid(True, which="minor")

plt.title("PM10 Validation prediction and Actual Value Plots",fontweight='bold')
#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
plt.ylabel("PM10 values",fontweight='bold')
plt.xlabel("Time/hr for Validation Sample",fontweight='bold')

#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))


#plt.xticks( rotation=25 )
#plt.tight_layout()

plt.show()
from scipy.interpolate import interp1d

xn_ax = np.linspace(0,48,48*2)


x_ax = np.linspace(0,50,48)
y_cor=np.array(network_output_test[-48:]).reshape(48,1)
y_cor=np.reshape(y_cor,(48))
# new x values
xn_ax = np.linspace(1,48,48*10)
 
# new y values
yn_cor = interp1d(x_ax,y_cor, kind='cubic')
plt.plot(xn_ax, yn_cor(xn_ax),color='deepskyblue',label='Smothened Validation set')
plt.xticks(x_ax, x_axis[-48:], size='small', rotation=70)
plt.tight_layout()
plt.xlabel('date',fontweight='bold')
plt.ylabel('PM Values', fontweight='bold')
plt.title('Smothened PM Value Prediction',fontweight='bold')
plt.show<jupyter_output><empty_output><jupyter_text>Testing<jupyter_code>n_list = 48;
Feature_eval=copy.deepcopy(Test_eval[-test_part:])


def get_pair_Test(Target):
    """
    Returns an (current, target) 
    Target is lag steps 24 ahead of current
    """
 
    
    global sliding_window_test
    #sliding_window_test.append(dataGetter(Target))
    #print(len(sliding_window))
    #sliding_window_featureset.append(dataGetter_target(Target2))
    
    #input_value = sliding_window_test
    
    
    input_value=np.array(sliding_window_test)
    #print(input_value.shape)
    
    #print(input_value.shape)
    input_value=np.reshape(input_value,(1,len(sliding_window_test)))
    mx = np.ma.masked_invalid(input_value)
    
    ##print(input_value)
    #output_value = sliding_window[-1]
    #my = np.ma.masked_invalid(output_value)
    ####remove last value from sliding_window
    #_ = sliding_window_featureset[-1]
    ##print(output_value)
   # sliding_window = sliding_window[1:]
    #sliding_window_featureset = sliding_window_featureset[1:]
    #print(len(sliding_window))
    # Process output
    mask_x = ~mx.mask # let mask return true for those non-nan values
    ## y output
    #mask_y =~my.mask
    #mask_yfloat = mask_y
    return input_value
counter_test=0;
def dataGetter_test(datalist):
    """
    Returns a single PM10 value.
    Input: PM10 Data list named Target_list above
    Output: single PM10 value
    """
    global counter_test;
    assert type(datalist) == list;
    
    
    value = datalist[counter_test]
    #print(counter)


    counter_test = counter_test +1
             
    return value
sliding_window_test = []
lag=25 # 24 hours time step from 0 to 24 plus the window_step_move (25) which is to be predicted
window_move_step = 1
for i in range(lag - window_move_step):
    temp=dataGetter_test(Feature_eval)
    
    sliding_window_test.append(temp);
previous_sliding_window_length=24;
time_step_shift = 24;
len(sliding_window_test)
##Testing
 

#Flush LSTM state
sess.run(lstm_state1.assign(tf.zeros([1, lstm_layer1.state_size])));
actual_output_test = []
network_output_test = []

 
for i in range(previous_sliding_window_length):
   
    input_v = get_pair_Test(sliding_window_test)
    _, network_output = sess.run([lstm_update_op1,
                                  final_output],
                                 feed_dict = {
                                     input_layer: input_v})
       
    array_store=np.array(Feature_eval[time_step_shift]) 
    #print(array_store)
    array_store[[0]] = network_output # replace real PM10 values with predicted values
    #print(array_store.shape)
    list_store = array_store.tolist() # convert to list
    #print(len(list_store))
    sliding_window_test.append( list_store) # append new values for prediction
    #print(len(sliding_window_test))
    sliding_window_test.pop(0) # remove the first value of the list 
    #print(len(sliding_window_test))
    time_step_shift = time_step_shift + 1; # increae the time step
    network_output_test.append(network_output)
import matplotlib.pyplot 


fig = plt.figure(figsize=(30, 2))

fig, ax = plt.subplots()
ax.plot( np.array(network_output_test).reshape(len(network_output_test),1), 'r-',color = 'deepskyblue',label=" Prediction")
ax.plot(np.array(Feature_eval[-24:]).reshape(len(Feature_eval[-24:]),1), 'b-',color = 'goldenrod',label="Actual Value")
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.,fontsize='x-large')
#ax.xaxis.set_minor_locator(dates.MonthLocator())
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%H:%M:%S'))
#ax.xaxis.set_minor_formatter(dates.DateFormatter('%d\n%a'))
#ax.xaxis.grid(True, which="minor")
#ax.yaxis.grid()
#ax.xaxis.set_major_locator(dates.DayLocator())
#ax.xaxis.set_major_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
#ax.xaxis.set_major_formatter(dates.DateFormatter('%Y-%b-%a %H:%M:%S'))
#ax.xaxis.grid(True, which="minor")

plt.title("PM10 Test prediction and Actual Value Plots (24hr Time Frame)",fontweight='bold')
#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))
plt.ylabel("PM10 values",fontweight='bold')
plt.xlabel("Time/hr for Test Sample",fontweight='bold')



#ax.xaxis.set_minor_formatter(dates.DateFormatter('\n\n%a\%b\%Y'))


#plt.xticks( rotation=25 )
#plt.tight_layout()

plt.show()
from sklearn.metrics import mean_squared_error
mean_squared_error(np.array(network_output_test).reshape(len(Target_list[-24:]),1),np.array(Target_list[-24:]).reshape(len(network_output_test),1) )**0.5 
import scipy
def rsquared(x, y):
    """ Return R^2 where x and y are array-like."""
    
    slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(x, y)
    return r_value**2
rsquared(np.array(network_output_test).reshape(len(Target_list[-24:]),),
         np.array(Target_list[-24:]).reshape(len(network_output_test),) ) 
rsquared(np.array(network_output_test).reshape(len(Target_list[-24:]),),
         np.array(Target_list[-24:]).reshape(len(network_output_test),) ) 
import scipy.stats as stats

stats.f_oneway(np.array(network_output_test).reshape(len(Target_list[-24:]),),
               np.array(Target_list[-24:]).reshape(len(network_output_test),))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13579416463465904, 'fraction_numerical': 0.021567308500798788, 'mean_word_length': 4.378023965634185, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '506919', 'license': 'MIT', 'n_tokens_mistral': 8704, 'n_tokens_neox': 8252, 'n_words': 1317, 'path': 'LSTM_RNN/2_prediction_using_only_PM10.ipynb', 'repo_name': 'adderbyte/Dynamic-Behaviour-Modelling-With-LSTM-.-AIR-POLLUTION-As-case-study', 'size': '23786'}