<jupyter_start><jupyter_text>**Data Pre Processing**<jupyter_code>DATA_DIR = "Beijing-Pollution-DataSet/"
from pandas import read_csv
from datetime import datetime
from random import randint
import random

def select_random_rows(seq_start=0, seq_end=12000, rand_percent=20):
    rand_num = int((20*(seq_end-seq_start))/100)
    rand_gen = random.sample(range(seq_start, seq_end), rand_num)
    return rand_gen

def apply_nan(sequence, seq_start=0, seq_end=12000, rand_percent=20):
    for i in range(0, 8):
        rand_r = select_random_rows(seq_start=seq_start, seq_end=seq_end, rand_percent=rand_percent)
        np.put(sequence[seq_start:seq_end, i], rand_r, np.nan)
    return sequence

# split a multivariate sequence into samples
def split_sequences(sequences, n_steps=11, n_samples=12000, start_from=0):
	X, y = list(), list()
	for i in range(start_from, (start_from + n_samples)):
        # find the end of this pattern
		end_ix = i + n_steps
        # check if we are beyond the dataset
        # gather input and output parts of the pattern
		seq_x = sequences[i:end_ix, :]
		seq_y = sequences[end_ix, 0]
		y.append(seq_y)
		X.append(seq_x)
  
	return array(X), array(y)


data = np.load(DATA_DIR + 'polution_dataSet.npy')
scaled_data = data
data1 = np.load(DATA_DIR + 'polution_dataSet.npy')

# specify the number of lag hours
n_hours = 11
n_features = 8





n = apply_nan(data, seq_start=0, seq_end=15000, rand_percent=20)
occurrences = np.count_nonzero(np.isnan(data1[:, 1]))
print(occurrences)
# example of evaluating a model after an imputer transform
from numpy import nan
from pandas import read_csv
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier

import numpy.ma as ma
data = np.where(np.isnan(data), ma.array(data, mask=np.isnan(data)).mean(axis=0), data)


occurrences = np.count_nonzero(np.isnan(data1[:, :]))
print("Number of Nan in Dataset : => ", occurrences)
criterion = nn.MSELoss()
loss1 = criterion(torch.tensor(data1[:, 0], dtype=torch.float32), torch.tensor(data[:, 0], dtype=torch.float32))
loss2 = criterion(torch.tensor(data1[:, 1], dtype=torch.float32), torch.tensor(data[:, 1], dtype=torch.float32))
loss3 = criterion(torch.tensor(data1[:, 2], dtype=torch.float32), torch.tensor(data[:, 2], dtype=torch.float32))
loss4 = criterion(torch.tensor(data1[:, 3], dtype=torch.float32), torch.tensor(data[:, 3], dtype=torch.float32))
loss5 = criterion(torch.tensor(data1[:, 4], dtype=torch.float32), torch.tensor(data[:, 4], dtype=torch.float32))
loss6 = criterion(torch.tensor(data1[:, 5], dtype=torch.float32), torch.tensor(data[:, 5], dtype=torch.float32))
loss7 = criterion(torch.tensor(data1[:, 6], dtype=torch.float32), torch.tensor(data[:, 6], dtype=torch.float32))
loss8 = criterion(torch.tensor(data1[:, 7], dtype=torch.float32), torch.tensor(data[:, 7], dtype=torch.float32))
print("MSE Loss Column 1 :=> {:.5f}".format(loss1.item()))
print("MSE Loss Column 2 :=> {:.5f}".format(loss2.item()))
print("MSE Loss Column 3 :=> {:.5f}".format(loss3.item()))
print("MSE Loss Column 4 :=> {:.5f}".format(loss4.item()))
print("MSE Loss Column 5 :=> {:.5f}".format(loss5.item()))
print("MSE Loss Column 6 :=> {:.5f}".format(loss6.item()))
print("MSE Loss Column 7 :=> {:.5f}".format(loss7.item()))
print("MSE Loss Column 8 :=> {:.5f}".format(loss8.item()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15246264526840067, 'fraction_numerical': 0.03763143331488655, 'mean_word_length': 5.965317919075145, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '402302', 'license': 'MIT', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1287, 'n_words': 292, 'path': 'Q2/MiniProj_Q2_PartE.ipynb', 'repo_name': 'parhamzm/Beijing-Weather-Prediction', 'size': '3614'}