<jupyter_start><jupyter_text>Importing<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.model_selection import train_test_split
from sklearn import metrics

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers 

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,Activation,Dropout,BatchNormalization
from tensorflow.keras.callbacks import EarlyStopping

import warnings
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Util Functions<jupyter_code>def statistics(train_dataset):
  train_stats = train_dataset.describe()
  train_stats = train_stats.transpose()
  return train_stats
def norm(x,train_stats):
  return (x - train_stats['mean']) / train_stats['std']
def preprocess(df):
  df.drop(columns=['Popuation [2001]'],inplace=True,axis=1)
  df.drop(columns=['City'],inplace=True,axis=1)
  df.drop(columns=['State','Type','SWM'],axis=1,inplace=True)
  
  for j in range(0,df.shape[0]):
    df['Population [2011]'][j]=str(df['Population [2011]'][j])
    df['Population [2011]'][j] = df['Population [2011]'][j].replace(',', '')
    df['Population [2011]'][j]=float(df['Population [2011]'][j])
  
  for col in df.columns:
    df[col] = df[col].fillna(df[col].median())

  return df
def num(x,y):
  diff=abs(x.flatten()-y.flatten())
  count=0;
  total=0;
  for item in diff:
    total+=1
    if item<=1000:
      count+=1

  return 100*(count/total)
# Regression chart 
def chart_regression(pred, y, sort=True):
    t = pd.DataFrame({'pred': pred, 'y': y.flatten()})
    if sort:
        t.sort_values(by=['y'], inplace=True)
    plt.plot(t['y'].tolist(), label='expected')
    plt.plot(t['pred'].tolist(), label='prediction')
    plt.ylabel('output')
    plt.legend()
    plt.show()<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>df_train=pd.read_csv(r"/content/Train_dataset - Train_data.csv")
df_train=preprocess(df_train)

y = df_train.pop('Covid Cases')
df_train.describe()
x=df_train.values
y=y.values
stats=statistics(df_train)
print(stats)
x = norm(df_train,stats)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=42)<jupyter_output><empty_output><jupyter_text>Model And Training<jupyter_code>model=Sequential()
model.add(Dropout(0.2))
model.add(Dense(512, input_dim=x.shape[1], activation='relu')) # Hidden 1
model.add(Dense(512, activation='relu')) # Hidden 2
model.add(Dense(1)) # Output

optim=tf.keras.optimizers.Adam(0.005)

model.compile(loss='mean_squared_error',optimizer=optim)
history=model.fit(x_train,y_train,validation_data=(x_test,y_test),verbose=0,epochs=1000)
# Predict
pred_valid = model.predict(x_test)
pred_train=model.predict(x_train)
# chcking error range and plotting
print(num(pred_train,y_train)) 
print(num(pred_valid,y_test))

plt.plot(pred_valid)
plt.show()
plt.plot(y_test)
plt.show()

plt.plot(pred_train)
plt.show()
plt.plot(y_train)
plt.show()
# Measure RMSE error-- see how they perform .. i don't think this overfits 
score = np.sqrt(metrics.mean_squared_error(pred_valid,y_test))
print("Final score (RMSE): {}".format(score))

score_train=np.sqrt(metrics.mean_squared_error(pred_train,y_train))
print("Final score (RMSE):Train {}".format(score_train))
chart_regression(pred_valid.flatten(),y_test)
chart_regression(pred_train.flatten(),y_train)
test_predictions = model.predict(x_test).flatten()

a = plt.axes(aspect='equal')
plt.scatter(test_predictions,y_test)
plt.xlabel('True Values ')
plt.ylabel('Predictions ')
plt.plot()
error = test_predictions - y_test.flatten()
plt.hist(error)
plt.xlabel("Prediction Error [MPG]")
_ = plt.ylabel("Count")<jupyter_output><empty_output><jupyter_text>SolutionThe error prediciton curve is gaussian and i don't think its overfitting.So I will go with this one<jupyter_code>df_test=pd.read_csv(r'/content/Test_dataset - Test_data.csv')
city_name=df_test['City']
df_test=preprocess(df_test)
df_test.shape
x_final = norm(df_test,stats)
print(x_final)
x_final.isna().sum()
print(x_final.shape)
result=[]

output=model.predict(x_final)
city=city_name.values
for i in range(0,501):
  result.append([city[i],output[i]])

df=pd.DataFrame(result,columns=['City','Covid Cases'])
df.to_csv('solution 01.csv',index=False)<jupyter_output><empty_output><jupyter_text>Task-2<jupyter_code># importing again

import math
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense, LSTM
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
df=pd.read_csv("/content/Test_dataset - Foreign_Visitors_TS.csv",header=None,names=['City','April','May','June','July','August'])
df.head()
df.drop(index=[0,1],axis=0,inplace=True)
city=df.pop('City')
df.fillna(0,inplace=True)
labels=df['August']

data = df.to_numpy()
city=city.values
labels=labels.values.astype('int')
new_pass=[]
for part in data:
  x_train=[]
  y_train=[]
  for i in range(4):
    a=int(part[i])
    b=int(part[i+1])
    x_train.append(a)
    y_train.append(b)
  x_train = np.array(x_train).reshape(4, 1, 1)
  y_train=np.array(y_train)
  model = Sequential()
  model.add(LSTM(50, activation='relu', input_shape=(1, 1)))
  model.add(Dense(1))
  model.compile(optimizer='adam', loss='mse')

  history = model.fit(x_train,y_train, epochs=1000, verbose=0)
  c=int(part[-1])
  test_input = np.array([c])
  test_input = test_input.reshape((1, 1, 1))
  test_output = model.predict(test_input, verbose=0)
  new_pass.append(test_output)
new_pass[0][0][0]
temp=pd.read_csv(r"/content/Test_dataset - Test_data.csv")
temp_2=temp.pop('Foreign Visitors')
temp_2[0]
foreign=[max(new_pass[i][0][0],temp_2[i]+500) for i in range(501)]
len(foreign)
temp
temp['Foreign Visitors']=foreign
temp
city_name=temp['City']
df_test=preprocess(temp)
x_final = norm(df_test,stats)
print(x_final)
x_final.isna().sum()
result=[]

output=model.predict(x_final)
city=city_name.values
for i in range(0,501):
  result.append([city[i],output[i]])

df=pd.DataFrame(result,columns=['City','Covid Cases'])
df.to_csv('solution 02.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1601675797615211, 'fraction_numerical': 0.019658395101514663, 'mean_word_length': 7.167105263157895, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '96827', 'license': 'Apache-2.0', 'n_tokens_mistral': 2478, 'n_tokens_neox': 2350, 'n_words': 386, 'path': 'Hackathon.ipynb', 'repo_name': 'KhunJahad/Flipr-Hackathon-6.0-ML', 'size': '6206'}