<jupyter_start><jupyter_text>Find population of a US CITY based on        ALAND    AWATER    LAT    LONG    STATE    UAtypesteps to train a DNN¶1. Prepare data_ clean data_ vectorize/encode data_ split data in train,validation and test data2. Train model_ create model and layers_ compile mode(optimizer , metrics, loss)_ fit model(train data, validation data, epochs, mini batch size)3. Validate model_ validate on test data4. Repeat<jupyter_code>import pandas as pd
import numpy as np

#prepare data
city_data = []
#00037	Abbeville, LA Urban Cluster	C	19824	8460	29222871	300497	      11.283	       0.116	 29.967602	 -92.098219  
def parseline(line):
    parts = line.split('\t')
    data={}
    namepart = parts[1].split(',')
    data['NAME']=namepart[0]
    data['STATE']=namepart[1].strip().split()[0].strip()
    data['UAtype']=parts[2]
    data['POP10']= int(parts[3])
    data['ALAND']= int(parts[5])
    data['AWATER']= int(parts[6])
    data['LAT']=float(parts[9])
    data['LONG']=float(parts[10])
    return data
with open('Gaz_ua_national.txt','r') as file:
    city_data = [ parseline(line.strip()) for line in file.readlines()[1:]]
city_data[-1]
len(city_data)
df = pd.DataFrame(city_data)
df = df.sample(len(df))#shuffle
df.head()
def to_numerical(labels):
    numerical_dict={}
    for label in labels:
        if label not in numerical_dict:
            numerical_dict[label]=len(numerical_dict)
    return numerical_dict

def to_one_hot(labels):
    numerical = to_numerical(labels)
    one_hot = np.zeros((len(labels),len(numerical)))
    for i,label in enumerate(labels):
        one_hot[i,numerical[label]]=1
    return np.asarray(one_hot)
#normalise data
train, validation, test = (0.6,0.2,0.2)
numeric_data = np.asarray(df[['ALAND','AWATER','LAT','LONG']].values)
print(numeric_data[0])
mean=[]
std=[]
def normalize_data(numeric_data,fraction):
    global mean, std
    train_size = int(len(numeric_data)*fraction)
    mean =  numeric_data[:train_size].mean(axis=0)
    std =  numeric_data[:train_size].std(axis=0)
    return (numeric_data - mean)/std
 
numeric_data = normalize_data(numeric_data,train)

#encode data
states = to_one_hot(df['STATE'].values)
UA = to_one_hot(df['UAtype'].values)
numeric_data = np.concatenate((numeric_data,states,UA),axis=1)
#merge all data
population = np.asarray(df['POP10'].values).astype(np.float32)/1000#population in thousands
population = population.reshape(len(population),1)
population.min()
numeric_data.shape
#split data
train_f, validation_f, test_f = (0.6,0.2,0.2)
train = int(train_f*len(numeric_data))
validation = int((train_f+validation_f)*len(numeric_data))
print(validation)
x_train,x_validate,x_test = (numeric_data[:train],numeric_data[train:(validation)],numeric_data[validation:])
y_train,y_validate,y_test = (population[:train],population[train:(validation)],population[validation:])
x_train.shape,x_validate.shape,x_test.shape
from keras import Sequential
from keras import layers
from keras import models
from keras import losses
from keras import optimizers
from keras import metrics

def createmodel(nodes,layers=2):
    model = Sequential()
    model.add(layers.Dense(nodes,activation='relu',input_shape=(x_train.shape[1],)))
    for layer in range(layers-1):
        model.add(layers.Dense(nodes,activation='relu'))
    model.add(layers.Dense(1))#no activation
    
    model.compile(optimizer='rmsprop',loss=['mse'],metrics=['mae'])
    return model
import matplotlib.pyplot as pp
%matplotlib inline
def plothistory(history):
    pp.figure(figsize=(14,4))
    pp.subplot(1,2,1)
    pp.plot(history.history['loss'],label='Training loss')
    pp.plot(history.history['val_loss'],label='Validation loss')
    pp.xlabel('epochs')
    pp.ylabel('loss')
    pp.legend()
    pp.title('Loss vs epoch')
    
    pp.subplot(1,2,2)
    pp.plot(history.history['mean_absolute_error'],label='mean absolute error')
    pp.plot(history.history['val_mean_absolute_error'],label='validation mean absolute error')
    pp.xlabel('epochs')
    pp.ylabel('mae')
    pp.legend()
    pp.title('Mean absolute error vs epoch')<jupyter_output><empty_output><jupyter_text>model = createmodel(64)history = model.fit(x_train,y_train,         validation_data=(x_validate,y_validate),         epochs=20,         batch_size = 10)2155/2155 [==============================] - 1s 242us/step - loss: 64388.7138 - mean_absolute_error: 32.4828 - val_loss: 17549.2027 - val_mean_absolute_error: 25.8405 model = createmodel(64)history = model.fit(x_train,y_train,         validation_data=(x_validate,y_validate),         epochs=20,        batch_size = 1)Epoch 20/202155/2155 [==============================] - 5s 2ms/step - loss: 41239.8373 - mean_absolute_error: 19.5487 - val_loss: 122755.2096 - val_mean_absolute_error: 32.1154 model = createmodel(16)history = model.fit(x_train,y_train,         validation_data=(x_validate,y_validate),         epochs=40,         batch_size = 1)2155/2155 [==============================] - 7s 3ms/step - loss: 44152.8033 - mean_absolute_error: 19.1785 - val_loss: 173393.0632 - val_mean_absolute_error: 33.0317 model = createmodel(32)history = model.fit(x_train,y_train,         validation_data=(x_validate,y_validate),         epochs=200,         batch_size = 50)2155/2155 [==============================] - 0s 51us/step - loss: 37556.1281 - mean_absolute_error: 23.7620 - val_loss: 17176.5076 - val_mean_absolute_error: 24.1855 model = createmodel(64)history = model.fit(x_train,y_train,         validation_data=(x_validate,y_validate),         epochs=200,         batch_size = 50)Epoch 200/2002155/2155 [==============================] - 0s 184us/step - loss: 31475.6261 - mean_absolute_error: 19.9625 - val_loss: 15004.8529 - val_mean_absolute_error: 20.4766<jupyter_code>model = createmodel(64)
history = model.fit(x_train,y_train,
         validation_data=(x_validate,y_validate),
         epochs=500,
         batch_size = 50)
plothistory(history)
def seepredictions(model):
    df_validation = df.loc[df.index[range(train,validation)]].copy()
    #df.tail(len(df)-train).head(validation) #this should work too
    pred = model.predict(x_validate)*1000
    df_validation['Predictions']=pred
    df_validation['DiffPercentage']=abs(df_validation['Predictions']-df_validation['POP10'])/df_validation['POP10']*100
    return df_validation
print('mean percentage diff = ', seepredictions(model)['DiffPercentage'].mean())
val_df = seepredictions(model)
val_df.head()
#analyze
l = len(val_df)
val_df.sort_values('Predictions').tail(1000).head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15777439024390244, 'fraction_numerical': 0.057926829268292686, 'mean_word_length': 5.019266055045872, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '71632', 'license': 'Apache-2.0', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2237, 'n_words': 403, 'path': 'Keras-train-analyse.ipynb', 'repo_name': 'krishansubudhi/DNN_USCityPopulation', 'size': '6560'}