<jupyter_start><jupyter_text>Vocalnic eruption data<jupyter_code>volc = pd.read_csv('data/txt/volerup.txt',delimiter = '\t', quoting = 3, encoding='utf-8')
volc.head()
volc.tail()
volc.isnull().sum()
len(volc)
Country_count = volc['Country'].value_counts()
sns.set(style="darkgrid")
sns.barplot(Country_count.index, Country_count.values, alpha=0.9)
plt.title('Frequency Distribution of Countries')
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel('Countries', fontsize=12)
plt.xticks(rotation = 90)
plt.show()  # So from what we can see there is alot of volcanic eruption in indonesia

labels = volc['Country'].astype('category').cat.categories.tolist()
counts = volc['Country'].value_counts()
sizes = [counts[var_cat] for var_cat in labels]
fig1, ax1 = plt.subplots()
ax1.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=True) #autopct is show the % on plot
ax1.axis('equal')
plt.show()
volc  = volc[['Year','Month','Day','Latitude','Longitude']]
len(volc)
volc.head()
volc  = volc.fillna(volc['Month'].value_counts().index[0])
volc  = volc.fillna(volc['Day'].value_counts().index[0])
volc.head()
# Repition shows that there were places that the eruption happend more than once.
print(volc.Latitude.duplicated().sum())
print(volc.Longitude.duplicated().sum())
volc.set_index("Year", inplace = True)<jupyter_output><empty_output><jupyter_text>Trying to see if there are trends in the coordinates.<jupyter_code>volc['Latitude'].plot()
volc['Longitude'].plot()
#volc['country_num'] = volc['Country'].astype('category').cat.codes
#volc['Location_num'] = volc['Location'].astype('category').cat.codes
#volc['Name_num'] = volc['Name'].astype('category').cat.codes
#volc.drop(['Country','Location','Name'],inplace = True, axis=1)
#volc.head()
#from sklearn.preprocessing import  OneHotEncoder
#from sklearn.compose import ColumnTransformer
#ct = ColumnTransformer(
#    [('one_hot_encoder', OneHotEncoder(categories='auto'), [2,3,4])],   # The column numbers to be transformed (here is [0] but can be [0, 1, 3])
#    remainder='passthrough'                                         # Leave the rest of the columns untouched
#)
#X = ct.fit_transform(volc)
#x = X.toarray()
#x
# ------------------------------------------------------------------------------------------------------------------------------------
volc = volc.reset_index()
volc.head()
#hold = volc['Latitude'].astype(str) + "," + volc['Longitude'].astype(str)
#volc['coor'] = hold
#volc.head()
from sklearn.preprocessing import MinMaxScaler
sc =MinMaxScaler(feature_range = (0,1))
volc_scaled_set = sc.fit_transform(volc)
volc_scaled_set
time_steps = 30 
check = len(volc_scaled_set)
check
# creating the time steps
x_train = []
y_train = []

for i in range(time_steps, check):
    x_train.append(volc_scaled_set[i-time_steps:i, :])  
    y_train.append(volc_scaled_set[i,:])
    
x_train, y_train = np.array(x_train), np.array(y_train)
len(x_train)
x_train = np.reshape(x_train,(x_train.shape[0], x_train.shape[1], 5))
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from keras import backend
# f beta metric, incase of imbalanced dataset ...... 
def f_beta(y_true, y_pred, beta =2):
    # clip prediction
    y_pred = backend.clip(y_pred, 0, 1)
    
    # calculate elements
    tp = backend.sum(backend.round(backend.clip(y_true * y_pred, 0,1)), axis = 1)
    fp = backend.sum(backend.round(backend.clip(y_pred - y_true, 0,1)), axis = 1)
    fn = backend.sum(backend.round(backend.clip(y_true - y_pred, 0,1)), axis = 1)
    
    #  precision
    p = tp / (tp + fp + backend.epsilon())
    
    #  recall
    r = tp / (tp +fn + backend.epsilon())
    
    # calculate fbeta
    
    bb = beta ** 2
    fbeta_score = backend.mean((1 + bb) * (p * r) / (bb * p + r + backend.epsilon()))
    return fbeta_score
    
detector.fit(x_train, y_train, epochs = 90, batch_size = 10)
# ---------------------------------------------------------------------------------------------------------<jupyter_output><empty_output><jupyter_text>Tsunmi data<jupyter_code>tsu = pd.read_csv('data/txt/tsrunup.txt',delimiter = '\t', quoting = 3, encoding='latin-1')
tsu.head()
tsu.isnull().sum()
tsu  = tsu[['YEAR', 'LOCATION_NAME','COUNTRY','LATITUDE','LONGITUDE']]
tsu.head()
# Repition shows that there were places that the eruption happend more than once.
print(tsu.LATITUDE.duplicated().sum())
print(tsu.LONGITUDE.duplicated().sum())
print(tsu.LOCATION_NAME.duplicated().sum())
### ------------------------------------------------------------------------------------------------------<jupyter_output><empty_output><jupyter_text>Earthquake data<jupyter_code>earth = pd.read_csv('data/txt/signif.txt',delimiter = '\t', quoting = 3, encoding='latin-1')
earth.head()
earth.isnull().sum()
earth  =earth[['YEAR', 'LOCATION_NAME','COUNTRY','LATITUDE','LONGITUDE']]
print(earth.LATITUDE.duplicated().sum())
print(earth.LONGITUDE.duplicated().sum())
print(earth.LOCATION_NAME.duplicated().sum())
# https://medium.com/@kasiarachuta/choosing-columns-in-pandas-dataframe-d0677b34a6ca
# https://towardsdatascience.com/time-series-forecasting-with-recurrent-neural-networks-74674e289816
# https://www.datacamp.com/community/tutorials/categorical-data
# https://cmdlinetips.com/2018/11/how-to-join-two-text-columns-into-a-single-column-in-pandas/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2200185356811863, 'fraction_numerical': 0.01464318813716404, 'mean_word_length': 3.741652021089631, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '5415', 'license': 'Apache-2.0', 'n_tokens_mistral': 1965, 'n_tokens_neox': 1886, 'n_words': 344, 'path': 'notebooks/Natural disaster pattern predictor/WO_Natural disaster pattern predictor.ipynb', 'repo_name': 'kene111/CFC2020', 'size': '5395'}