<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/thonic92/chal_TM/blob/master/model_tweets.ipynb)<jupyter_code>import json
import numpy as np
import pandas as pd
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
import sys
import re
import unicodedata
from collections import Counter
import nltk
with open("/content/gdrive/My Drive/json_datas_full.json", "r", encoding="latin1",errors='ignore' ) as read_file:
    data = json.load(read_file)
tweets = []
for i in range(len(data)):
    tweets.append(data[i]['text'].lower())
print(tweets[0:2])
tweets_str = ' '.join(tweets)
tweets_str=unicodedata.normalize('NFD',tweets_str).encode('ascii', 'ignore').decode("utf-8")
print(tweets_str[0:1000])
tweets_words = tweets_str.split(' ')
#print(tweets_words[1:100])
type(tweets_words)
#print(sorted(set(tweets_words)))
print(Counter(tweets_words).most_common()[0:100])
print(Counter(list(nltk.bigrams(tweets_words))).most_common()[0:100])
# on retire les urls 
tweets_str = re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ", tweets_str)
tweets_str = re.sub("paris2024"," ", tweets_str)
tweets_str = re.sub("jo2024"," ", tweets_str)
tweets_str = re.sub("jo 2024"," ", tweets_str)
tweets_str = re.sub("jo de 2024"," ", tweets_str)
tweets_str = re.sub("paris"," ", tweets_str)
tweets_str = re.sub("sport"," ", tweets_str)
tweets_str = re.sub("olympicday"," ", tweets_str)
tweets_str = re.sub("enmodejo"," ", tweets_str)
tweets_str = re.sub("grandparis"," ", tweets_str)
tweets_str = re.sub("cio"," ", tweets_str)
tweets_str = re.sub("jouerlejeu"," ", tweets_str)
tweets_str = re.sub("jeuxolympiques"," ", tweets_str)
tweets_str = re.sub("venezpartager"," ", tweets_str)
tweets_str = re.sub("jo2024paris"," ", tweets_str)
tweets_str = re.sub("jerevedesjeux"," ", tweets_str)
tweets_str = re.sub("france"," ", tweets_str)
tweets_str = re.sub("madeforsharing"," ", tweets_str)
tweets_str = re.sub("rio2016"," ", tweets_str)
tweets_str = re.sub("generation2024"," ", tweets_str)
tweets_str = re.sub("gagnonsensemble"," ", tweets_str)
tweets_str = re.sub("engager"," ", tweets_str)
tweets_str = re.sub("pleinement"," ", tweets_str)
tweets_str = re.sub("candidature"," ", tweets_str)
tweets_str = re.sub("nouvelle etape"," ", tweets_str)
tweets_str = re.sub("hidalgo veut"," ", tweets_str)
tweets_str = re.sub("favorable"," ", tweets_str)
tweets_str = re.sub("s engage"," ", tweets_str)
tweets_str = ' '.join(tweets_str.split())
tweets_str = tweets_str[0:300000]
print(tweets_str[0:1000])
len(tweets_str)


chars = sorted(list(set(tweets_str)))
char_to_int = dict((c, i) for i, c in enumerate(chars))
print(chars)
len(char_to_int)
n_chars = len(tweets_str.split(sep=' '))
n_vocab = len(chars)
print(n_chars)
print(n_vocab)
seq_length = 100
dataX = []
dataY = []
for i in range(0, n_chars - seq_length, 1):
    seq_in = tweets_str[i:i + seq_length]
    seq_out = tweets_str[i + seq_length]
    dataX.append([char_to_int[char] for char in seq_in])
    dataY.append(char_to_int[seq_out])
n_patterns = len(dataX)
print(n_patterns)


# reshape X to be [samples, time steps, features]
X = np.reshape(dataX, (n_patterns, seq_length, 1))
# normalize
X = X / float(n_vocab)
# one hot encode the output variable
y = np_utils.to_categorical(dataY)
model = Sequential()
model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(256))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
filepath="/content/gdrive/My Drive/weights-improvement-{epoch:02d}-{loss:.4f}-bigger3.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
callbacks_list = [checkpoint]
model.fit(X, y, epochs=30, batch_size=128, callbacks=callbacks_list)<jupyter_output>Epoch 1/30
53998/53998 [==============================] - 204s 4ms/step - loss: 2.8107

Epoch 00001: loss improved from inf to 2.81069, saving model to /content/gdrive/My Drive/weights-improvement-01-2.8107-bigger3.hdf5
Epoch 2/30
53998/53998 [==============================] - 203s 4ms/step - loss: 2.6043

Epoch 00002: loss improved from 2.81069 to 2.60432, saving model to /content/gdrive/My Drive/weights-improvement-02-2.6043-bigger3.hdf5
Epoch 3/30
53998/53998 [==============================] - 203s 4ms/step - loss: 2.4907

Epoch 00003: loss improved from 2.60432 to 2.49072, saving model to /content/gdrive/My Drive/weights-improvement-03-2.4907-bigger3.hdf5
Epoch 4/30
53998/53998 [==============================] - 203s 4ms/step - loss: 2.3632

Epoch 00004: loss improved from 2.49072 to 2.36319, saving model to /content/gdrive/My Drive/weights-improvement-04-2.3632-bigger3.hdf5
Epoch 5/30
53998/53998 [==============================] - 202s 4ms/step - loss: 2.2129

Epoch 00005: loss im[...]
{'fraction_non_alphanumeric': 0.18860083118939244, 'fraction_numerical': 0.06134969325153374, 'mean_word_length': 6.240687679083095, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '7828', 'license': 'MIT', 'n_tokens_mistral': 2225, 'n_tokens_neox': 1969, 'n_words': 368, 'path': 'notebooks/model_tweets.ipynb', 'repo_name': 'thonic92/chal_TM', 'size': '5053'}