<jupyter_start><jupyter_text>Load framework and TPU environment<jupyter_code>%pylab inline
import tensorflow as tf
import pandas as pd
import jieba
from gensim import corpora
from keras.preprocessing import sequence
from scipy import stats
import ast
import os
from keras.models import Model,Sequential
from keras.layers.embeddings import Embedding
from keras.layers import Dense, Reshape,Input, LSTM,Bidirectional,GRU, concatenate
from keras.layers.normalization import BatchNormalization
from keras.callbacks import ReduceLROnPlateau
from keras import optimizers
from sklearn.preprocessing import OneHotEncoder

resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.TPUStrategy(resolver)<jupyter_output>Populating the interactive namespace from numpy and matplotlib
INFO:tensorflow:Initializing the TPU system: grpc://10.56.1.34:8470<jupyter_text>Load the saved training set, model, and so on. But I only need the training set. The file includes the model because I packaged everything in a zip for convenience.<jupyter_code>!gdown https://drive.google.com/uc?id=1nbWJO9wGMODfnmYGCASi9kEozpgtIRjo
!unzip weibo_baselines.zip
cd weibo0.85split_testAcc98.84%<jupyter_output>/content/weibo0.85split_testAcc98.84%<jupyter_text>Split the data set into training set and test set. The split_pos in the third line of code means that 85% will be the training set and 15% will be the test set. This code cell is responsible for processing the training set.<jupyter_code>dataset=pd.read_csv('weibo_dataset.csv',sep=',',keep_default_na=False)
split_rate=0.85
split_pos=int(len(dataset)*split_rate)
training_dataset=dataset[:split_pos]
n_weibos=training_dataset['n_weibos'].values
labels=training_dataset['label'].values
num_weibo=20
def merge_two_list(a,b):
  if isinstance(a[0],ndarray) == False and isinstance(a[0],list)==False:
    return np.concatenate((array(a).reshape(-1,1),array(b).reshape(-1,1)),axis=1).reshape(len(a)*2)
  return np.concatenate((a,b),axis=1).reshape(len(a)*2,len(a[0]),-1)
def chinese_cut_token(sentences):
  sentences_cut=[]
  for input in sentences:
    input=ast.literal_eval(input)
    sentence_cut=[["None"]]*num_weibo
    for i,weibo in enumerate(input):
      if (i>=num_weibo): break
      tokens=jieba.cut(weibo, cut_all=True)
      sentence_cut[i]=[word for word in tokens]
    sentences_cut.append(sentence_cut[:num_weibo])
  dic=corpora.Dictionary([token for sentence_cut in sentences_cut for token in sentence_cut]).token2id
  X=[]
  X_mask=[]
  for user in sentences_cut:
    x,x_mask=[],[]
    for weibo_sentence in user:
      x.append([dic[token]+2 for token in weibo_sentence])
      ids=array([dic[token]+2 for token in weibo_sentence])
      if len(ids)>=2:
        ids[randint(len(ids)-1,size=int(ceil(len(ids)/(10 if rand()>0.68 else 8))))]=1
      x_mask.append(ids)
    X.append(x)
    X_mask.append(x_mask)
  maxLen=max(len(max(x,key=len)) for x in X)
  wordsSize =len(dic)+1
  return sentences_cut,dic,merge_two_list(X,X_mask),maxLen,wordsSize
_,dicW,XW,maxLenW,wordsSizeW=chinese_cut_token(n_weibos)
n_weibos_seq = sequence.pad_sequences(XW.reshape(-1), maxLenW).reshape(len(n_weibos)*2,num_weibo,maxLenW)
n_topicsAndRetwefeatuers=np.concatenate((training_dataset['n_topics'],training_dataset['n_retwefeatuers']))
_,dicTR,XTR,maxLenTR,wordsSizeTR=chinese_cut_token(n_topicsAndRetwefeatuers)
n_TR_seq = sequence.pad_sequences(XTR.reshape(-1), maxLenTR).reshape(len(n_topicsAndRetwefeatuers)*2,num_weibo,maxLenTR)
gap=len(training_dataset['n_retwefeatuers'])*2
n_topics_seq=n_TR_seq[:gap]
n_retwefeatuers_seq=n_TR_seq[gap:]
user_metrics_dataset=training_dataset[['follow_count', 'followers_count', 'mbrank', 'mbtype', 'statuses_count', 'urank', 'verified', 'verified_type', 'verified_type_ext']]
umd_mean=user_metrics_dataset.mean(axis=0).values
umd_std=user_metrics_dataset.std(axis=0).values
user_metrics_dataset=(user_metrics_dataset.values-umd_mean)/umd_std
user_metrics_dataset=merge_two_list(user_metrics_dataset,user_metrics_dataset)
tmp=[ast.literal_eval(row) for row in training_dataset['n_weibo_infos']]
weibo_metrics_dataset=ones((len(tmp),num_weibo,len(tmp[0][0])))*-2#e.g if some users has less than num_weibo, then -2 means empty weibo
for i in range(weibo_metrics_dataset.shape[0]):
  weibo_metrics_dataset[i,:len(tmp[i])]=tmp[i][:num_weibo]
wmd_mean=weibo_metrics_dataset.reshape(-1,5).mean(axis=0)
wmd_std=weibo_metrics_dataset.reshape(-1,5).std(axis=0)
weibo_metrics_dataset=array([(wi-wmd_mean)/wmd_std for wi in weibo_metrics_dataset])
weibo_metrics_dataset=merge_two_list(weibo_metrics_dataset,weibo_metrics_dataset)
trainInputs=[*[(n_weibos_seq[:,i],n_retwefeatuers_seq[:,i],n_topics_seq[:,i],weibo_metrics_dataset[:,i])for i in range(num_weibo)],user_metrics_dataset]
trainOutputs=OneHotEncoder().fit_transform(merge_two_list(labels,labels).reshape((-1,1))).toarray()<jupyter_output>Building prefix dict from the default dictionary ...
DEBUG:jieba:Building prefix dict from the default dictionary ...
Dumping model to file cache /tmp/jieba.cache
DEBUG:jieba:Dumping model to file cache /tmp/jieba.cache
Loading model cost 1.017 seconds.
DEBUG:jieba:Loading model cost 1.017 seconds.
Prefix dict has been built successfully.
DEBUG:jieba:Prefix dict has been built successfully.
<string>:6: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray<jupyter_text>This code cell is responsible for processing the test set.<jupyter_code>testset_dataset=dataset[split_pos:]
n_weibos=testset_dataset['n_weibos'].values
labels=testset_dataset['label'].values
def chinese_cut_token(sentences,dic=None):
  sentences_cut=[]
  for input in sentences:
    input=ast.literal_eval(input)
    sentence_cut=[["None"]]*num_weibo
    for i,weibo in enumerate(input):
      if (i>=num_weibo): break
      tokens=jieba.cut(weibo, cut_all=True)
      sentence_cut[i]=[word for word in tokens]
    sentences_cut.append(sentence_cut[:num_weibo])
  dic=corpora.Dictionary([token for sentence_cut in sentences_cut for token in sentence_cut]).token2id if dic==None else dic
  X=[]
  for user in sentences_cut:
    x=[]
    for weibo_sentence in user:
      x.append([dic.get(token,-1)+2 for token in weibo_sentence])
    X.append(x)
  maxLen=max(len(max(x,key=len)) for x in X)
  wordsSize =len(dic)+1
  return sentences_cut,dic,array(X),maxLen,wordsSize
_,_,XW,_,_=chinese_cut_token(n_weibos,dicW)
n_weibos_seq2 = sequence.pad_sequences(XW.reshape(-1), maxLenW).reshape(len(n_weibos),num_weibo,maxLenW)
n_topicsAndRetwefeatuers2=append(testset_dataset['n_retwefeatuers'],testset_dataset['n_topics'])
_,_,XTR,_,_=chinese_cut_token(n_topicsAndRetwefeatuers2,dicTR)
n_topicsAndRetwefeatuers_seq2 = sequence.pad_sequences(XTR.reshape(-1), maxLenTR).reshape(len(n_topicsAndRetwefeatuers2),num_weibo,maxLenTR)
n_retwefeatuers_seq2=n_topicsAndRetwefeatuers_seq2[:len(testset_dataset['n_retwefeatuers'])]
n_topics_seq2=n_topicsAndRetwefeatuers_seq2[len(testset_dataset['n_retwefeatuers']):]
user_metrics_dataset2=testset_dataset[['follow_count', 'followers_count', 'mbrank', 'mbtype', 'statuses_count', 'urank', 'verified', 'verified_type', 'verified_type_ext']]
user_metrics_dataset2=(user_metrics_dataset2-umd_mean)/umd_std
tmp=[ast.literal_eval(row) for row in testset_dataset['n_weibo_infos']]
weibo_metrics_dataset2=ones((len(tmp),num_weibo,len(tmp[0][0])))*-2
for i in range(weibo_metrics_dataset2.shape[0]):
  weibo_metrics_dataset2[i,:len(tmp[i])]=tmp[i][:num_weibo]
weibo_metrics_dataset2=array([(wi-wmd_mean)/wmd_std for wi in weibo_metrics_dataset2])
testInputs=[*[(n_weibos_seq2[:,i],n_retwefeatuers_seq2[:,i],n_topics_seq2[:,i],weibo_metrics_dataset2[:,i])for i in range(num_weibo)],user_metrics_dataset2]
testOutputs=OneHotEncoder().fit_transform(append(labels,0).reshape((-1,1))).toarray()[:-1]<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:23: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray<jupyter_text>Model design, build the model<jupyter_code>model = Sequential()
def biLSTM(wordsSize,maxLen):
  modelBiLSTM=Sequential()#81
  modelBiLSTM.add(Embedding(wordsSize+1,32,input_length=maxLen))
  modelBiLSTM.add(Bidirectional(LSTM(32,dropout=0.4,return_sequences=True)))
  modelBiLSTM.add(Bidirectional(LSTM(64,dropout=0.4,return_sequences=False)))
  modelBiLSTM.add(Dense(1,activation='tanh'))
  return modelBiLSTM
N=num_weibo
inputs=[Input(shape=n_weibos_seq[0,0].shape) for i in range(N)]
inputs_t=[Input(shape=n_topics_seq[0,0].shape) for i in range(N)]
inputs_r=[Input(shape=n_retwefeatuers_seq[0,0].shape) for i in range(N)]
inputsm=[Input(shape=(len(weibo_metrics_dataset[0][0]),)) for i in range(N)]
b=biLSTM(wordsSizeW,maxLenW)
b_t=biLSTM(wordsSizeTR,maxLenTR)
b_r=biLSTM(wordsSizeTR,maxLenTR)
weibo_metrics_importance=2
weibo_metrics = Sequential()
weibo_metrics.add(Dense(16, activation='tanh', input_shape=(len(weibo_metrics_dataset[0][0]),)))
weibo_metrics.add(Dense(weibo_metrics_importance, activation='tanh'))
concat = concatenate([b.output,b_t.output,b_r.output,weibo_metrics.output], axis=-1)
output = Dense(16, activation='tanh')(concat)
output = Dense(1, activation='tanh')(output)
model = Model(inputs=[b.inputs,b_t.inputs,b_r.inputs,weibo_metrics.inputs], outputs=output)
biLSTMs=[model([inputs[i],inputs_t[i],inputs_r[i],inputsm[i]]) for i in range(N)]
concat = concatenate(biLSTMs, axis=-1)
output = Reshape((-1,1))(concat)
output=LSTM(32,dropout=0.4,return_sequences=False)(output)
output = Dense(1, activation='tanh')(output)
user_metrics_importance=5
user_metrics = Sequential()
user_metrics.add(Dense(32, activation='tanh', input_shape=(user_metrics_dataset.shape[1],)))
user_metrics.add(Dense(user_metrics_importance, activation='tanh'))
concat = concatenate([output,user_metrics.output], axis=-1)
output = Dense(16, activation='tanh')(concat)
output = Dense(2, activation='softmax')(output)
model = Model(inputs=[*list(zip(inputs,inputs_t,inputs_r,inputsm)),user_metrics.inputs], outputs=output)
from keras.utils import plot_model
plot_model(model,show_shapes=True,show_layer_names=True,expand_nested=True)<jupyter_output><empty_output><jupyter_text>Train the model, a little slow in the first epoch, please wait.<jupyter_code>model.compile(loss='mse', optimizer='adam',metrics=['accuracy'])
class AccStop(tf.keras.callbacks.Callback):
    def __init__(self, threshold):
        super(AccStop, self).__init__()
        self.threshold = threshold
    def on_epoch_end(self, epoch, logs=None): 
        if logs["val_accuracy"] >= self.threshold and logs["accuracy"] >= self.threshold:
            self.model.stop_training = True
reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=10, mode='auto')
acc_stop=AccStop(0.95)
model.fit(trainInputs, array(trainOutputs).astype(int), batch_size=32, epochs=1000,shuffle=True,validation_data=(testInputs, array(testOutputs).astype(int)),callbacks=[reduce_lr,acc_stop])<jupyter_output><empty_output><jupyter_text>Test the model, test set accuracy.<jupyter_code>pred=model.predict(testInputs).argmax(axis=1)
acc=np.count_nonzero((pred.round().astype(int).reshape(-1)==testOutputs.argmax(axis=1))==True)/len(pred)
print("%d test set accuracy = %.2f%% (test set includes tokens which have never been trained. Model was trained with %d training set)"%(len(testOutputs),acc*100,len(trainOutputs)//2))
#This is for those who understand Chinese language or use weibo.
predict_wrong=where((pred.round().astype(int).reshape(-1)==testOutputs.argmax(axis=1))==False)
print("The ids of users who were identified incorrectly: "+str(dataset[split_pos:]['uid'].values[predict_wrong]))
#these are wrong prediction when test set accuracy is 95.35%, some interesting facts:
#3977649527 posted a tweet "Why my weibo was hacked?"
#6432141741 and 3977649527 were incorrectly predicted in different models.<jupyter_output>The ids of users who were identified incorrectly: [3977649527 3308066363 1236000223 6432141741]<jupyter_text>Save metrics' means and standard deviations (for input normalization), dictionaries, as well as the model and its weights. which will be the baseline file (not a necessary step, just if you want to save the model in case you will have to retrain it later)<jupyter_code>cd ..
from shutil import make_archive
!mkdir weibo_model
pd.DataFrame.from_dict(dicW, orient='index', columns=['id']).to_csv('weibo_model/token2idW.csv', columns=['id'],encoding="utf_8_sig") 
pd.DataFrame.from_dict(dicTR, orient='index', columns=['id']).to_csv('weibo_model/token2idTR.csv', columns=['id'],encoding="utf_8_sig") 
metrics_means_and_stds=pd.DataFrame([[str(wmd_mean.tolist()),str(wmd_std.tolist()),str(umd_mean.tolist()),str(umd_std.tolist())]],columns=["weibo_metrics_dataset_mean","weibo_metrics_dataset_std","user_metrics_dataset_mean","user_metrics_dataset_std"])
metrics_means_and_stds.to_csv('weibo_model/metrics_means_and_stds.csv') 
dataset.to_csv('weibo_model/weibo_dataset.csv',index=False,encoding="utf_8_sig") 
save_options = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')
model.save('weibo_model/weibo_model.h5',options=save_options)
make_archive("weibo%.2fsplit_test%.2f%%"%(split_rate,acc*100),'zip',root_dir="weibo_model",base_dir=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13006962888522, 'fraction_numerical': 0.020386189074725433, 'mean_word_length': 7.890874282067645, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '612890', 'license': 'MIT', 'n_tokens_mistral': 5262, 'n_tokens_neox': 5065, 'n_words': 839, 'path': 'Weibo_sentiment_analysis_training_and_testing.ipynb', 'repo_name': 'timmmGZ/Weibo-sentiment-analysis-and-crawler', 'size': '13931'}