<jupyter_start><jupyter_text>訓練情感模型分析這篇文章中的李＋賢是否有ＡＭＬ風險＿將2-2處理好的300個字資料集進行訓練<jupyter_code>import transformers
from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup
import torch
import numpy as np
import pandas as pd
import seaborn as sns
from pylab import rcParams
import matplotlib.pyplot as plt
from matplotlib import rc
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from collections import defaultdict
from textwrap import wrap
from torch import nn, optim
from torch.utils.data import Dataset, DataLoader<jupyter_output><empty_output><jupyter_text>情感分析模型<jupyter_code>df = pd.read_csv('step2_2_output_train_data_300.csv')
# 正負資料比較及
class_names = ['negative', 'positive']
ax = sns.countplot(df.article_tags)
plt.xlabel('review sentiment')
ax.set_xticklabels(class_names);
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")#, do_lower_case=False)
sample_txt = df.iloc[0].article_truncate
tokens = tokenizer.tokenize(sample_txt)
token_ids = tokenizer.convert_tokens_to_ids(tokens)
print(f' Sentence: {sample_txt}')
print(f'   Tokens: {tokens}')
print(f'Token IDs: {token_ids}')
MAX_LEN = 300
class GPReviewDataset(Dataset):
    def __init__(self, reviews, targets, tokenizer, max_len):
        self.reviews = reviews
        self.targets = targets
        self.tokenizer = tokenizer
        self.max_len = max_len
    def __len__(self):
        return len(self.reviews)
    def __getitem__(self, item):
        review = str(self.reviews[item])
        target = self.targets[item]
        encoding = self.tokenizer.encode_plus(
          review,
          add_special_tokens=True,
          max_length=self.max_len,
          return_token_type_ids=False,
          pad_to_max_length=True,
          return_attention_mask=True,
          return_tensors='pt',
        )
        return {
          'review_text': review,
          'input_ids': encoding['input_ids'].flatten(),
          'attention_mask': encoding['attention_mask'].flatten(),
          'targets': torch.tensor(target, dtype=torch.long)
        }
df_train, df_test = train_test_split(
  df,
  test_size=0.1,
  random_state=38
)
df_val = df_test.copy()
df_train.article_tags.value_counts()#0.08869431270014999

df_test.article_tags.value_counts() #0.09140767824497258

df_test.head()
def create_data_loader(df, tokenizer, max_len, batch_size):
    ds = GPReviewDataset(
    reviews=df.article_truncate.to_numpy(),
    targets=df.article_tags.to_numpy(),
    tokenizer=tokenizer,
    max_len=max_len
    )
    return DataLoader(
    ds,
    batch_size=batch_size,
    num_workers=0
    )
BATCH_SIZE = 16
train_data_loader = create_data_loader(df_train, tokenizer, MAX_LEN, BATCH_SIZE)
val_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)
test_data_loader = create_data_loader(df_test, tokenizer, MAX_LEN, BATCH_SIZE)
PRE_TRAINED_MODEL_NAME = "bert-base-chinese"

bert_model = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)

class SentimentClassifier(nn.Module):
    def __init__(self, n_classes):
        super(SentimentClassifier, self).__init__()
        self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)
        self.drop = nn.Dropout(p=0.3)
        self.out = nn.Linear(self.bert.config.hidden_size, n_classes)
    def forward(self, input_ids, attention_mask):
        _, pooled_output = self.bert(
          input_ids=input_ids,
          attention_mask=attention_mask
        )
        output = self.drop(pooled_output)
        return self.out(output)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

model = SentimentClassifier(len(class_names))
model = model.to(device)
EPOCHS = 10
optimizer = AdamW(model.parameters(), lr=2e-5, correct_bias=False)
total_steps = len(train_data_loader) * EPOCHS
scheduler = get_linear_schedule_with_warmup(
  optimizer,
  num_warmup_steps=0,
  num_training_steps=total_steps
)
loss_fn = nn.CrossEntropyLoss().to(device)
def train_epoch(
  model,
  data_loader,
  loss_fn,
  optimizer,
  device,
  scheduler,
  n_examples
):
    model = model.train()
    losses = []
    correct_predictions = 0
    for d in data_loader:
        input_ids = d["input_ids"].to(device)
        attention_mask = d["attention_mask"].to(device)
        targets = d["targets"].to(device)
        outputs = model(
          input_ids=input_ids,
          attention_mask=attention_mask
        )
        _, preds = torch.max(outputs, dim=1)
        loss = loss_fn(outputs, targets)
        correct_predictions += torch.sum(preds == targets)
        losses.append(loss.item())
        loss.backward()
        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()
    return correct_predictions.double() / n_examples, np.mean(losses)
def eval_model(model, data_loader, loss_fn, device, n_examples):
    model = model.eval()
    losses = []
    correct_predictions = 0
    with torch.no_grad():
        for d in data_loader:
            input_ids = d["input_ids"].to(device)
            attention_mask = d["attention_mask"].to(device)
            targets = d["targets"].to(device)
            outputs = model(
            input_ids=input_ids,
            attention_mask=attention_mask
            )
            _, preds = torch.max(outputs, dim=1)
            loss = loss_fn(outputs, targets)
            correct_predictions += torch.sum(preds == targets)
            losses.append(loss.item())
    return correct_predictions.double() / n_examples, np.mean(losses)
%%time
history = defaultdict(list)
best_accuracy = 0
for epoch in range(EPOCHS):
    print(f'Epoch {epoch + 1}/{EPOCHS}')
    print('-' * 10)
    train_acc, train_loss = train_epoch(
    model,
    train_data_loader,
    loss_fn,
    optimizer,
    device,
    scheduler,
    len(df_train)
    )
    print(f'Train loss {train_loss} accuracy {train_acc}')
    val_acc, val_loss = eval_model(
    model,
    val_data_loader,
    loss_fn,
    device,
    len(df_val)
    )
    print(f'Val   loss {val_loss} accuracy {val_acc}')
    print()
    history['train_acc'].append(train_acc)
    history['train_loss'].append(train_loss)
    history['val_acc'].append(val_acc)
    history['val_loss'].append(val_loss)
    if val_acc > best_accuracy:
        torch.save(model.state_dict(), 'best_model_state.bin')
        best_accuracy = val_acc
# save model
import os 
PATH = os.getcwd()+str('/step3_output_bert_senti.pth')
torch.save(model, PATH)
PATH = os.getcwd()+str('/step3_output_bert_senti.pth')

model2 = torch.load(PATH)
model2.eval()
plt.plot(history['train_acc'], label='train accuracy')
plt.plot(history['val_acc'], label='validation accuracy')
plt.title('Training history')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend()
plt.ylim([0, 1]);
test_acc, _ = eval_model(
  model,
  test_data_loader,
  loss_fn,
  device,
  len(df_test)
)
#test_acc.item()
test_acc.item() #0.9728643216080403

def get_predictions(model, data_loader):
    model = model.eval()
    review_texts = []
    predictions = []
    prediction_probs = []
    real_values = []
    with torch.no_grad():
        for d in data_loader:
            texts = d["review_text"]
            input_ids = d["input_ids"].to(device)
            attention_mask = d["attention_mask"].to(device)
            targets = d["targets"].to(device)
            outputs = model(
            input_ids=input_ids,
            attention_mask=attention_mask
            )
            _, preds = torch.max(outputs, dim=1)
            review_texts.extend(texts)
            predictions.extend(preds)
            prediction_probs.extend(outputs)
            real_values.extend(targets)
    predictions = torch.stack(predictions).cpu()
    prediction_probs = torch.stack(prediction_probs).cpu()
    real_values = torch.stack(real_values).cpu()
    return review_texts, predictions, prediction_probs, real_values
y_review_texts, y_pred, y_pred_probs, y_test = get_predictions(
  model,
  test_data_loader
)
print(classification_report(y_test, y_pred, target_names=class_names))

def show_confusion_matrix(confusion_matrix):
    hmap = sns.heatmap(confusion_matrix, annot=True, fmt="d", cmap="Blues")
    hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')
    hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')
    plt.ylabel('True sentiment')
    plt.xlabel('Predicted sentiment');
cm = confusion_matrix(y_test, y_pred)
df_cm = pd.DataFrame(cm, index=class_names, columns=class_names)
show_confusion_matrix(df_cm)
idx = 7
review_text = y_review_texts[idx]
true_sentiment = y_test[idx]
pred_df = pd.DataFrame({
  'class_names': class_names,
  'values': y_pred_probs[idx]
})
print("\n".join(wrap(review_text)))
print()
print(f'True sentiment: {class_names[true_sentiment]}')
df_test2 = df_test.reset_index()#[df_test.article_tags==1]
df_test2[df_test2.article_tags==1]
pd.Series(np.array(y_test)).value_counts()
y_pred_probs
pd.DataFrame(np.array(y_pred_probs.tolist())).min()
pd.DataFrame(np.array(y_pred_probs.tolist())).max()<jupyter_output><empty_output><jupyter_text>預測<jupyter_code># Model class must be defined somewhere
PATH = 'step3_output_bert_senti.pth'
model2 = torch.load(PATH)
model2.eval()
review_text = "富二代網路警察李家賢查獲一間詐騙集團，協助玉山銀行進行洗錢防制"
encoded_review = tokenizer.encode_plus(
  review_text,
  max_length=100,#MAX_LEN,
  add_special_tokens=True,
  return_token_type_ids=False,
  pad_to_max_length=True,
  return_attention_mask=True,
  return_tensors='pt',
)
input_ids = encoded_review['input_ids'].to(device)
attention_mask = encoded_review['attention_mask'].to(device)
output = model2(input_ids, attention_mask)
_, prediction = torch.max(output, dim=1)
print(f'Review text: {review_text}')
print(f'Sentiment  : {class_names[prediction]}')
prediction.tolist()[0]
np.sum(output)
np.sum(np.array(output.tolist()).squeeze())
a, prediction = torch.max(output, dim=1)
df_test2.head()
predict=[]

for i in range(len(df_test2)):
    review_text = df_test2.iloc[i].article_truncate
    encoded_review = tokenizer.encode_plus(
      review_text,
      max_length=MAX_LEN,
      add_special_tokens=True,
      return_token_type_ids=False,
      pad_to_max_length=True,
      return_attention_mask=True,
      return_tensors='pt',
    )
    input_ids = encoded_review['input_ids'].to(device)
    attention_mask = encoded_review['attention_mask'].to(device)
    output = model(input_ids, attention_mask)
    _, prediction = torch.max(output, dim=1)
    predict.append( prediction.tolist()[0] ) 
    #predict.append( np.sum(np.array(output.tolist()).squeeze()) )
len(predict)
len(df_test2)
df_test2['predict'] = predict
group_test =df_test2[['article_index','article_tags','article_tags_name','predict']].groupby(['article_index','article_tags_name']).mean()
group_test.predict.value_counts()
group_test.reset_index(inplace=True)
uniq_article = np.unique(group_test.article_index.values) # 文章獨一無二的量
group_test[(group_test.article_index==2)&(group_test.article_tags==1)].article_tags_name.tolist()
group_test
predd_list = []
answer_list = []
for i in uniq_article:
    print('article:{f}'.format(f = i))
    predd = group_test[(group_test.article_index==i)&(group_test.predict>=0.5)].article_tags_name.tolist()
    predd_list.append(predd)
    print('預測 = '+str(predd))
    
    
    answer = group_test[(group_test.article_index==i)&(group_test.article_tags==1)].article_tags_name.tolist()
    print('答案 = '+str(answer))
    answer_list.append(answer)
predd_list = []
answer_list = []
for i in uniq_article:
    print('article:{f}'.format(f = i))
    predd = group_test[(group_test.article_index==i)&(group_test.predict>=0.2)].article_tags_name.tolist()
    predd_list.append(predd)
    print('預測 = '+str(predd))
    
    
    answer = group_test[(group_test.article_index==i)&(group_test.article_tags==1)].article_tags_name.tolist()
    print('答案 = '+str(answer))
    answer_list.append(answer)
predd_list = []
answer_list = []
for i in uniq_article:
    print('article:{f}'.format(f = i))
    predd = group_test[(group_test.article_index==i)&(group_test.predict>=1.0)].article_tags_name.tolist()
    predd_list.append(predd)
    print('預測 = '+str(predd))
    
    
    answer = group_test[(group_test.article_index==i)&(group_test.article_tags==1)].article_tags_name.tolist()
    print('答案 = '+str(answer))
    answer_list.append(answer)
def score(truth, predict):
    if truth == [] and predict != []:
        return 0
    
    if truth != [] and predict == []:
        return 0
    
    if truth == [] and predict == []:
        return 1
    
    recall = len([i for i in truth if i in predict])/len(truth)
    precision = len([i for i in predict if i in truth])/len(predict)
    
    try:
        return 2/((1/recall)+(1/precision))
    except:
        return 0
score(answer_list, predd_list)0.6
score(answer_list, predd_list)0.2
score(answer_list, predd_list)#0.8
predict=[]

for i in range(len(df_test2)):
    review_text = df_test2.iloc[i].article_truncate
    encoded_review = tokenizer.encode_plus(
      review_text,
      max_length=MAX_LEN,
      add_special_tokens=True,
      return_token_type_ids=False,
      pad_to_max_length=True,
      return_attention_mask=True,
      return_tensors='pt',
    )
    input_ids = encoded_review['input_ids'].to(device)
    attention_mask = encoded_review['attention_mask'].to(device)
    output = model2(input_ids, attention_mask)
    _, prediction = torch.max(output, dim=1)
    predict.append( prediction.tolist()[0] ) 
    #predict.append( np.sum(np.array(output.tolist()).squeeze()) )
df_test2['predict'] = predict
group_test =df_test2[['article_index','article_tags','article_tags_name','predict']].groupby(['article_index','article_tags_name']).mean()
group_test.reset_index(inplace=True)
uniq_article = np.unique(group_test.article_index.values) # 文章獨一無二的量
predd_list = []
answer_list = []
for i in uniq_article:
    print('article:{f}'.format(f = i))
    predd = group_test[(group_test.article_index==i)&(group_test.predict>=0.5)].article_tags_name.tolist()
    predd_list.append(predd)
    print('預測 = '+str(predd))
    
    
    answer = group_test[(group_test.article_index==i)&(group_test.article_tags==1)].article_tags_name.tolist()
    print('答案 = '+str(answer))
    answer_list.append(answer)
score(answer_list, predd_list)#0.2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12818736141906872, 'fraction_numerical': 0.011225055432372505, 'mean_word_length': 4.236937590711175, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '497241', 'license': 'MIT', 'n_tokens_mistral': 5255, 'n_tokens_neox': 5164, 'n_words': 775, 'path': 'step3_bertmodel_300.ipynb', 'repo_name': 'TimKuo38/AML-Blacklist-Prediction', 'size': '14432'}