<jupyter_start><jupyter_text>$$ \textbf{ EANN: Event Adversarial Neural Networks for Multi-Modal Fake News Detection } $$$$ <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME> $$$$ KDD, 2018 $$ $ \textbf{ Scenario }$- Fake news detection    - Input - Text with image    - Output - Fake or not$ \textbf{ Challenges }$- To identify fake news on newly emerged events.    - Most existing works tend to learn event-specific features     - which cannot be transferred to unseen events.$ \textbf{ Methods }$1. Feature extractor    - Text, Image - CNN   2. Fake news detector    - Predict fake or not     3. Event discriminator    - Event classification        $ \textbf{ Contributions }$- Rather than event-specific feature, invariant features learned to help predict new events.$ \textbf{ Results }$ $\textbf{ Source }$$\textbf{ Code }$- aims to debug  Define parserParameters are set by the parser.<jupyter_code>#=========#
# EANN.py #
#=========#
import argparse

def parse_arguments(parser):
    parser.add_argument('training_file', type = str, metavar = '<training_file>', help = '')
    #parser.add_argument('validation_file', type = str, metavar='<validation_file>', help = '')
    parser.add_argument('testing_file', type = str, metavar = '<testing_file>', help = '')
    parser.add_argument('output_file', type = str, metavar = '<output_file>', help = '')

    parse.add_argument('--static', type = bool, default=True, help='')
    parser.add_argument('--sequence_length', type = int, default = 28, help = '')
    parser.add_argument('--class_num', type = int, default = 2, help = '')
    parser.add_argument('--hidden_dim', type = int, default = 32, help = '')
    parser.add_argument('--embed_dim', type = int, default = 32, help = '')
    parser.add_argument('--vocab_size', type = int, default = 300, help = '')
    parser.add_argument('--dropout', type = int, default = 0.5, help = '')
    parser.add_argument('--filter_num', type = int, default = 5, help = '')
    parser.add_argument('--lambd', type = int, default = 1, help = '')
    parser.add_argument('--text_only', type = bool, default = False, help = '')

    #    parser.add_argument('--sequence_length', type = int, default = 28, help = '')
    #    parser.add_argument('--input_size', type = int, default = 28, help = '')
    #    parser.add_argument('--hidden_size', type = int, default = 128, help = '')
    #    parser.add_argument('--num_layers', type = int, default = 2, help = '')
    #    parser.add_argument('--num_classes', type = int, default = 10, help = '')
    parser.add_argument('--d_iter', type = int, default = 3, help = '')
    parser.add_argument('--batch_size', type = int, default = 15, help = '') # 100
    parser.add_argument('--num_epochs', type = int, default = 10, help = '')
    parser.add_argument('--learning_rate', type = float, default = 0.001, help = '')
    parser.add_argument('--event_num', type = int, default = 10, help = '')

    #    args = parser.parse_args()
    return parser
if __name__ == '__main__':
    parse = argparse.ArgumentParser()
    parser = parse_arguments(parse)
    train = '' 
    test = ''
    output = '../Data/weibo/RESULT/'
    args = parser.parse_args([train, test, output])
    
    print(args)
    
    # main(args)<jupyter_output>Namespace(batch_size=15, class_num=2, d_iter=3, dropout=0.5, embed_dim=32, event_num=10, filter_num=5, hidden_dim=32, lambd=1, learning_rate=0.001, num_epochs=10, output_file='../Data/weibo/RESULT/', sequence_length=28, static=True, testing_file='', text_only=False, training_file='', vocab_size=300)<jupyter_text>Preprocess dataData is from the Weibo.<jupyter_code>#=======================#
# process_data_weibo.py #
#=======================#

import os
from torchvision import transforms

def read_image():
    image_list = {}
    file_list = ['../Data/weibo/nonrumor_images/', '../Data/weibo/rumor_images/']
    
    data_transforms = transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
            ])
    
    for path in file_list:
        for i, filename in enumerate(os.listdir(path)):  # assuming gif

            # print(filename)
            
            try:
                im = Image.open(path + filename).convert('RGB')
                im = data_transforms(im)
                image_list[filename.split('/')[-1].split(".")[0].lower()] = im  # format the file name
            except:
                print(filename)
            
    print("image length " + str(len(image_list)))
    #print("image names are " + str(image_list.keys()))
    return image_list
'''Check function - read_image()''' 

from PIL import Image
import matplotlib.pyplot as plt

path = '../Data/weibo/nonrumor_images/'

data_transforms = transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
            ])
filename = os.listdir(path)[0]
print(filename)

im = Image.open(path + filename)
#plt.imshow(im)

im = im.convert('RGB')
#plt.imshow(im)

im = data_transforms(im)
#plt.imshow(im)

print(filename.split('/')[-1].split(".")[0].lower())

image = read_image()
# image
def stopwordslist(filepath = '../Data/weibo/stop_words.txt'):
    stopwords = {}
    for line in open(filepath, 'r', encoding = 'UTF-8').readlines():
        stopwords[line] = 1
    return stopwords
'''Check function - stopwordslist(filepath = '../Data/weibo/stop_words.txt')''' 
# stopwordslist(filepath = '../Data/weibo/stop_words.txt')
import re

def clean_str_sst(string):
    """
    Tokenization/string cleaning for the SST dataset
    """
    string = re.sub(u"[，。 :,.；|-“”——_/nbsp+&;@、《》～（）())#O！：【】]", "", string)
    return string.strip().lower()

import jieba
'''Check clean_str_sst(string)'''

l = clean_str_sst('【每日美图】  西西里的希腊古城塞利侬特（Selinunte)')

jieba.cut_for_search(l)
'''Check clean text'''

stop_words = stopwordslist()

seg_list = jieba.cut_for_search(l)
new_seg_list = []
for word in seg_list:
    if word not in stop_words:
        new_seg_list.append(word)

new_seg_list
def write_txt(data):
    f = open("../Data/weibo/top_n_data.txt", 'wb')
    for line in data:
        for l in line:
            f.write(l+"\n")
        f.write("\n")
        f.write("\n")
    f.close()
import _pickle as pickle
import pandas as pd
import numpy as np

def write_data(flag, image, text_only):

    def read_post(flag):
        stop_words = stopwordslist() # dict = {'stop_word': 1}
        pre_path = "../Data/weibo/tweets/"
        file_list = [pre_path + "test_nonrumor.txt", pre_path + "test_rumor.txt", \
                         pre_path + "train_nonrumor.txt", pre_path + "train_rumor.txt"]
        if flag == "train":
            id = pickle.load(open("../Data/weibo/train_id.pickle", 'rb')) # dict = {'img_num': id}
        elif flag == "validate":
            id = pickle.load(open("../Data/weibo/validate_id.pickle", 'rb'))
        elif flag == "test":
            id = pickle.load(open("../Data/weibo/test_id.pickle", 'rb'))


        post_content = []
        # labels = []
        # image_ids = []
        # twitter_ids = []
        data = []
        column = ['post_id', 'image_id', 'original_post', 'post_text', 'label', 'event_label']
        # key = -1
        map_id = {}
        # top_data = []
        for k, f in enumerate(file_list):

            f = open(f, 'r')
            if k % 2 == 0:
                label = 0  ### real is 0  # images in "test_nonrumor.txt" and "train_nonrumor.txt" labeled as 0
            else:
                label = 1  ### fake is 1

            twitter_id = 0
            line_data = []
            # top_line_data = []

            for i, l in enumerate(f.readlines()):

                if i % 3 == 0:  # twitter_id '''3907393559904503|null|null|null|1447078683000|true'''
                    line_data = []
                    twitter_id = l.split('|')[0]
                    line_data.append(twitter_id)

                if i % 3 == 1:  # http://ww3.sinaimg.cn/large/62b31d36gw1exv2qu0x0ij20hm0bn0v8.jpg|http://ww1.sinaimg.cn/large...
                    line_data.append(l.lower())

                if i % 3 == 2: # 【每日美图】西西里的希腊古城塞利侬特（Selinunte）
                    ## Remove the punctuation marks
                    l = clean_str_sst(l)
                    
                    ## Cut the chinese sentence as words
                    seg_list = jieba.cut_for_search(l)
                    
                    ## Keep the words not in stop words
                    new_seg_list = []
                    for word in seg_list:
                        if word not in stop_words:
                            new_seg_list.append(word)
                            
                    ## Merge as a new / clean sentence 
                    clean_l = " ".join(new_seg_list)
                    
                    if len(clean_l) > 10 and line_data[0] in id: # len(sentence) > 10 & twitter_id is in img_num
                        post_content.append(l) # before the jieba cut and stop wordss removed
                        line_data.append(l)
                        line_data.append(clean_l) # after the jieba cut and stop wordss removed
                        line_data.append(label) # fake or not
                        
                        event = int(id[line_data[0]]) # img_num to id as event
                        
                        ## map id to 0 ~ ..
                        if event not in map_id:
                            map_id[event] = len(map_id)
                            event = map_id[event]
                        else:
                            event = map_id[event]

                        line_data.append(event) # id from 0

                        data.append(line_data)


            f.close()
            # print(data)
            #     return post_content
        
        data_df = pd.DataFrame(np.array(data), columns = column)
        # write_txt(top_data)

        return post_content, data_df

    post_content, post = read_post(flag)
    print("Original post length is " + str(len(post_content)))
    print("Original data frame is " + str(post.shape))

    '''
    def find_most(db):
        maxcount = max(len(v) for v in db.values())
        return [k for k, v in db.items() if len(v) == maxcount]

    def select(train, selec_indices):
        temp = []
        for i in range(len(train)):
            ele = list(train[i])
            temp.append([ele[i] for i in selec_indices])
        return temp
    '''


    def paired(text_only = False):
        ordered_image = []
        ordered_text = []
        ordered_post = []
        ordered_event= []
        label = []
        post_id = []
        image_id_list = []
        #image = []

        image_id = ""
        for i, id in enumerate(post['post_id']):
            for image_id in post.iloc[i]['image_id'].split('|'):
                image_id = image_id.split("/")[-1].split(".")[0]
                if image_id in image:
                    break

            if text_only or image_id in image:
                if not text_only:
                    image_name = image_id
                    image_id_list.append(image_name)
                    ordered_image.append(image[image_name])
                ordered_text.append(post.iloc[i]['original_post'])
                ordered_post.append(post.iloc[i]['post_text'])
                ordered_event.append(post.iloc[i]['event_label'])
                post_id.append(id)


                label.append(post.iloc[i]['label'])

        label = np.array(label, dtype = np.int)
        ordered_event = np.array(ordered_event, dtype = np.int)

        print("Label number is " + str(len(label)))
        print("Rumor number is " + str(sum(label)))
        print("Non rumor is " + str(len(label) - sum(label)))



        #
        if flag == "test":
            y = np.zeros(len(ordered_post))
        else:
            y = []


        data = {"post_text": np.array(ordered_post),
                "original_post": np.array(ordered_text),
                "image": ordered_image, "social_feature": [],
                "label": np.array(label), \
                "event_label": ordered_event, "post_id":np.array(post_id),
                "image_id":image_id_list}
        #print(data['image'][0])


        print("data size is " + str(len(data["post_text"])))
        
        return data

    paired_data = paired(text_only)

    print("paired post length is "+str(len(paired_data["post_text"])))
    print("paried data has " + str(len(paired_data)) + " dimension")
    return paired_data

'''Check write_data() // read_post(flag)'''
import _pickle as pickle

# pickle.load(open("../Data/weibo/train_id.pickle", 'rb'))
'''Check post_content, post in write_data() // read_post(flag)'''

def write_data_(flag):

    def read_post(flag):
        stop_words = stopwordslist() # dict = {'stop_word': 1}
        pre_path = "../Data/weibo/tweets/"
        file_list = [pre_path + "test_nonrumor.txt", pre_path + "test_rumor.txt", \
                         pre_path + "train_nonrumor.txt", pre_path + "train_rumor.txt"]
        if flag == "train":
            id = pickle.load(open("../Data/weibo/train_id.pickle", 'rb')) # dict = {'img_num': id}
        elif flag == "validate":
            id = pickle.load(open("../Data/weibo/validate_id.pickle", 'rb'))
        elif flag == "test":
            id = pickle.load(open("../Data/weibo/test_id.pickle", 'rb'))


        post_content = []
        # labels = []
        # image_ids = []
        # twitter_ids = []
        data = []
        column = ['post_id', 'image_id', 'original_post', 'post_text', 'label', 'event_label']
        # key = -1
        map_id = {}
        # top_data = []
        for k, f in enumerate(file_list):

            f = open(f, 'r')
            if k % 2 == 0:
                label = 0  ### real is 0  # images in "test_nonrumor.txt" and "train_nonrumor.txt" labeled as 0
            else:
                label = 1  ### fake is 1

            twitter_id = 0
            line_data = []
            # top_line_data = []

            for i, l in enumerate(f.readlines()):

                if i % 3 == 0:  # twitter_id '''3907393559904503|null|null|null|1447078683000|true'''
                    line_data = []
                    twitter_id = l.split('|')[0]
                    line_data.append(twitter_id)

                if i % 3 == 1:  # http://ww3.sinaimg.cn/large/62b31d36gw1exv2qu0x0ij20hm0bn0v8.jpg|http://ww1.sinaimg.cn/large...
                    line_data.append(l.lower())

                if i % 3 == 2: # 【每日美图】西西里的希腊古城塞利侬特（Selinunte）
                    ## Remove the punctuation marks
                    l = clean_str_sst(l)
                    
                    ## Cut the chinese sentence as words
                    seg_list = jieba.cut_for_search(l)
                    
                    ## Keep the words not in stop words
                    new_seg_list = []
                    for word in seg_list:
                        if word not in stop_words:
                            new_seg_list.append(word)
                            
                    ## Merge as a new / clean sentence 
                    clean_l = " ".join(new_seg_list)
                    
                    if len(clean_l) > 10 and line_data[0] in id: # len(sentence) > 10 & twitter_id is in img_num
                        post_content.append(l) # before the jieba cut and stop wordss removed
                        line_data.append(l)
                        line_data.append(clean_l) # after the jieba cut and stop wordss removed
                        line_data.append(label) # fake or not
                        
                        event = int(id[line_data[0]]) # img_num to id as event
                        
                        ## map id to 0 ~ ..
                        if event not in map_id:
                            map_id[event] = len(map_id)
                            event = map_id[event]
                        else:
                            event = map_id[event]

                        line_data.append(event) # id from 0

                        data.append(line_data)


            f.close()
            # print(data)
            #     return post_content
        
        data_df = pd.DataFrame(np.array(data), columns = column)
        # write_txt(top_data)

        return post_content, data_df

    post_content, post = read_post(flag)
    print("Original post length is " + str(len(post_content)))
    print("Original data frame is " + str(post.shape))
    
    return post_content, post

post_content, post = write_data_('train')

print(post_content[:2])
print(post.head())
'''Check paired(text_only = False)'''
text_only = False
image = read_image()
flag = 'train'

ordered_image = []
ordered_text = []
ordered_post = []
ordered_event= []
label = []
post_id = []
image_id_list = []


image_id = ""
for i, id in enumerate(post['post_id']):
    for image_id in post.iloc[i]['image_id'].split('|'):
        image_id = image_id.split("/")[-1].split(".")[0]
        if image_id in image:
            break

    if text_only or image_id in image:
        if not text_only:
            image_name = image_id
            image_id_list.append(image_name)
            ordered_image.append(image[image_name])
        ordered_text.append(post.iloc[i]['original_post'])
        ordered_post.append(post.iloc[i]['post_text'])
        ordered_event.append(post.iloc[i]['event_label'])
        post_id.append(id)


        label.append(post.iloc[i]['label'])

label = np.array(label, dtype = np.int)
ordered_event = np.array(ordered_event, dtype = np.int)

print("Label number is " + str(len(label)))
print("Rumor number is " + str(sum(label)))
print("Non rumor is " + str(len(label) - sum(label)))



#
if flag == "test":
    y = np.zeros(len(ordered_post))
else:
    y = []


data = {"post_text": np.array(ordered_post),
        "original_post": np.array(ordered_text),
        "image": ordered_image, "social_feature": [],
        "label": np.array(label), \
        "event_label": ordered_event, "post_id":np.array(post_id),
        "image_id":image_id_list}
#print(data['image'][0])


print("data size is " + str(len(data["post_text"])))


paired_data = data

print("paired post length is "+str(len(paired_data["post_text"])))
print("paried data has " + str(len(paired_data)) + " dimension")
# paired_data

post.iloc[0]['image_id'].split('|')
post.iloc[0]['image_id'].split('|')[0]
post.iloc[0]['image_id'].split('|')[0].split("/")
post.iloc[0]['image_id'].split('|')[0].split("/")[-1].split(".")[0]
# write_data("train", image, False)
'''Finish checking write_data("train", image_list, text_only) in get_data(text_only)'''
from collections import defaultdict

def load_data(train, validate, test):
    vocab = defaultdict(float)
    all_text = list(train['post_text']) + list(validate['post_text']) + list(test['post_text'])
    for sentence in all_text:
        for word in sentence:
            vocab[word] += 1
    return vocab, all_text
'''Check load_data'''
vocab = defaultdict(float)
vocab['tmp'] += 3
vocab
word_embedding_path = "../Data/weibo/w2v.pickle"
w2v = pickle.load(open(word_embedding_path, 'rb'), encoding = 'bytes')
# w2v
def add_unknown_words(word_vecs, vocab, min_df = 1, k = 32):
    """
    For words that occur in at least min_df documents, create a separate word vector.
    0.25 is chosen so the unknown vectors have (approximately) same variance as pre-trained ones
    """
    for word in vocab:
        if word not in word_vecs and vocab[word] >= min_df:
            word_vecs[word] = np.random.uniform(-0.25, 0.25, k)

def get_W(word_vecs, k = 32):
    """
    Get word matrix. W[i] is the vector for word indexed by i
    """
    # vocab_size = len(word_vecs)
    word_idx_map = dict()
    W = np.zeros(shape = (len(word_vecs) + 1, k), dtype = 'float32')
    W[0] = np.zeros(k, dtype  ='float32')
    i = 1
    for word in word_vecs:
        W[i] = word_vecs[word]
        word_idx_map[word] = i
        i += 1
    return W, word_idx_map
import pickle
# 也可以这样：
# import cPickle as pickle

obj = {"a": 1, "b": 2, "c": 3}

# 将 obj 持久化保存到文件 tmp.txt 中
pickle.dump(obj, open("tmp.txt", "wb"))

# do something else ...

# 从 tmp.txt 中读取并恢复 obj 对象
obj2 = pickle.load(open("tmp.txt", "rb"))

obj2
def get_data(text_only):
    # text_only = False

    if text_only:
        print("Text only")
        image_list = []
    else:
        print("Text and image")
        image_list = read_image() # dict = {'img_file_name': img_tensor}

    train_data = write_data("train", image_list, text_only) # dict
    valiate_data = write_data("validate", image_list, text_only)
    test_data = write_data("test", image_list, text_only)

    print("loading data...")
    # w2v_file = '../Data/GoogleNews-vectors-negative300.bin'
    vocab, all_text = load_data(train_data, valiate_data, test_data)
    # print(str(len(all_text)))

    print("number of sentences: " + str(len(all_text)))
    print("vocab size: " + str(len(vocab)))
    max_l = len(max(all_text, key=len))
    print("max sentence length: " + str(max_l))

    #
    #
    word_embedding_path = "../Data/weibo/w2v.pickle"

    w2v = pickle.load(open(word_embedding_path, 'rb'), encoding = 'bytes')
 
    print("word2vec loaded!")
    print("num words already in word2vec: " + str(len(w2v)))
    
    print("word2vec loaded!")
    print("num words already in word2vec: " + str(len(w2v)))
    add_unknown_words(w2v, vocab)
    W, word_idx_map = get_W(w2v)
    # # rand_vecs = {}
    # # add_unknown_words(rand_vecs, vocab)
    W2 = rand_vecs = {}
    w_file = open("../Data/weibo/word_embedding.pickle", "wb")
    pickle.dump([W, W2, word_idx_map, vocab, max_l], w_file)
    w_file.close()
    return train_data, valiate_data, test_data<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>#=======================#
# EANN.py #
#=======================#
import copy

def word2vec(post, word_id_map, W):
    word_embedding = []
    mask = []
    #length = []

    for sentence in post:
        sen_embedding = []
        seq_len = len(sentence) -1
        mask_seq = np.zeros(args.sequence_len, dtype = np.float32)
        mask_seq[:len(sentence)] = 1.0
        for i, word in enumerate(sentence):
            sen_embedding.append(word_id_map[word])

        while len(sen_embedding) < args.sequence_len:
            sen_embedding.append(0)

        word_embedding.append(copy.deepcopy(sen_embedding))
        mask.append(copy.deepcopy(mask_seq))
        #length.append(seq_len)
    return word_embedding, mask
import process_data_weibo as process_data

def load_data_(args):
    train, validate, test = process_data.get_data(args.text_only) # get_data(args.text_only)
    #print(train[4][0])
    word_vector_path = '../Data/weibo/word_embedding.pickle'
    f = open(word_vector_path, 'rb')
    weight = pickle.load(f)  # W, W2, word_idx_map, vocab
    W, W2, word_idx_map, vocab, max_len = weight[0], weight[1], weight[2], weight[3], weight[4]
    args.vocab_size = len(vocab)
    args.sequence_len = max_len
    print("translate data to embedding")

    word_embedding, mask = word2vec(validate['post_text'], word_idx_map, W)
    validate['post_text'] = word_embedding
    validate['mask'] = mask


    print("translate test data to embedding")
    word_embedding, mask = word2vec(test['post_text'], word_idx_map, W)
    test['post_text'] = word_embedding
    test['mask'] = mask
    #test[-2]= transform(test[-2])
    
    word_embedding, mask = word2vec(train['post_text'], word_idx_map, W)
    train['post_text'] = word_embedding
    train['mask'] = mask
    
    print("sequence length " + str(args.sequence_length))
    print("Train Data Size is "+str(len(train['post_text'])))
    print("Finished loading data ")
    return train, validate, test, W
'''Check load_data_(args)'''
train, validate, test = get_data(args.text_only)
'''Check load_data_(args)'''
type(validate)
list(validate.keys())
# validate['post_text']
'''Check load_data_(args)'''
word_vector_path = '../Data/weibo/word_embedding.pickle'
f = open(word_vector_path, 'rb')
weight = pickle.load(f)  # W, W2, word_idx_map, vocab
W, W2, word_idx_map, vocab, max_len = weight[0], weight[1], weight[2], weight[3], weight[4]
args.vocab_size = len(vocab)
args.sequence_len = max_len
#print("translate data to embedding")
args
'''Check load_data_(args)'''
'''
train, validate, test = get_data(args.text_only)

word_embedding, mask = word2vec(validate['post_text'], word_idx_map, W)
validate['post_text'] = word_embedding
validate['mask'] = mask
validate['post_text']
validate['mask']
'''

'''Check word2vec(post, word_id_map, W)'''
train, validate, test = get_data(args.text_only)

post = validate['post_text']
word_id_map = word_idx_map

'''
word_embedding = []
mask = []

for sentence in post:
    sen_embedding = []
    seq_len = len(sentence) -1
    mask_seq = np.zeros(args.sequence_len, dtype = np.float32)
    mask_seq[:len(sentence)] = 1.0
    for i, word in enumerate(sentence):
        sen_embedding.append(word_id_map[word])

    while len(sen_embedding) < args.sequence_len:
        sen_embedding.append(0)

    word_embedding.append(copy.deepcopy(sen_embedding))
    mask.append(copy.deepcopy(mask_seq))
'''

sentence = post[0]

sen_embedding = []
seq_len = len(sentence) -1

mask_seq = np.zeros(args.sequence_len, dtype = np.float32)
mask_seq[:len(sentence)] = 1.0
for i, word in enumerate(sentence):
    sen_embedding.append(word_id_map[word])

while len(sen_embedding) < args.sequence_len:
    sen_embedding.append(0)

# sen_embedding<jupyter_output>Text and image
image length 268
Original post length is 2329
Original data frame is (2329, 6)
Label number is 43
Rumor number is 15
Non rumor is 28
data size is 43
paired post length is 43
paried data has 8 dimension
Original post length is 340
Original data frame is (340, 6)
Label number is 11
Rumor number is 7
Non rumor is 4
data size is 11
paired post length is 11
paried data has 8 dimension
Original post length is 686
Original data frame is (686, 6)
Label number is 20
Rumor number is 5
Non rumor is 15
data size is 20
paired post length is 20
paried data has 8 dimension
loading data...
number of sentences: 74
vocab size: 1348
max sentence length: 283
word2vec loaded!
num words already in word2vec: 4626
word2vec loaded!
num words already in word2vec: 4626<jupyter_text>Main function - Train and Test<jupyter_code>args
'''Load data'''
print('loading data')

train, validation, test, W = load_data_(args)
test_id = test['post_id']
import torch
from torch.utils.data import Dataset, DataLoader

class Rumor_Data(Dataset):
    def __init__(self, dataset):
        self.text = torch.from_numpy(np.array(dataset['post_text']))
        self.image = list(dataset['image'])
        #self.social_context = torch.from_numpy(np.array(dataset['social_feature']))
        self.mask = torch.from_numpy(np.array(dataset['mask']))
        self.label = torch.from_numpy(np.array(dataset['label']))
        self.event_label = torch.from_numpy(np.array(dataset['event_label']))
        print('Text: %d, Image: %d, Label: %d, Event: %d'
               % (len(self.text), len(self.image), len(self.label), len(self.event_label)))

    def __len__(self):
        return len(self.label)

    def __getitem__(self, idx):
        return (self.text[idx], self.image[idx], self.mask[idx]), self.label[idx], self.event_label[idx]

'''Format data by a class, Rumor_Data(Dataset)'''

train_dataset = Rumor_Data(train)

validate_dataset = Rumor_Data(validation)

test_dataset = Rumor_Data(test) 


'''Check Rumor_Data(Dataset)'''
train['post_text']
np.array(train['post_text']).shape
train_dataset.text

train_dataset.image
train_dataset
from torch.autograd import Variable, Function

class ReverseLayerF(Function):
    '''
    def __init__(self, lambd):
        self.lambd = lambd
    '''
    @staticmethod
    def forward(self, x):
        self.lambd = args.lambd
        return x.view_as(x)

    @staticmethod
    def backward(self, grad_output):
        return (grad_output * -self.lambd)
def grad_reverse(x):
    return ReverseLayerF.apply(x)

import torch.nn as nn
import torchvision
import torch.nn.functional as F

class CNN_Fusion(nn.Module):
    def __init__(self, args, W):
        super(CNN_Fusion, self).__init__()
        self.args = args

        self.event_num = args.event_num

        vocab_size = args.vocab_size
        emb_dim = args.embed_dim

        C = args.class_num
        self.hidden_size = args.hidden_dim
        self.lstm_size = args.embed_dim
        self.social_size = 19

        # TEXT RNN

        self.embed = nn.Embedding(vocab_size, emb_dim)
        self.embed.weight = nn.Parameter(torch.from_numpy(W))
        self.lstm = nn.LSTM(self.lstm_size, self.lstm_size) # embedding_dim, hidden_dim
        self.text_fc = nn.Linear(self.lstm_size, self.hidden_size)
        self.text_encoder = nn.Linear(emb_dim, self.hidden_size)

        ### TEXT CNN
        channel_in = 1
        filter_num = 20
        window_size = [1, 2, 3, 4]
        self.convs = nn.ModuleList([nn.Conv2d(channel_in, filter_num, (K, emb_dim)) for K in window_size])
        self.fc1 = nn.Linear(len(window_size) * filter_num, self.hidden_size)

        self.dropout = nn.Dropout(args.dropout)

        # IMAGE
        # hidden_size = args.hidden_dim
        vgg_19 = torchvision.models.vgg19(pretrained=True)
        for param in vgg_19.parameters():
            param.requires_grad = False
        # visual model
        num_ftrs = vgg_19.classifier._modules['6'].out_features
        self.vgg = vgg_19
        self.image_fc1 = nn.Linear(num_ftrs,  self.hidden_size)
        # self.image_fc2 = nn.Linear(512, self.hidden_size)
        self.image_adv = nn.Linear(self.hidden_size,  int(self.hidden_size))
        self.image_encoder = nn.Linear(self.hidden_size, self.hidden_size)



        ### social context
        self.social = nn.Linear(self.social_size, self.hidden_size)


        ##ATTENTION
        self.attention_layer = nn.Linear(self.hidden_size, emb_dim)



        ## Class  Classifier
        self.class_classifier = nn.Sequential()
        self.class_classifier.add_module('c_fc1', nn.Linear(2 * self.hidden_size, 2))
        #self.class_classifier.add_module('c_bn1', nn.BatchNorm2d(100))
        #self.class_classifier.add_module('c_relu1', nn.ReLU(True))
        #self.class_classifier.add_module('c_drop1', nn.Dropout2d())
        #self.class_classifier.add_module('c_fc2', nn.Linear(self.hidden_size, 2))
        #self.class_classifier.add_module('c_bn2', nn.BatchNorm2d(self.hidden_size))
        #self.class_classifier.add_module('c_relu2', nn.ReLU(True))
        #self.class_classifier.add_module('c_fc3', nn.Linear(100, 10))
        self.class_classifier.add_module('c_softmax', nn.Softmax(dim = 1))


        ## Event Classifier
        self.domain_classifier = nn.Sequential()
        self.domain_classifier.add_module('d_fc1', nn.Linear(2 * self.hidden_size, self.hidden_size))
        #self.domain_classifier.add_module('d_bn1', nn.BatchNorm2d(self.hidden_size))
        self.domain_classifier.add_module('d_relu1', nn.LeakyReLU(True))
        self.domain_classifier.add_module('d_fc2', nn.Linear(self.hidden_size, self.event_num))
        self.domain_classifier.add_module('d_softmax', nn.Softmax(dim = 1))

        
    def init_hidden(self, batch_size):
        # Before we've done anything, we don't have any hidden state.
        # Refer to the Pytorch documentation to see exactly
        # why they have this dimensionality.
        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
        return (to_var(torch.zeros(1, batch_size, self.lstm_size)),
                to_var(torch.zeros(1, batch_size, self.lstm_size)))

    def conv_and_pool(self, x, conv):
        x = F.relu(conv(x)).squeeze(3)  # (sample number,hidden_dim, length)
        #x = F.avg_pool1d(x, x.size(2)).squeeze(2)
        x = F.max_pool1d(x, x.size(2)).squeeze(2)

        return x

    def forward(self, text, image, mask):

        ### IMAGE #####
        image = self.vgg(image) #[N, 512]
        image = F.leaky_relu(self.image_fc1(image))

        ########## CNN ##################
        text = self.embed(text)
        text = text * mask.unsqueeze(2).expand_as(text)
        text = text.unsqueeze(1)
        text = [F.leaky_relu(conv(text)).squeeze(3) for conv in self.convs]  # [(N,hidden_dim,W), ...]*len(window_size)
        #text = [F.avg_pool1d(i, i.size(2)).squeeze(2) for i in text]  # [(N,hidden_dim), ...]*len(window_size)
        text = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in text]
        text = torch.cat(text, 1)
        text = F.leaky_relu(self.fc1(text))

        text_image = torch.cat((text, image), 1)

        ## Fake or real
        class_output = self.class_classifier(text_image)
        
        ## Domain (which Event)
        reverse_feature = grad_reverse(text_image)
        domain_output = self.domain_classifier(reverse_feature)

        # ### Multimodal
        # text_reverse_feature = grad_reverse(text)
        # image_reverse_feature = grad_reverse(image)
        # text_output = self.modal_classifier(text_reverse_feature)
        # image_output = self.modal_classifier(image_reverse_feature)

        return class_output, domain_output

model = CNN_Fusion(args, W)

model.embed
model.embed.weight

torch.from_numpy(W)
nn.Parameter(torch.from_numpy(W))

W.shape # (4630, 32) ## 4630 words with 32 dim.
model.embed


text = torch.rand([15, 283]).long() # 15 data, each 283 words at most
text
image = torch.rand([15, 3, 224, 224])
mask = torch.rand([15, 283])

# model.forward(text, image, mask)


'''Image'''
image = model.vgg(image) #[N, 512]
image = F.leaky_relu(model.image_fc1(image))

print(model.vgg)
print(model.image_fc1)

print(image.shape)

'''Text'''
text = model.embed(text) # torch.Size([15, 283, 32]) ## 15 data, each 283 words, each 32-dim
text = text * mask.unsqueeze(2).expand_as(text) # torch.Size([15, 283, 32]) ## weighted 
text = text.unsqueeze(1) # torch.Size([15, 1, 283, 32]) ## 15 data, 1 channel,  each 283 words, each 32-dim

text = [F.leaky_relu(conv(text)).squeeze(3) for conv in model.convs]  
    # [torch.Size([batch_size, out_channel, dim]), ..., torch.Size([batch_size, out_channel, dim])]

print(model.embed)
print(model.convs)
    
'''Check tensor size'''
print([i.shape for i in text])
print([F.max_pool1d(i, i.size(2)).shape for i in text])
print([F.max_pool1d(i, i.size(2)).squeeze(2).shape for i in text])
print(torch.cat([F.max_pool1d(i, i.size(2)).squeeze(2) for i in text], 1).shape)
print(model.fc1)
    
#text = [F.avg_pool1d(i, i.size(2)).squeeze(2) for i in text] 
text = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in text]
    # [torch.Size([batch_size, out_channel]), ..., torch.Size([batch_size, out_channel])]
    
text = torch.cat(text, 1)  # concat tensors in the list along the dim. 1
    # torch.Size([batch_size, out_channel * window_size]) ## window_size: len(num_type_of_filter) = 4 here
    
text = F.leaky_relu(model.fc1(text))

print(text.shape)
# mask.shape
# mask.unsqueeze(2).shape
# mask.unsqueeze(2).expand_as(text).shape

text_image = torch.cat((text, image), 1)

print(text_image.shape)

## Fake or real
class_output = model.class_classifier(text_image)

print(model.class_classifier)

## Domain (which Event)
reverse_feature = grad_reverse(text_image)
domain_output = model.domain_classifier(reverse_feature)

print(text_image.shape)
print(text_image.view_as(text_image).shape)
print(model.domain_classifier)

print(class_output.shape)
print(domain_output.shape)
# Data Loader (Input Pipeline)
train_loader = DataLoader(dataset = train_dataset,
                          batch_size = args.batch_size,
                          shuffle = True)

validate_loader = DataLoader(dataset = validate_dataset,
                             batch_size = args.batch_size,
                             shuffle=False)

test_loader = DataLoader(dataset=test_dataset,
                         batch_size=args.batch_size,
                         shuffle=False)

# print('building model')
model = CNN_Fusion(args, W)

if torch.cuda.is_available():
    print("CUDA")
    model.cuda()

    
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()


optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, list(model.parameters())),
                             lr= args.learning_rate) 
    ## gradient descent only on those parameters which requires_grad = True

filter(lambda p: p.requires_grad, list(model.parameters()))
list(model.parameters())[0].requires_grad

# len(list(iter(train_loader))[0])
# list(iter(train_loader))[0][1] ## self.label[idx]
# list(iter(train_loader))[0][2] ## self.event_label[idx]
# list(iter(train_loader))[0][0] ## (self.text[idx], self.image[idx], self.mask[idx])
iter_per_epoch = len(train_loader)

print("loader size " + str(len(train_loader)))
best_validate_acc = 0.000
best_test_acc = 0.000
best_loss = 100
best_validate_dir = ''
best_list = [0,0]

print('training model')
adversarial = True
def to_var(x):
    if torch.cuda.is_available():
        x = x.cuda()
    return Variable(x)
import time
from torch.autograd import Variable, Function
import warnings
warnings.filterwarnings("ignore")

# Train
epoch = 0

p = float(epoch) / 100

lr = 0.001 / (1. + 10 * p) ** 0.75

optimizer.lr = lr
#rgs.lambd = lambd



start_time = time.time()
cost_vector = []
class_cost_vector = []
domain_cost_vector = []
acc_vector = []
valid_acc_vector = []
test_acc_vector = []
vali_cost_vector = []
test_cost_vector = []

# for i, (train_data, train_labels, event_labels) in enumerate(train_loader):
i = 0
train_data, train_labels, event_labels = list(iter(train_loader))[0]

train_text, train_image,  train_mask, train_labels, event_labels = \
    to_var(train_data[0]), to_var(train_data[1]), to_var(train_data[2]), \
    to_var(train_labels), to_var(event_labels)

print(type(train_data[0]))
print(type(train_text))
print(train_text.shape)
print(train_image.shape)
print(train_mask.shape)
print(train_labels.shape)

# Forward + Backward + Optimize
optimizer.zero_grad()



model = CNN_Fusion(args, W)

if torch.cuda.is_available():
    print("CUDA")
    model.cuda()
    
    
    
# print(model)
# class_outputs, domain_outputs = model(train_text, train_image, train_mask)
model(train_text, train_image, train_mask)
'''
## Fake or Real loss
class_loss = criterion(class_outputs, train_labels)
# Event Loss
domain_loss = criterion(domain_outputs, event_labels)
loss = class_loss + domain_loss
loss.backward()

optimizer.step()
_, argmax = torch.max(class_outputs, 1)

cross_entropy = True

if True:
    accuracy = (train_labels == argmax.squeeze()).float().mean()
else:
    _, labels = torch.max(train_labels, 1)
    accuracy = (labels.squeeze() == argmax.squeeze()).float().mean()

class_cost_vector.append(class_loss.item()) # lass_loss.data[0]
domain_cost_vector.append(domain_loss.item())
cost_vector.append(loss.item())
acc_vector.append(accuracy.item())
'''
#class_outputs, domain_outputs = model(train_text, train_image, train_mask)
args
def to_np(x):
    return x.data.cpu().numpy()
import time 
from torch.autograd import Variable, Function
from sklearn import metrics

def main(args):
    print('loading data')
    #    dataset = DiabetesDataset(root=args.training_file)
    #    train_loader = DataLoader(dataset=dataset,
    #                              batch_size=32,
    #                              shuffle=True,
    #                              num_workers=2)

    # MNIST Dataset
    train, validation, test, W = load_data_(args)
    test_id = test['post_id']

    #train, validation = split_train_validation(train,  1)

    #weights = make_weights_for_balanced_classes(train[-1], 15)
    #weights = torch.DoubleTensor(weights)
    #sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights))


    train_dataset = Rumor_Data(train)

    validate_dataset = Rumor_Data(validation)

    test_dataset = Rumor_Data(test) 




    # Data Loader (Input Pipeline)
    train_loader = DataLoader(dataset = train_dataset,
                              batch_size = args.batch_size,
                              shuffle = True)

    validate_loader = DataLoader(dataset = validate_dataset,
                                 batch_size = args.batch_size,
                                 shuffle = False)

    test_loader = DataLoader(dataset = test_dataset,
                             batch_size = args.batch_size,
                             shuffle = False)

    print('building model')
    model = CNN_Fusion(args, W)

    if torch.cuda.is_available():
        print("CUDA")
        model.cuda()

    # Loss and Optimizer
    criterion = nn.CrossEntropyLoss()


    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, list(model.parameters())),
                                 lr = args.learning_rate)
    #optimizer = torch.optim.RMSprop(filter(lambda p: p.requires_grad, list(model.parameters())),
                                 #lr=args.learning_rate)
    #scheduler = StepLR(optimizer, step_size= 10, gamma= 1)


    iter_per_epoch = len(train_loader)
    print("loader size " + str(len(train_loader)))
    best_validate_acc = 0.000
    best_test_acc = 0.000
    best_loss = 100
    best_validate_dir = ''
    best_list = [0,0]

    print('training model')
    adversarial = True
    
    # Train the Model
    for epoch in range(args.num_epochs):

        p = float(epoch) / 100
        #lambd = 2. / (1. + np.exp(-10. * p)) - 1
        lr = 0.001 / (1. + 10 * p) ** 0.75

        optimizer.lr = lr
        #rgs.lambd = lambd



        start_time = time.time()
        cost_vector = []
        class_cost_vector = []
        domain_cost_vector = []
        acc_vector = []
        valid_acc_vector = []
        test_acc_vector = []
        vali_cost_vector = []
        test_cost_vector = []

        for i, (train_data, train_labels, event_labels) in enumerate(train_loader):
            train_text, train_image,  train_mask, train_labels, event_labels = \
                to_var(train_data[0]), to_var(train_data[1]), to_var(train_data[2]), \
                to_var(train_labels), to_var(event_labels)

            # Forward + Backward + Optimize
            optimizer.zero_grad()

            class_outputs, domain_outputs = model(train_text, train_image, train_mask)

            ## Fake or Real loss
            class_loss = criterion(class_outputs, train_labels)
            # Event Loss
            domain_loss = criterion(domain_outputs, event_labels)
            loss = class_loss + domain_loss
            loss.backward()
            optimizer.step()
            _, argmax = torch.max(class_outputs, 1)

            cross_entropy = True

            if True:
                accuracy = (train_labels == argmax.squeeze()).float().mean()
            else:
                _, labels = torch.max(train_labels, 1)
                accuracy = (labels.squeeze() == argmax.squeeze()).float().mean()

            class_cost_vector.append(class_loss.item()) # class_loss.data[0]
            domain_cost_vector.append(domain_loss.item())
            cost_vector.append(loss.item())
            acc_vector.append(accuracy.item())
            # if i == 0:
            #     train_score = to_np(class_outputs.squeeze())
            #     train_pred = to_np(argmax.squeeze())
            #     train_true = to_np(train_labels.squeeze())
            # else:
            #     class_score = np.concatenate((train_score, to_np(class_outputs.squeeze())), axis=0)
            #     train_pred = np.concatenate((train_pred, to_np(argmax.squeeze())), axis=0)
            #     train_true = np.concatenate((train_true, to_np(train_labels.squeeze())), axis=0)



        model.eval()
        validate_acc_vector_temp = []
        for i, (validate_data, validate_labels, event_labels) in enumerate(validate_loader):
            validate_text, validate_image,  validate_mask, validate_labels, event_labels = \
                to_var(validate_data[0]), to_var(validate_data[1]), to_var(validate_data[2]), \
                to_var(validate_labels), to_var(event_labels)
            validate_outputs, domain_outputs = model(validate_text, validate_image, validate_mask)
            _, validate_argmax = torch.max(validate_outputs, 1)
            vali_loss = criterion(validate_outputs, validate_labels)
            #domain_loss = criterion(domain_outputs, event_labels)
                #_, labels = torch.max(validate_labels, 1)
            validate_accuracy = (validate_labels == validate_argmax.squeeze()).float().mean()
            vali_cost_vector.append(vali_loss.item())
                #validate_accuracy = (validate_labels == validate_argmax.squeeze()).float().mean()
            validate_acc_vector_temp.append(validate_accuracy.item())
        validate_acc = np.mean(validate_acc_vector_temp)
        valid_acc_vector.append(validate_acc)
        model.train()


        print ('Epoch [%d/%d],  Loss: %.4f, Class Loss: %.4f, domain loss: %.4f, Train_Acc: %.4f,  Validate_Acc: %.4f.'
                % (
                epoch + 1, args.num_epochs,  np.mean(cost_vector), np.mean(class_cost_vector),  np.mean(domain_cost_vector),
                    np.mean(acc_vector),   validate_acc))


        if validate_acc > best_validate_acc:
            best_validate_acc = validate_acc
            if not os.path.exists(args.output_file):
                os.mkdir(args.output_file)

            best_validate_dir = args.output_file + str(epoch + 1) + '.pkl'
            torch.save(model.state_dict(), best_validate_dir)


        duration = time.time() - start_time
        # print ('Epoch: %d, Mean_Cost: %.4f, Duration: %.4f, Mean_Train_Acc: %.4f, Mean_Test_Acc: %.4f'
        # % (epoch + 1, np.mean(cost_vector), duration, np.mean(acc_vector), np.mean(test_acc_vector)))
        # best_validate_dir = args.output_file + 'weibo_GPU2_out.' + str(52) + '.pkl'
    


    # Test the Model
    print('testing model')
    model = CNN_Fusion(args, W)
    model.load_state_dict(torch.load(best_validate_dir))
    #    print(torch.cuda.is_available())
    if torch.cuda.is_available():
        model.cuda()
    model.eval()
    test_score = []
    test_pred = []
    test_true = []
    for i, (test_data, test_labels, event_labels) in enumerate(test_loader):
        test_text, test_image, test_mask, test_labels = to_var(
            test_data[0]), to_var(test_data[1]), to_var(test_data[2]), to_var(test_labels)
        test_outputs, domain_outputs = model(test_text, test_image, test_mask)
        _, test_argmax = torch.max(test_outputs, 1)
        
        if i == 0:
            test_score = to_np(test_outputs.squeeze())
            test_pred = to_np(test_argmax.squeeze())
            test_true = to_np(test_labels.squeeze())
        else:
            test_score = np.concatenate((test_score, to_np(test_outputs.squeeze())), axis = 0)
            test_pred = np.concatenate((test_pred, to_np(test_argmax.squeeze())), axis = 0)
            test_true = np.concatenate((test_true, to_np(test_labels.squeeze())), axis = 0)

    test_accuracy = metrics.accuracy_score(test_true, test_pred)
    test_f1 = metrics.f1_score(test_true, test_pred, average = 'macro')
    test_precision = metrics.precision_score(test_true, test_pred, average = 'macro')
    test_recall = metrics.recall_score(test_true, test_pred, average = 'macro')

    
    test_score_convert = [x[1] for x in test_score]
    test_aucroc = metrics.roc_auc_score(test_true, test_score_convert, average = 'macro')
    
    test_confusion_matrix = metrics.confusion_matrix(test_true, test_pred)

    print("Classification Acc: %.4f, AUC-ROC: %.4f"
          % (test_accuracy, test_aucroc))
    print("Classification report:\n%s\n"
          % (metrics.classification_report(test_true, test_pred)))
    print("Classification confusion matrix:\n%s\n"
          % (test_confusion_matrix))

import warnings
warnings.filterwarnings("ignore") 

if __name__ == '__main__':
    parse = argparse.ArgumentParser()
    parser = parse_arguments(parse)
    train = '' 
    test = ''
    output = '../Data/weibo/RESULT/'
    args = parser.parse_args([train, test, output])
    
    main(args)<jupyter_output>loading data
Text and image
image length 268
Original post length is 2329
Original data frame is (2329, 6)
Label number is 43
Rumor number is 15
Non rumor is 28
data size is 43
paired post length is 43
paried data has 8 dimension
Original post length is 340
Original data frame is (340, 6)
Label number is 11
Rumor number is 7
Non rumor is 4
data size is 11
paired post length is 11
paried data has 8 dimension
Original post length is 686
Original data frame is (686, 6)
Label number is 20
Rumor number is 5
Non rumor is 15
data size is 20
paired post length is 20
paried data has 8 dimension
loading data...
number of sentences: 74
vocab size: 1348
max sentence length: 283
word2vec loaded!
num words already in word2vec: 4626
word2vec loaded!
num words already in word2vec: 4626
translate data to embedding
translate test data to embedding
sequence length 28
Train Data Size is 43
Finished loading data 
Text: 43, Image: 43, Label: 43, Event: 43
Text: 11, Image: 11, Label: 11, Event: 11
Text: 20, [...]
{'fraction_non_alphanumeric': 0.13420184896728907, 'fraction_numerical': 0.017720955636922702, 'mean_word_length': 2.6833320914983982, 'pattern_counts': {'":': 27, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '199394', 'license': 'MIT', 'n_tokens_mistral': 17111, 'n_tokens_neox': 16190, 'n_words': 3461, 'path': 'src/EANN.ipynb', 'repo_name': 'tinganchen/Pytorch-EANN', 'size': '49459'}