<jupyter_start><jupyter_text>BaselineFactify is a Multi-Modal Fact Verification dataset released for a shared task as part of the De-Factify workshop in AAAI-21.<jupyter_code>import os
import io
import requests
from PIL import Image
import pandas as pd
import zipfile<jupyter_output><empty_output><jupyter_text>Installation Download the training data from the codalab competition page from the Participate > Files > Public Dataor use the drive library below<jupyter_code>from google_drive_downloader import GoogleDriveDownloader as gdd
gdd.download_file_from_google_drive(file_id='1ig7XEYU1UKDHrHgDYgqiARWvNdswgFEX',
                                    dest_path='/content/public_folder.zip')
#fill password
pswd=""
with zipfile.ZipFile("public_folder.zip") as file:
  file.extractall(pwd = bytes(pswd, 'utf-8'))
download_path = "public_folder/"
train_df = pd.read_csv(download_path + "train.csv", index_col="Id")
train_df.head()
train_df=train_df.head()<jupyter_output><empty_output><jupyter_text>You can save images as you like, but we provide a template to store claim and support images seperately in directories created class-wise.<jupyter_code>image_directory = "images"
if not os.path.isdir(image_directory):
  os.makedirs(image_directory)
for i in ["claim", "document"]:
  if not os.path.isdir(image_directory + "/" + i):
    os.makedirs(image_directory + "/" + i)
  for cls in ["Support_Multimodal", "Support_Text", "Insufficient_Multimodal", "Insufficient_Text", "Refute"]:
    if not os.path.isdir(image_directory + "/" + i + "/" + cls):
      os.makedirs(image_directory + "/" + i + "/" + cls)
for n, row in train_df.iterrows():
  headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
  response = requests.get(row["claim_image"], headers=headers)
  img = Image.open(io.BytesIO(response.content))
  img = img.convert('RGB')
  img.save(image_directory + "/claim/" + row["Category"] + "/" + str(n) + ".jpg")

  response = requests.get(row["document_image"], headers=headers)
  img = Image.open(io.BytesIO(response.content))
  img = img.convert('RGB')
  img.save(image_directory + "/document/" + row["Category"] + "/" + str(n) + ".jpg")<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># Model definition and training
!pip install sentence-transformers
!pip install keras_applications
import nltk
import numpy as np
import cv2
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from sentence_transformers import SentenceTransformer
from resnet50 import ResNet50
import tensorflow
from keras.layers import Input
from keras.preprocessing import image
from keras.applications.imagenet_utils import preprocess_input
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
from tqdm import tqdm
import nltk
nltk.download('stopwords')
nltk.download('punkt')
def get_feature_vector_fromPIL(img):
    feature_vector = feature_model.predict(img)
    a, b, c, n = feature_vector.shape
    feature_vector= feature_vector.reshape(b,n)
    return feature_vector

def calculate_similarity_cosine(vector1, vector2):
    #return 1 - distance.cosine(vector1, vector2)
    return cosine_similarity(vector1, vector2)

# This distance can be in range of [0,∞]. And this distance is converted to a [0,1]
def calculate_similarity_euclidean(vector1, vector2):
    return 1/(1 + np.linalg.norm(vector1- vector2))  

#Use ResNet-50 model as an image feature extractor
image_input = Input(shape=(224, 224, 3))
feature_model = ResNet50(input_tensor=image_input, include_top=False,weights='imagenet')
def cosinesimilarity_Resnet50(a1,a2):
    x = image.load_img(a1, target_size=(224, 224))
    x = image.img_to_array(x)
    x = np.expand_dims(x, axis=0)
    x = preprocess_input(x)
    x2 = image.load_img(a2, target_size=(224, 224))
    x2 = image.img_to_array(x2)
    x2 = np.expand_dims(x2, axis=0)
    x2 = preprocess_input(x2)
    image_similarity_cosine = calculate_similarity_cosine(get_feature_vector_fromPIL(x), get_feature_vector_fromPIL(x2))[0][0]
    return image_similarity_cosine
def cosine(u, v):
    return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))
sbert_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
em1 = sbert_model.encode(list(train_df['claim']))
em2 = sbert_model.encode(list(train_df['document']))
ps = PorterStemmer()
sw = list(stopwords.words('english'))
punc = '''!()-[]{};:'"\, <>./?@#$%^&*_~'''
text_sim = []
img_sim = []
for i in tqdm(range(len(em1))):
    sim = cosine(em1[i], em2[i])
    text_sim.append(sim)
    
    sim2 = cosinesimilarity_Resnet50(
        'images/claim/' + train_df["Category"].values[i] + "/" + str(train_df.index.values[i]) + ".jpg",
        'images/document/' + train_df["Category"].values[i] + "/" + str(train_df.index.values[i]) + ".jpg",
        )
    img_sim.append(sim2)

train_df['Text_Sim'] = text_sim
train_df['Img_Sim'] = img_sim
train_df.head()<jupyter_output><empty_output><jupyter_text>Validation<jupyter_code>val_df = pd.read_csv(download_path + "val.csv", index_col="Id")
val_df.head()
image_directory_val = "images_val"
if not os.path.isdir(image_directory_val):
  os.makedirs(image_directory_val)
for i in ["claim", "document"]:
  if not os.path.isdir(image_directory_val + "/" + i):
    os.makedirs(image_directory_val + "/" + i)
for n, row in val_df.iterrows():
  headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
  response = requests.get(row["claim_image"], headers=headers)
  img = Image.open(io.BytesIO(response.content))
  img = img.convert('RGB')
  img.save(image_directory_val + "/claim/" + str(n) + ".jpg")

  response = requests.get(row["document_image"], headers=headers)
  img = Image.open(io.BytesIO(response.content))
  img = img.convert('RGB')
  img.save(image_directory_val + "/document/" + str(n) + ".jpg")
sbert_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
em1 = sbert_model.encode(list(val_df['claim']))
em2 = sbert_model.encode(list(val_df['document']))
ps = PorterStemmer()
sw = list(stopwords.words('english'))
punc = '''!()-[]{};:'"\, <>./?@#$%^&*_~'''
# Validation data prediction using trained model
text_sim = []
img_sim = []
for i in tqdm(range(len(em1))):
    sim = cosine(em1[i], em2[i])
    text_sim.append(sim)
    
    sim2 = cosinesimilarity_Resnet50(
        'images_val/claim/' + str(val_df.index.values[i]) + ".jpg",
        'images_val/document/' + str(val_df.index.values[i]) + ".jpg",
        )
    img_sim.append(sim2)

val_df['Text_Sim'] = text_sim
val_df['Img_Sim'] = img_sim<jupyter_output><empty_output><jupyter_text>Text-only Model<jupyter_code>from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score
X_train = train_df[["Text_Sim"]]
y_train = train_df["Category"]

X_test = val_df[["Text_Sim"]]
y_test = val_df["Category"]
neigh = KNeighborsClassifier(n_neighbors=3)
neigh.fit(X_train, y_train)
predictions = neigh.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = make_pipeline(StandardScaler(), SVC(gamma='auto', kernel='rbf'))
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = LogisticRegression(random_state=0).fit(X_train, y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = DecisionTreeClassifier()
clf = clf.fit(X_train,y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = RandomForestClassifier(n_estimators=500,max_depth=10, random_state=16)
clf = clf.fit(X_train,y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)<jupyter_output><empty_output><jupyter_text>Multimodal<jupyter_code>X_train = train_df[["Text_Sim", "Img_Sim"]]
y_train = train_df["Category"]

X_test = val_df[["Text_Sim", "Img_Sim"]]
y_test = val_df["Category"]
neigh = KNeighborsClassifier(n_neighbors=3)
neigh.fit(X_train, y_train)
predictions = neigh.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = make_pipeline(StandardScaler(), SVC(gamma='auto', kernel='rbf'))
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = LogisticRegression(random_state=0).fit(X_train, y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = DecisionTreeClassifier()
clf = clf.fit(X_train,y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)
clf = RandomForestClassifier(n_estimators=500,max_depth=10, random_state=16)
clf = clf.fit(X_train,y_train)
predictions = clf.predict(X_test)
score = f1_score(y_test, predictions, average='weighted')
print(score)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>from google_drive_downloader import GoogleDriveDownloader as gdd
gdd.download_file_from_google_drive(file_id='1dAlXZVDc09gmaEwPHHw8H2BCBYtDH4gz',
                                    dest_path='/content/test.zip')
#fill password
pswd=""
with zipfile.ZipFile("test.zip") as file:
  file.extractall(pwd = bytes(pswd, 'utf-8'))
download_path = ""
test_df = pd.read_csv(download_path + "test.csv", index_col="Id")
test_df.head()
image_directory_test = "images_test"
if not os.path.isdir(image_directory_test):
  os.makedirs(image_directory_test)
for i in ["claim", "document"]:
  if not os.path.isdir(image_directory_test + "/" + i):
    os.makedirs(image_directory_test + "/" + i)
for n, row in test_df.iterrows():
  headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
  response = requests.get(row["claim_image"], headers=headers)
  img = Image.open(io.BytesIO(response.content))
  img = img.convert('RGB')
  img.save(image_directory_test + "/claim/" + str(n) + ".jpg")

  response = requests.get(row["document_image"], headers=headers)
  img = Image.open(io.BytesIO(response.content))
  img = img.convert('RGB')
  img.save(image_directory_test + "/document/" + str(n) + ".jpg")
sbert_model = SentenceTransformer('paraphrase-MiniLM-L6-v2')
em1 = sbert_model.encode(list(test_df['claim']))
em2 = sbert_model.encode(list(test_df['document']))
ps = PorterStemmer()
sw = list(stopwords.words('english'))
punc = '''!()-[]{};:'"\, <>./?@#$%^&*_~'''
# Test data prediction using trained model
text_sim = []
img_sim = []
for i in tqdm(range(len(em1))):
    sim = cosine(em1[i], em2[i])
    text_sim.append(sim)
    
    sim2 = cosinesimilarity_Resnet50(
        'images_test/claim/' + str(test_df.index.values[i]) + ".jpg",
        'images_test/document/' + str(test_df.index.values[i]) + ".jpg",
        )
    img_sim.append(sim2)

test_df['Text_Sim'] = text_sim
test_df['Img_Sim'] = img_sim
# Multimodal Model

X_train = train_df[["Text_Sim", "Img_Sim"]]
y_train = train_df["Category"]

X_test = test_df[["Text_Sim", "Img_Sim"]]
clf = RandomForestClassifier(n_estimators=500,max_depth=10, random_state=16)
clf = clf.fit(X_train,y_train)
predictions = clf.predict(X_test)
res = {"Id": test_df.index, "Category": predictions}
answer = pd.DataFrame(res)
answer.to_csv("answer.csv",index=False)
!zip answer.zip answer.csv<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14552680534415693, 'fraction_numerical': 0.01902587519025875, 'mean_word_length': 5.892191142191142, 'pattern_counts': {'":': 2, '<': 36, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '190930', 'license': 'MIT', 'n_tokens_mistral': 4492, 'n_tokens_neox': 4339, 'n_words': 798, 'path': 'Baseline.ipynb', 'repo_name': 'Shreyashm16/Factify', 'size': '11826'}