<jupyter_start><jupyter_text>Read from a local copy of the ClaimReview Database  Installation script```conda create -n ner python=3.6source activate nerconda install -c conda-forge spacy -yconda install ipython jupyter nb_conda nltk numpy -yconda install -c conda-forge rake_nltk -ypython -m spacy download enpython -m spacy download en_core_web_smpython -m spacy download en_core_web_lg```<jupyter_code>import spacy
nlp = spacy.load('en')
doc = nlp(u'Apple is looking at buying U.K. startup for $1 billion')

for ent in doc.ents:
    print(ent.text, ent.start_char, ent.end_char, ent.label_)
import spacy
from spacy.symbols import nsubj, VERB

nlp = spacy.load('en_core_web_sm')
doc = nlp(u"Autonomous cars shift insurance liability toward manufacturers")

# Finding a verb with a subject from below — good
verbs = set()
for possible_subject in doc:
    if possible_subject.dep == nsubj and possible_subject.head.pos == VERB:
        verbs.add(possible_subject.head)
print(verbs)
fc_path = "fact_checks_20180502.txt"

with open(fc_path) as f:
    fc_raw = f.readlines()
print("No. of Claims:", len(fc_raw))<jupyter_output>No. of Claims: 8902<jupyter_text>Functions to normalise the text<jupyter_code>import re
import json
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet

lemmatizer = WordNetLemmatizer()

def strip_html(data):
    p = re.compile(r'<.*?>')
    return p.sub('', data)

def clean_text(data):
    text = re.sub(r'[^\w\s]', ' ', data.lower()).replace("\n", "").replace("  ", " ")
    text = "".join([c for c in text if (c.isalpha() or c == " ")])
    text = text.split(" ")
    output = ""
    for word in text:
        if word not in stopwords.words("english"):
            output = output + " " + word
    return output.strip().replace("  ", " ")

def nltk2wn_tag(nltk_tag):
    if nltk_tag.startswith('J'):
        return wordnet.ADJ
    elif nltk_tag.startswith('V'):
        return wordnet.VERB
    elif nltk_tag.startswith('N'):
        return wordnet.NOUN
    elif nltk_tag.startswith('R'):
        return wordnet.ADV
    else:          
        return None
    
def lemmatize_sentence(sentence):
    nltk_tagged = nltk.pos_tag(nltk.word_tokenize(sentence))  
    wn_tagged = map(lambda x: (x[0], nltk2wn_tag(x[1])), nltk_tagged)
    res_words = []
    for word, tag in wn_tagged:
        if tag is None:            
            res_words.append(word)
        else:
            res_words.append(lemmatizer.lemmatize(word, tag))
    return " ".join(res_words)

def norm_text(data,):
    raw = strip_html(data)
    text = clean_text(raw)
    norm_text = lemmatize_sentence(text)
    return norm_text
from rake_nltk import Rake

# Uses stopwords for english from NLTK, and all puntuation characters by
# default
r = Rake()

# Extraction given the text.
r.extract_keywords_from_text("In August 2018, adhesive replicas of President <NAME>'s recently-destroyed star were placed on the Walk of Fame on Hollywood Boulevard.")

# # Extraction given the list of strings where each string is a sentence.
# r.extract_keywords_from_sentences(<list of sentences>)

# To get keyword phrases ranked highest to lowest.
print(r.get_ranked_phrases())

# To get keyword phrases ranked highest to lowest with scores.
print(r.get_ranked_phrases_with_scores())
# # ATTEMPTED TO EXTRACT THE VERBS WHICH IS NOT VERY HELPFUL

# import numpy as np
# random_point = np.random.randint(len(fc_raw)-200)
# print(random_point)

# for fc in fc_raw[random_point:random_point+10]:
#     fc = fc.strip("\n")
#     fc = fc.replace("</script>", "").replace('<script type="application/ld+json">', "")
#     fc = json.loads(fc)
#     claim = fc["claimReviewed"]
# #     date_published = fc["datePublished"]
#     review_url = fc["url"]
    
#     print("Claim:", claim)
#     r.extract_keywords_from_text(claim)
#     r.get_ranked_phrases()
#     print(r.get_ranked_phrases_with_scores())
#     doc = nlp(claim)
    
#     # EXTRACTING VERBS WHICH IS NOT VERY HELPFUL
#     print([(e.text, e.start_char, e.end_char, e.label_) for e in doc.ents])
#     verbs = set()
#     for possible_subject in doc:
#         if possible_subject.dep == nsubj and possible_subject.head.pos == VERB:
#             verbs.add(possible_subject.head)
#     print(verbs)
#     print()<jupyter_output><empty_output><jupyter_text>DEMO<jupyter_code>sentence_array = [
# verbatim 
'Actor <NAME> said that the United States is "morally upside down."',
'Liberty University students "were mandated" to attend Sen. Ted Cruz’s presidential \
announcement "or they would be fined.',
'A photograph shows a baseball stadium repurposed as a residential neighborhood.',
'Folding a $5 bill in a specific pattern will reveal a secret image of a stack of pancakes.',
# paraphrase
# <NAME> mocked "someone with a disability."
'The fact that President Trump mocked someone with a disability shows how idiotic he is.',
'We can see how <NAME> is a coward when he blocked all incoming calls and faxes.',
'Being tough on drug is stupid when you know that more people die from prescription \
narcotic painkillers than from heroin and cocaine combined.',
'<NAME> was arrested for bringing drugs to Japan.'
]
# here I am comparing the text with every claim. 
# If the keyword is spread across sentences they need to be retested again.
text = "".join(sentence_array)
from rake_nltk import Rake
r1 = Rake()
r1.extract_keywords_from_text(text)
r1_scored = r1.get_ranked_phrases_with_scores()
print(r1_scored)
import argparse
import sys

from google.cloud import language
from google.cloud.language import enums
from google.cloud.language import types
import six

# from snippets import do_everything

# [START language_entities_text]
def entities_text(text, debug=False):
    """Detects entities in the text."""
    client = language.LanguageServiceClient()

    if isinstance(text, six.binary_type):
        text = text.decode('utf-8')

    # Instantiates a plain text document.
    # [START language_python_migration_entities_text]
    document = types.Document(
        content=text,
        type=enums.Document.Type.PLAIN_TEXT)

    # Detects entities in the document. You can also analyze HTML with:
    #   document.type == enums.Document.Type.HTML
    entities = client.analyze_entities(document).entities

    # entity types from enums.Entity.Type
    entity_type = ('UNKNOWN', 'PERSON', 'LOCATION', 'ORGANIZATION',
                   'EVENT', 'WORK_OF_ART', 'CONSUMER_GOOD', 'OTHER')

    entity_name_list = []
    
    for entity in entities:
        entity_name_list.append([entity.salience, entity.name])
        if debug:
            print('=' * 20)
            print(u'{:<16}: {}'.format('name', entity.name))
            print(u'{:<16}: {}'.format('type', entity_type[entity.type]))
            print(u'{:<16}: {}'.format('metadata', entity.metadata))
            print(u'{:<16}: {}'.format('salience', entity.salience))
            print(u'{:<16}: {}'.format('wikipedia_url',
                  entity.metadata.get('wikipedia_url', '-')))
    
    return entity_name_list
    
    # [END language_python_migration_entities_text]
# [END language_entities_text]
import spacy
import numpy as np
import matplotlib.pyplot as plt
from rake_nltk import Rake

nlp = spacy.load('en_core_web_lg')

def calculate_similarity_score(value_mesh, r1_scored, r2_scored):
    similarity_array = np.maximum.reduce([row for row in value_mesh[0]])
    weighted_similarity = np.multiply(similarity_array, [keyword_tuple[0] for keyword_tuple in r1_scored])
    weighted_similarity_sum = np.sum(weighted_similarity)
    sum_of_weights = np.sum([keyword_tuple[0] for keyword_tuple in r1_scored])
    weighted_similarity_sum_over_weights = weighted_similarity_sum/sum_of_weights
    return weighted_similarity_sum_over_weights

def phrase_vect(text):
    tokens = nlp(text)
    maxpool = np.maximum.reduce([token.vector for token in tokens])
    return maxpool
#     minpool = np.minimum.reduce([token.vector for token in tokens])
#     # https://stackoverflow.com/questions/21816433/
#     result = np.concatenate((minpool,maxpool))
#     return result

def meshgrid(x,y): # DW code !!!
    return (
        [[x_ for x_ in x] for  _ in y],
        [[y_ for  _ in x] for y_ in y])

def cosine(u,v):
#     print(np.shape(u))
    return np.dot(u,v)/(np.linalg.norm(u)*np.linalg.norm(v))

def calculate_claim_text(claim, text):
#     r1 = Rake()
#     r1.extract_keywords_from_text(claim)
    r1_scored = entities_text(claim)
    print(r1_scored)

#     r2 = Rake()
#     r2.extract_keywords_from_text(text)
    r2_scored = entities_text(text)
#     print(r2_scored)
    
    # assign vectors
    r1_scored_vec = [[claim_keyword_tuple[0], 
                      claim_keyword_tuple[1], 
                      phrase_vect(claim_keyword_tuple[1])] for claim_keyword_tuple in r1_scored]
#     print(len(r1_scored_vec))
#     print(len(r1_scored_vec[0]))

    r2_scored_vec = [[text_keyword_tuple[0], 
                      text_keyword_tuple[1], 
                      phrase_vect(text_keyword_tuple[1])] for text_keyword_tuple in r2_scored]
#     print(len(r1_scored_vec))
#     print(len(r1_scored_vec[0]))
    
    
    # compare statements

    pair_mesh = meshgrid(r1_scored_vec, r2_scored_vec)
    value_mesh = meshgrid([0]*len(r1_scored_vec), [0]*len(r2_scored_vec))

    import numpy as np
    for i,_ in enumerate(pair_mesh[0]):
        for j,_ in enumerate(pair_mesh[0][0]): 
            vector1 = pair_mesh[0][i][j][2]
            vector2 = pair_mesh[1][i][j][2]

            value_mesh[0][i][j] = cosine(vector1, vector2)
            
    return calculate_similarity_score(value_mesh, r1_scored, r2_scored)
import numpy as np
random_point = np.random.randint(len(fc_raw)-200)
random_point = 100
print(random_point)

similarity_array = []

for fc in fc_raw[random_point:random_point+200]:
    fc = fc.strip("\n")
    fc = fc.replace("</script>", "").replace('<script type="application/ld+json">', "")
    fc = json.loads(fc)
    claim = fc["claimReviewed"]
    print(claim)
#     date_published = fc["datePublished"]
    review_url = fc["url"]
    
    similarity = calculate_claim_text(claim, text)
    print(similarity)
    similarity_array.append(similarity)
    
plt.figure(figsize=(15,4))
plt.plot(similarity_array)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11570407571952869, 'fraction_numerical': 0.011396561715279118, 'mean_word_length': 3.66021602160216, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '239100', 'license': 'MIT', 'n_tokens_mistral': 3645, 'n_tokens_neox': 3488, 'n_words': 863, 'path': 'browse-gcp_ent_keyword-wordvec_compare.ipynb', 'repo_name': 'tonghuikang/fake-news-claimreview-search', 'size': '10379'}