|
import streamlit as st |
|
import spacy |
|
import wikipediaapi |
|
import wikipedia |
|
from wikipedia.exceptions import DisambiguationError |
|
from transformers import TFAutoModel, AutoTokenizer |
|
import numpy as np |
|
import pandas as pd |
|
import faiss |
|
import datetime |
|
import time |
|
|
|
|
|
try: |
|
nlp = spacy.load("en_core_web_sm") |
|
except: |
|
spacy.cli.download("en_core_web_sm") |
|
nlp = spacy.load("en_core_web_sm") |
|
|
|
wh_words = ['what', 'who', 'how', 'when', 'which'] |
|
|
|
def get_concepts(text): |
|
text = text.lower() |
|
doc = nlp(text) |
|
concepts = [] |
|
for chunk in doc.noun_chunks: |
|
if chunk.text not in wh_words: |
|
concepts.append(chunk.text) |
|
return concepts |
|
|
|
def get_passages(text, k=1000): |
|
doc = nlp(text) |
|
passages = [] |
|
passage_len = 0 |
|
passage = "" |
|
sents = list(doc.sents) |
|
for i in range(len(sents)): |
|
sen = sents[i] |
|
passage_len += len(sen) |
|
if passage_len >= k: |
|
passages.append(passage) |
|
passage = sen.text |
|
passage_len = len(sen) |
|
continue |
|
elif i == (len(sents) - 1): |
|
passage += " " + sen.text |
|
passages.append(passage) |
|
passage = "" |
|
passage_len = 0 |
|
continue |
|
passage += " " + sen.text |
|
return passages |
|
|
|
def get_dicts_for_dpr(concepts, n_results=200, k=1000): |
|
dicts = [] |
|
for concept in concepts: |
|
wikis = wikipedia.search(concept, results=n_results) |
|
st.write(f"{concept} No of Wikis: {len(wikis)}") |
|
for wiki in wikis: |
|
try: |
|
html_page = wikipedia.page(title=wiki, auto_suggest=False) |
|
except DisambiguationError: |
|
continue |
|
htmlResults = html_page.content |
|
passages = get_passages(htmlResults, k=k) |
|
for passage in passages: |
|
i_dicts = {} |
|
i_dicts['text'] = passage |
|
i_dicts['title'] = wiki |
|
dicts.append(i_dicts) |
|
return dicts |
|
|
|
passage_encoder = TFAutoModel.from_pretrained("nlpconnect/dpr-ctx_encoder_bert_uncased_L-2_H-128_A-2") |
|
query_encoder = TFAutoModel.from_pretrained("nlpconnect/dpr-question_encoder_bert_uncased_L-2_H-128_A-2") |
|
p_tokenizer = AutoTokenizer.from_pretrained("nlpconnect/dpr-ctx_encoder_bert_uncased_L-2_H-128_A-2") |
|
q_tokenizer = AutoTokenizer.from_pretrained("nlpconnect/dpr-question_encoder_bert_uncased_L-2_H-128_A-2") |
|
|
|
def get_title_text_combined(passage_dicts): |
|
res = [] |
|
for p in passage_dicts: |
|
res.append(tuple((p['title'], p['text']))) |
|
return res |
|
|
|
def extracted_passage_embeddings(processed_passages, max_length=156): |
|
passage_inputs = p_tokenizer.batch_encode_plus( |
|
processed_passages, |
|
add_special_tokens=True, |
|
truncation=True, |
|
padding="max_length", |
|
max_length=max_length, |
|
return_token_type_ids=True |
|
) |
|
passage_embeddings = passage_encoder.predict([np.array(passage_inputs['input_ids']), np.array(passage_inputs['attention_mask']), |
|
np.array(passage_inputs['token_type_ids'])], |
|
batch_size=64, |
|
verbose=1) |
|
return passage_embeddings |
|
|
|
def extracted_query_embeddings(queries, max_length=64): |
|
query_inputs = q_tokenizer.batch_encode_plus( |
|
queries, |
|
add_special_tokens=True, |
|
truncation=True, |
|
padding="max_length", |
|
max_length=max_length, |
|
return_token_type_ids=True |
|
) |
|
|
|
query_embeddings = query_encoder.predict([np.array(query_inputs['input_ids']), |
|
np.array(query_inputs['attention_mask']), |
|
np.array(query_inputs['token_type_ids'])], |
|
batch_size=1, |
|
verbose=1) |
|
return query_embeddings |
|
|
|
def get_pagetext(page): |
|
s = str(page).replace("/t","") |
|
return s |
|
|
|
def get_wiki_summary(search): |
|
wiki_wiki = wikipediaapi.Wikipedia('en') |
|
page = wiki_wiki.page(search) |
|
|
|
|
|
def get_wiki_summaryDF(search): |
|
wiki_wiki = wikipediaapi.Wikipedia('en') |
|
page = wiki_wiki.page(search) |
|
|
|
isExist = page.exists() |
|
if not isExist: |
|
return isExist, "Not found", "Not found", "Not found", "Not found" |
|
|
|
pageurl = page.fullurl |
|
pagetitle = page.title |
|
pagesummary = page.summary[0:60] |
|
pagetext = get_pagetext(page.text) |
|
|
|
backlinks = page.backlinks |
|
linklist = "" |
|
for link in backlinks.items(): |
|
pui = link[0] |
|
linklist += pui + " , " |
|
a=1 |
|
|
|
categories = page.categories |
|
categorylist = "" |
|
for category in categories.items(): |
|
pui = category[0] |
|
categorylist += pui + " , " |
|
a=1 |
|
|
|
links = page.links |
|
linklist2 = "" |
|
for link in links.items(): |
|
pui = link[0] |
|
linklist2 += pui + " , " |
|
a=1 |
|
|
|
sections = page.sections |
|
|
|
ex_dic = { |
|
'Entity' : ["URL","Title","Summary", "Text", "Backlinks", "Links", "Categories"], |
|
'Value': [pageurl, pagetitle, pagesummary, pagetext, linklist,linklist2, categorylist ] |
|
} |
|
|
|
df = pd.DataFrame(ex_dic) |
|
|
|
return df |
|
|
|
|
|
def save_message_old1(name, message): |
|
now = datetime.datetime.now() |
|
timestamp = now.strftime("%Y-%m-%d %H:%M:%S") |
|
with open("chat.txt", "a") as f: |
|
f.write(f"{timestamp} - {name}: {message}\n") |
|
|
|
def press_release(): |
|
st.markdown("""🎉🎊 Breaking News! 📢📣 |
|
|
|
Introducing StreamlitWikipediaChat - the ultimate way to chat with Wikipedia and the whole world at the same time! 🌎📚👋 |
|
|
|
Are you tired of reading boring articles on Wikipedia? Do you want to have some fun while learning new things? Then StreamlitWikipediaChat is just the thing for you! 😃💻 |
|
|
|
With StreamlitWikipediaChat, you can ask Wikipedia anything you want and get instant responses! Whether you want to know the capital of Madagascar or how to make a delicious chocolate cake, Wikipedia has got you covered. 🍰🌍 |
|
|
|
But that's not all! You can also chat with other people from around the world who are using StreamlitWikipediaChat at the same time. It's like a virtual classroom where you can learn from and teach others. 🌐👨🏫👩🏫 |
|
|
|
And the best part? StreamlitWikipediaChat is super easy to use! All you have to do is type in your question and hit send. That's it! 🤯🙌 |
|
|
|
So, what are you waiting for? Join the fun and start chatting with Wikipedia and the world today! 😎🎉 |
|
|
|
StreamlitWikipediaChat - where learning meets fun! 🤓🎈""") |
|
|
|
|
|
def main_old1(): |
|
st.title("Streamlit Chat") |
|
|
|
name = st.text_input("Enter your name") |
|
message = st.text_input("Enter a topic to share from Wikipedia") |
|
if st.button("Submit"): |
|
|
|
|
|
df = get_wiki_summaryDF(message) |
|
|
|
save_message(name, message) |
|
save_message(name, df) |
|
|
|
st.text("Message sent!") |
|
|
|
|
|
st.text("Chat history:") |
|
with open("chat.txt", "a+") as f: |
|
f.seek(0) |
|
chat_history = f.read() |
|
|
|
st.markdown(chat_history) |
|
|
|
countdown = st.empty() |
|
t = 60 |
|
while t: |
|
mins, secs = divmod(t, 60) |
|
countdown.text(f"Time remaining: {mins:02d}:{secs:02d}") |
|
time.sleep(1) |
|
t -= 1 |
|
if t == 0: |
|
countdown.text("Time's up!") |
|
with open("chat.txt", "a+") as f: |
|
f.seek(0) |
|
chat_history = f.read() |
|
|
|
st.markdown(chat_history) |
|
|
|
press_release() |
|
|
|
t = 60 |
|
|
|
def save_message(name, message): |
|
with open("chat.txt", "a") as f: |
|
f.write(f"{name}: {message}\n") |
|
|
|
def main(): |
|
st.title("Streamlit Chat") |
|
|
|
name = st.text_input("Enter your name") |
|
message = st.text_input("Enter a topic to share from Wikipedia") |
|
if st.button("Submit"): |
|
|
|
|
|
df = get_wiki_summaryDF(message) |
|
|
|
save_message(name, message) |
|
save_message(name, df) |
|
|
|
st.text("Message sent!") |
|
|
|
st.text("Chat history:") |
|
with open("chat.txt", "a+") as f: |
|
f.seek(0) |
|
chat_history = f.read() |
|
st.markdown(chat_history) |
|
|
|
countdown = st.empty() |
|
t = 60 |
|
while t: |
|
mins, secs = divmod(t, 60) |
|
countdown.text(f"Time remaining: {mins:02d}:{secs:02d}") |
|
time.sleep(1) |
|
t -= 1 |
|
if t == 0: |
|
countdown.text("Time's up!") |
|
with open("chat.txt", "a+") as f: |
|
f.seek(0) |
|
chat_history = f.read() |
|
st.markdown(chat_history) |
|
|
|
press_release() |
|
|
|
t = 60 |
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
main() |
|
|
|
|
