<jupyter_start><jupyter_text>AI Chatbot Test<jupyter_code>import nltk
import numpy as np
import random
import string # process standard python strings<jupyter_output><empty_output><jupyter_text>Read the raw txt file<jupyter_code>f = open('chatbot.txt', 'r', errors = 'ignore')
raw = f.read()
raw = raw.lower()
# 1st time use only
# nltk.download('punkt')
# nltk.download('wordnet')<jupyter_output><empty_output><jupyter_text>Tokenize the raw data<jupyter_code># converts to list of sentences
sent_tokens = nltk.sent_tokenize(raw)
# converts to list of words
word_tokens = nltk.word_tokenize(raw)
sent_tokens
word_tokens[:2]<jupyter_output><empty_output><jupyter_text>Pre-processing the Raw Text<jupyter_code>'''
define function called LemTokens which will
take in the tokens as inputs
& return normalised tokens
'''
lemmer = nltk.stem.WordNetLemmatizer()
# Wordnet is a semantically-oriented dictionary of English included 
# in NLTK
def LemTokens(tokens):
    return [lemmer.lemmatize(token) for token in tokens]

remove_punct_dict = dict((ord(punct), None) for punct in string.punctuation)
def LemNormalize(text):
    return LemTokens(nltk.word_tokenize(text.lower().translate(remove_punct_dict)))<jupyter_output><empty_output><jupyter_text>Keyword Matching<jupyter_code>'''
define function for greeting by bot
Eg. If user input greeting, bot reply with greeting
ELIZA uses simple keyword matching for greetings
'''
GREETING_INPUTS = ('hello', 'hi', 'greetings', 'sup', "what's up", 'hey')
GREETING_RESPONSES = ['hi', 'hey', '*nods', 'hi there', 'hello', 
                      "I'm glad! You're talking to me!"]
def greeting(sentence):
    for word in sentence.split():
        if word.lower() in GREETING_INPUTS:
            return random.choice(GREETING_RESPONSES)<jupyter_output><empty_output><jupyter_text>Generating Response<jupyter_code>'''
To generate response from our bot for input qns,
concept of codument similarity used
We'll begin by importing the necessary modules
'''
# From scikit learn libary, import TFidf vectorizer to convert collection of 
# raw documents to matrix of TF-IDF features
from sklearn.feature_extraction.text import TfidfVectorizer
# import cosine similary module from scikit learn lib
from sklearn.metrics.pairwise import cosine_similarity
# used to find similarity between words entered by user & owrds in corpus
# This is the simplest implementation of a chatbot
# define function response which searches user's utterance for >1 known keywords
# & returns several possible responses
# if no matches, return "I'm sorry! Don't understand you!"

def response(user_resp):
    robo_response = ''
    sent_tokens.append(user_resp)
    
    tfidf_vec = TfidfVectorizer(tokenizer = LemNormalize, stop_words = 'english')
    tfidf = tfidf_vec.fit_transform(sent_tokens)
    
    vals = cosine_similarity(tfidf[-1], tfidf)
    idx = vals.argsort()[0][-2]
    
    flat = vals.flatten()
    flat.sort()
    
    req_tfidf = flat[-2]
    
    robo_response += "Sorry! Don't understand" if (req_tfidf == 0) else sent_tokens[idx]
    return robo_response
# feed lines we want our bot to say while starting & ending conversation
# depending on user's input

flag = True
print('BOI: Howdy. Ask me Anything :)')
robo_text: str = ''

while flag:
    user_resp = input('>>>').lower()
    
    if user_resp not in ['bye', 'exit']:
        if user_resp in ['thanks', 'thank you']:
            flag = False
            robo_text = 'No Problem'
        else:
            if greeting(user_resp) != None:
                robo_text = greeting(user_resp)
            else:
                print(response(user_resp))
                sent_tokens.remove(user_resp)
    else:
        flag = False
        robo_text = 'Cya!'
    
    print(f"BOI: {robo_text}")<jupyter_output>BOI: Howdy. Ask me Anything :)
>>>what is an ai
{'fraction_non_alphanumeric': 0.10180109631949882, 'fraction_numerical': 0.0020882276168102325, 'mean_word_length': 4.185385656292287, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '5872', 'license': 'MIT', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1224, 'n_words': 360, 'path': 'Test Notebooks/SP Chatbot Test/chatbot_test.ipynb', 'repo_name': 'Asienwald/SmartPoly-AI-Chatbot', 'size': '3831'}