<jupyter_start><jupyter_text>Chatbot data preparationIn this notebook we are going to prepare the data for a simple chatbot using the text classification technique of nlp. Imports<jupyter_code>import os, json
from google.colab import drive<jupyter_output><empty_output><jupyter_text>DataWe are going to load the data from our google drive, we are going to load the `intent.json` file<jupyter_code>drive.mount("/content/drive")<jupyter_output>Mounted at /content/drive<jupyter_text>File name<jupyter_code>file_name = "/content/drive/My Drive/NLP Data/chatbot/Intent.json"<jupyter_output><empty_output><jupyter_text>Reading the file<jupyter_code>with open(file_name, "r") as f:
  data = json.load(f)
print(data)<jupyter_output>{'intents': [{'intent': 'Greeting', 'text': ['Hi', 'Hi there', 'Hola', 'Hello', 'Hello there', 'Hya', 'Hya there'], 'responses': ['Hi human, please tell me your GeniSys user', 'Hello human, please tell me your GeniSys user', 'Hola human, please tell me your GeniSys user'], 'extension': {'function': '', 'entities': False, 'responses': []}, 'context': {'in': '', 'out': 'GreetingUserRequest', 'clear': False}, 'entityType': 'NA', 'entities': []}, {'intent': 'GreetingResponse', 'text': ['My user is Adam', 'This is Adam', 'I am Adam', 'It is Adam', 'My user is Bella', 'This is Bella', 'I am Bella', 'It is Bella'], 'responses': ['Great! Hi <HUMAN>! How can I help?', 'Good! Hi <HUMAN>, how can I help you?', 'Cool! Hello <HUMAN>, what can I do for you?', 'OK! Hola <HUMAN>, how can I help you?', 'OK! hi <HUMAN>, what can I do for you?'], 'extension': {'function': 'extensions.gHumans.updateHuman', 'entities': True, 'responses': ['Hi %%HUMAN%%! How can I help?', 'Hi %%HUMAN%%, how can I help you?'[...]<jupyter_text>Getting the intents<jupyter_code>intents = data["intents"]<jupyter_output><empty_output><jupyter_text>Creating the datasetWe are going to use the text classsification aproach to classify the intents for the text that will be provided.<jupyter_code># we are interested in the intent and their text.

dataset =[]
for intent in intents:
  texts = intent["text"]
  intent_ = intent["intent"].lower()
  for text in texts:
    dataset.append((text, intent_))<jupyter_output><empty_output><jupyter_text>Checking the first 2 exmples<jupyter_code>dataset[:2]<jupyter_output><empty_output><jupyter_text>Checking the length of the dataset<jupyter_code>len(dataset)<jupyter_output><empty_output><jupyter_text>Checking the last 10 examples<jupyter_code>dataset[-10:]<jupyter_output><empty_output><jupyter_text>Saving `csv` files.<jupyter_code>import pandas as pd
import random
random.shuffle(dataset)
train_df = pd.DataFrame(dataset, columns=["text", "label" ])
random.shuffle(dataset)
test_df = pd.DataFrame(dataset[:40], columns=["text", "label" ])
random.shuffle(dataset)
val_df = pd.DataFrame(dataset[:60], columns=["text", "label" ])
train_df.head(10)<jupyter_output><empty_output><jupyter_text>Saving 1. train.csv2. test.csv3. val.csv<jupyter_code>base_path = "/content/drive/My Drive/NLP Data/chatbot"
train_df.to_csv(os.path.join(base_path, "train.csv"),  index = False, header = True)
test_df.to_csv(os.path.join(base_path, "test.csv"),  index = False, header = True)
val_df.to_csv(os.path.join(base_path, "val.csv"),  index = False, header = True)

print("Done")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1563156315631563, 'fraction_numerical': 0.004500450045004501, 'mean_word_length': 7.092233009708738, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '546350', 'license': 'MIT', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1161, 'n_words': 315, 'path': '08_Simple_Chatbot/00_Data_Preparation.ipynb', 'repo_name': 'CrispenGari/NLP-PyTorch', 'size': '3333'}