<jupyter_start><jupyter_text>NLP Based Tools<jupyter_code>import nltk
import re
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer<jupyter_output><empty_output><jupyter_text>Amazon Review Dataset<jupyter_code>import tensorflow_datasets as tfds
ds = tfds.load('amazon_us_reviews/Mobile_Electronics_v1_00', split='train', shuffle_files=True)

assert isinstance(ds, tf.data.Dataset) # Check if dataset if Wrapped under tf.data.Dataset
print(ds)
df = tfds.as_dataframe(ds)
df.head()
# Giving negative and positive to Sentiment
df["Sentiment"] = df["data/star_rating"].apply(lambda score: "positive" if score >= 3 else "negative")
df['Sentiment'] = df['Sentiment'].map({'positive':1, 'negative':0})
df['short_review'] = df['data/review_body'].str.decode('utf-8')
df = df[['short_review', 'Sentiment']]
df.head()
# Drop Last N rows 
N = 54975
df.drop(df.tail(N).index, inplace=True)
index = df.index
no_of_rows = len(index)
print(no_of_rows)
df.tail()
# Split into reviews and labels
reviews = df['short_review'].values.tolist()
labels = df['Sentiment'].tolist()
print(reviews[:2])
print(labels[:2])
# Split into train and test
from sklearn.model_selection import train_test_split
train_texts, val_texts, train_labels, val_labels = train_test_split(reviews, labels, test_size=0.2)<jupyter_output><empty_output><jupyter_text>TokenizerWe are using `DistilBERTTokenizer` as from pretrained `DistilBert` Model.<jupyter_code>tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
tokenizer([train_texts[0]], truncation=True, padding=True, max_length=128)<jupyter_output><empty_output><jupyter_text>Prepare Encodings<jupyter_code>train_encodings = tokenizer(train_texts, truncation=True, padding=True)
val_encodings = tokenizer(val_texts, truncation=True, padding=True)
# Make Dataset from these Encodings
train_dataset = tf.data.Dataset.from_tensor_slices((dict(train_encodings), train_labels))

val_dataset = tf.data.Dataset.from_tensor_slices((dict(val_encodings), val_labels))<jupyter_output><empty_output><jupyter_text>ModelA Pretrained `DistilBERT` is used for this. From the Transformers module we use `TFDistilBertForSequenceClassification`.<jupyter_code>model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels = 2)
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5, epsilon=1e-08)
model.compile(optimizer=optimizer, loss=model.compute_loss, metrics=['accuracy'])

model.fit(train_dataset.shuffle(100).batch(16), epochs=2, batch_size=16, validation_data=val_dataset.shuffle(100).batch(16))
model.save_pretrained('./sentiment')
loaded_model = TFDistilBertForSequenceClassification.from_pretrained('./sentiment')
test_sentence = "This is a really good product. I really love it."

predict_input = tokenizer.encode(test_sentence, truncation=True, padding=True, return_tensors="tf")
tf_output = loaded_model.predict(predict_input)[0]

tf_prediction = tf.nn.softmax(tf_output, axis=1)
labels = ['Negative', 'Positive']
label = tf.argmax(tf_prediction, axis=1)
label = label.numpy()
print(labels[label[0]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12844909609895339, 'fraction_numerical': 0.01363780526482715, 'mean_word_length': 9.30718954248366, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '431140', 'license': 'MIT', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1093, 'n_words': 194, 'path': 'NLP Amazon Review Dataset/Sentiment Analysis on Amazon Reviews TFDS.ipynb', 'repo_name': 'krishnapalS/ML-Gallery', 'size': '3153'}