<jupyter_start><jupyter_text>New Section<jupyter_code># Confirm that we are using GPU's which will speed up our processing
num_gpus_available = len(tf.config.experimental.list_physical_devices('GPU'))
print("Num GPUs Available: ", num_gpus_available)
assert num_gpus_available > 0
# Load the dataset 
ds = tfds.load('amazon_us_reviews/Mobile_Electronics_v1_00', split='train', shuffle_files=True)
df = tfds.as_dataframe(ds)
assert isinstance(ds, tf.data.Dataset)
df.head()
# Create a feature that indicates if a review has a good user rating
df["Sentiment"] = df["data/star_rating"].apply(lambda score: 1 if score >= 3 else 0)

# Clean data types
df['short_review'] = df['data/review_body'].str.decode("utf-8")
df['product_id'] = df['data/product_id'].str.decode("utf-8")
df['product_parent'] = df['data/product_parent'].str.decode("utf-8")
df['product_title'] = df['data/product_title'].str.decode("utf-8")
df['rating'] = df['data/star_rating']
# Drop unneeded features
df = df[["short_review",  "Sentiment",  'product_id', 'product_parent', 'product_title', 'rating']]
# Load tokenizer model for tokenizing review text
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
import os

# This function loads and primes a pre-trained Bert model to predict the attention of amazon reviews based on their ratings 
def load_and_prime_bert_attention_model(df):
  # If a primed model already is saved, load it instead of re-training the model
  if os.path.isfile("./sentiment/tf_model.h5"):
    return TFDistilBertForSequenceClassification.from_pretrained("./sentiment", output_attentions=True)
  else:
    # Create a subset of the data to prime the pre-trained bert model 
    df_subset = df.head(25000)
    reviews = df_subset['short_review'].values.tolist()
    labels = df_subset['Sentiment'].tolist()
    training_sentences, validation_sentences, training_labels, validation_labels = train_test_split(reviews, labels, test_size=.2)

    # Tokenize training and testing data
    train_encodings = tokenizer(training_sentences, truncation=True, padding=True)
    val_encodings = tokenizer(validation_sentences, truncation=True, padding=True)

    # Create the training and testing datasets from the encoded data and their corresponding rating labels
    train_dataset = tf.data.Dataset.from_tensor_slices((
        dict(train_encodings),
        training_labels
    ))
    val_dataset = tf.data.Dataset.from_tensor_slices((
        dict(val_encodings),
        validation_labels
    ))

    # Load the pre-trained bert model
    model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=2)

    # Prime the model with review data to make it work for our purposes
    optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5, epsilon=1e-08)
    model.compile(optimizer=optimizer, loss=model.compute_loss, metrics=['accuracy'])
    model.fit(train_dataset.shuffle(100).batch(16),
              epochs=2,
              batch_size=16,
              validation_data=val_dataset.shuffle(100).batch(16))
    
    # Save the model for future use
    model.save_pretrained("./sentiment")
    return model
# Get models for analysis
attention_model = load_and_prime_bert_attention_model(df)
keyword_model = KeyBERT()
## Define helper functions for classifications for created models

# Gets the attention value of a review using the bert model primed above
def get_attention_value(input, model):
  predict_input = tokenizer.encode(input,
                                 truncation=True,
                                 padding=True,
                                 return_tensors="tf")

  outputs = model.predict(predict_input)
  attention_vectors = []
  for layer in outputs.attentions:
    X = layer.flatten()
    attention_vectors.append(X)
  attention_vectors = np.array(attention_vectors)
  attention_vectors = np.mean(attention_vectors)
  return attention_vectors

# Gets review keywords using the pretrained bert model
def get_keywords(input, model):
  keywords = model.extract_keywords(input, keyphrase_ngram_range=(1, 3), stop_words='english',top_n=2)
  if len(keywords) > 0:
    return keywords[0][0]
  else:
    return 'None'
from time import time

# Define what data we want to run analysis on
project_data = df.head(10000000)
# Get the keyword for each review
start = time()
project_data['keyword'] = project_data["short_review"].apply(lambda x: get_keywords(x, keyword_model))

time_elapsed = time() - start
m, s = divmod(time_elapsed, 60)
h, m = divmod(m, 60)
print("Hours: " + str(h))
print("Minutes: " + str(m))
print("Seconds: " + str(s))
# Get the attention for each review
start = time()
project_data['attention'] = project_data["keyword"].apply(lambda x: get_attention_value(x, attention_model))

time_elapsed = time() - start
m, s = divmod(time_elapsed, 60)
h, m = divmod(m, 60)
print("Hours: " + str(h))
print("Minutes: " + str(m))
print("Seconds: " + str(s))
# Display results
project_data.head()
# Tokenize key phrases so they can be clustered 
keywords = project_data['keyword'].tolist()

# Create frame to cluster
attention = project_data[['attention']]
embeded_keywords = pd.DataFrame(tokenizer(keywords, truncation=True, padding=True)['input_ids'])
clustering_data = pd.concat([attention, embeded_keywords], axis=1)

# Save what we have so far just in case
backup = pd.concat([project_data, clustering_data], axis=1)
backup.to_csv("./drive/MyDrive/backup_results.csv",index=False)

# Display data
clustering_data.head()
# Clustering and finding the optimal number of clusters
cluster_step = 25
X = clustering_data
maximum_number_of_clusters = int(X.shape[0]/4)
davies_boulding_scores = []

for num_clusters in range(cluster_step, maximum_number_of_clusters, cluster_step):
  k_means = KMeans(n_clusters=num_clusters)
  labels = k_means.fit_predict(X)
  db_index = davies_bouldin_score(X, labels)
  davies_boulding_scores.append(db_index)

plt.plot(list(i * cluster_step for i in range(1, len(davies_boulding_scores)+1)), davies_boulding_scores)
plt.show()
# Clustering and finding the optimal number of clusters (using backedup data)
project_data = pd.read_csv('./backup_results.csv')
project_data.head()

X = project_data[["attention",  "0",  '1', '2', '3', '4', '5', '6', '7', '8']]

cluster_sizes = [50, 100, 200, 450, 750, 1000]
davies_boulding_scores = []

for num_clusters in cluster_sizes:
  print(num_clusters)
  k_means = KMeans(n_clusters=num_clusters)
  labels = k_means.fit_predict(X)
  db_index = davies_bouldin_score(X, labels)
  davies_boulding_scores.append(db_index)

plt.plot(cluster_sizes, davies_boulding_scores)
plt.show()
# Determine the optimal number of clusters
optimal_number_of_clusters = cluster_sizes[davies_boulding_scores.index(min(davies_boulding_scores[1:]))]
print(optimal_number_of_clusters)
# Cluster Again
k_means = KMeans(n_clusters=optimal_number_of_clusters)
labels = k_means.fit_predict(X)
# Save the cluster results
pickle.dump(k_means, open("./k_means.pickle", "wb"))
# Put each product into a cluster
project_data['label'] = labels
project_data.head()
# Export cluster results
X_export = project_data.reset_index()
X_export = project_data[["attention", "Sentiment", "product_id", "product_title", "rating", "keyword", "label"]]

X_export.to_csv("cluster.csv",index=False)
X_export.to_json("cluster.json")
# Number of products per cluster
X_export['label'].value_counts()
# Products in a cluster
filter = X_export[X_export['label'] == 0]
filter.head()
X_export.to_csv("./drive/MyDrive/cluster.csv",index=False)
X_export.to_json("./drive/MyDrive/cluster.json")
pickle.dump(k_means, open("./drive/MyDrive/k_means.pickle", "wb"))
pickle.dump(k_means, open("./drive/MyDrive/k_means.pickle", "wb"))
X_export.to_csv("./drive/MyDrive/cluster.csv",index=False)
X_export.to_json("./drive/MyDrive/cluster.json")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12000509035377958, 'fraction_numerical': 0.012216849071010435, 'mean_word_length': 5.353274050121261, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '652648', 'license': 'MIT', 'n_tokens_mistral': 2709, 'n_tokens_neox': 2558, 'n_words': 588, 'path': 'review_analysis/code/Amazon_product_analysis.ipynb', 'repo_name': 'bopas2/amazon-review-analysis', 'size': '7858'}