<jupyter_start><jupyter_text>Video Games Analysis with Tensorflow and Keras  Notes- My tf_env contains the necessary packages and dependencies for this notebook Data from> Justifying recommendations using distantly-labeled reviews and fined-grained aspects<NAME>, <NAME>, <NAME>Empirical Methods in Natural Language Processing (EMNLP), 2019<jupyter_code># import packages
import numpy as np
import pandas as pd
from keras.preprocessing.text import Tokenizer, one_hot, text_to_word_sequence
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Embedding
from keras.layers import LSTM

from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())
from keras import backend as K
K.tensorflow_backend._get_available_gpus()
import tensorflow as tf
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))


# max_features = 20000
# # cut texts after this number of words (among top max_features most common words)
# maxlen = 80
# batch_size = 32




!wget http://deepyeti.ucsd.edu/jianmo/amazon/categoryFilesSmall/Video_Games_5.json.gz
vg = pd.read_json('Video_Games_5.json.gz', lines=True, compression='gzip')
vg.head()
vg = vg.loc[:,['overall', 'reviewText']]
vg.info()
# clean up nan values and change datatype
vg = vg.dropna(how='any')
vg.loc[:,'overall'] = vg.overall.astype('int16')
vg.overall.value_counts()
# map sentiment for two-class pretrained model
vg.loc[:,'pt_sentiment'] = vg.overall.map({1: 0, 2: 0, 3: 1, 4: 1, 5: 1}).astype('int16')
vg.info()
vg.pt_sentiment.value_counts()
# import resample
from sklearn.utils import resample

# down-sample to balance classes
vg_class1 = vg[vg.pt_sentiment == 1]
vg_class0 = vg[vg.pt_sentiment == 0]

# downsample majority class
vg_class1_down = resample(vg_class1, replace=False, n_samples=vg_class0.shape[0], random_state=42)
# concat the dfs back together
vg_down = pd.concat([vg_class1_down, vg_class0])
vg_down.pt_sentiment.value_counts()
X = vg_down.reviewText.values
y = vg_down.pt_sentiment.values
X[0]
t = Tokenizer(num_words=None, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower=True, 
          split=' ', char_level=False, oov_token=None, document_count=2)
type(t)
help(t)
# split the data into train/test

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42, stratify=y)

len(X_train)
t.fit_on_texts(X_train)
# summarize what was learned
print('Number of training documents {}'.format(t.document_count))
encoded_docs = t.texts_to_matrix(X_train, mode='count')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12246431254695718, 'fraction_numerical': 0.018031555221637866, 'mean_word_length': 6.336088154269972, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '172740', 'license': 'MIT', 'n_tokens_mistral': 988, 'n_tokens_neox': 922, 'n_words': 232, 'path': 'zz__Colab_TF_Keras_VG.ipynb', 'repo_name': 'chuktuk/Amazon_Customer_Data', 'size': '2678'}