<jupyter_start><jupyter_text>label=1<jupyter_code>wkd = all_value[all_value['attribute_value'].apply(lambda x: 'weekend' in x)]
wkd = pd.merge(wkd, data, how='left', on='product_id')
wkd = wkd.iloc[:,3:]
wkd.columns = ['label','brand','description','details','name']
wkd = wkd[wkd.label=='weekend']
wkd.drop_duplicates(inplace=True)
dtn = all_value[all_value['attribute_value'].apply(lambda x: 'daytonight' in x)]
dtn = pd.merge(dtn, data, how='left', on='product_id')
dtn = dtn.iloc[:,3:]
dtn.columns = ['label','brand','description','details','name']
dtn = dtn[dtn.label=='daytonight']
dtn.drop_duplicates(inplace=True)
work = all_value[all_value['attribute_value'].apply(lambda x: 'work' in x)]
work = pd.merge(work, data, how='left', on='product_id')
work = work.iloc[:,3:]
work.columns = ['label','brand','description','details','name']
work = work[work.label=='work']
work.drop_duplicates(inplace=True)
ntot = all_value[all_value['attribute_value'].apply(lambda x: 'nightout' in x)]
ntot = pd.merge(ntot, data, how='left', on='product_id')
ntot = ntot.iloc[:,3:]
ntot.columns = ['label','brand','description','details','name']
ntot = ntot[ntot.label=='nightout']
ntot.drop_duplicates(inplace=True)
vc = all_value[all_value['attribute_value'].apply(lambda x: 'vacation' in x)]
vc = pd.merge(vc, data, how='left', on='product_id')
vc = vc.iloc[:,3:]
vc.columns = ['label','brand','description','details','name']
vc = vc[vc.label=='vacation']
vc.drop_duplicates(inplace=True)
wkot = all_value[all_value['attribute_value'].apply(lambda x: 'workout' in x)]
wkot = pd.merge(wkot, data, how='left', on='product_id')
wkot = wkot.iloc[:,3:]
wkot.columns = ['label','brand','description','details','name']
wkot = wkot[wkot.label=='workout']
wkot.drop_duplicates(inplace=True)
cw = all_value[all_value['attribute_value'].apply(lambda x: 'coldweather' in x)]
cw = pd.merge(cw, data, how='left', on='product_id')
cw = cw.iloc[:,3:]
cw.columns = ['label','brand','description','details','name']
cw = cw[cw.label=='coldweather']
cw.drop_duplicates(inplace=True)<jupyter_output><empty_output><jupyter_text>label = 0<jupyter_code>nwkd = all_value[all_value['attribute_value'].apply(lambda x: 'weekend' not in x)]
nwkd = pd.merge(nwkd, data, how='left', on='product_id')
nwkd = nwkd.iloc[:,3:]
nwkd.columns = ['label','brand','description','details','name']
nwkd['label'] = 0
nwkd.drop_duplicates(inplace=True)
ndtn = all_value[all_value['attribute_value'].apply(lambda x: 'daytonight' not in x)]
ndtn = pd.merge(ndtn, data, how='left', on='product_id')
ndtn = ndtn.iloc[:,3:]
ndtn.columns = ['label','brand','description','details','name']
ndtn['label'] = 0
ndtn.drop_duplicates(inplace=True)
nvc = all_value[all_value['attribute_value'].apply(lambda x: 'vacation' not in x)]
nvc = pd.merge(nvc, data, how='left', on='product_id')
nvc = nvc.iloc[:,3:]
nvc.columns = ['label','brand','description','details','name']
nvc['label'] = 0
nvc.drop_duplicates(inplace=True)
nwkot = all_value[all_value['attribute_value'].apply(lambda x: 'workout' not in x)]
nwkot = pd.merge(nwkot, data, how='left', on='product_id')
nwkot = nwkot.iloc[:,3:]
nwkot.columns = ['label','brand','description','details','name']
nwkot['label'] = 0
nwkot.drop_duplicates(inplace=True)
ncw = all_value[all_value['attribute_value'].apply(lambda x: 'coldweather' not in x)]
ncw = pd.merge(ncw, data, how='left', on='product_id')
ncw = ncw.iloc[:,3:]
ncw.columns = ['label','brand','description','details','name']
ncw['label'] = 0
ncw.drop_duplicates(inplace=True)
ncw<jupyter_output><empty_output><jupyter_text>Sample data  wkd<jupyter_code>from numpy import array
from keras.preprocessing.text import one_hot
from keras.preprocessing.sequence import pad_sequences
from numpy import asarray
from numpy import zeros
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import Embedding
from keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder
import spacy
import en_core_web_md
from sklearn.model_selection import train_test_split
num_1 = len(wkd['description'])
num_0 = len(nwkd['description'])
docs = wkd['description'].tolist() + nwkd['description'].tolist()
labels = np.concatenate([np.ones(num_1), np.zeros(num_0)])
nlp = en_core_web_md.load()
stopwords_removed_docs = list(
    map(lambda doc: " ".join([token.text for token in nlp(doc) if not token.is_stop]), docs))
tokenizer = Tokenizer(num_words=5000, oov_token="<PASSWORD>")
tokenizer.fit_on_texts(stopwords_removed_docs)
def integer_encode_documents(docs, tokenizer):
    return tokenizer.texts_to_sequences(docs)

# integer encode the documents
encoded_docs = integer_encode_documents(stopwords_removed_docs, tokenizer)
# this is a list of lists, the numbers represent the index position of that word.
MAX_SEQUENCE_LENGTH = len(max(stopwords_removed_docs,key=len))
padded_docs = pad_sequences(encoded_docs, maxlen=MAX_SEQUENCE_LENGTH, padding='post')
encoder = LabelEncoder()
labels = to_categorical(encoder.fit_transform(labels))
X_train, X_test, y_train, y_test = train_test_split(padded_docs, labels, test_size=0.2)
X_train.shape<jupyter_output><empty_output><jupyter_text>Glove<jupyter_code>from random import randint
from numpy import array, argmax, asarray, zeros
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Embedding
from keras import optimizers
def load_glove_vectors():
    embeddings_index = {}
    with open('glove.6B.100d.txt') as f:
        for line in f:
            values = line.split()
            word = values[0]
            coefs = asarray(values[1:], dtype='float32')
            embeddings_index[word] = coefs
    print('Loaded %s word vectors.' % len(embeddings_index))
    return embeddings_index


embeddings_index = load_glove_vectors()
VOCAB_SIZE = int(len(tokenizer.word_index) * 1.1)
MAX_SEQUENCE_LENGTH = len(max(stopwords_removed_docs,key=len))
MAX_SEQUENCE_LENGTH
embedding_matrix = zeros((VOCAB_SIZE, 100))
for word, i in tokenizer.word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None: # check that it is an actual word that we have embeddings for
        embedding_matrix[i] = embedding_vector
embedding_matrix.shape
from keras.layers.recurrent import SimpleRNN, LSTM
from keras.layers import Flatten, Masking
# define model

def make_binary_classification_rnn_model(plot=False):
    model = Sequential()
    # embedding(input_dim, output_dim)
    model.add(Embedding(VOCAB_SIZE, 100, weights=[embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=False))
    model.add(Masking(mask_value=0.0)) # masking layer, masks any words that don't have an embedding as 0s.
    model.add(SimpleRNN(units=32, input_shape=(1, MAX_SEQUENCE_LENGTH))) # set hidden state dimensionality to 64
    model.add(Dense(16))
    model.add(Dense(2, activation='softmax')) # 2 nodes, prediction for positive/negative, two classes
    
    # Compile the model
    model.compile(
    optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
    # summarize the model
    model.summary()
    
    if plot:
        plot_model(model, to_file='model.png', show_shapes=True)
    return model

def make_lstm_classification_model(plot=False):
    model = Sequential()
    model.add(Embedding(VOCAB_SIZE, 100, weights=[embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=False))
    model.add(Masking(mask_value=0.0)) # masking layer, masks any words that don't have an embedding as 0s.
    model.add(LSTM(units=32, input_shape=(1, MAX_SEQUENCE_LENGTH)))
    model.add(Dense(16))
    model.add(Dense(2, activation='softmax'))
    
    opt = optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)
    # Compile the model
    model.compile(
    optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
    # summarize the model
    model.summary()
    
    if plot:
        plot_model(model, to_file='model.png', show_shapes=True)
    return model
MAX_SEQUENCE_LENGTH = len(max(stopwords_removed_docs,key=len))
model = make_lstm_classification_model()
#model = make_binary_classification_rnn_model()
history = model.fit(X_train, y_train,validation_split = 0.1, epochs=25, verbose=1)
import keras
from matplotlib import pyplot as plt

def plot_fit_history(history):
    plt.plot(history.history['accuracy'])
    plt.plot(history.history['val_accuracy'])
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(['train', 'val'], loc='upper left')
    plt.show()
plot_fit_history(history)
loss, accuracy = model.evaluate(X_test, y_test, verbose=1)
print('Accuracy: %f' % (accuracy*100))<jupyter_output>798/798 [==============================] - 4s 5ms/step
Accuracy: 75.939852<jupyter_text>Train own embedding<jupyter_code>num_1 = len(wkd['description'])
num_0 = len(nwkd['description'])
docs = wkd['description'].tolist() + nwkd['description'].tolist()
labels = np.concatenate([np.ones(num_1), np.zeros(num_0)])
nlp = en_core_web_md.load()
stopwords_removed_docs = list(
    map(lambda doc: " ".join([token.text for token in nlp(doc) if not token.is_stop]), docs))
def integer_encode_documents(docs, tokenizer):
    return tokenizer.texts_to_sequences(docs)


vocab_size = 5000
MAX_SEQUENCE_LENGTH = len(max(stopwords_removed_docs,key=len))
EMBEDDING_SIZE = 100


tokenizer = Tokenizer()
tokenizer.fit_on_texts(stopwords_removed_docs)
encoded_docs = integer_encode_documents(stopwords_removed_docs, tokenizer)
padded_docs = pad_sequences(encoded_docs, maxlen=MAX_SEQUENCE_LENGTH, padding='post')

encoder = LabelEncoder()
labels = to_categorical(encoder.fit_transform(labels))
X_train, X_test, y_train, y_test = train_test_split(padded_docs, labels, test_size=0.2)
model = make_lstm_classification_model()
history = model.fit(X_train, y_train,validation_split = 0.1, epochs=25, verbose=1)
plot_fit_history(history)
loss, accuracy = model.evaluate(X_test, y_test, verbose=1)
print('Accuracy: %f' % (accuracy*100))<jupyter_output>798/798 [==============================] - 3s 4ms/step
Accuracy: 72.180450
{'fraction_non_alphanumeric': 0.15152409046214355, 'fraction_numerical': 0.01415929203539823, 'mean_word_length': 6.841942945258288, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '391627', 'license': 'MIT', 'n_tokens_mistral': 3740, 'n_tokens_neox': 3564, 'n_words': 693, 'path': 'project/Model - description.ipynb', 'repo_name': 'Emmayyyyy/dso-560-nlp-and-text-analytics', 'size': '10173'}