<jupyter_start><jupyter_text>preprocess<jupyter_code>import warnings
warnings.filterwarnings('ignore')
import sys
sys.path.insert(0, '..')
import os
import json
import jieba
import gensim
import collections
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from wordcloud import WordCloud
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
%matplotlib inline
tf.enable_eager_execution()<jupyter_output><empty_output><jupyter_text>Load dataset<jupyter_code>from senti_analysis import config
train_dataset = pd.read_csv(config.TRAIN_SET_PATH)
validation_dataset = pd.read_csv(config.VALIDATION_SET_PATH)
test_dataset = pd.read_csv(config.TEST_SET_PATH)
train_dataset.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 105000 entries, 0 to 104999
Data columns (total 22 columns):
id                                          105000 non-null int64
content                                     105000 non-null object
location_traffic_convenience                105000 non-null int64
location_distance_from_business_district    105000 non-null int64
location_easy_to_find                       105000 non-null int64
service_wait_time                           105000 non-null int64
service_waiters_attitude                    105000 non-null int64
service_parking_convenience                 105000 non-null int64
service_serving_speed                       105000 non-null int64
price_level                                 105000 non-null int64
price_cost_effective                        105000 non-null int64
price_discount                              105000 non-null int64
environment_decoration                      105000 non-null int64
environment_noise             [...]<jupyter_text>Build Vocabulary<jupyter_code>def cut(text):
    return jieba.lcut(text)
contents = np.concatenate([np.array(train_dataset['content']),
                          np.array(test_dataset['content']),
                          np.array(validation_dataset['content'])])
def content2sentence(contents):
    sentences = []
    for content in contents:
        sentences.append(' '.join(cut(content)))

    return sentences
def dump(json_obj, filepath):
    with open(filepath, 'w+') as f:
        json.dump(json_obj, f, ensure_ascii=False)
def load(filepath):
    with open(filepath) as f:
        return json.load(f)
train_sentences = content2sentence(train_dataset['content'])
dump(train_sentences, 'train_sents.json')
train_sents = load('train_sents.json')
validation_sents = content2sentence(validation_dataset['content'])
dump(validation_sents, 'validation_sents.json')
validation_sents = load('validation_sents.json')
test_sents = content2sentence(test_dataset['content'])
dump(test_sents, 'test_sents.json')
test_sents = load('test_sents.json')
# def stats_word(contents):
#     counter = collections.Counter()
#     for content in contents:
#         tokens = cut(content)
#         counter.update(tokens)

#     return counter
# def dump_word_stats(word_stats, filepath):
#     word_stats_dict = dict(word_stats)

#     with open(filepath, 'w+') as f:
#         f.write(json.dumps(word_stats_dict, ensure_ascii=False))

# def load_word_stats(filepath):
#     with open(filepath) as f:
#         return collections.Counter(json.load(f))

# def build_vocab_from_contents(contents):
#     counter = stats_word(contents)
#     return list(dict(counter).keys())
# def build_vocab_from_word_stats(word_stats):
#     return list(dict(word_stats).keys())
# def dump_vocab(vocab, filepath):
#     with open(filepath, 'w') as f:
#         return json.dump(vocab, f, ensure_ascii=False)
# def load_vocab(filepath):
#     with open(filepath) as f:
#         return json.load(f)
# vocab = build_vocab_from_contents(contents)
# dump_vocab(vocab, 'vocab.json')
# vocab = load_vocab('vocab.json')
# len(vocab)
# def get_sentence_max_length(sentences):
#     max_length = 0
#     for sentence in sentences:
#         max_length = max([max_length, len(cut(sentence))])

#     return max_length
# get_sentence_max_length(contents[:1000])
# MAX_SEQUENCE_LENGTH = 1500
tokenizer = Tokenizer()
tokenizer.fit_on_texts(train_sents)
tokenizer.fit_on_texts(validation_sents)
tokenizer.fit_on_texts(test_sents)
def dump_tokenizer(tokenizer, filepath):
    with open(filepath, 'w') as f:
        f.write(tokenizer.to_json(ensure_ascii=False))
from keras.preprocessing.text import tokenizer_from_json
def load_tokenizer(filepath):
    with open(filepath) as f:
        return tokenizer_from_json(f.read())
dump_tokenizer(tokenizer, 'tokenizer.json')
tokenizer = load_tokenizer('tokenizer.json')
vocab_set = set(tokenizer.word_index.keys())
len(vocab_set)
from tensorflow.keras.preprocessing.sequence import pad_sequences
sequences = tokenizer.texts_to_sequences(train_sentences)
train_data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)<jupyter_output><empty_output><jupyter_text>Vocab encode<jupyter_code># encoder = tfds.features.text.TokenTextEncoder(vocab)
# train_dataset['content'][0]
# ' '.join(cut(original_string))
# original_string = '中了大众点评的霸王餐'
# encoded_string = encoder.encode(' '.join(cut(original_string)))
# decoded_string = encoder.decode(encoded_string)
# original_string, encoded_string, ''.join(decoded_string.split(' '))
# '"' in encoder.tokens
'霸王餐' in vocab_set
original_string = '中了大众点评的霸王餐'
' '.join(cut(original_string))
sequences = tokenizer.texts_to_sequences([' '.join(cut(original_string))])
sequences
tokenizer.sequences_to_texts(sequences)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>VOCAB_SIZE = len(vocab_set)
EMBEDDING_DIM = 100

MAX_SEQUENCE_LENGTH = 1500
wv =  gensim.models.Word2Vec.load('w2v.model')
def init_embedding_matrix(word_index:dict, wv):
    """
    Initialize embedding matrix.
    """
    embedding_matrix = np.zeros((VOCAB_SIZE+1, EMBEDDING_DIM))

    for word, index in word_index.items():
        if word not in wv:
            continue
        embedding_matrix[index] = wv[word]

    return embedding_matrix

embedding_matrix = init_embedding_matrix(word_index, wv)
embedding_matrix.shape
# encoding: utf8

import tensorflow as tf

from senti_analysis import config

num_class = 4

inputs = tf.keras.layers.Input(shape=(MAX_SEQUENCE_LENGTH,))
embedding = tf.keras.layers.Embedding(VOCAB_SIZE, EMBEDDING_DIM)(inputs)
hidden = tf.keras.layers.GRU(64, activation='relu', return_sequences=True)(embedding)
hidden = tf.keras.layers.GRU(32, activation='relu')(hidden)
outputs = tf.keras.layers.Dense(num_class, activation='softmax')(hidden)

model = tf.keras.Model(inputs=inputs, outputs=outputs, name='gru_model')
model.summary()
tf.keras.utils.plot_model(model, show_shapes=True)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>分类任务<jupyter_code>train_dataset['service_waiters_attitude'].hist()
num_class = 4

inputs = tf.keras.layers.Input(shape=(MAX_SEQUENCE_LENGTH,))
embedding = tf.keras.layers.Embedding(VOCAB_SIZE, EMBEDDING_DIM)(inputs)
hidden = tf.keras.layers.GRU(64, activation='relu', return_sequences=True)(embedding)
hidden = tf.keras.layers.GRU(32, activation='relu')(hidden)
outputs = tf.keras.layers.Dense(num_class, activation='softmax')(hidden)

model = tf.keras.Model(inputs=inputs, outputs=outputs, name='gru_model')
model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
tf.keras.utils.plot_model(model, show_shapes=True)
train_dataset['service_waiters_attitude'].shape, train_data.shape
hitory = model.fit?
service_waiters_attitude = train_dataset['service_waiters_attitude'].copy()
print(len(service_waiters_attitude[service_waiters_attitude==1]))
print(len(service_waiters_attitude[service_waiters_attitude==0]))
print(len(service_waiters_attitude[service_waiters_attitude==-1]))
print(len(service_waiters_attitude[service_waiters_attitude==-2]))
service_waiters_attitude.hist()
service_waiters_attitude[service_waiters_attitude[service_waiters_attitude==1].index] = 3
service_waiters_attitude[service_waiters_attitude[service_waiters_attitude==0].index] = 2
service_waiters_attitude[service_waiters_attitude[service_waiters_attitude==-1].index] = 1
service_waiters_attitude[service_waiters_attitude[service_waiters_attitude==-2].index] = 0
service_waiters_attitude.hist()
X_train, y_train = train_data, service_waiters_attitude
X_train.shape, y_train.shape
validation_sequences = tokenizer.texts_to_sequences(validation_sents)
validation_data = pad_sequences(validation_sequences, maxlen=MAX_SEQUENCE_LENGTH)
val_service_waiters_attitude = validation_dataset['service_waiters_attitude'].copy()
val_service_waiters_attitude.hist()
val_service_waiters_attitude[val_service_waiters_attitude[val_service_waiters_attitude==1].index] = 3
val_service_waiters_attitude[val_service_waiters_attitude[val_service_waiters_attitude==0].index] = 2
val_service_waiters_attitude[val_service_waiters_attitude[val_service_waiters_attitude==-1].index] = 1
val_service_waiters_attitude[val_service_waiters_attitude[val_service_waiters_attitude==-2].index] = 0
X_val, y_val = validation_data, val_service_waiters_attitude
X_val.shape, y_val.shape
hitory = model.fit(train_data, y_train, batch_size=32, epochs=10, validation_data=(X_val, y_val))
len({'key': 'value'})
from gensim.models import Word2Vec
wv_model = Word2Vec.load('w2v.model')
wv_model.vector_size<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10865874363327674, 'fraction_numerical': 0.019100169779286927, 'mean_word_length': 5.665487977369166, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '435467', 'license': 'MIT', 'n_tokens_mistral': 3327, 'n_tokens_neox': 3177, 'n_words': 494, 'path': 'notebooks/.ipynb_checkpoints/preprocess-checkpoint.ipynb', 'repo_name': 'hotbaby/sentiment-analysis', 'size': '9424'}