<jupyter_start><jupyter_text>Load Dataset<jupyter_code>dir_path = '../Dataset/tripadvisor/'
# Load cleaned train
train_clean_file = os.path.join(dir_path, 'train_review_filtered.json')
train_review = []
with open(train_clean_file, 'r') as f:
    for line in f:
        line_data = json.loads(line)
        item_id = line_data['item']
        user_id = line_data['user']
        rating = line_data['rating']
        review = line_data['review']
        feature = line_data['feature']
        train_review.append([item_id, user_id, rating, review, feature])
print("Number of reviews on trainset: {}".format(len(train_review)))
# Load cleaned test
test_clean_file = os.path.join(dir_path, 'test_review_filtered_clean.json')
test_review = []
with open(test_clean_file, 'r') as f:
    for line in f:
        line_data = json.loads(line)
        item_id = line_data['item']
        user_id = line_data['user']
        rating = line_data['rating']
        review = line_data['review']
        feature = line_data['feature']
        test_review.append([item_id, user_id, rating, review, feature])
print("Number of reviews on testset: {}".format(len(test_review)))<jupyter_output>Number of reviews on testset: 19444<jupyter_text>Convert List Data to Pandas Dataframe<jupyter_code>df_train_data = pd.DataFrame(train_review, columns=['item', 'user', 'rating', 'review', 'feature'])
df_test_data = pd.DataFrame(test_review, columns=['item', 'user', 'rating', 'review', 'feature'])
df_train_data
df_test_data
print("number of user in trainset: {}".format(len(list(df_train_data['user'].unique()))))
print("number of item in trainset: {}".format(len(list(df_train_data['item'].unique()))))<jupyter_output>number of user in trainset: 4950
number of item in trainset: 4493<jupyter_text>Check the already labeled features<jupyter_code># Load attributes
attribute_file = os.path.join(dir_path, 'ta_output/attributes.json')
with open(attribute_file, 'r') as f:
    print("Load file: {}".format(attribute_file))
    attributes_vocab = json.load(f)
# Load aspects
aspect_file = os.path.join(dir_path, 'ta_output/aspects.json')
with open(aspect_file, 'r') as f:
    print("Load file: {}".format(aspect_file))
    aspect_dict = json.load(f)
feature_set = set()
for attr in attributes_vocab:
    if attr in punct:
        print("Remove: ", attr)
    else:
        feature_set.add(attr)
for aspect in aspect_dict:
    if aspect in punct:
        print("Remove: ", aspect)
    else:
        feature_set.add(aspect)
print("Number of attributes: {}".format(len(attributes_vocab)))
print("Number of aspects: {}".format(len(aspect_dict)))
print("--"*10+"Features are a combination of attributes and aspects"+"--"*10)
print("Number of features: {}".format(len(feature_set)))
# from spacy.lang.en import English

# nlp = English()
# nlp.add_pipe(nlp.create_pipe('sentencizer'))
# doc = nlp("the pork chop itself is moist and tender . everything was great .")
# assert len(list(doc.sents)) == 2
# print(list(doc.sents))
# print(list(doc.sents)[0].text)
cnt_line_without_features = 0
cnt_rvw_features = list()
for row in df_train_data.iterrows():
    item_id = row[1]['item']
    user_id = row[1]['user']
    rating = row[1]['rating']
    rvw_text = row[1]['review']
    rvw_features = row[1]['feature']
    # rvw_sents = sent_tokenize(rvw_text)
    rvw_tokens = word_tokenize(rvw_text)
    num_rvw_features = 0
    for feat_word in feature_set:
        if feat_word in rvw_tokens:
            num_rvw_features += 1
    if num_rvw_features == 0:
        cnt_line_without_features += 1
    else:
        cnt_rvw_features.append(num_rvw_features)

    if (row[0]+1) % 50000 == 0:
        print("{} lines".format(row[0]+1))

print("Number of reviews that has no feature words: {}".format(cnt_line_without_features))
print("Avg number of features per review: {}".format(np.mean(cnt_rvw_features)))

len(cnt_rvw_features)
# Extract Features
feature_count = dict()
cnt_features = 0
for row in df_train_data.iterrows():
    row_sentences = row[1]['review']
    row_tokens = word_tokenize(row_sentences)
    for row_token in row_tokens:
        if row_token in feature_set:
            if row_token not in feature_count:
                feature_count[row_token] = 1
                cnt_features += 1
            else:
                feature_count[row_token] += 1
# assert cnt_features == len(feature_count)
print("Totally {} features appear in train-set reviews.".format(cnt_features))

# sort the feature based on counts
sorted_feat_counts = sorted(feature_count.items(), key = lambda x: -x[1])
# build feature word to feature ids mappings
feature2id_vocab = {entry[0]: str(id) for (id, entry) in enumerate(sorted_feat_counts)}
id2feature_vocab = {v: u for u, v in feature2id_vocab.items()}
feature2id_file = "../Dataset/tripadvisor/train/feature/feature2id.json"
with open(feature2id_file, 'w') as f:
    print("Write file: {}".format(feature2id_file))
    json.dump(feature2id_vocab, f)

id2feature_file = "../Dataset/tripadvisor/train/feature/id2feature.json"
with open(id2feature_file, 'w') as f:
    print("Write file: {}".format(id2feature_file))
    json.dump(id2feature_vocab, f)

feature2df = dict()
last_df = -1
for feat_df in sorted_feat_counts:
    assert feat_df[0] not in feature2df
    if last_df > 0:
        assert last_df >= feat_df[1]
    last_df = feat_df[1]
    feature2df[feat_df[0]] = feat_df[1]
feature2df_file = "../Dataset/tripadvisor/train/feature/feature2df.json"
with open(feature2df_file, 'w') as f:
    print("Write file: {}".format(feature2df_file))
    json.dump(feature2df, f)<jupyter_output>Write file: ../Dataset/tripadvisor/train/feature/feature2df.json<jupyter_text>Check Whether Test Features All Appears in Train Feature Set<jupyter_code>cnt_line_without_features_testset = 0
cnt_rvw_features_testset = list()
for row in df_test_data.iterrows():
    item_id = row[1]['item']
    user_id = row[1]['user']
    rating = row[1]['rating']
    rvw_text = row[1]['review']
    # rvw_features = row[1]['feature']
    rvw_tokens = word_tokenize(rvw_text)
    num_rvw_features = 0
    for feat_word in feature2id_vocab.keys():
        if feat_word in rvw_tokens:
            num_rvw_features += 1
    if num_rvw_features == 0:
        cnt_line_without_features_testset += 1
    else:
        cnt_rvw_features_testset.append(num_rvw_features)
    if (row[0]+1) % 10000 == 0:
        print("{} lines".format(row[0]+1))
print("Finish! Totally {} lines.".format(row[0]+1))
print("Number of reviews that has no feature words: {}".format(cnt_line_without_features_testset))
print("Avg number of features per review: {}".format(np.mean(cnt_rvw_features_testset)))
len(cnt_rvw_features_testset)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1336116910229645, 'fraction_numerical': 0.014613778705636743, 'mean_word_length': 4.426375404530744, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '562276', 'license': 'MIT', 'n_tokens_mistral': 2355, 'n_tokens_neox': 2303, 'n_words': 509, 'path': '2-FeatureWords_ta.ipynb', 'repo_name': 'stillarrow/BuildDataset', 'size': '6706'}