<jupyter_start><jupyter_text>Data Loading  Load annotated data<jupyter_code>trend_type = None

annotated = data_seperation.get_data(trend_type)

trend_type = annotated[['id', 'type']]
trend_hash = annotated['id'].tolist()
trend_name = annotated['name'].tolist()
trend_dict = dict(zip(trend_hash, trend_name))<jupyter_output>Traceback (most recent call last):
  File "/Users/khoanguyen/Workspace/git/twitter-trend/jsonl_parser.py", line 12, in load_jsonl
    data.append(json.loads(line.rstrip('\n|\r')))
  File "/Users/khoanguyen/miniconda3/envs/twitter/lib/python3.8/json/__init__.py", line 357, in loads
    return _default_decoder.decode(s)
  File "/Users/khoanguyen/miniconda3/envs/twitter/lib/python3.8/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/Users/khoanguyen/miniconda3/envs/twitter/lib/python3.8/json/decoder.py", line 353, in raw_decode
    obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 662 (char 661)<jupyter_text>Load Tweet Data<jupyter_code>data = load_raw_data('/Users/khoanguyen/Workspace/dataset/twitter-trending/TT-classification/dataset-full/', trend_hash, 'en', False)

df = pd.DataFrame(data)<jupyter_output><empty_output><jupyter_text>Mapping Trend from annotated dataframe into Tweet data<jupyter_code>df['trend_name'] = df['trend_hash'].map(trend_dict)

df['trend_hash'].value_counts()

df['text'] = df['text'].apply(lambda x: text_preprocess.remove_hyperlink(x))

df_trend = df[['text', 'trend_hash', 'trend_name']]

trend_label = text_features.trend_mapping(set(df_trend['trend_hash'].tolist()))

df_trend = (df_trend.merge(trend_type, left_on='trend_hash', right_on='id').reindex(columns=['text', 'trend_hash', 'trend_name', 'type']))

df_trend['label'] = df_trend.apply(lambda row: trend_label[row.trend_hash], axis=1)<jupyter_output><empty_output><jupyter_text>Removing trend with less than 10 tweets (if neccessary)<jupyter_code>df_trend = df_trend.groupby('label')
df_trend = df_trend.filter(lambda x: len(x) > 10)<jupyter_output><empty_output><jupyter_text>Text Features  Unigram<jupyter_code>tf_uni, features_uni = text_features.generate_term_freq(df['text'], 1)

tf_uni_array = tf_uni.toarray()<jupyter_output><empty_output><jupyter_text>Bigram<jupyter_code>tf_bi, features_bi = text_features.generate_term_freq(df_trend['text'], 2)

tf_bi_array = tf_bi.toarray()<jupyter_output><empty_output><jupyter_text>TF-IDF<jupyter_code>tfidf, terms = text_features.generate_tfidf(df_trend['text'])

tfidf_array = tfidf.toarray()<jupyter_output><empty_output><jupyter_text>TF-IDF without removing special character and keep true casing - Loose TF-IDF<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vector = TfidfVectorizer(lowercase=False,
                                token_pattern=r'\S+',
                                stop_words='english')

loose_tfidf = tfidf_vector.fit_transform(df_trend['text'])<jupyter_output><empty_output><jupyter_text>Tweet Topology Classification Linear SVM using loose TF-IDF<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split

class_name = df_trend['type'].tolist()

X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(loose_tfidf, class_name, df_trend.index, test_size=0.25, random_state=0, stratify=class_name)

from sklearn.svm import LinearSVC
new_svc_model = LinearSVC()

new_svc_model.fit(X_train, y_train)

y_pred = new_svc_model.predict(X_test)

from sklearn import metrics

print(metrics.classification_report(y_test, y_pred, target_names=df_trend['type'].unique()))<jupyter_output>precision    recall  f1-score   support

ongoing-event       0.99      0.96      0.98      2524
         meme       0.99      0.96      0.97     18618
         news       0.98      0.96      0.97     18757
commemorative       0.97      0.99      0.98     44677

     accuracy                           0.98     84576
    macro avg       0.98      0.97      0.97     84576
 weighted avg       0.98      0.98      0.98     84576<jupyter_text>Mutinominal Nayes Bay with loose TF-IDF<jupyter_code>from sklearn.naive_bayes import MultinomialNB

new_MNB_model = MultinomialNB()
new_MNB_model.fit(X_train, y_train)

y_pred = new_MNB_model.predict(X_test)

print(metrics.classification_report(y_test, y_pred, target_names=df_trend['type'].unique()))<jupyter_output>precision    recall  f1-score   support

ongoing-event       1.00      0.38      0.55      2524
         meme       0.98      0.80      0.88     18618
         news       0.97      0.81      0.88     18757
commemorative       0.84      0.99      0.91     44677

     accuracy                           0.89     84576
    macro avg       0.95      0.75      0.81     84576
 weighted avg       0.91      0.89      0.89     84576
{'fraction_non_alphanumeric': 0.11399918133442488, 'fraction_numerical': 0.04564060581252558, 'mean_word_length': 4.043343653250774, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '524291', 'license': 'MIT', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1819, 'n_words': 295, 'path': 'trend_topology.ipynb', 'repo_name': 'nnkhoa/twitter-trend', 'size': '4886'}