<jupyter_start><jupyter_text>- import lib<jupyter_code># What version of Python do you have?
import sys

from collections import Counter

import tensorflow.keras
import pandas as pd
import sklearn as sk
from imblearn.over_sampling import SMOTE
from imblearn.over_sampling import SMOTENC
import tensorflow as tf
import seaborn as sns
import math
import matplotlib.pyplot as plt
import numpy as np
import re
import demoji
import datetime

print(f"Tensor Flow Version: {tf.__version__}")
print(f"Keras Version: {tensorflow.keras.__version__}")
print()
print(f"Python {sys.version}")
print(f"Pandas {pd.__version__}")
print(f"Scikit-Learn {sk.__version__}")
print("GPU is", "available" if tf.test.is_gpu_available() else "NOT AVAILABLE")<jupyter_output>Tensor Flow Version: 1.13.1
Keras Version: 2.2.4-tf

Python 3.6.10 |Anaconda, Inc.| (default, Jan  7 2020, 15:01:53) 
[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]
Pandas 1.0.1
Scikit-Learn 0.22.1
GPU is NOT AVAILABLE<jupyter_text>- read data<jupyter_code>tweet_vector_df = pd.read_csv("./data_vectors/vectors-100-2019-02-06.csv")<jupyter_output><empty_output><jupyter_text>- SVD<jupyter_code>u, s, vh = np.linalg.svd(tweet_vector_df, full_matrices=True)
num_of_bars = 50
_ = plt.bar(np.arange(num_of_bars), s[:num_of_bars])
threshold = 15

basis_events = []
for i in range(len(s)):
    if s[i] < threshold: break
    if s[i] >= threshold: basis_events.append(vh[i])
len(basis_events)<jupyter_output><empty_output><jupyter_text>- project each non-basis vectors onto the basis vectors and aggregate them<jupyter_code>aggregated_vector = [0 for i in range(768)]
for idx, tweet in tweet_vector_df.iterrows():
    # iterate through each basis event
    for basis_event in basis_events:
        norm = np.linalg.norm(basis_event)
        aggregated_vector = np.add(aggregated_vector, np.multiply(np.dot(tweet.to_list(), basis_event) / norm, basis_event))
#         print(np.dot(tweet.to_list(), basis_event))
#         print("==============")
# print(aggregated_vector)<jupyter_output><empty_output><jupyter_text>- assign sentiment class<jupyter_code>def sentiment_class(num):
    if num >= 1/3: 
        return 1
    elif num >=-1/3: 
        return 0
    else: 
        return -1
def sentiment_class2(num):
    if num >= 1/15 or num < -1/15: 
        return 11
    else: 
        return 22<jupyter_output><empty_output><jupyter_text>- function to aggregate vector with projection<jupyter_code>def aggregator(date):
    # read date
    tweet_vector_df = pd.read_csv("./data_vectors/vectors-10000-" + date +".csv")
    tweet_vector_df['sentiment_class'] = tweet_vector_df.sentiment.apply(sentiment_class)
    
    # SMOTE oversampling
    all_rows = tweet_vector_df.values.tolist()
    X = [row[: -1] for row in all_rows]
    y = [row[-1] for row in all_rows]
    sentiment_count = Counter(y)
    if sentiment_count[0] > sentiment_count[1] + sentiment_count[-1]:
        if min(sentiment_count.values()) > 3:
            sm = SMOTE(random_state=2, k_neighbors=3)
            X_res, y_res = sm.fit_resample(X, y)

            # re-assemble back to dataframe
            tweet_vector_df = pd.DataFrame(X_res)
            tweet_vector_df = tweet_vector_df.rename(columns={768: "sentiment_score"})
            tweet_vector_df['sentiment_class'] = y_res
    
    # print(tweet_vector_df.shape, tweet_vector_df.columns)
    # svd
    u, s, vh = np.linalg.svd(tweet_vector_df, full_matrices=True)
    
    # generate basis events
    threshold = 15

    basis_events = []
    for i in range(len(s)):
        if s[i] < threshold: break
        if s[i] >= threshold: basis_events.append(vh[i])
    
    # projection
    aggregated_vector = [0 for i in range(797)]
    for idx, tweet in tweet_vector_df.iterrows():
        # iterate through each basis event
        for basis_event in basis_events:
            norm = np.linalg.norm(basis_event)
            aggregated_vector = np.add(aggregated_vector, np.multiply(np.dot(tweet.to_list(), basis_event) / norm, basis_event))
    return np.concatenate((np.array([date]), aggregated_vector))
# plain addition version of aggregator
def aggregator(date):
    # read date
    tweet_vector_df = pd.read_csv("./data_vectors/vectors-10000-" + date +".csv")
    tweet_vector_df['sentiment_class'] = tweet_vector_df.sentiment.apply(sentiment_class)
    
    # trigger SMOTE oversampling iff neutral ratio too large
    all_rows = tweet_vector_df.values.tolist()
    X = [row[: -1] for row in all_rows]
    y = [row[-1] for row in all_rows]
    sentiment_count = Counter(y)
    if sentiment_count[0] > sentiment_count[1] + sentiment_count[-1]:
        if min(sentiment_count.values()) > 3:
            sm = SMOTE(random_state=2, k_neighbors=3)
            X_res, y_res = sm.fit_resample(X, y)

            # re-assemble back to dataframe
            tweet_vector_df = pd.DataFrame(X_res)
            tweet_vector_df = tweet_vector_df.rename(columns={768: "sentiment_score"})
            tweet_vector_df['sentiment_class'] = y_res
    
    '''if sentiment_count[1] <= 3 or sentiment_count[-1] <= 3:
        # read date
        tweet_vector_df['sentiment_class'] = tweet_vector_df.sentiment.apply(sentiment_class2)

        # SMOTE oversampling
        all_rows = tweet_vector_df.values.tolist()
        X = [row[: -1] for row in all_rows]
        y = [row[-1] for row in all_rows]
    sm = SMOTE(random_state=2, k_neighbors=3)
    X_res, y_res = sm.fit_resample(X, y)
    
    # re-assemble back to dataframe
    tweet_vector_df = pd.DataFrame(X_res)
    tweet_vector_df = tweet_vector_df.rename(columns={768: "sentiment_score"})
    tweet_vector_df['sentiment_class'] = y_res'''
    
    # print(tweet_vector_df.shape, tweet_vector_df.columns)
    # svd
    u, s, vh = np.linalg.svd(tweet_vector_df, full_matrices=True)
    
    # generate basis events
    threshold = 15

    basis_events = []
    for i in range(len(s)):
        if s[i] < threshold: break
        if s[i] >= threshold: basis_events.append(vh[i])
    
    # projection
    aggregated_vector = [0 for i in range(tweet_vector_df.shape[1])]
    for idx, tweet in tweet_vector_df.iterrows():
        aggregated_vector = np.add(aggregated_vector, tweet.to_list())
    return np.concatenate((np.array([date]), aggregated_vector))<jupyter_output><empty_output><jupyter_text>- wrap up<jupyter_code># Inclusive on both sides
start_date = datetime.date(2019, 2, 6)
end_date = datetime.date(2020, 2, 6)
time_window = (end_date - start_date).days + 1

sample_df = pd.read_csv("./data_vectors/vectors-10000-2019-02-06.csv")

feature_by_date = pd.DataFrame(columns=['date'] + list(sample_df.columns) + ['sentiment_class'])

for single_date in (start_date + datetime.timedelta(n) for n in range(time_window)): 
    this_date = single_date.strftime("%Y-%m-%d")
    new_row_df = pd.DataFrame(aggregator(this_date)).transpose()
    new_row_df = new_row_df.rename(columns={0: "Date"})
    new_row_df.columns = ['date'] + list(sample_df.columns) + ['sentiment_class']
    feature_by_date = pd.concat([feature_by_date, new_row_df], ignore_index=True)
    sys.stdout.write('\r')
    # the exact output you're looking for:
    sys.stdout.write("finished %s" % this_date)
    sys.stdout.flush()
print("\nall finished")
feature_by_date = feature_by_date.rename(columns={"date": "Date"})
feature_by_date.drop(['sentiment_class'], axis=1).to_csv("aggregated_vector.csv", index=False)
feature_by_date<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12323990881051361, 'fraction_numerical': 0.02105404318090385, 'mean_word_length': 3.8397144711226474, 'pattern_counts': {'":': 1, '<': 34, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '28006', 'license': 'MIT', 'n_tokens_mistral': 2801, 'n_tokens_neox': 2623, 'n_words': 548, 'path': '2. feature_aggregation + sentiment.ipynb', 'repo_name': 'Sapphirine/202005-Event-Linkage-and-Impact-Prediction', 'size': '7457'}