<commit_before>__author__ = '<NAME> (<EMAIL>)'

import numpy as np

try:
    from reveal_user_classification.embedding.arcte.cython_opt.arcte import arcte_and_centrality
except ImportError:
    from reveal_user_classification.embedding.arcte.arcte import arcte_and_centrality
from reveal_user_classification.classification import model_fit, classify_users
from reveal_user_annotation.common.config_package import get_threads_number
from reveal_user_annotation.mongo.mongo_util import start_local_mongo_daemon, establish_mongo_connection
from reveal_user_annotation.mongo.preprocess_data import get_collection_documents_generator,\
    extract_graphs_and_lemmas_from_tweets,  store_user_documents, read_user_documents_generator,\
    extract_connected_components
from reveal_user_annotation.twitter.user_annotate import decide_which_users_to_annotate,\
    fetch_twitter_lists_for_user_ids_generator, extract_user_keywords_generator, form_user_label_matrix


def user_network_profile_classifier(assessment_id):
    """
    Performs Online Social Network user classification.

    Specifically:
           - Establishes a connection with a Mongo database and gets the newest tweets.
           - Forms graphs and text-based vector representation for the users involved.
           - Fetches Twitter lists for influential users.
           - Extracts keywords from Twitter lists and thus annotates these users as experts in these topics.
           - Extracts graph-based features using the ARCTE algorithm.
           - Performs user classification for the rest of the users.
           - Stores the results at PServer.

    Input: - assessment_id: The connection details for making a connection with a MongoDB instance.
    """
    # Start a local mongo daemon.
    daemon = start_local_mongo_daemon()
    daemon.start()

    ####################################################################################################################
    # Establish MongoDB connection.
    ####################################################################################################################
    external_client, database_name, collection_name = establish_mongo_connection(assessment_id)

    ####################################################################################################################
    # Preprocess tweets.
    ####################################################################################################################
    mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, lemma_to_attribute =\
        get_graphs_and_lemma_matrix(external_client, database_name, collection_name)

    adjacency_matrix, node_to_id, features, centrality = integrate_graphs(mention_graph,
                                                                          retweet_graph,
                                                                          user_lemma_matrix)

    ####################################################################################################################
    # Annotate users.
    ####################################################################################################################
    # Establish connection with a local MongoDB instance.
    local_client = establish_mongo_connection(mongodb_host_name="localhost",
                                              mongodb_port=27017)

    twitter_lists_gen, user_ids_to_annotate = fetch_twitter_lists(local_client, centrality)

    user_label_matrix, annotated_user_ids, label_to_topic = annotate_users(local_client,
                                                                           twitter_lists_gen,
                                                                           user_ids_to_annotate)

    ####################################################################################################################
    # Perform user classification.
    ####################################################################################################################
    prediction = user_classification(features, user_label_matrix, annotated_user_ids, node_to_id)

    # Stop the local mongo daemon.
    daemon.join()


def translate_assessment_id_to_mongo_access(assessment_id):
    """
    The assessment id is translated to MongoDB host, port, database and collection names.

    INTEGRATION NOTE: For testing purposes, I hard-code these configuration parameters to a local MongoDB setting.

    Input:   - assessment_id: The connection details for making a connection with a MongoDB instance.

    Outputs: - mongodb_host_name: The name of the MongoDB host in string format.
             - mongodb_port: The port of the MongoDB in integer format.
             - database_name: The name of the Mongo database in string format.
             - collection_name: The name of the collection of tweets to read in string format.
    """
    mongodb_host_name = "localhost"
    mongodb_port = 27017
    database_name = "snow_tweets_database"
    collection_name = "snow_tweets_collection"

    return mongodb_host_name, mongodb_port, database_name, collection_name


def establish_mongodb_connection(assessment_id):
    """
    Establish connection with external MongoDB.

    Input:   - assessment_id: The connection details for making a connection with a MongoDB instance.

    Outputs: - external_client: A MongoDB client.
             - database_name: The name of the Mongo database in string format.
             - collection_name: The name of the collection of tweets to read in string format.
    """
    # Get external MongoDB connection details.
    mongodb_host_name,\
    mongodb_port,\
    database_name,\
    collection_name = translate_assessment_id_to_mongo_access(assessment_id)

    # Connect to MongoDB.
    external_client = establish_mongo_connection(mongodb_host_name=mongodb_host_name,
                                                 mongodb_port=mongodb_port)
    return external_client, database_name, collection_name


def get_graphs_and_lemma_matrix(external_client, database_name, collection_name):
    """
    Processes a set of tweets and extracts interaction graphs and a user-lemma vector representation matrix.

    Inputs:  - external_client: A MongoDB client.
             - database_name: The name of a Mongo database.
             - collection_name: The name of the collection of tweets.

    Outputs: - mention_graph: The mention graph as a SciPy sparse matrix.
             - retweet_graph: The retweet graph as a SciPy sparse matrix.
             - user_lemma_matrix: The user lemma vector representation matrix as a SciPy sparse matrix.
             - tweet_id_set: A python set containing the Twitter ids for all the dataset tweets.
             - user_id_set: A python set containing the Twitter ids for all the dataset users.
             - lemma_to_attribute: A map from lemmas to numbers in python dictionary format.
    """
    # Form the mention and retweet graphs, as well as the attribute matrix.
    tweet_gen = get_collection_documents_generator(mongodb_client=external_client,
                                                   database_name=database_name,
                                                   collection_name=collection_name)

    mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, lemma_to_attribute =\
        extract_graphs_and_lemmas_from_tweets(tweet_gen)

    return mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, lemma_to_attribute


def integrate_graphs(mention_graph, retweet_graph, user_lemma_matrix):<commit_msg>Additional console script arguments.<commit_after>__author__ = '<NAME> (<EMAIL>)'

import numpy as np
import datetime

try:
    from reveal_user_classification.embedding.arcte.cython_opt.arcte import arcte_and_centrality
except ImportError:
    from reveal_user_classification.embedding.arcte.arcte import arcte_and_centrality
from reveal_user_classification.classification import model_fit, classify_users
from reveal_user_annotation.common.config_package import get_threads_number
from reveal_user_annotation.mongo.mongo_util import start_local_mongo_daemon, establish_mongo_connection
from reveal_user_annotation.mongo.preprocess_data import get_collection_documents_generator,\
    extract_graphs_and_lemmas_from_tweets,  store_user_documents, read_user_documents_generator,\
    extract_connected_components
from reveal_user_annotation.twitter.user_annotate import decide_which_users_to_annotate,\
    fetch_twitter_lists_for_user_ids_generator, extract_user_keywords_generator, form_user_label_matrix


def user_network_profile_classifier(assessment_id, **kwargs):
    """
    Performs Online Social Network user classification.

    Specifically:
           - Establishes a connection with a Mongo database and gets the newest tweets.
           - Forms graphs and text-based vector representation for the users involved.
           - Fetches Twitter lists for influential users.
           - Extracts keywords from Twitter lists and thus annotates these users as experts in these topics.
           - Extracts graph-based features using the ARCTE algorithm.
           - Performs user classification for the rest of the users.
           - Stores the results at PServer.

    Input: - assessment_id: The connection details for making a connection with a MongoDB instance.
           - **kwargs: 1) + number_of_latest_tweets
                       2) + lower_timestamp
                          + upper_timestamp
                       3) + number_of_latest_tweets
                          + lower_timestamp
                          + upper_timestamp
    """
    # Manage argument input.
    if "number_of_latest_tweets" not in kwargs.keys():
        number_of_latest_tweets = None
    else:
        number_of_latest_tweets = kwargs["number_of_latest_tweets"]

    if ("lower_timestamp" in kwargs.keys()) and ("upper_timestamp" in kwargs.keys()):
        lower_datetime = datetime.datetime.strftime(datetime.datetime.utcfromtimestamp(kwargs["lower_timestamp"]),
                                                    "%b %d %Y %H:%M:%S")
        upper_datetime = datetime.datetime.strftime(datetime.datetime.utcfromtimestamp(kwargs["upper_timestamp"]),
                                                    "%b %d %Y %H:%M:%S")
        spec = {'time': {'$gte': lower_datetime, '$lt': upper_datetime}}
    else:
        spec = None

    # Start a local mongo daemon.
    daemon = start_local_mongo_daemon()
    daemon.start()

    ####################################################################################################################
    # Establish MongoDB connection.
    ####################################################################################################################
    external_client, database_name, collection_name = establish_mongo_connection(assessment_id)

    ####################################################################################################################
    # Preprocess tweets.
    ####################################################################################################################
    mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, lemma_to_attribute =\
        get_graphs_and_lemma_matrix(external_client,
                                    database_name,
                                    collection_name,
                                    spec,
                                    number_of_latest_tweets)

    adjacency_matrix, node_to_id, features, centrality = integrate_graphs(mention_graph,
                                                                          retweet_graph,
                                                                          user_lemma_matrix)

    ####################################################################################################################
    # Annotate users.
    ####################################################################################################################
    # Establish connection with a local MongoDB instance.
    local_client = establish_mongo_connection(mongodb_host_name="localhost",
                                              mongodb_port=27017)

    twitter_lists_gen, user_ids_to_annotate = fetch_twitter_lists(local_client, centrality)

    user_label_matrix, annotated_user_ids, label_to_topic = annotate_users(local_client,
                                                                           twitter_lists_gen,
                                                                           user_ids_to_annotate)

    ####################################################################################################################
    # Perform user classification.
    ####################################################################################################################
    prediction = user_classification(features, user_label_matrix, annotated_user_ids, node_to_id)

    # Stop the local mongo daemon.
    daemon.join()


def translate_assessment_id_to_mongo_access(assessment_id):
    """
    The assessment id is translated to MongoDB host, port, database and collection names.

    INTEGRATION NOTE: For testing purposes, I hard-code these configuration parameters to a local MongoDB setting.

    Input:   - assessment_id: The connection details for making a connection with a MongoDB instance.

    Outputs: - mongodb_host_name: The name of the MongoDB host in string format.
             - mongodb_port: The port of the MongoDB in integer format.
             - database_name: The name of the Mongo database in string format.
             - collection_name: The name of the collection of tweets to read in string format.
    """
    mongodb_host_name = "localhost"
    mongodb_port = 27017
    database_name = "snow_tweets_database"
    collection_name = "snow_tweets_collection"

    return mongodb_host_name, mongodb_port, database_name, collection_name


def establish_mongodb_connection(assessment_id):
    """
    Establish connection with external MongoDB.

    Input:   - assessment_id: The connection details for making a connection with a MongoDB instance.

    Outputs: - external_client: A MongoDB client.
             - database_name: The name of the Mongo database in string format.
             - collection_name: The name of the collection of tweets to read in string format.
    """
    # Get external MongoDB connection details.
    mongodb_host_name,\
    mongodb_port,\
    database_name,\
    collection_name = translate_assessment_id_to_mongo_access(assessment_id)

    # Connect to MongoDB.
    external_client = establish_mongo_connection(mongodb_host_name=mongodb_host_name,
                                                 mongodb_port=mongodb_port)
    return external_client, database_name, collection_name


def get_graphs_and_lemma_matrix(external_client,
                                database_name,
                                collection_name,
                                spec,
                                number_of_latest_tweets):
    """
    Processes a set of tweets and extracts interaction graphs and a user-lemma vector representation matrix.

    Inputs:  - external_client: A MongoDB client.
             - database_name: The name of a Mongo database.
             - collection_name: The name of the collection of tweets.
             - spec: A python dictionary that defines higher query arguments.
             - number_of_latest_tweets: The number of latest results we require from the mongo document collection.

    Outputs: - mention_graph: The mention graph as a SciPy sparse matrix.
             - retweet_graph: The retweet graph as a SciPy sparse matrix.
             - user_lemma_matrix: The user lemma vector representation matrix as a SciPy sparse matrix.
             - tweet_id_set: A python set containing the Twitter ids for all the dataset tweets.
             - user_id_set: A python set containing the Twitter ids for all the dataset users.
             - lemma_to_attribute: A map from lemmas to numbers in python dictionary format.
    """
    # Form the mention and retweet graphs, as well as the attribute matrix.
    tweet_gen = get_collection_documents_generator(mongodb_client=external_client,
                                                   database_name=database_name,
                                                   collection_name=collection_name,
                                                   spec=spec,
                                                   latest_n=number_of_latest_tweets,
                                                   sort_key="created_at")

    mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, lemma_to_attribute =\
        extract_graphs_and_lemmas_from_tweets(tweet_gen)

    return mention_graph, retweet_graph, user_lemma_matrix, tweet_id_set, user_id_set, lemma_to_attribute


def integrate_graphs(mention_graph, retweet_graph, user_lemma_matrix):
{'fraction_non_alphanumeric': 0.1526297294131428, 'fraction_numerical': 0.001347077427667799, 'mean_word_length': 2.932519576232151, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6948164', 'n_tokens_mistral': 4191, 'n_tokens_neox': 3882, 'n_words': 1106}