<commit_before>from pymongo import MongoClient
from pymongo import ReadPreference
import json as _json
import os
import mysql.connector as mysql
import requests

requests.packages.urllib3.disable_warnings()

# NOTE get_user_info_from_auth2 sets up the initial dict.
# The following functions update certain fields in the dict.
# So get_user_info_from_auth2 must be called before get_internal_users and get_user_orgs_count

metrics_mysql_password = os.environ["METRICS_MYSQL_PWD"]
mongoDB_metrics_connection = os.environ["MONGO_PATH"]

profile_url = os.environ["PROFILE_URL"]
kb_internal_user_url = os.environ["KB_INTERNAL_USER_URL"]
sql_host = os.environ["SQL_HOST"]
query_on = os.environ["QUERY_ON"]

to_auth2 = os.environ["AUTH2_SUFFIX"]
to_groups = os.environ["GRP_SUFFIX"]
to_workspace = os.environ["WRK_SUFFIX"]

_CT = "content-type"
_AJ = "application/json"


def get_user_info_from_auth2():
    """ get auth2 info and kbase_internal_users. Creates initial dict for the data. """

    client_auth2 = MongoClient(mongoDB_metrics_connection + to_auth2)
    db_auth2 = client_auth2.auth2

    user_stats_dict = {}  # dict that will have userid as the key,
    # value is a dict with name, signup_date, last_signin_date,
    # and email (that gets values from this function)
    # orcid may be present and populated by this function.
    # later called functions will populate kbase_internal_user, num_orgs and ...

    user_info_query = db_auth2.users.find(
        {}, {"_id": 0, "user": 1, "email": 1, "display": 1, "create": 1, "login": 1}
    )
    for record in user_info_query:
        if record["user"] == "***ROOT***":
            continue
        user_stats_dict[record["user"]] = {
            "name": record["display"],
            "signup_date": record["create"],
            "last_signin_date": record["login"],
            "email": record["email"],
            "kbase_internal_user": False,
            "institution": None,
            "country": None,
            "orcid": None,
            "num_orgs": 0,
            "narrative_count": 0,
            "shared_count": 0,
            "narratives_shared": 0,
        }

    # Get all users with an ORCID authentication set up.
    users_orcid_query = db_auth2.users.find(
        {"idents.prov": "OrcID"},
        {"user": 1, "idents.prov": 1, "idents.prov_id": 1, "_id": 0},
    )
    for record in users_orcid_query:
        for ident in record["idents"]:
            if ident["prov"] == "OrcID":
                # just use the first orcid seen.
                user_stats_dict[record["user"]]["orcid"] = ident["prov_id"]
                continue

    client_auth2.close()
    return user_stats_dict


def get_internal_users(user_stats_dict):
    """
    Gets the internal users from the kb_internal_staff google sheet that Roy maintains.
    """
    params = (("tqx", "out:csv"), ("sheet", "KBaseStaffAccounts"))
    response = requests.get(kb_internal_user_url, params=params)
    if response.status_code != 200:
        print(
            "ERROR - KB INTERNAL USER GOOGLE SHEET RESPONSE STATUS CODE : "
            + str(response.status_code)
        )
        print(
            "KB INTERNAL USER will not get updated until this is fixed. Rest of the uuser upload should work."
        )
        return user_stats_dict
    lines = response.text.split("\n")
    if len(lines) < 390:
        print(
            "SOMETHING IS WRONG WITH KBASE INTERNAL USERS LIST: "
            + str(response.status_code)
        )
    users_not_found_count = 0
    for line in lines:
        elements = line.split(",")
        user = elements[0][1:-1]
        if user in user_stats_dict:
            user_stats_dict[user]["kbase_internal_user"] = True
        else:
            users_not_found_count += 1
    if users_not_found_count > 0:
        print(
            "NUMBER OF USERS FOUND IN KB_INTERNAL GOOGLE SHEET THAT WERE NOT FOUND IN THE AUTH2 RECORDS : "
            + str(users_not_found_count)<commit_msg>minor fixes for KBase staff list to deal with leading or trainling spaces<commit_after>from pymongo import MongoClient
from pymongo import ReadPreference
import json as _json
import os
import mysql.connector as mysql
import requests

requests.packages.urllib3.disable_warnings()

# NOTE get_user_info_from_auth2 sets up the initial dict.
# The following functions update certain fields in the dict.
# So get_user_info_from_auth2 must be called before get_internal_users and get_user_orgs_count

metrics_mysql_password = os.environ["METRICS_MYSQL_PWD"]
mongoDB_metrics_connection = os.environ["MONGO_PATH"]

profile_url = os.environ["PROFILE_URL"]
kb_internal_user_url = os.environ["KB_INTERNAL_USER_URL"]
sql_host = os.environ["SQL_HOST"]
query_on = os.environ["QUERY_ON"]

to_auth2 = os.environ["AUTH2_SUFFIX"]
to_groups = os.environ["GRP_SUFFIX"]
to_workspace = os.environ["WRK_SUFFIX"]

_CT = "content-type"
_AJ = "application/json"


def get_user_info_from_auth2():
    """ get auth2 info and kbase_internal_users. Creates initial dict for the data. """

    client_auth2 = MongoClient(mongoDB_metrics_connection + to_auth2)
    db_auth2 = client_auth2.auth2

    user_stats_dict = {}  # dict that will have userid as the key,
    # value is a dict with name, signup_date, last_signin_date,
    # and email (that gets values from this function)
    # orcid may be present and populated by this function.
    # later called functions will populate kbase_internal_user, num_orgs and ...

    user_info_query = db_auth2.users.find(
        {}, {"_id": 0, "user": 1, "email": 1, "display": 1, "create": 1, "login": 1}
    )
    for record in user_info_query:
        if record["user"] == "***ROOT***":
            continue
        user_stats_dict[record["user"]] = {
            "name": record["display"],
            "signup_date": record["create"],
            "last_signin_date": record["login"],
            "email": record["email"],
            "kbase_internal_user": False,
            "institution": None,
            "country": None,
            "orcid": None,
            "num_orgs": 0,
            "narrative_count": 0,
            "shared_count": 0,
            "narratives_shared": 0,
        }

    # Get all users with an ORCID authentication set up.
    users_orcid_query = db_auth2.users.find(
        {"idents.prov": "OrcID"},
        {"user": 1, "idents.prov": 1, "idents.prov_id": 1, "_id": 0},
    )
    for record in users_orcid_query:
        for ident in record["idents"]:
            if ident["prov"] == "OrcID":
                # just use the first orcid seen.
                user_stats_dict[record["user"]]["orcid"] = ident["prov_id"]
                continue

    client_auth2.close()
    return user_stats_dict


def get_internal_users(user_stats_dict):
    """
    Gets the internal users from the kb_internal_staff google sheet that Roy maintains.
    """
    params = (("tqx", "out:csv"), ("sheet", "KBaseStaffAccounts"))
    response = requests.get(kb_internal_user_url, params=params)
    if response.status_code != 200:
        print(
            "ERROR - KB INTERNAL USER GOOGLE SHEET RESPONSE STATUS CODE : "
            + str(response.status_code)
        )
        print(
            "KB INTERNAL USER will not get updated until this is fixed. Rest of the uuser upload should work."
        )
        return user_stats_dict
    lines = response.text.split("\n")
    if len(lines) < 390:
        print(
            "SOMETHING IS WRONG WITH KBASE INTERNAL USERS LIST: "
            + str(response.status_code)
        )
    users_not_found_count = 0
    for line in lines:
        elements = line.split(",")
        user = elements[0][1:-1].strip()
        if user in user_stats_dict:
            user_stats_dict[user]["kbase_internal_user"] = True
        else:
            users_not_found_count += 1
            print(
                "KBase Username ::"
                + str(user)
                + "::  was not found in the DB"
            )
    if users_not_found_count > 0:
        print(
            "NUMBER OF USERS FOUND IN KB_INTERNAL GOOGLE SHEET THAT WERE NOT FOUND IN THE AUTH2 RECORDS : "
            + str(users_not_found_count)
{'fraction_non_alphanumeric': 0.10442999140998896, 'fraction_numerical': 0.010308013253159897, 'mean_word_length': 2.714676390154968, 'pattern_counts': {'":': 51, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3133359', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2539, 'n_words': 683}