<commit_before> 

import requests
import os
requests.packages.urllib3.disable_warnings()
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
from category_to_app_dict import create_app_dictionary
catalog = Catalog(url = "https://kbase.us/services/catalog", token = os.environ['METRICS_USER_TOKEN'])
nms = NarrativeMethodStore(url = "https://kbase.us/services/narrative_method_store/rpc")

import pandas as pd
import datetime, time
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 500)
from operator import itemgetter


# Get User_App_Stats: Main Function

"""Requires a unique list of username as input and runs it 3-4min on average.
outputs a list of user dictionaries with app statistics for each userm example user : [{app1_usage_info}, {app2_usage_info}]

If you would like to generate user-app stats for all of KBase time please choose: start = "2016-03-04" end = "todays-date"""

# Get User_App_Stats

yesterday = (datetime.date.today() - datetime.timedelta(days=1))
             
def user_app_stats(unique_usernames, start_date= datetime.datetime.combine(yesterday, datetime.datetime.min.time()), end_date=datetime.datetime.combine(yesterday, datetime.datetime.max.time()) ):
    
    now = int(time.time())
    time_interval = {'begin': 1, 'end': now}
    
    # From str to datetime:
    if type(start_date) == str:
        start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        start_date = datetime.datetime.combine(start_date, datetime.datetime.min.time())
        end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
        end_date = datetime.datetime.combine(end_date, datetime.datetime.max.time())

    stats = catalog.get_exec_raw_stats(time_interval)
    stats = sorted(stats, key=itemgetter('exec_start_time'), reverse=True)
    catalog_data_all = pd.DataFrame.from_dict(stats)

    catalog_data_all.creation_time = pd.to_datetime(catalog_data_all['creation_time'], unit='s')
    catalog_data_all.exec_start_time = pd.to_datetime(catalog_data_all['exec_start_time'], unit='s')
    catalog_data_all.finish_time = pd.to_datetime(catalog_data_all['finish_time'], unit='s')
    catalog_data_all.drop( ['git_commit_hash','func_module_name', 'func_name', 'job_id'], axis=1, inplace=True)
    catalog_data_all = catalog_data_all.assign(run_time = (catalog_data_all['finish_time'] - catalog_data_all['exec_start_time']).astype('timedelta64[s]') )
    catalog_data_all["full_app_id"] = catalog_data_all["app_module_name"] + "/" + catalog_data_all["app_id"]

    # Slice catalog data by data range 
    catalog_data_all.index = catalog_data_all['creation_time']
    catalog_data_all.sort_index(inplace=True)
    catalog_data = catalog_data_all[start_date:end_date]
    # Filter null values in app_id
    catalog_data["full_app_id"].fillna("Not Specified", inplace=True)
    catalog_data = catalog_data[catalog_data.full_app_id != "Not Specified"]
    # Initiate dictionaries and arrays

    # Need app_dict here!
    app_dict = create_app_dictionary()
    total_user_app_stats = []
    values = app_dict.values()
    KBase_apps = [item for sublist in values for item in sublist]

    # Iterate over all KBase users
    for user in unique_usernames:
        # Get catalog data for specific user
        user_condition = catalog_data.user_id == user
        user_specific_catdata = catalog_data[user_condition]
        # Get all apps used by user
        app_lst = list(set(list(user_specific_catdata.full_app_id)))

        if not app_lst:
            continue

        user_app_dict_lst = []

        # for app in user's app list get stats
        for app in app_lst:
            # if app name in app dictionary get category of app, make app_stats dict and append dict to users app dictionary
            if app in KBase_apps:

                # Initiate app and error condition
                app_condition = user_specific_catdata.full_app_id == app
                # Count app usage and errors
                catalog_at_app = user_specific_catdata[app_condition]

                for i in catalog_at_app.index.values:
                    start_date = catalog_at_app.exec_start_time[i]
                    finish_date = catalog_at_app.finish_time[i]
                    run_time = catalog_at_app.run_time[i]
                    is_error = catalog_at_app.is_error[i]

                    # Make app_stats dictionary for app and append to array
                    app_stats = {"user_name": user, "app_name": app, "start_date": start_date,
                                 "finish_date": finish_date, "run_time": run_time, "is_error": is_error}
                    user_app_dict_lst.append(app_stats)

        # user_app_dict_lst = [item for sublist in user_app_dict_lst for item in sublist]
        # print(user_app_dict_lst)

        total_user_app_stats.extend(user_app_dict_lst)

    return total_user_app_stats
<commit_msg>Fixed string-date bug.<commit_after> 

import requests
import os
requests.packages.urllib3.disable_warnings()
from biokbase.catalog.Client import Catalog
from biokbase.narrative_method_store.client import NarrativeMethodStore
from category_to_app_dict import create_app_dictionary
catalog = Catalog(url = "https://kbase.us/services/catalog", token = os.environ['METRICS_USER_TOKEN'])
nms = NarrativeMethodStore(url = "https://kbase.us/services/narrative_method_store/rpc")

import pandas as pd
import datetime, time
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 500)
from operator import itemgetter


# Get User_App_Stats: Main Function

"""Requires a unique list of username as input and runs it 3-4min on average.
outputs a list of user dictionaries with app statistics for each userm example user : [{app1_usage_info}, {app2_usage_info}]

If you would like to generate user-app stats for all of KBase time please choose: start = "2016-03-04" end = "todays-date"""

# Get User_App_Stats

yesterday = (datetime.date.today() - datetime.timedelta(days=1))
             
def user_app_stats(unique_usernames, start_date= datetime.datetime.combine(yesterday, datetime.datetime.min.time()), end_date=datetime.datetime.combine(yesterday, datetime.datetime.max.time()) ):
    
    now = int(time.time())
    time_interval = {'begin': 1, 'end': now}
    
    # From str to datetime:
    if type(start_date) == str:
        start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
        end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')


    stats = catalog.get_exec_raw_stats(time_interval)
    stats = sorted(stats, key=itemgetter('exec_start_time'), reverse=True)
    catalog_data_all = pd.DataFrame.from_dict(stats)

    catalog_data_all.creation_time = pd.to_datetime(catalog_data_all['creation_time'], unit='s')
    catalog_data_all.exec_start_time = pd.to_datetime(catalog_data_all['exec_start_time'], unit='s')
    catalog_data_all.finish_time = pd.to_datetime(catalog_data_all['finish_time'], unit='s')
    catalog_data_all.drop( ['git_commit_hash','func_module_name', 'func_name', 'job_id'], axis=1, inplace=True)
    catalog_data_all = catalog_data_all.assign(run_time = (catalog_data_all['finish_time'] - catalog_data_all['exec_start_time']).astype('timedelta64[s]') )
    catalog_data_all["full_app_id"] = catalog_data_all["app_module_name"] + "/" + catalog_data_all["app_id"]

    # Slice catalog data by data range 
    catalog_data_all.index = catalog_data_all['creation_time']
    catalog_data_all.sort_index(inplace=True)
    catalog_data = catalog_data_all[start_date:end_date]
    # Filter null values in app_id
    catalog_data["full_app_id"].fillna("Not Specified", inplace=True)
    catalog_data = catalog_data[catalog_data.full_app_id != "Not Specified"]
    # Initiate dictionaries and arrays

    # Need app_dict here!
    app_dict = create_app_dictionary()
    total_user_app_stats = []
    values = app_dict.values()
    KBase_apps = [item for sublist in values for item in sublist]

    # Iterate over all KBase users
    for user in unique_usernames:
        # Get catalog data for specific user
        user_condition = catalog_data.user_id == user
        user_specific_catdata = catalog_data[user_condition]
        # Get all apps used by user
        app_lst = list(set(list(user_specific_catdata.full_app_id)))

        if not app_lst:
            continue

        user_app_dict_lst = []

        # for app in user's app list get stats
        for app in app_lst:
            # if app name in app dictionary get category of app, make app_stats dict and append dict to users app dictionary
            if app in KBase_apps:

                # Initiate app and error condition
                app_condition = user_specific_catdata.full_app_id == app
                # Count app usage and errors
                catalog_at_app = user_specific_catdata[app_condition]

                for i in catalog_at_app.index.values:
                    start_date = catalog_at_app.exec_start_time[i]
                    finish_date = catalog_at_app.finish_time[i]
                    run_time = catalog_at_app.run_time[i]
                    is_error = catalog_at_app.is_error[i]

                    # Make app_stats dictionary for app and append to array
                    app_stats = {"user_name": user, "app_name": app, "start_date": start_date,
                                 "finish_date": finish_date, "run_time": run_time, "is_error": is_error}
                    user_app_dict_lst.append(app_stats)

        # user_app_dict_lst = [item for sublist in user_app_dict_lst for item in sublist]
        # print(user_app_dict_lst)

        total_user_app_stats.extend(user_app_dict_lst)

    return total_user_app_stats

{'fraction_non_alphanumeric': 0.09357685204186333, 'fraction_numerical': 0.004309460291401601, 'mean_word_length': 3.546175373134328, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2845785', 'n_tokens_mistral': 3108, 'n_tokens_neox': 3006, 'n_words': 676}