<commit_before>from totalimpactwebapp.card import Card
from totalimpactwebapp.util import as_int_or_float_if_possible
from totalimpactwebapp import products_list
from totalimpactwebapp import thresholds as thresh

import requests
import json
import datetime
import arrow


def products_above_threshold(product_dicts, metric_name, threshold):
    above_threshold = []
    for product in product_dicts:
        if (metric_name in product["metrics"]):
            value = product["metrics"][metric_name]["values"]["raw"]
            if value >= threshold:
                above_threshold.append(product)
    return above_threshold


def get_percentile(metric_dict):
    for value_type in metric_dict["values"]:
        # the keys that aren't "raw" are reference ests
        if value_type != "raw":
            return (metric_dict["values"][value_type]["estimate_lower"])
    return None


def get_threshold_just_crossed(current_value, diff_value, thresholds):
    try:
        previous_value = current_value - diff_value
    except TypeError:
        # not numeric
        return None

    for threshold in thresholds:
        if (current_value >= threshold) and (previous_value < threshold):
            return threshold
    return None

def get_median(metric_dict, medians_lookup):
    try:
        refset_year = metric_dict["refset_year"]
        refset_genre = metric_dict["refset_genre"]
        refset_name = "unknown"
        for (key, val) in metric_dict["values"].iteritems():
            try:
                if "CI95_lower" in val.keys():
                    refset_name = key
            except AttributeError:
                pass
        metric_name = metric_dict["name"]

        median = medians_lookup[refset_genre][refset_name][refset_year][metric_name]
    except KeyError:
        median = None
    return median


def populate_card(user_id, tiid, metric_dict, metric_name, thresholds_lookup=[], medians_lookup={}):
    hist = metric_dict["historical_values"]
    current_value = as_int_or_float_if_possible(hist["current"]["raw"])
    diff_value = as_int_or_float_if_possible(hist["diff"]["raw"])
    thresholds = thresholds_lookup.get(metric_name, [])
    newest_diff_timestamp = arrow.get(hist["current"]["collected_date"]).datetime
    oldest_diff_timestamp = arrow.get(hist["previous"]["collected_date"]).datetime

    my_card = Card(
        card_type="new metrics",
        granularity="product",
        metric_name=metric_name,
        user_id=user_id,
        tiid=tiid,
        diff_value=diff_value,
        current_value=current_value,
        newest_diff_timestamp=newest_diff_timestamp,
        oldest_diff_timestamp=oldest_diff_timestamp, 
        diff_window_days = (newest_diff_timestamp - oldest_diff_timestamp).days,
        percentile_current_value=get_percentile(metric_dict),
        median=get_median(metric_dict, medians_lookup),
        threshold_awarded=get_threshold_just_crossed(current_value, diff_value, thresholds),
        weight=0.7
    )

    return my_card


def get_product_list_for_cards(user):
    product_dicts = products_list.prep(
            user.products,
            include_headings=False,
            display_debug=True
        )
    return product_dicts


def get_medians_lookup():
    url = "http://total-impact-core-staging.herokuapp.com/collections/reference-sets-medians"
    resp = requests.get(url)
    return json.loads(resp.text)


class CardGenerator:
    pass

<commit_msg>point to production not staging<commit_after>from totalimpactwebapp.card import Card
from totalimpactwebapp.util import as_int_or_float_if_possible
from totalimpactwebapp import products_list
from totalimpactwebapp import thresholds as thresh

import requests
import json
import datetime
import arrow


def products_above_threshold(product_dicts, metric_name, threshold):
    above_threshold = []
    for product in product_dicts:
        if (metric_name in product["metrics"]):
            value = product["metrics"][metric_name]["values"]["raw"]
            if value >= threshold:
                above_threshold.append(product)
    return above_threshold


def get_percentile(metric_dict):
    for value_type in metric_dict["values"]:
        # the keys that aren't "raw" are reference ests
        if value_type != "raw":
            return (metric_dict["values"][value_type]["estimate_lower"])
    return None


def get_threshold_just_crossed(current_value, diff_value, thresholds):
    try:
        previous_value = current_value - diff_value
    except TypeError:
        # not numeric
        return None

    for threshold in thresholds:
        if (current_value >= threshold) and (previous_value < threshold):
            return threshold
    return None

def get_median(metric_dict, medians_lookup):
    try:
        refset_year = metric_dict["refset_year"]
        refset_genre = metric_dict["refset_genre"]
        refset_name = "unknown"
        for (key, val) in metric_dict["values"].iteritems():
            try:
                if "CI95_lower" in val.keys():
                    refset_name = key
            except AttributeError:
                pass
        metric_name = metric_dict["name"]

        median = medians_lookup[refset_genre][refset_name][refset_year][metric_name]
    except KeyError:
        median = None
    return median


def populate_card(user_id, tiid, metric_dict, metric_name, thresholds_lookup=[], medians_lookup={}):
    hist = metric_dict["historical_values"]
    current_value = as_int_or_float_if_possible(hist["current"]["raw"])
    diff_value = as_int_or_float_if_possible(hist["diff"]["raw"])
    thresholds = thresholds_lookup.get(metric_name, [])
    newest_diff_timestamp = arrow.get(hist["current"]["collected_date"]).datetime
    oldest_diff_timestamp = arrow.get(hist["previous"]["collected_date"]).datetime

    my_card = Card(
        card_type="new metrics",
        granularity="product",
        metric_name=metric_name,
        user_id=user_id,
        tiid=tiid,
        diff_value=diff_value,
        current_value=current_value,
        newest_diff_timestamp=newest_diff_timestamp,
        oldest_diff_timestamp=oldest_diff_timestamp, 
        diff_window_days = (newest_diff_timestamp - oldest_diff_timestamp).days,
        percentile_current_value=get_percentile(metric_dict),
        median=get_median(metric_dict, medians_lookup),
        threshold_awarded=get_threshold_just_crossed(current_value, diff_value, thresholds),
        weight=0.7
    )

    return my_card


def get_product_list_for_cards(user):
    product_dicts = products_list.prep(
            user.products,
            include_headings=False,
            display_debug=True
        )
    return product_dicts


def get_medians_lookup():
    url = "http://total-impact-core.herokuapp.com/collections/reference-sets-medians"
    resp = requests.get(url)
    return json.loads(resp.text)


class CardGenerator:
    pass


{'fraction_non_alphanumeric': 0.08815865662999421, 'fraction_numerical': 0.0011580775911986102, 'mean_word_length': 3.334378920953576, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5918169', 'n_tokens_mistral': 2079, 'n_tokens_neox': 2020, 'n_words': 392}