<commit_before>from werkzeug import generate_password_hash, check_password_hash
from couchdb import ResourceNotFound, ResourceConflict
import shortuuid, datetime, hashlib, threading, json, time, copy

from totalimpact.providers.provider import ProviderFactory
from totalimpact.providers.provider import ProviderTimeout
from totalimpact import default_settings
from totalimpact.utils import Retry

# Master lock to ensure that only a single thread can write
# to the DB at one time to avoid document conflicts

import logging
logger = logging.getLogger('ti.models')

class NotAuthenticatedError(Exception):
    pass


def closest(target, collection) :
    return min((abs(target - i), i) for i in collection)[1]

class ItemFactory():

    all_static_meta = ProviderFactory.get_all_static_meta()


    @classmethod
    def get_item(cls, tiid, myrefsets, dao):
        res = dao.view("queues/by_tiid_with_snaps")
        rows = res[[tiid,0]:[tiid,"zzzzzzzzzzzzzz"]].rows

        if not rows:
            return None
        else:
            item = rows[0]["value"]
            most_recent_snaps = {}
            for row in rows[1:]:
                snap = row["value"]
                # they are in reverse order of creation, so overwrite
                # and most recent one will be on top.
                most_recent_snaps[snap["metric_name"]] = snap

            snaps = most_recent_snaps.values()

            try:
                item = cls.build_item_for_client(item, snaps, myrefsets)
            except Exception, e:
                item = None
                logger.error("Exception %s: Unable to build item %s, %s, %s" % (e.__repr__(), tiid, str(item), str(snaps)))
                raise
        return item

    @classmethod
    def build_item_for_client(cls, item, snaps, myrefsets):
        item["biblio"]['genre'] = cls.decide_genre(item['aliases'])
        #item["metrics"] = {} #not using what is in stored item for this

        # need year to calculate normalization below
        try:
            year = item["biblio"]["year"]
            if year < 2002:
                year = 2002
        except KeyError:
            year = 99 # hack so that it won't match anything.  what else to do?

        for snap in snaps:
            metric_name = snap["metric_name"]
            if metric_name in cls.all_static_meta.keys():
                # add snap values. this line can go away once all snap values migrated into items in db.  
                item = cls.add_snap_data(item, snap)

                #delete the raw history from what we return to the client for now
                del item["metrics"][metric_name]["values"]["raw_history"]

                # add static data
                item["metrics"][metric_name]["static_meta"] = cls.all_static_meta[metric_name]            

                # add normalization values
                normalized_values = cls.get_normalized_values(item["biblio"]['genre'], year, metric_name, snap["value"], myrefsets)
                item["metrics"][metric_name]["values"].update(normalized_values)

        return item

    @classmethod
    def add_snap_data(cls, item, snap):
        metrics = item.setdefault("metrics", {})
        
        this_metric = metrics.setdefault(snap["metric_name"], {})
        this_metric["provenance_url"] = snap["drilldown_url"]

        this_metric_values = this_metric.setdefault("values", {})
        this_metric_values["raw"] = snap["value"]

        this_metric_values_raw_history = this_metric_values.setdefault("raw_history", {})
        this_metric_values_raw_history[snap["created"]] = snap["value"]
        return item
<commit_msg>use snaps in items if they are there<commit_after>from werkzeug import generate_password_hash, check_password_hash
from couchdb import ResourceNotFound, ResourceConflict
import shortuuid, datetime, hashlib, threading, json, time, copy

from totalimpact.providers.provider import ProviderFactory
from totalimpact.providers.provider import ProviderTimeout
from totalimpact import default_settings
from totalimpact.utils import Retry

# Master lock to ensure that only a single thread can write
# to the DB at one time to avoid document conflicts

import logging
logger = logging.getLogger('ti.models')

class NotAuthenticatedError(Exception):
    pass


def closest(target, collection) :
    return min((abs(target - i), i) for i in collection)[1]

class ItemFactory():

    all_static_meta = ProviderFactory.get_all_static_meta()


    @classmethod
    def get_item(cls, tiid, myrefsets, dao):
        res = dao.view("queues/by_tiid_with_snaps")
        rows = res[[tiid,0]:[tiid,"zzzzzzzzzzzzzz"]].rows

        if not rows:
            return None
        else:
            item = rows[0]["value"]
            most_recent_snaps = {}
            for row in rows[1:]:
                snap = row["value"]
                # they are in reverse order of creation, so overwrite
                # and most recent one will be on top.
                most_recent_snaps[snap["metric_name"]] = snap

            snaps = most_recent_snaps.values()

            try:
                item = cls.build_item_for_client(item, snaps, myrefsets)
            except Exception, e:
                item = None
                logger.error("Exception %s: Unable to build item %s, %s, %s" % (e.__repr__(), tiid, str(item), str(snaps)))
                raise
        return item

    @classmethod
    def build_item_for_client(cls, item, snaps, myrefsets):
        item["biblio"]['genre'] = cls.decide_genre(item['aliases'])
        #item["metrics"] = {} #not using what is in stored item for this

        # need year to calculate normalization below
        try:
            year = item["biblio"]["year"]
            if year < 2002:
                year = 2002
        except KeyError:
            year = 99 # hack so that it won't match anything.  what else to do?

        # temporary while we still have snaps
        for snap in snaps:
            metric_name = snap["metric_name"]
            if metric_name in cls.all_static_meta.keys():
                # add snap values. this line can go away once all snap values migrated into items in db.  
                item = cls.add_snap_data(item, snap)

        for metric_name in item["metrics"]:
            if metric_name in cls.all_static_meta.keys():  # make sure we still support this metrics type
                #delete the raw history from what we return to the client for now
                del item["metrics"][metric_name]["values"]["raw_history"]

                # add static data
                item["metrics"][metric_name]["static_meta"] = cls.all_static_meta[metric_name]            

                # add normalization values
                normalized_values = cls.get_normalized_values(item["biblio"]['genre'], year, metric_name, snap["value"], myrefsets)
                item["metrics"][metric_name]["values"].update(normalized_values)

        return item

    @classmethod
    def add_snap_data(cls, item, snap):
        metrics = item.setdefault("metrics", {})
        
        this_metric = metrics.setdefault(snap["metric_name"], {})
        this_metric["provenance_url"] = snap["drilldown_url"]

        this_metric_values = this_metric.setdefault("values", {})
        this_metric_values["raw"] = snap["value"]

        this_metric_values_raw_history = this_metric_values.setdefault("raw_history", {})
        this_metric_values_raw_history[snap["created"]] = snap["value"]
        return item

{'fraction_non_alphanumeric': 0.0964338186189395, 'fraction_numerical': 0.003739815680512889, 'mean_word_length': 2.523764705882353, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6235971', 'n_tokens_mistral': 2158, 'n_tokens_neox': 2052, 'n_words': 587}