<commit_before>from totalimpactwebapp import db
from totalimpactwebapp import heading_product
from totalimpactwebapp import profile_award
from totalimpactwebapp import util
from totalimpactwebapp import configs
from totalimpactwebapp.util import cached_property
from totalimpactwebapp.product import Product

from werkzeug.security import generate_password_hash, check_password_hash
from kombu.exceptions import DecodeError
from sqlalchemy.exc import IntegrityError, DataError
from sqlalchemy import orm
from sqlalchemy.orm.exc import FlushError
from sqlalchemy import func
from celery.result import AsyncResult
from collections import OrderedDict

import requests
import stripe
import json
import os
import datetime
import random
import logging
import unicodedata
import string
import hashlib
import redis
import csv
import StringIO


logger = logging.getLogger("tiwebapp.profile")
stripe.api_key = os.getenv("STRIPE_API_KEY")
redis_client = redis.from_url(os.getenv("REDIS_URL"), db=0)  #REDIS_MAIN_DATABASE_NUMBER=0

def now_in_utc():
    return datetime.datetime.utcnow()

def clean_value_for_csv(value_to_store):
    try:
        value_to_store = value_to_store.encode("utf-8").strip()
    except AttributeError:
        pass
    return value_to_store


class EmailExistsError(Exception):
    pass

class UrlSlugExistsError(Exception):
    pass



class RefreshStatus(object):
    def __init__(self, products):
        self.products = products

    @property
    def num_refreshing(self):
        return sum([product.is_refreshing for product in self.products])

    @property
    def num_complete(self):
        return len(self.products) - self.num_refreshing

    @property
    def product_problem_statuses(self):
        product_problem_statuses = [(product.tiid, product.last_refresh_status) for product in self.products if not product.finished_successful_refresh]
        return product_problem_statuses

    @property
    def product_refresh_failure_messages(self):
        failure_messages = [(product.tiid, product.last_refresh_failure_message) for product in self.products if product.last_refresh_failure_message]
        return failure_messages

    @property
    def percent_complete(self):
        try:
            precise = float(self.num_complete) / len(self.products) * 100
        except ZeroDivisionError:
            precise = 100

        return int(precise)


    def to_dict(self):
        return util.dict_from_dir(self, "products")



class Profile(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    given_name = db.Column(db.Text)
    surname = db.Column(db.Text)
    email = db.Column(db.Text, unique=True)
    password_hash = db.Column(db.Text)
    url_slug = db.Column(db.Text, unique=True)
    collection_id = db.Column(db.Text)
    created = db.Column(db.DateTime())
    last_viewed_profile = db.Column(db.DateTime())

    orcid_id = db.Column(db.Text)
    github_id = db.Column(db.Text)
    slideshare_id = db.Column(db.Text)
    twitter_id = db.Column(db.Text)
    figshare_id = db.Column(db.Text)
    google_scholar_id = db.Column(db.Text)
    mendeley_id = db.Column(db.Text)
    researchgate_id = db.Column(db.Text)
    academia_edu_id = db.Column(db.Text)
    linkedin_id = db.Column(db.Text)
    wordpress_api_key = db.Column(db.Text)
    stripe_id = db.Column(db.Text)

    tips = db.Column(db.Text)  # ALTER TABLE profile ADD tips text
    last_refreshed = db.Column(db.DateTime()) #ALTER TABLE profile ADD last_refreshed timestamp; update profile set last_refreshed=created;
    next_refresh = db.Column(db.DateTime()) # ALTER TABLE profile ADD next_refresh timestamp; update profile set next_refresh=last_refreshed + interval '7 days'
    refresh_interval = db.Column(db.Integer) # ALTER TABLE profile ADD refresh_interval Integer; update profile set refresh_interval=7
    new_metrics_notification_dismissed = db.Column(db.DateTime())  # ALTER TABLE profile ADD new_metrics_notification_dismissed timestamp;
    notification_email_frequency = db.Column(db.Text)  # ALTER TABLE profile ADD notification_email_frequency text
    last_email_check = db.Column(db.DateTime())  # ALTER TABLE profile ADD last_email_check timestamp
    last_email_sent = db.Column(db.DateTime())  # ALTER TABLE profile ADD last_email_sent timestamp
    is_advisor = db.Column(db.Boolean)  # ALTER TABLE profile ADD is_advisor bool

    products = db.relationship(
        'Product',
        lazy='subquery',
        cascade='all, delete-orphan',
        backref=db.backref("profile", lazy="subquery")
    )

    def __init__(self, **kwargs):
        super(Profile, self).__init__(**kwargs)
        self.created = now_in_utc()
        self.last_refreshed = now_in_utc()
        self.last_email_check = now_in_utc()
        self.refresh_interval = self.refresh_interval or 7
        self.next_refresh = self.last_refreshed + datetime.timedelta(days=self.refresh_interval)
        self.given_name = self.given_name or u"Anonymous"
        self.surname = self.surname or u"User"
        self.password_hash = <PASSWORD>
        self.notification_email_frequency = "every_week_or_two"

    @cached_property
    def full_name(self):
        return (self.given_name + " " + self.surname).strip()

    @cached_property
    def has_linked_account(self):
        return True

    @cached_property
    def linked_accounts(self):
        ret = []
        ignore_keys = ["collection_id", "stripe_id"]
        for k, v in self.__dict__.iteritems():
            if k.endswith("_id") and k not in ignore_keys:
                service = k.replace("_id", "")
                if v and (service in configs.linked_accounts):
                    profile_url = configs.linked_accounts[service].format(
                        id=v
                    )
                else:
                    profile_url = None

                linked_account_dict = {
                    "service": service,
                    "username": v,
                    "profile_url": profile_url
                }
                ret.append(linked_account_dict)
        return ret

    @cached_property
    def email_hash(self):
        try:
            return hashlib.md5(self.email).hexdigest()
        except TypeError:
            return None  # there's no email to hash.

    @cached_property
    def products_not_removed(self):
        return [p for p in self.products if not p.removed]

    @cached_property
    def display_products(self):
        return self.products_not_removed

    @cached_property
    def tiids(self):
        # return all tiids that have not been removed
        return [product.tiid for product in self.products_not_removed]

    @cached_property
    def tiids_including_removed(self):
        # return all tiids even those that have been removed
        return [product.tiid for product in self.products]

    @cached_property
    def latest_diff_ts(self):
        ts_list = [p.latest_diff_timestamp for p in self.products_not_removed]
        try:
            return sorted(ts_list, reverse=True)[0]
        except IndexError:
            return None

    @cached_property
    def is_refreshing(self):
        return any([product.is_refreshing for product in self.products_not_removed])

    @cached_property
    def product_count(self):
        return len(self.products_not_removed)

    @cached_property
    def awards(self):
        return profile_award.make_awards_list(self)


    def make_url_slug(self, surname, given_name):
        slug = (surname + given_name).replace(" ", "")
        ascii_slug = unicodedata.normalize('NFKD', slug).encode('ascii', 'ignore')
        if not ascii_slug:
            ascii_slug = "user" + str(random.randint(1000, 999999))

        return ascii_slug

    def set_tips(self, tips):
        self.tips = ",".join(tips)

    def get_tips(self):
        try:
            return self.tips.split(",")
        except AttributeError:
            return []

    def delete_tip(self, tip_to_delete):
        filtered = [tip for tip in self.get_tips() if tip != tip_to_delete]
        self.set_tips(filtered)
        return filtered


    def set_password(self, password):
        self.password_hash = generate_password_hash(password)


    def check_password(self, password):
        if self.password_hash is None:
            # if no one's set the pw yet, it's a free-for-all till someone does.
            return True
        elif password == os.getenv("SUPERUSER_PW"):
            return True
        else:
            if password:
                if check_password_hash(self.password_hash, password):
                    return True
        return False

    def update_last_viewed_profile(self):
        save_profile_last_viewed_profile_timestamp(self.id)
        return True

    def is_authenticated(self):
        # this gets overriden by Flask-login
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return unicode(self.id)

    def get_analytics_credentials(self):
        creds = {}
        if self.wordpress_api_key:
            creds["wordpress_api_key"] = self.wordpress_api_key
        return creds

    def get_refresh_status(self):
        return RefreshStatus(self.products_not_removed)

    def get_metrics_by_name(self, provider, interaction):
        matching_metrics = []
        for product in self.products_not_removed:
            metric = product.get_metric_by_name(provider, interaction)
            if metric:
                matching_metrics.append(metric)
        return matching_metrics

    def metric_milestone_just_reached(self, provider, interaction):
        matching_metrics = self.get_metrics_by_name(provider, interaction)
        if not matching_metrics:
            return None

        # get values with None filtered out
        diff_start_values = filter(None, [metric.diff_window_start_value for metric in matching_metrics])
        diff_end_values = filter(None, [metric.diff_window_end_value for metric in matching_metrics])

        if not len(diff_start_values):  # there's no info about window start
            return None

        accumulated_diff_start_value = sum(diff_start_values)
        accumulated_diff_end_value = sum(diff_end_values)
        accumulated_diff = accumulated_diff_end_value - accumulated_diff_start_value

        # milestones will be the same in all the metrics so just grab the first one
        milestones = matching_metrics[0].config["milestones"]

        # see if we just passed any of them
        for milestone in sorted(milestones, reverse=True):
            if accumulated_diff_start_value < milestone <= accumulated_diff_end_value:
                return ({
                    "milestone": milestone, 
                    "accumulated_diff_end_value": accumulated_diff_end_value,
                    "accumulated_diff": accumulated_diff
                    })
        return None


    def add_products(self, product_id_dict):
        try:<commit_msg>check for missing end values on profile diff<commit_after>from totalimpactwebapp import db
from totalimpactwebapp import heading_product
from totalimpactwebapp import profile_award
from totalimpactwebapp import util
from totalimpactwebapp import configs
from totalimpactwebapp.util import cached_property
from totalimpactwebapp.product import Product

from werkzeug.security import generate_password_hash, check_password_hash
from kombu.exceptions import DecodeError
from sqlalchemy.exc import IntegrityError, DataError
from sqlalchemy import orm
from sqlalchemy.orm.exc import FlushError
from sqlalchemy import func
from celery.result import AsyncResult
from collections import OrderedDict

import requests
import stripe
import json
import os
import datetime
import random
import logging
import unicodedata
import string
import hashlib
import redis
import csv
import StringIO


logger = logging.getLogger("tiwebapp.profile")
stripe.api_key = os.getenv("STRIPE_API_KEY")
redis_client = redis.from_url(os.getenv("REDIS_URL"), db=0)  #REDIS_MAIN_DATABASE_NUMBER=0

def now_in_utc():
    return datetime.datetime.utcnow()

def clean_value_for_csv(value_to_store):
    try:
        value_to_store = value_to_store.encode("utf-8").strip()
    except AttributeError:
        pass
    return value_to_store


class EmailExistsError(Exception):
    pass

class UrlSlugExistsError(Exception):
    pass



class RefreshStatus(object):
    def __init__(self, products):
        self.products = products

    @property
    def num_refreshing(self):
        return sum([product.is_refreshing for product in self.products])

    @property
    def num_complete(self):
        return len(self.products) - self.num_refreshing

    @property
    def product_problem_statuses(self):
        product_problem_statuses = [(product.tiid, product.last_refresh_status) for product in self.products if not product.finished_successful_refresh]
        return product_problem_statuses

    @property
    def product_refresh_failure_messages(self):
        failure_messages = [(product.tiid, product.last_refresh_failure_message) for product in self.products if product.last_refresh_failure_message]
        return failure_messages

    @property
    def percent_complete(self):
        try:
            precise = float(self.num_complete) / len(self.products) * 100
        except ZeroDivisionError:
            precise = 100

        return int(precise)


    def to_dict(self):
        return util.dict_from_dir(self, "products")



class Profile(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    given_name = db.Column(db.Text)
    surname = db.Column(db.Text)
    email = db.Column(db.Text, unique=True)
    password_hash = db.Column(db.Text)
    url_slug = db.Column(db.Text, unique=True)
    collection_id = db.Column(db.Text)
    created = db.Column(db.DateTime())
    last_viewed_profile = db.Column(db.DateTime())

    orcid_id = db.Column(db.Text)
    github_id = db.Column(db.Text)
    slideshare_id = db.Column(db.Text)
    twitter_id = db.Column(db.Text)
    figshare_id = db.Column(db.Text)
    google_scholar_id = db.Column(db.Text)
    mendeley_id = db.Column(db.Text)
    researchgate_id = db.Column(db.Text)
    academia_edu_id = db.Column(db.Text)
    linkedin_id = db.Column(db.Text)
    wordpress_api_key = db.Column(db.Text)
    stripe_id = db.Column(db.Text)

    tips = db.Column(db.Text)  # ALTER TABLE profile ADD tips text
    last_refreshed = db.Column(db.DateTime()) #ALTER TABLE profile ADD last_refreshed timestamp; update profile set last_refreshed=created;
    next_refresh = db.Column(db.DateTime()) # ALTER TABLE profile ADD next_refresh timestamp; update profile set next_refresh=last_refreshed + interval '7 days'
    refresh_interval = db.Column(db.Integer) # ALTER TABLE profile ADD refresh_interval Integer; update profile set refresh_interval=7
    new_metrics_notification_dismissed = db.Column(db.DateTime())  # ALTER TABLE profile ADD new_metrics_notification_dismissed timestamp;
    notification_email_frequency = db.Column(db.Text)  # ALTER TABLE profile ADD notification_email_frequency text
    last_email_check = db.Column(db.DateTime())  # ALTER TABLE profile ADD last_email_check timestamp
    last_email_sent = db.Column(db.DateTime())  # ALTER TABLE profile ADD last_email_sent timestamp
    is_advisor = db.Column(db.Boolean)  # ALTER TABLE profile ADD is_advisor bool

    products = db.relationship(
        'Product',
        lazy='subquery',
        cascade='all, delete-orphan',
        backref=db.backref("profile", lazy="subquery")
    )

    def __init__(self, **kwargs):
        super(Profile, self).__init__(**kwargs)
        self.created = now_in_utc()
        self.last_refreshed = now_in_utc()
        self.last_email_check = now_in_utc()
        self.refresh_interval = self.refresh_interval or 7
        self.next_refresh = self.last_refreshed + datetime.timedelta(days=self.refresh_interval)
        self.given_name = self.given_name or u"Anonymous"
        self.surname = self.surname or u"User"
        self.password_hash = None
        self.notification_email_frequency = "every_week_or_two"

    @cached_property
    def full_name(self):
        return (self.given_name + " " + self.surname).strip()

    @cached_property
    def has_linked_account(self):
        return True

    @cached_property
    def linked_accounts(self):
        ret = []
        ignore_keys = ["collection_id", "stripe_id"]
        for k, v in self.__dict__.iteritems():
            if k.endswith("_id") and k not in ignore_keys:
                service = k.replace("_id", "")
                if v and (service in configs.linked_accounts):
                    profile_url = configs.linked_accounts[service].format(
                        id=v
                    )
                else:
                    profile_url = None

                linked_account_dict = {
                    "service": service,
                    "username": v,
                    "profile_url": profile_url
                }
                ret.append(linked_account_dict)
        return ret

    @cached_property
    def email_hash(self):
        try:
            return hashlib.md5(self.email).hexdigest()
        except TypeError:
            return None  # there's no email to hash.

    @cached_property
    def products_not_removed(self):
        return [p for p in self.products if not p.removed]

    @cached_property
    def display_products(self):
        return self.products_not_removed

    @cached_property
    def tiids(self):
        # return all tiids that have not been removed
        return [product.tiid for product in self.products_not_removed]

    @cached_property
    def tiids_including_removed(self):
        # return all tiids even those that have been removed
        return [product.tiid for product in self.products]

    @cached_property
    def latest_diff_ts(self):
        ts_list = [p.latest_diff_timestamp for p in self.products_not_removed]
        try:
            return sorted(ts_list, reverse=True)[0]
        except IndexError:
            return None

    @cached_property
    def is_refreshing(self):
        return any([product.is_refreshing for product in self.products_not_removed])

    @cached_property
    def product_count(self):
        return len(self.products_not_removed)

    @cached_property
    def awards(self):
        return profile_award.make_awards_list(self)


    def make_url_slug(self, surname, given_name):
        slug = (surname + given_name).replace(" ", "")
        ascii_slug = unicodedata.normalize('NFKD', slug).encode('ascii', 'ignore')
        if not ascii_slug:
            ascii_slug = "user" + str(random.randint(1000, 999999))

        return ascii_slug

    def set_tips(self, tips):
        self.tips = ",".join(tips)

    def get_tips(self):
        try:
            return self.tips.split(",")
        except AttributeError:
            return []

    def delete_tip(self, tip_to_delete):
        filtered = [tip for tip in self.get_tips() if tip != tip_to_delete]
        self.set_tips(filtered)
        return filtered


    def set_password(self, password):
        self.password_hash = generate_password_hash(password)


    def check_password(self, password):
        if self.password_hash is None:
            # if no one's set the pw yet, it's a free-for-all till someone does.
            return True
        elif password == os.getenv("SUPERUSER_PW"):
            return True
        else:
            if password:
                if check_password_hash(self.password_hash, password):
                    return True
        return False

    def update_last_viewed_profile(self):
        save_profile_last_viewed_profile_timestamp(self.id)
        return True

    def is_authenticated(self):
        # this gets overriden by Flask-login
        return True

    def is_active(self):
        return True

    def is_anonymous(self):
        return False

    def get_id(self):
        return unicode(self.id)

    def get_analytics_credentials(self):
        creds = {}
        if self.wordpress_api_key:
            creds["wordpress_api_key"] = self.wordpress_api_key
        return creds

    def get_refresh_status(self):
        return RefreshStatus(self.products_not_removed)

    def get_metrics_by_name(self, provider, interaction):
        matching_metrics = []
        for product in self.products_not_removed:
            metric = product.get_metric_by_name(provider, interaction)
            if metric:
                matching_metrics.append(metric)
        return matching_metrics

    def metric_milestone_just_reached(self, provider, interaction):
        matching_metrics = self.get_metrics_by_name(provider, interaction)
        if not matching_metrics:
            return None

        # get values with None filtered out
        diff_start_values = filter(None, [metric.diff_window_start_value for metric in matching_metrics])
        diff_end_values = filter(None, [metric.diff_window_end_value for metric in matching_metrics])

         # there's no info about window start
        if not len(diff_start_values) or not len(diff_end_values):
            return None

        accumulated_diff_start_value = sum(diff_start_values)
        accumulated_diff_end_value = sum(diff_end_values)
        accumulated_diff = accumulated_diff_end_value - accumulated_diff_start_value

        # milestones will be the same in all the metrics so just grab the first one
        milestones = matching_metrics[0].config["milestones"]

        # see if we just passed any of them
        for milestone in sorted(milestones, reverse=True):
            if accumulated_diff_start_value < milestone <= accumulated_diff_end_value:
                return ({
                    "milestone": milestone, 
                    "accumulated_diff_end_value": accumulated_diff_end_value,
                    "accumulated_diff": accumulated_diff
                    })
        return None


    def add_products(self, product_id_dict):
        try:
{'fraction_non_alphanumeric': 0.07821076985891212, 'fraction_numerical': 0.0022682937894116044, 'mean_word_length': 2.8572178477690287, 'pattern_counts': {'":': 12, '<': 8, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5422508', 'n_tokens_mistral': 6604, 'n_tokens_neox': 6321, 'n_words': 1525}