<commit_before>from totalimpactwebapp.profile import Profile
from totalimpactwebapp.profile import ProductsFromCore
from totalimpactwebapp import db
import tasks

from sqlalchemy import func
from celery import chain
import time
import datetime
import argparse
import logging

logger = logging.getLogger("webapp.daily")



"""
requires these env vars be set in this environment:
DATABASE_URL
"""

try:
    # set jason's env variables for local running.
    import config
    config.set_env_vars_from_dot_env()
except ImportError:
    pass


def page_query(q):
    offset = 0
    while True:
        r = False
        for elem in q.limit(5).offset(offset):
           r = True
           yield elem
        offset += 5
        if not r:
            break

def add_profile_deets_for_everyone():
    for user in page_query(Profile.query.order_by(Profile.url_slug.asc())):
        ProductsFromCore.clear_cache()
        logger.info(u"add_profile_deets_for_everyone: {url_slug}".format(url_slug=user.url_slug))
        response = tasks.add_profile_deets.delay(user)


def deduplicate_everyone():
    for user in page_query(Profile.query.order_by(Profile.url_slug.asc())):
        ProductsFromCore.clear_cache()
        logger.info(u"deduplicate_everyone: {url_slug}".format(url_slug=user.url_slug))
        response = tasks.deduplicate.delay(user)



def create_cards_for_everyone(url_slug=None):
    cards = []
    if url_slug:
        user = Profile.query.filter(func.lower(Profile.url_slug) == func.lower(url_slug)).first()
        ProductsFromCore.clear_cache()
        # print user.url_slug        
        cards = tasks.create_cards(user)
    else:    
        for user in page_query(Profile.query.order_by(Profile.url_slug.asc())):
            ProductsFromCore.clear_cache()
            # print user.url_slug        
            cards = tasks.create_cards.delay(user)
    return cards



def email_report_to_url_slug(url_slug=None):
    if url_slug:
        user = Profile.query.filter(func.lower(Profile.url_slug) == func.lower(url_slug)).first()
        ProductsFromCore.clear_cache()
        # print user.url_slug        
        tasks.send_email_report(user)


def email_report_to_everyone_who_needs_one():
    for user in page_query(Profile.query.order_by(Profile.url_slug.asc())):

        logger.info(u"clearing user cache for {url_slug}".format(
            url_slug=user.url_slug))

        ProductsFromCore.clear_cache()

        try:
            if not user.email or (u"@" not in user.email):
                logger.info(u"not sending, no email address for {url_slug}".format(url_slug=user.url_slug))
            elif user.notification_email_frequency == "none":
                logger.info(u"not sending, {url_slug} is unsubscribed".format(url_slug=user.url_slug))
            elif user.last_email_sent and ((datetime.datetime.utcnow() - user.last_email_sent).days < 7):
                logger.info(u"not sending, {url_slug} already got email this week".format(url_slug=user.url_slug))
            else:
                logger.info(u"adding ASYNC notification check to celery for {url_slug}".format(url_slug=user.url_slug))
                status = tasks.send_email_if_new_diffs.delay(user.id)
        except Exception as e:
            logger.warning(u"EXCEPTION in email_report_to_everyone_who_needs_one for {url_slug}, skipping to next user.  Error {e}".format(
                url_slug=user.url_slug, e=e))
            pass
    return


def main(function, url_slug):
    if function=="email_report":
        if url_slug:
            email_report_to_url_slug(url_slug)
        else:    
            email_report_to_everyone_who_needs_one()
    elif function=="dedup":
        deduplicate_everyone()
    elif function=="profile_deets":
        add_profile_deets_for_everyone()<commit_msg>replace "user" with "profile" vars in daily.py<commit_after>from totalimpactwebapp.profile import Profile
from totalimpactwebapp.profile import ProductsFromCore
from totalimpactwebapp import db
import tasks

from sqlalchemy import func
import datetime
import argparse
import logging

logger = logging.getLogger("webapp.daily")



"""
requires these env vars be set in this environment:
DATABASE_URL
"""

try:
    # set jason's env variables for local running.
    import config
    config.set_env_vars_from_dot_env()
except ImportError:
    pass


def page_query(q):
    offset = 0
    while True:
        r = False
        for elem in q.limit(5).offset(offset):
           r = True
           yield elem
        offset += 5
        if not r:
            break

def add_profile_deets_for_everyone():
    for profile in page_query(Profile.query.order_by(Profile.url_slug.asc())):
        ProductsFromCore.clear_cache()
        logger.info(u"add_profile_deets_for_everyone: {url_slug}".format(url_slug=profile.url_slug))
        response = tasks.add_profile_deets.delay(profile)


def deduplicate_everyone():
    for profile in page_query(Profile.query.order_by(Profile.url_slug.asc())):
        ProductsFromCore.clear_cache()
        logger.info(u"deduplicate_everyone: {url_slug}".format(url_slug=profile.url_slug))
        response = tasks.deduplicate.delay(profile)



def create_cards_for_everyone(url_slug=None):
    cards = []
    if url_slug:
        profile = Profile.query.filter(func.lower(Profile.url_slug) == func.lower(url_slug)).first()
        ProductsFromCore.clear_cache()
        # print profile.url_slug
        cards = tasks.create_cards(profile)
    else:    
        for profile in page_query(Profile.query.order_by(Profile.url_slug.asc())):
            ProductsFromCore.clear_cache()
            # print profile.url_slug
            cards = tasks.create_cards.delay(profile)
    return cards



def email_report_to_url_slug(url_slug=None):
    if url_slug:
        profile = Profile.query.filter(func.lower(Profile.url_slug) == func.lower(url_slug)).first()
        ProductsFromCore.clear_cache()
        # print profile.url_slug
        tasks.send_email_report(profile)


def email_report_to_everyone_who_needs_one():
    for profile in page_query(Profile.query.order_by(Profile.url_slug.asc())):

        logger.info(u"clearing profile cache for {url_slug}".format(
            url_slug=profile.url_slug))

        ProductsFromCore.clear_cache()

        try:
            if not profile.email or (u"@" not in profile.email):
                logger.info(u"not sending, no email address for {url_slug}".format(url_slug=profile.url_slug))
            elif profile.notification_email_frequency == "none":
                logger.info(u"not sending, {url_slug} is unsubscribed".format(url_slug=profile.url_slug))
            elif profile.last_email_sent and ((datetime.datetime.utcnow() - profile.last_email_sent).days < 7):
                logger.info(u"not sending, {url_slug} already got email this week".format(url_slug=profile.url_slug))
            else:
                logger.info(u"adding ASYNC notification check to celery for {url_slug}".format(url_slug=profile.url_slug))
                status = tasks.send_email_if_new_diffs.delay(profile.id)
        except Exception as e:
            logger.warning(u"EXCEPTION in email_report_to_everyone_who_needs_one for {url_slug}, skipping to next profile.  Error {e}".format(
                url_slug=profile.url_slug, e=e))
            pass
    return


def main(function, url_slug):
    if function=="email_report":
        if url_slug:
            email_report_to_url_slug(url_slug)
        else:    
            email_report_to_everyone_who_needs_one()
    elif function=="dedup":
        deduplicate_everyone()
    elif function=="profile_deets":
        add_profile_deets_for_everyone()
{'fraction_non_alphanumeric': 0.09559207094418362, 'fraction_numerical': 0.0010432968179447052, 'mean_word_length': 3.0151832460732986, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1473145', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2405, 'n_words': 465}