<commit_before>from __future__ import absolute_import, unicode_literals

from django.utils import timezone

from celery.utils.log import get_task_logger
from djcelery_transactions import task

from dash.orgs.models import Org
from dash.utils import datetime_to_ms
from dash.utils.sync import sync_pull_contacts, sync_push_contact

from tracpro.orgs_ext.utils import run_org_task


logger = get_task_logger(__name__)


@task
def push_contact_change(contact_id, change_type):
    """
    Task to push a local contact change to RapidPro
    """
    from tracpro.groups.models import Group, Region
    from .models import Contact

    contact = Contact.objects.select_related('org', 'region').get(pk=contact_id)
    org = contact.org

    logger.info("Pushing %s change to contact %s" % (change_type.name.upper(), contact.uuid))

    region_uuids = set([r.uuid for r in Region.get_all(org)])
    group_uuids = set([r.uuid for r in Group.get_all(org)])

    sync_push_contact(org, contact, change_type, [region_uuids, group_uuids])


@task
def sync_org_contacts(org_id):
    """
    Syncs all contacts for the given org
    """
    from tracpro.groups.models import Region, Group
    from tracpro.orgs_ext.constants import TaskType
    from .models import Contact

    org = Org.objects.get(pk=org_id)

    logger.info('Starting contact sync task for org #%d' % org.id)

    sync_groups = [r.uuid for r in Region.get_all(org)] + [g.uuid for g in Group.get_all(org)]

    created, updated, deleted, failed = sync_pull_contacts(
        org, Contact, fields=(), groups=sync_groups)

    task_result = dict(time=datetime_to_ms(timezone.now()),
                       counts=dict(created=len(created),
                                   updated=len(updated),
                                   deleted=len(deleted),
                                   failed=len(failed)))
    org.set_task_result(TaskType.sync_contacts, task_result)

    logger.info("Finished contact sync for org #%d (%d created, "
                "%d updated, %d deleted, %d failed)" %
                (org.id, len(created), len(updated), len(deleted), len(failed)))


@task
def sync_all_contacts():
    """
    Syncs all contacts for all orgs
    """
    logger.info("Starting contact sync for all orgs...")
    for org in Org.objects.filter(is_active=True):
        run_org_task(org, sync_org_contacts)<commit_msg>New cached redis timestamp for last contact sync. Call dash sync_pull_contacts with the timestamp to only get updated contacts.<commit_after>from __future__ import absolute_import, unicode_literals

from django.utils import timezone

from celery.utils.log import get_task_logger
from djcelery_transactions import task
from django_redis import get_redis_connection

from temba_client.utils import parse_iso8601, format_iso8601

from dash.orgs.models import Org
from dash.utils import datetime_to_ms
from dash.utils.sync import sync_pull_contacts, sync_push_contact

from tracpro.orgs_ext.utils import run_org_task


logger = get_task_logger(__name__)

LAST_FETCHED_CONTACTS_RUN_TIME_KEY = 'org:%d:last_fetched_contact_run_time'


@task
def push_contact_change(contact_id, change_type):
    """
    Task to push a local contact change to RapidPro
    """
    from tracpro.groups.models import Group, Region
    from .models import Contact

    contact = Contact.objects.select_related('org', 'region').get(pk=contact_id)
    org = contact.org

    logger.info("Pushing %s change to contact %s" % (change_type.name.upper(), contact.uuid))

    region_uuids = set([r.uuid for r in Region.get_all(org)])
    group_uuids = set([r.uuid for r in Group.get_all(org)])

    sync_push_contact(org, contact, change_type, [region_uuids, group_uuids])


@task
def sync_org_contacts(org_id):
    """
    Syncs all contacts for the given org
    """
    from tracpro.groups.models import Region, Group
    from tracpro.orgs_ext.constants import TaskType
    from .models import Contact

    org = Org.objects.get(pk=org_id)

    logger.info('Starting contact sync task for org #%d' % org.id)

    sync_groups = [r.uuid for r in Region.get_all(org)] + [g.uuid for g in Group.get_all(org)]

    redis_connection = get_redis_connection()
    last_time_key = LAST_FETCHED_CONTACTS_RUN_TIME_KEY % org.pk
    last_time = redis_connection.get(last_time_key)

    created, updated, deleted, failed = sync_pull_contacts(
        org, Contact, fields=(), groups=sync_groups, last_time=last_time)

    task_result = dict(time=datetime_to_ms(timezone.now()),
                       counts=dict(created=len(created),
                                   updated=len(updated),
                                   deleted=len(deleted),
                                   failed=len(failed)))
    org.set_task_result(TaskType.sync_contacts, task_result)

    logger.info("Finished contact sync for org #%d (%d created, "
                "%d updated, %d deleted, %d failed)" %
                (org.id, len(created), len(updated), len(deleted), len(failed)))

    redis_connection.set(last_time_key, format_iso8601(timezone.now()))


@task
def sync_all_contacts():
    """
    Syncs all contacts for all orgs
    """
    logger.info("Starting contact sync for all orgs...")
    for org in Org.objects.filter(is_active=True):
        run_org_task(org, sync_org_contacts)
{'fraction_non_alphanumeric': 0.09740015071590052, 'fraction_numerical': 0.002260738507912585, 'mean_word_length': 3.3731466227347613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6928267', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1633, 'n_words': 416}