<commit_before>'''
Created on May 23, 2014

@author: <NAME>
'''

from celery.canvas import chord
import flask

import superdesk
from superdesk.io import providers

from .utc import utc, utcnow
from superdesk.celery_app import celery, finish_task_for_progress,\
    finish_subtask_from_progress, add_subtask_to_progress
from celery.result import AsyncResult
from flask.globals import current_app as app
from .items import import_rendition, import_media
from superdesk.base_model import BaseModel
from settings import CELERY_ALWAYS_EAGER


def update_status(task_id, current, total):
    if current is None:
        current = 0
    archive_item.update_state(task_id, state='PROGRESS', meta={'current': int(current), 'total': int(total)})


@celery.task(bind=True, max_retries=3)
def archive_media(self, task_id, guid, href):
    try:
        if self.request.retries == 0:
            update_status(*add_subtask_to_progress(task_id))
        import_media(guid, href)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)


@celery.task(bind=True, max_retries=3)
def archive_rendition(self, task_id, guid, name, href):
    try:
        if self.request.retries == 0:
            update_status(*add_subtask_to_progress(task_id))
        import_rendition(guid, name, href)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)


@celery.task()
def update_item(result, is_main_task, task_id, guid):
    # update import status as done
    data = app.data
    data.update('archive', guid, {"task_id": ""})

    if is_main_task:
        update_status(*finish_task_for_progress(task_id))


@celery.task(bind=True, max_retries=3)
def archive_item(self, guid, provider_id, user, task_id=None):
    data = app.data

    # I CELERY_ALWAYS_EAGER=True the created request context is empty and already initialized one is on request_stack
    if CELERY_ALWAYS_EAGER:
        self.request_stack.pop()

    crt_task_id = self.request.id
    if not task_id:
        task_id = crt_task_id

    if self.request.retries == 0:
        update_status(*add_subtask_to_progress(task_id))

    provider = data.find_one('ingest_providers', _id=provider_id, req=None)
    if provider is None:
        payload = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (guid, provider_id)
        raise superdesk.SuperdeskError(payload=payload)
    service_provider = providers[provider.get('type')]
    service_provider.provider = provider

    item = None
    try:
        items = service_provider.get_items(guid)
    except Exception:
        raise self.retry(countdown=2)

    for item_it in items:
        if item_it['guid'] == guid:
            item = item_it<commit_msg>SD-404 : Write more behave tests, especially for error flows<commit_after>'''
Created on May 23, 2014

@author: <NAME>
'''

from celery.canvas import chord
import flask

import superdesk
from superdesk.io import providers

from .utc import utc, utcnow
from superdesk.celery_app import celery, finish_task_for_progress,\
    finish_subtask_from_progress, add_subtask_to_progress
from celery.result import AsyncResult
from flask.globals import current_app as app
from .items import import_rendition, import_media
from superdesk.base_model import BaseModel
from settings import CELERY_ALWAYS_EAGER


def update_status(task_id, current, total):
    if current is None:
        current = 0
    archive_item.update_state(task_id, state='PROGRESS', meta={'current': int(current), 'total': int(total)})


@celery.task(bind=True, max_retries=3)
def archive_media(self, task_id, guid, href):
    try:
        if self.request.retries == 0:
            update_status(*add_subtask_to_progress(task_id))
        import_media(guid, href)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)


@celery.task(bind=True, max_retries=3)
def archive_rendition(self, task_id, guid, name, href):
    try:
        if self.request.retries == 0:
            update_status(*add_subtask_to_progress(task_id))
        import_rendition(guid, name, href)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)


@celery.task()
def update_item(result, is_main_task, task_id, guid):
    # update import status as done
    data = app.data
    data.update('archive', guid, {"task_id": ""})

    if is_main_task:
        update_status(*finish_task_for_progress(task_id))


@celery.task(bind=True, max_retries=3)
def archive_item(self, guid, provider_id, user, task_id=None):
    data = app.data

    # For CELERY_ALWAYS_EAGER=True the current request context is empty but already initialized one is on request_stack
    if CELERY_ALWAYS_EAGER:
        self.request_stack.pop()

    crt_task_id = self.request.id
    if not task_id:
        task_id = crt_task_id

    if self.request.retries == 0:
        update_status(*add_subtask_to_progress(task_id))

    provider = data.find_one('ingest_providers', _id=provider_id, req=None)
    if provider is None:
        payload = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (guid, provider_id)
        raise superdesk.SuperdeskError(payload=payload)
    service_provider = providers[provider.get('type')]
    service_provider.provider = provider

    item = None
    try:
        items = service_provider.get_items(guid)
    except Exception:
        raise self.retry(countdown=2)

    for item_it in items:
        if item_it['guid'] == guid:
            item = item_it
{'fraction_non_alphanumeric': 0.09358906055762742, 'fraction_numerical': 0.006215592257147931, 'mean_word_length': 3.8846487424111014, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3073568', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1806, 'n_words': 437}