<commit_before>from __future__ import absolute_import
import os

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oclapi.settings.local')
os.environ.setdefault('DJANGO_CONFIGURATION', 'Local')

# order of imports seems to matter. Do the django-configuration
# import first
from configurations import importer
importer.install()

from celery import Celery
from celery.utils.log import get_task_logger
from celery_once import QueueOnce
from concepts.models import ConceptVersion
from mappings.models import Mapping
from oclapi.utils import update_all_in_index, write_export_file
from sources.models import SourceVersion

celery = Celery('tasks', backend='redis://redis.openconceptlab.org:6379/0', broker='django://')
celery.config_from_object('django.conf:settings')

logger = get_task_logger('celery.worker')

@celery.task(base=QueueOnce)
def export_source(version_id):
    logger.info('Finding source version...')
    version = SourceVersion.objects.get(id=version_id)
    logger.info('Found source version %s.  Beginning export...' % version.mnemonic)
    write_export_file(version, logger)
    logger.info('Export complete!')


@celery.task
def update_children_for_source_version(version_id):
    sv = SourceVersion.objects.get(id=version_id)
    sv._ocl_processing = True
    sv.save()
    versions = ConceptVersion.objects.filter(id__in=sv.concepts)
    update_all_in_index(ConceptVersion, versions)
    mappings = Mapping.objects.filter(id__in=sv.mappings)
    update_all_in_index(Mapping, mappings)
    sv._ocl_processing = False
    sv.save()<commit_msg>+Anshu | 153 | celery_once url for redis in tasks<commit_after>from __future__ import absolute_import
import os

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oclapi.settings.local')
os.environ.setdefault('DJANGO_CONFIGURATION', 'Local')

# order of imports seems to matter. Do the django-configuration
# import first
from configurations import importer
importer.install()

from celery import Celery
from celery.utils.log import get_task_logger
from celery_once import QueueOnce
from concepts.models import ConceptVersion
from mappings.models import Mapping
from oclapi.utils import update_all_in_index, write_export_file
from sources.models import SourceVersion

celery = Celery('tasks', backend='redis://redis.openconceptlab.org:6379/0', broker='django://')
celery.config_from_object('django.conf:settings')

logger = get_task_logger('celery.worker')
celery.conf.ONCE_REDIS_URL = 'redis://redis.openconceptlab.org:6379/0'

@celery.task(base=QueueOnce)
def export_source(version_id):
    logger.info('Finding source version...')
    version = SourceVersion.objects.get(id=version_id)
    logger.info('Found source version %s.  Beginning export...' % version.mnemonic)
    write_export_file(version, logger)
    logger.info('Export complete!')


@celery.task
def update_children_for_source_version(version_id):
    sv = SourceVersion.objects.get(id=version_id)
    sv._ocl_processing = True
    sv.save()
    versions = ConceptVersion.objects.filter(id__in=sv.concepts)
    update_all_in_index(ConceptVersion, versions)
    mappings = Mapping.objects.filter(id__in=sv.mappings)
    update_all_in_index(Mapping, mappings)
    sv._ocl_processing = False
    sv.save()
{'fraction_non_alphanumeric': 0.10188912976153608, 'fraction_numerical': 0.005574481263549086, 'mean_word_length': 5.814345991561181, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2069792', 'n_tokens_mistral': 1035, 'n_tokens_neox': 997, 'n_words': 208}