<commit_before>from __future__ import absolute_import

#Because nertag.py imports below, but isn't available in context of workers
import re
import os
from nltk.tag.stanford import NERTagger
import time
import pickle

from itertools import izip
from collections import defaultdict

from celery import chord, group
from articurate.celery import celery
from articurate.nertagger import nertag
import articurate.utils.loader as article_loader
import articurate
from articurate.metrics import metrics
from celery import current_task
from celery.utils.log import get_task_logger

from articurate.utils.config import *
from articurate.pymotherlode import api

import simplejson as json

logger = get_task_logger(__name__)

@celery.task
def run_nertag():

    # get latest dump of articles
    articles = article_loader.get_latest_dump()

    print "Got so many articles: ", len(articles)

    ner_types = ['ORGANIZATION', 'LOCATION', 'PERSON']    

    try:
        print "run_nertag: starting ", ner_types
        
        if config['nertag.content']:
            all_content = [article.content for count, article in enumerate(articles)]
        else:
            all_content = [article.title for count, article in enumerate(articles)]            

        result = chord(parse_NER_celery.s(article, count, ner_types) for count, article in enumerate(all_content))(save_celery.s(kwargs={'ner_types': ner_types}))
        
        print "run_nertag: done! ", ner_types
        return 'True'

    except:
        return 'False'

@celery.task
@metrics.track
def save_celery(results, **kwargs):
    print "save_celery: starting "
        
    ner_types = ['ORGANIZATION', 'LOCATION', 'PERSON']
   
    # get what is currently present in redis db
    try:
        final_dict = json.loads(api.getMetric("articurate.nertagger.celery_tasks.save_celery"))
    except:
        pass

    if final_dict == None:
        final_dict = {}

    for item in ner_types:
        value = final_dict[item] if item in final_dict else []
        for dictionary in results:
            value.extend(dictionary[item])
        value = list(set(value))
        final_dict[item] = value <commit_msg>Fix for UnboundLocalError<commit_after>from __future__ import absolute_import

#Because nertag.py imports below, but isn't available in context of workers
import re
import os
from nltk.tag.stanford import NERTagger
import time
import pickle

from itertools import izip
from collections import defaultdict

from celery import chord, group
from articurate.celery import celery
from articurate.nertagger import nertag
import articurate.utils.loader as article_loader
import articurate
from articurate.metrics import metrics
from celery import current_task
from celery.utils.log import get_task_logger

from articurate.utils.config import *
from articurate.pymotherlode import api

import simplejson as json

logger = get_task_logger(__name__)

@celery.task
def run_nertag():

    # get latest dump of articles
    articles = article_loader.get_latest_dump()

    print "Got so many articles: ", len(articles)

    ner_types = ['ORGANIZATION', 'LOCATION', 'PERSON']    

    try:
        print "run_nertag: starting ", ner_types
        
        if config['nertag.content']:
            all_content = [article.content for count, article in enumerate(articles)]
        else:
            all_content = [article.title for count, article in enumerate(articles)]            

        result = chord(parse_NER_celery.s(article, count, ner_types) for count, article in enumerate(all_content))(save_celery.s(kwargs={'ner_types': ner_types}))
        
        print "run_nertag: done! ", ner_types
        return 'True'

    except:
        return 'False'

@celery.task
@metrics.track
def save_celery(results, **kwargs):
    print "save_celery: starting "
        
    ner_types = ['ORGANIZATION', 'LOCATION', 'PERSON']
   
    # get what is currently present in redis db
    try:
        final_dict = json.loads(api.getMetric("articurate.nertagger.celery_tasks.save_celery"))
    except:
        final_dict = None
        pass

    if final_dict == None:
        final_dict = {}

    for item in ner_types:
        value = final_dict[item] if item in final_dict else []
        for dictionary in results:
            value.extend(dictionary[item])
        value = list(set(value))
        final_dict[item] = value 
{'fraction_non_alphanumeric': 0.07989926739926739, 'fraction_numerical': 0.0, 'mean_word_length': 3.291748526522593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5199183', 'n_tokens_mistral': 1324, 'n_tokens_neox': 1259, 'n_words': 382}