<commit_before>
from __future__ import absolute_import

import alog
import boto
import os
import mimetypes
import requests
from ast import literal_eval
from celery import Celery
from django.conf import settings

from firecares.settings.base import REDIS_URL
from firecares.utils.s3put import singlepart_upload
from celery.task import current

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'firecares.settings')

app = Celery('firecares')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
config = app.config_from_object('django.conf:settings')

app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

app.conf.BROKER_URL = REDIS_URL

def download_file(url, download_to=None):

    if not download_to:
        download_to = url.split('/')[-1]

    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(os.path.join(download_to, download_to), 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()
    return download_to

def task_exists(name, args=None, kwargs=None):
    args = args or tuple()
    kwargs = kwargs or {}

    from celery.task.control import inspect
    inspector = inspect()

    print(inspector.app.backend.url)

    def matches(task):
        if name not in task['name']:
            return False

        if args != literal_eval(task['args']):
            return False

        if kwargs != literal_eval(task['kwargs']):
            return False
<commit_msg>Use the right print statement<commit_after>
from __future__ import absolute_import

import alog
import boto
import os
import mimetypes
import requests
from ast import literal_eval
from celery import Celery
from django.conf import settings

from firecares.settings.base import REDIS_URL
from firecares.utils.s3put import singlepart_upload
from celery.task import current

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'firecares.settings')

app = Celery('firecares')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
config = app.config_from_object('django.conf:settings')

app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

app.conf.BROKER_URL = REDIS_URL

def download_file(url, download_to=None):

    if not download_to:
        download_to = url.split('/')[-1]

    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(os.path.join(download_to, download_to), 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()
    return download_to

def task_exists(name, args=None, kwargs=None):
    args = args or tuple()
    kwargs = kwargs or {}

    from celery.task.control import inspect
    inspector = inspect()

    print 'inspector.app.backend.url: {}'.format(inspector.app.backend.url)

    def matches(task):
        if name not in task['name']:
            return False

        if args != literal_eval(task['args']):
            return False

        if kwargs != literal_eval(task['kwargs']):
            return False

{'fraction_non_alphanumeric': 0.08901458767490325, 'fraction_numerical': 0.003572491813039595, 'mean_word_length': 3.380704041720991, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4968232', 'n_tokens_mistral': 1029, 'n_tokens_neox': 993, 'n_words': 304}