<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

from django.conf import settings

from collections import namedtuple
import logging
import os

import django_rq


logger = logging.getLogger(__name__)


AsyncDjangoFile = namedtuple('AsyncDjangoFile', ['data', 'kwargs'])


@django_rq.job
def process_file(raw_file):
    return [AsyncDjangoFile(raw_file, {})]


@django_rq.job
def save_files(model, process, job_id, queue='default'):
    queue = django_rq.get_queue(queue)
    logger.debug('Retrieving results for job {} from queue {}'.format(
        job_id, queue.name))
    processed_files = queue.fetch_job(job_id).result

    for f, kwargs in processed_files:
        logger.debug('Saving file \'{}\' for process {}'.format(
            f.name, process.uuid_full))

        try:
            content_type = f.content_type
        except AttributeError:
            content_type = 'application/octet-stream'

        obj = model.objects.create(
            data=None, process=process, content_type=content_type, **kwargs)
        obj.data = f
        obj.save()
        _save_sample_files(process, obj)


def _save_sample_files(process, file_object):
    """
    Create hard links in sample-specific folders to the datafile in the
    process-specific folder.

    Create sample-specific directory structure:
      samples/<sample.uuid>/<process.slug>/<process.uuid_full.hex>/
    and create a hard link to the file in the process-specific
    directory structure:
      processes/<process.uuid_full.hex>/
    """
    for sample in process.samples(unique=True):
        sample_dir = os.path.abspath(os.path.join(
            settings.MEDIA_ROOT, 'samples', sample.uuid, process.slug))
        target_dir = os.path.join(sample_dir, process.uuid_full.hex)
        try:
            os.makedirs(target_dir)
            logger.debug('created directory {}'.format(target_dir))<commit_msg>Added handling of removing temporary files.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

from django.conf import settings

from collections import namedtuple
import logging
import os

import django_rq


logger = logging.getLogger(__name__)


AsyncDjangoFile = namedtuple('AsyncDjangoFile', ['data', 'kwargs'])


@django_rq.job
def process_file(raw_file):
    return [AsyncDjangoFile(raw_file, {})]


@django_rq.job
def save_files(model, process, job_id, queue='default'):
    queue = django_rq.get_queue(queue)
    logger.debug('Retrieving results for job {} from queue {}'.format(
        job_id, queue.name))
    processed_files = queue.fetch_job(job_id).result

    for f, kwargs in processed_files:
        logger.debug('Saving file \'{}\' for process {}'.format(
            f.name, process.uuid_full))

        try:
            content_type = f.content_type
        except AttributeError:
            content_type = 'application/octet-stream'

        obj = model.objects.create(
            data=None, process=process, content_type=content_type, **kwargs)
        obj.data = f
        obj.save()
        f.close_and_delete()
        _save_sample_files(process, obj)


def _save_sample_files(process, file_object):
    """
    Create hard links in sample-specific folders to the datafile in the
    process-specific folder.

    Create sample-specific directory structure:
      samples/<sample.uuid>/<process.slug>/<process.uuid_full.hex>/
    and create a hard link to the file in the process-specific
    directory structure:
      processes/<process.uuid_full.hex>/
    """
    for sample in process.samples(unique=True):
        sample_dir = os.path.abspath(os.path.join(
            settings.MEDIA_ROOT, 'samples', sample.uuid, process.slug))
        target_dir = os.path.join(sample_dir, process.uuid_full.hex)
        try:
            os.makedirs(target_dir)
            logger.debug('created directory {}'.format(target_dir))
{'fraction_non_alphanumeric': 0.10762560571282836, 'fraction_numerical': 0.000510073960724305, 'mean_word_length': 3.1502645502645503, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5471007', 'n_tokens_mistral': 1162, 'n_tokens_neox': 1145, 'n_words': 275}