<commit_before>import os
import random
import string
import subprocess
import time
import warnings

from django.utils import timezone
from typing import Dict

from data_refinery_common.logging import get_and_configure_logger
from data_refinery_common.models import (
    ComputationalResult,
    ComputationalResultAnnotation,
    ComputedFile,
    OriginalFile,
    Pipeline,
    Processor,
    SampleComputedFileAssociation,
    SampleResultAssociation,
)
from data_refinery_common.utils import get_env_variable
from data_refinery_workers.processors import utils, smasher


S3_BUCKET_NAME = get_env_variable("S3_BUCKET_NAME", "data-refinery")
logger = get_and_configure_logger(__name__)


def _prepare_input(job_context: Dict) -> Dict:

    # We're going to use the smasher outside of the smasher.
    # I'm not crazy about this yet. Maybe refactor later,
    # but I need the data now.

    job_context = smasher._prepare_files(job_context)
    job_context = smasher._smash(job_context)

    if not 'final_frame' in job_context.keys():
        logger.error("Unable to prepare files for creating QN target.",
            job_id=job_context['job'].id)
        job_context["job"].failure_reason = "Couldn't prepare files creating QN target (no final_frame)."
        job_context['success'] = False
        return job_context

    # We only need the resulting frame, not the entire archive
    os.makedirs(job_context["work_dir"])
    outfile_base = job_context['work_dir'] + str(time.time()).split('.')[0]
    outfile = outfile_base + '.tsv'
    job_context['final_frame'].to_csv(outfile, sep='\t', encoding='utf-8')
    job_context['smashed_file'] = outfile
    job_context['target_file'] = outfile_base + '_target.tsv'

    return job_context

def _quantile_normalize(job_context: Dict) -> Dict:
    """Run the R script we have to create the reference for QN.
    """
    try:
        job_context['time_start'] = timezone.now()

        job_context['formatted_command'] = [
            "/usr/bin/Rscript",
            "--vanilla",
            "/home/user/data_refinery_workers/processors/qn_reference.R",
            "--inputFile", job_context['smashed_file'],
            "--outputFile", job_context['target_file']
        ]
<commit_msg>fix #615 probably<commit_after>import os
import random
import string
import subprocess
import time
import warnings

from django.utils import timezone
from typing import Dict

from data_refinery_common.logging import get_and_configure_logger
from data_refinery_common.models import (
    ComputationalResult,
    ComputationalResultAnnotation,
    ComputedFile,
    OriginalFile,
    Pipeline,
    Processor,
    SampleComputedFileAssociation,
    SampleResultAssociation,
)
from data_refinery_common.utils import get_env_variable
from data_refinery_workers.processors import utils, smasher


S3_BUCKET_NAME = get_env_variable("S3_BUCKET_NAME", "data-refinery")
logger = get_and_configure_logger(__name__)


def _prepare_input(job_context: Dict) -> Dict:

    # We're going to use the smasher outside of the smasher.
    # I'm not crazy about this yet. Maybe refactor later,
    # but I need the data now.

    job_context = smasher._prepare_files(job_context)
    job_context = smasher._smash(job_context)

    if not 'final_frame' in job_context.keys():
        logger.error("Unable to prepare files for creating QN target.",
            job_id=job_context['job'].id)
        job_context["job"].failure_reason = "Couldn't prepare files creating QN target (no final_frame)."
        job_context['success'] = False
        return job_context

    # work_dir is already created by smasher._prepare_files
    outfile_base = job_context['work_dir'] + str(time.time()).split('.')[0]
    outfile = outfile_base + '.tsv'
    job_context['final_frame'].to_csv(outfile, sep='\t', encoding='utf-8')
    job_context['smashed_file'] = outfile
    job_context['target_file'] = outfile_base + '_target.tsv'

    return job_context

def _quantile_normalize(job_context: Dict) -> Dict:
    """Run the R script we have to create the reference for QN.
    """
    try:
        job_context['time_start'] = timezone.now()

        job_context['formatted_command'] = [
            "/usr/bin/Rscript",
            "--vanilla",
            "/home/user/data_refinery_workers/processors/qn_reference.R",
            "--inputFile", job_context['smashed_file'],
            "--outputFile", job_context['target_file']
        ]

{'fraction_non_alphanumeric': 0.09864864864864865, 'fraction_numerical': 0.0024774774774774773, 'mean_word_length': 3.704449152542373, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6611202', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1342, 'n_words': 323}