<commit_before>########################################################################
#
# University of Southampton IT Innovation Centre, 2011
#
# Copyright in this library belongs to the University of Southampton
# University Road, Highfield, Southampton, UK, SO17 1BJ
#
# This software may not be used, sold, licensed, transferred, copied
# or reproduced in whole or in part in any manner or form or in or
# on any media by any person other than in accordance with the terms
# of the Licence Agreement supplied with the software, or otherwise
# without the prior written consent of the copyright owners.
#
# This software is distributed WITHOUT ANY WARRANTY, without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE, except where stated in the Licence Agreement supplied with
# the software.
#
#	Created By :			<NAME>
#	Created Date :			2011-07-26
#	Created for Project :		Postmark
#
########################################################################
from celery.task import task
import pycurl
import logging
import os
import re
import tempfile
import datetime
import shutil
import os.path
import time
import dataservice.utils as utils
import settings as settings
import paramiko
import uuid
import tarfile
import simplejson as json
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from ssh import MultiSSHClient
from dataservice.models import MFile
from jobservice.models import JobOutput
from django.shortcuts import render_to_response

def tar_files(temp_tarfile, files):
    tar = tarfile.open(temp_tarfile, "w:gz")
    for name in files:
        (head,tail) = os.path.split(name)
        tar.add(name,arcname=tail)
    tar.close()


def get_files(directory, prefix=None, suffix=None, after=None):
    _files=[]
    for fname in os.listdir(directory):
        file = os.path.join(directory,fname)
        if not os.path.isfile(file):
            continue
        if after:
            mtime = os.path.getmtime(file)
            if datetime.datetime.fromtimestamp(mtime) <= after:
                continue
        if prefix:
            if not re.match(prefix,fname):
                continue
        if suffix:
            if not re.search(suffix+"$",fname):
                continue
        _files.append(file)
    return _files


def _ssh_r2d(file, export_type, tmpimage, start_frame=1, end_frame=2):

    ssh = MultiSSHClient()

    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(settings.R3D_HOST, username=settings.R3D_USER, password=settings.R3D_<PASSWORD>)
    output_file_dir,output_file_name = os.path.split(tmpimage)

    command = "redline --i %s --outDir %s -o %s --exportPreset %s -s %s -e %s" % (file, output_file_dir, output_file_name, export_type, start_frame, end_frame)
    logging.info(command)

    stdin, stdout, stderr = ssh.exec_command(command)

    result = {}
    result["stdout"] = stdout.readlines()
    result["sdterr"] = stderr.readlines()

    return result

@task
def red2dtranscode(inputs,outputs,options={},callbacks=[]):

    started = datetime.datetime.now()

    logging.info("Inputs %s"% (inputs))
    if len(inputs) > 0:
        input_mfile  = MFile.objects.get(id=inputs[0])
        logging.info("R2D input file %s" % input_mfile)
        remote_mount = "/Volumes/ifs/mserve/"

        start_frame = 1
        if "start_frame" in options:
            start_frame = options["start_frame"]

        end_frame = 2
        if "end_frame" in options:
            end_frame = options["end_frame"]

        export_type = "tiff"
        if "export_type" in options and options["export_type"] != "":
            export_type = options["export_type"]
        
        file_local_mount = os.path.join(settings.STORAGE_ROOT)
        if input_mfile.service.container.default_path:
            file_local_mount = os.path.join(settings.STORAGE_ROOT, input_mfile.service.container.default_path)

        file_path=input_mfile.file.path
        file_relative= os.path.join(file_path.replace(file_local_mount,remote_mount))
        tfile_uuid = "r2d-image-"+str(uuid.uuid4())
        remoteimage = os.path.join(remote_mount,tfile_uuid)

        if export_type == "tiff":
            suffix = "."+("0".zfill(6))+".tif"
            localimage = os.path.join(file_local_mount,tfile_uuid,tfile_uuid+suffix)
        elif export_type == "avid":
            input_name = input_mfile.name.split(".")[0]
            fname = input_name+".mxf"
            localimage = os.path.join(file_local_mount,"MXF",fname)
        elif export_type == "fcp":
            fname = tfile_uuid+".mov"
            localimage = os.path.join(file_local_mount,fname)
        else:
            raise Exception("Unknown export type '%s'" % export_type)

        logging.info("file_local_mount %s" % file_local_mount)
        logging.info("file_path %s" % file_path)
        logging.info("file_relative %s" % file_relative)
        logging.info("remoteimage %s" % remoteimage)
        logging.info("localimage %s" % localimage)

        result = _ssh_r2d(file_relative,export_type,remoteimage,start_frame=start_frame,end_frame=end_frame)
        outputfile = None

        if export_type == "tiff":
            localdir = os.path.join(file_local_mount,tfile_uuid)
            temp_tarfile = tempfile.NamedTemporaryFile('wb')
            tar_files(temp_tarfile.name, [localdir] )
            outputfile = open(temp_tarfile.name, 'r')

        if export_type == "avid":
            localdir = os.path.join(file_local_mount,"MXF")
            files = get_files(localdir, prefix=input_mfile.name[:8], after=started)
            logging.info(files)
            temp_tarfile = tempfile.NamedTemporaryFile('wb')
            tar_files(temp_tarfile.name, files )
            outputfile = open(temp_tarfile.name, 'r')

        if export_type == "fcp":
            files = [localimage]
            logging.info(files)
            temp_tarfile = tempfile.NamedTemporaryFile('wb')
            tar_files(temp_tarfile.name, files )
            outputfile = open(temp_tarfile.name, 'r')

        logging.info("outputfile %s", outputfile)

        if outputfile:
            suf = SimpleUploadedFile("mfile",outputfile.read(), content_type='image/tiff')

            if len(outputs)>0:
                jo = JobOutput.objects.get(id=outputs[0])
                jo.file.save('results.tar.gz', suf, save=True)
            else:
                logging.error("Nowhere to save output")

            outputfile.close()
        else:
            raise Exception("Unable to get outputfile location")
<commit_msg>Fix red2dtranscode tars. We now output into a new uuid named folder and return all resulting files as the output.<commit_after>########################################################################
#
# University of Southampton IT Innovation Centre, 2011
#
# Copyright in this library belongs to the University of Southampton
# University Road, Highfield, Southampton, UK, SO17 1BJ
#
# This software may not be used, sold, licensed, transferred, copied
# or reproduced in whole or in part in any manner or form or in or
# on any media by any person other than in accordance with the terms
# of the Licence Agreement supplied with the software, or otherwise
# without the prior written consent of the copyright owners.
#
# This software is distributed WITHOUT ANY WARRANTY, without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE, except where stated in the Licence Agreement supplied with
# the software.
#
#	Created By :			<NAME>
#	Created Date :			2011-07-26
#	Created for Project :		Postmark
#
########################################################################
from celery.task import task
import pycurl
import logging
import os
import re
import tempfile
import datetime
import shutil
import os.path
import time
import dataservice.utils as utils
import settings as settings
import paramiko
import uuid
import tarfile
import simplejson as json
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from ssh import MultiSSHClient
from dataservice.models import MFile
from jobservice.models import JobOutput
from django.shortcuts import render_to_response

def tar_files(temp_tarfile, base_dir, files):
    tar = tarfile.open(temp_tarfile, "w:gz")
    for name in files:
        fname = os.path.join(base_dir, name)
        aname = os.path.relpath(fname, base_dir)
        tar.add(fname, arcname=aname)
    tar.close()


def get_files(directory, prefix=None, suffix=None, after=None):
    _files=[]
    for fname in os.listdir(directory):
        file = os.path.join(directory,fname)
        if not os.path.isfile(file):
            continue
        if after:
            mtime = os.path.getmtime(file)
            if datetime.datetime.fromtimestamp(mtime) <= after:
                continue
        if prefix:
            if not re.match(prefix,fname):
                continue
        if suffix:
            if not re.search(suffix+"$",fname):
                continue
        _files.append(file)
    return _files


def _ssh_r2d(file, export_type, tmp_folder, start_frame=1, end_frame=2):

    ssh = MultiSSHClient()

    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(settings.R3D_HOST, username=settings.R3D_USER, password=settings.R3D_PASS)
    #output_file_dir,output_file_name = os.path.split(tmpimage)

    command = "redline --i %s --outDir %s/%s --exportPreset %s -s %s -e %s" % (file, tmp_folder, export_type, start_frame, end_frame)
    logging.info(command)

    stdin, stdout, stderr = ssh.exec_command(command)

    result = {}
    result["stdout"] = stdout.readlines()
    result["sdterr"] = stderr.readlines()

    return result

@task
def red2dtranscode(inputs,outputs,options={},callbacks=[]):

    started = datetime.datetime.now()

    logging.info("Inputs %s"% (inputs))
    if len(inputs) > 0:
        input_mfile  = MFile.objects.get(id=inputs[0])
        logging.info("R2D input file %s" % input_mfile)
        remote_mount = "/Volumes/ifs/mserve/"

        start_frame = 1
        if "start_frame" in options:
            start_frame = options["start_frame"]

        end_frame = 2
        if "end_frame" in options:
            end_frame = options["end_frame"]

        export_type = "tiff"
        if "export_type" in options and options["export_type"] != "":
            export_type = options["export_type"]
        
        file_local_mount = os.path.join(settings.STORAGE_ROOT)
        if input_mfile.service.container.default_path:
            file_local_mount = os.path.join(settings.STORAGE_ROOT, input_mfile.service.container.default_path)

        file_path=input_mfile.file.path
        file_relative= os.path.join(file_path.replace(file_local_mount,remote_mount))
        tfile_uuid = "r2d-transcode-"+str(uuid.uuid4())
        remoteimage = os.path.join(remote_mount,tfile_uuid)
        localimage = os.path.join(file_local_mount,tfile_uuid)

        logging.info("file_local_mount %s" % file_local_mount)
        logging.info("file_path %s" % file_path)
        logging.info("file_relative %s" % file_relative)
        logging.info("remoteimage %s" % remoteimage)
        logging.info("localimage %s" % localimage)

        result = _ssh_r2d(file_relative,export_type,remoteimage,start_frame=start_frame,end_frame=end_frame)

        localdir = os.path.join(file_local_mount,tfile_uuid)
        files = os.listdir(localdir)
        temp_tarfile = tempfile.NamedTemporaryFile('wb')
        tar_files(temp_tarfile.name, localdir, files)
        outputfile = open(temp_tarfile.name, 'r')

        logging.info("outputfile %s", outputfile)

        if outputfile:
            suf = SimpleUploadedFile("mfile",outputfile.read(), content_type='image/tiff')

            if len(outputs)>0:
                jo = JobOutput.objects.get(id=outputs[0])
                jo.file.save('results.tar.gz', suf, save=True)
            else:
                logging.error("Nowhere to save output")

            outputfile.close()
        else:
            raise Exception("Unable to get outputfile location")

{'fraction_non_alphanumeric': 0.12063570487483531, 'fraction_numerical': 0.005681818181818182, 'mean_word_length': 3.07824042981867, 'pattern_counts': {'":': 8, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1055141', 'n_tokens_mistral': 3700, 'n_tokens_neox': 3574, 'n_words': 892}