<commit_before>from __future__ import print_function

import json

from django.conf import settings
from django.http import HttpResponse

from catmaid.control.authentication import requires_user_role
from catmaid.control.common import get_relation_to_id_map, get_class_to_id_map
from catmaid.control.common import json_error_response, id_generator
from catmaid.control.cropping import CropJob, extract_substack, process_crop_job
from catmaid.models import ClassInstanceClassInstance, TreenodeConnector
from catmaid.models import UserRole

from celery.task import task

import os.path
import shutil
import tarfile


# Prefix for stored archive files
connector_file_prefix = "connector_archive_"
# The path were archive files get stored in
connector_output_path = os.path.join(settings.MEDIA_ROOT,
    settings.MEDIA_CONNECTOR_SUBDIRECTORY)

class ConnectorExportJob:
    """ A container with data needed for the creation of a connector archive.
    """
    def __init__(self, user, project_id, stack_id, skeleton_ids,
            x_radius, y_radius, z_radius, sample):
        # Sanity checks
        if not skeleton_ids:
            raise Exception("Please specify at least on skeleton ID")
        try:
            project_id = int(project_id)
            stack_id = int(stack_id)
        except (ValueError, TypeError) as e:
            raise Exception("Couldn't determine stack or project ID")
        try:
            x_radius = int(x_radius)
            y_radius = int(y_radius)
            z_radius = int(z_radius)
        except (ValueError, TypeError) as e:
            raise Exception("The x_radius, y_radius and z_radius parameters have " \
                    "to be numbers!")
        try:
            # Expect a boolean or a number
            sample = bool(int(sample))
        except (ValueError, TypeError) as e:
            raise ValueError("The sample parameter has to be a number or a" \
                    "boolean!")

        # Store data
        self.user = user
        self.project_id = project_id
        self.stack_id = stack_id
        self.skeleton_ids = skeleton_ids
        self.x_radius = x_radius
        self.y_radius = y_radius
        self.z_radius = z_radius
        self.sample = sample

        # Output path for this job will be initialized, when needed
        self.output_path = None

        # Cache for neuron and relation folder names
        self.skid_to_neuron_folder = {}
        self.relid_to_rel_folder = {}

        # Get relation map
        self.relation_map = get_relation_to_id_map(project_id)

    def create_basic_output_path(self):
        """ Will create a random <connector_file_prefix> prefixed output folder
        name as well as the actual directory.
        """
        # Find non-existing random folder name
        while True:
            folder_name = connector_file_prefix + id_generator()
            output_path = os.path.join(connector_output_path, folder_name)
            if not os.path.exists(output_path):
                break
        # Create folder and store path as field
        os.makedirs(output_path)
        self.output_path = output_path

    def create_connector_path(self, connector):
        """ Based on the output path, this function will create a folder
        structure for a particular connector. Things that are supposedly
        needed multiple times, will be cached. This function will also make
        sure the path exists and is ready to be written to.
        """
        # Get (and create if needed) cache entry for string of neuron id
        if connector.skeleton_id not in self.skid_to_neuron_folder:
            neuron_cici = ClassInstanceClassInstance.objects.get(
                    relation_id=self.relation_map['model_of'],
                    project_id=self.project_id,
                    class_instance_a=connector.skeleton.id)
            self.skid_to_neuron_folder[connector.skeleton.id] = \
                    str(neuron_cici.class_instance_b_id)
        neuron_folder = self.skid_to_neuron_folder[connector.skeleton.id]

        # get (and create if needed) cache entry for string of relation name
        if connector.relation_id not in self.relid_to_rel_folder:
            if connector.relation_id == self.relation_map['presynaptic_to']:
                rel_folder = "presynaptic"
            elif connector.relation_id == self.relation_map['postsynaptic_to']:
                rel_folder = "postsynaptic"
            else:
                rel_folder = "unknown_" + str(connector.relation_id)
            self.relid_to_rel_folder[connector.relation_id] = rel_folder
        relation_folder =  self.relid_to_rel_folder[connector.relation_id]

        # Create path output_path/neuron_id/relation_name/connector_id
        connector_path = os.path.join(self.output_path, neuron_folder,
                relation_folder, str(connector.id))
        try:
            os.makedirs(connector_path)
        except OSError as e:
            # Everything is fine if the path exists and is writable
            if not os.path.exists(connector_path) or not \
                    os.access(connector_path, os.W_OK):
                raise e

        return connector_path

def export_single_connector(job, connector_link):
    """ Exports a single connector and expects the output path to be existing
    and writable.
    """
    connector_path = job.create_connector_path(connector_link)
    connector_image_path = os.path.join(connector_path, 'connector.tiff')
    connector = connector_link.connector

    # Calculate bounding box for current connector
    x_min = connector.location.x - job.x_radius
    x_max = connector.location.x + job.x_radius
    y_min = connector.location.y - job.y_radius
    y_max = connector.location.y + job.y_radius
    z_min = connector.location.z - job.z_radius
    z_max = connector.location.z + job.z_radius
    rotation_cw = 0
    zoom_level = 0

    # Crop images as single multi-page TIFF
    crop_job = CropJob(job.user, job.project_id, job.stack_id, x_min, x_max,
            y_min, y_max, z_min, z_max, rotation_cw, zoom_level,
            single_channel=True, output_path=connector_image_path)
    error = process_crop_job(crop_job, create_message=False)
    if error:
        process_connector_export_job.get_logger().info("An error occured: " + e)

@task()
def process_connector_export_job(job):
    """ This method does the actual archive creation. It controls the data
    extraction and the creation of all sub-stacks. It can be executed as Celery
    task. If the job asks only for a sample, the first pre-synaptic connector
    of the first skeleton will be used. If such a connector doesn't exist, the
    first one found is used. Otherwise, if no sample should be taken, all
    connectors of all skeletons are exported.
    """
    if job.sample:
        # First try to get a pre-synaptic connector, because these are usually a
        # larger than the post-synaptic ones.
        try:<commit_msg>Connector export: export single images instread of multipage TIFFs<commit_after>from __future__ import print_function

import json

from django.conf import settings
from django.http import HttpResponse

from catmaid.control.authentication import requires_user_role
from catmaid.control.common import get_relation_to_id_map, get_class_to_id_map
from catmaid.control.common import json_error_response, id_generator
from catmaid.control.cropping import CropJob, extract_substack, process_crop_job
from catmaid.models import ClassInstanceClassInstance, TreenodeConnector
from catmaid.models import UserRole

from celery.task import task

import os.path
import shutil
import tarfile


# Prefix for stored archive files
connector_file_prefix = "connector_archive_"
# The path were archive files get stored in
connector_output_path = os.path.join(settings.MEDIA_ROOT,
    settings.MEDIA_CONNECTOR_SUBDIRECTORY)

class ConnectorExportJob:
    """ A container with data needed for the creation of a connector archive.
    """
    def __init__(self, user, project_id, stack_id, skeleton_ids,
            x_radius, y_radius, z_radius, sample):
        # Sanity checks
        if not skeleton_ids:
            raise Exception("Please specify at least on skeleton ID")
        try:
            project_id = int(project_id)
            stack_id = int(stack_id)
        except (ValueError, TypeError) as e:
            raise Exception("Couldn't determine stack or project ID")
        try:
            x_radius = int(x_radius)
            y_radius = int(y_radius)
            z_radius = int(z_radius)
        except (ValueError, TypeError) as e:
            raise Exception("The x_radius, y_radius and z_radius parameters have " \
                    "to be numbers!")
        try:
            # Expect a boolean or a number
            sample = bool(int(sample))
        except (ValueError, TypeError) as e:
            raise ValueError("The sample parameter has to be a number or a" \
                    "boolean!")

        # Store data
        self.user = user
        self.project_id = project_id
        self.stack_id = stack_id
        self.skeleton_ids = skeleton_ids
        self.x_radius = x_radius
        self.y_radius = y_radius
        self.z_radius = z_radius
        self.sample = sample

        # Output path for this job will be initialized, when needed
        self.output_path = None

        # Cache for neuron and relation folder names
        self.skid_to_neuron_folder = {}
        self.relid_to_rel_folder = {}

        # Get relation map
        self.relation_map = get_relation_to_id_map(project_id)

    def create_basic_output_path(self):
        """ Will create a random <connector_file_prefix> prefixed output folder
        name as well as the actual directory.
        """
        # Find non-existing random folder name
        while True:
            folder_name = connector_file_prefix + id_generator()
            output_path = os.path.join(connector_output_path, folder_name)
            if not os.path.exists(output_path):
                break
        # Create folder and store path as field
        os.makedirs(output_path)
        self.output_path = output_path

    def create_connector_path(self, connector):
        """ Based on the output path, this function will create a folder
        structure for a particular connector. Things that are supposedly
        needed multiple times, will be cached. This function will also make
        sure the path exists and is ready to be written to.
        """
        # Get (and create if needed) cache entry for string of neuron id
        if connector.skeleton_id not in self.skid_to_neuron_folder:
            neuron_cici = ClassInstanceClassInstance.objects.get(
                    relation_id=self.relation_map['model_of'],
                    project_id=self.project_id,
                    class_instance_a=connector.skeleton.id)
            self.skid_to_neuron_folder[connector.skeleton.id] = \
                    str(neuron_cici.class_instance_b_id)
        neuron_folder = self.skid_to_neuron_folder[connector.skeleton.id]

        # get (and create if needed) cache entry for string of relation name
        if connector.relation_id not in self.relid_to_rel_folder:
            if connector.relation_id == self.relation_map['presynaptic_to']:
                rel_folder = "presynaptic"
            elif connector.relation_id == self.relation_map['postsynaptic_to']:
                rel_folder = "postsynaptic"
            else:
                rel_folder = "unknown_" + str(connector.relation_id)
            self.relid_to_rel_folder[connector.relation_id] = rel_folder
        relation_folder =  self.relid_to_rel_folder[connector.relation_id]

        # Create path output_path/neuron_id/relation_name/connector_id
        connector_path = os.path.join(self.output_path, neuron_folder,
                relation_folder, str(connector.id))
        try:
            os.makedirs(connector_path)
        except OSError as e:
            # Everything is fine if the path exists and is writable
            if not os.path.exists(connector_path) or not \
                    os.access(connector_path, os.W_OK):
                raise e

        return connector_path

def export_single_connector(job, connector_link):
    """ Exports a single connector and expects the output path to be existing
    and writable.
    """
    connector_path = job.create_connector_path(connector_link)
    connector_image_path = os.path.join(connector_path, 'connector.tiff')
    connector = connector_link.connector

    # Calculate bounding box for current connector
    x_min = connector.location.x - job.x_radius
    x_max = connector.location.x + job.x_radius
    y_min = connector.location.y - job.y_radius
    y_max = connector.location.y + job.y_radius
    z_min = connector.location.z - job.z_radius
    z_max = connector.location.z + job.z_radius
    rotation_cw = 0
    zoom_level = 0

    # Create a single file for each section (instead of a mulipage TIFF)
    crop_job = CropJob(job.user, job.project_id, job.stack_id, x_min, x_max,
            y_min, y_max, z_min, z_max, rotation_cw, zoom_level,
            single_channel=True)
    cropped_stack = extract_substack(crop_job)
    # Save each file in output path
    connector_path = job.create_connector_path(connector_link)
    for i, img in enumerate(cropped_stack):
        # Save image in output path, named after the image center's coordinates,
        # rounded to full integers.
        x = int(connector.location.x + 0.5)
        y = int(connector.location.y + 0.5)
        z = int(z_min + i * crop_job.stacks[0].resolution.z  + 0.5)
        image_name = "%s_%s_%s.tiff" % (x, y, z)
        connector_image_path = os.path.join(connector_path, image_name)
        img.write(connector_image_path)

@task()
def process_connector_export_job(job):
    """ This method does the actual archive creation. It controls the data
    extraction and the creation of all sub-stacks. It can be executed as Celery
    task. If the job asks only for a sample, the first pre-synaptic connector
    of the first skeleton will be used. If such a connector doesn't exist, the
    first one found is used. Otherwise, if no sample should be taken, all
    connectors of all skeletons are exported.
    """
    if job.sample:
        # First try to get a pre-synaptic connector, because these are usually a
        # larger than the post-synaptic ones.
        try:
{'fraction_non_alphanumeric': 0.06656046543842638, 'fraction_numerical': 0.0007618783765064414, 'mean_word_length': 2.631539235412475, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6536757', 'n_tokens_mistral': 4190, 'n_tokens_neox': 4011, 'n_words': 1207}