<commit_before>import json
import glob
import os.path
import yaml

from django import forms
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponse, HttpResponseRedirect
from django.template import Context, loader
from django.contrib.formtools.wizard.views import SessionWizardView
from django.shortcuts import render_to_response
from django.utils.datastructures import SortedDict

from guardian.models import Permission, User, Group
from guardian.shortcuts import get_perms_for_model, assign

import urllib2 as urllib

from catmaid.models import Project, Stack, ProjectStack, Overlay, Double3D

from catmaid.control.common import urljoin

TEMPLATES = {"pathsettings": "catmaid/import/setup_path.html",
             "projectselection": "catmaid/import/setup_projects.html",
             "confirmation": "catmaid/import/confirmation.html"}

info_file_name = "project.yaml"
datafolder_setting = "CATMAID_IMPORT_PATH"
base_url_setting = "CATMAID_IMPORT_URL"

class ImageBaseMixin:
    def set_image_fields(self, info_object, project_url, data_folder, needs_zoom):
        """ Sets the image_base, num_zoom_levels, file_extension fields
        of the calling object. Favor a URL field, if there is one. A URL
        field, however, also requires the existence of the
        'fileextension' and and 'zoomlevels' field.
        """
        if 'url' in info_object:
            self.image_base = info_object['url']
            if needs_zoom:
                self.num_zoom_levels = info_object['zoomlevels']
            self.file_extension = info_object['fileextension']
        else:
            # The image base of a stack is the combination of the
            # project URL and the stack's folder name.
            folder = info_object['folder']
            self.image_base = urljoin(project_url, folder)
            # Favor 'zoomlevel' and 'fileextension' fields, if
            # available, but try to find this information if thosa
            # fields are not present.
            zoom_available = 'zoomlevels' in info_object
            ext_available = 'fileextension' in info_object
            if zoom_available and needs_zoom:
                self.num_zoom_levels = info_object['zoomlevels']
            if ext_available:
                self.file_extension = info_object['fileextension']
            # Only retrieve file extension and zoom level if one of
            # them is not available in the stack definition.
            if (not zoom_available and needs_zoom) or not ext_available:
                file_ext, zoom_levels = find_zoom_levels_and_file_ext(
                    data_folder, folder, needs_zoom )
                # If there is no zoom level provided, use the found one
                if not zoom_available and needs_zoom:
                    self.num_zoom_levels = zoom_levels
                # If there is no file extension level provided, use the
                # found one
                if not ext_available:
                    self.file_extension = file_ext
        # Make sure the image base has a trailing slash, because this is expected
        if self.image_base[-1] != '/':
            self.image_base = self.image_base + '/'

        # Test if the data is accessible through HTTP
        self.accessible = check_http_accessibility( self.image_base )

class PreOverlay(ImageBaseMixin):
    def __init__(self, info_object, project_url, data_folder):
        self.name = info_object['name']
        # Set default opacity, if available, defaulting to 0
        if 'defaultopacity' in info_object:
            self.default_opacity = info_object['defaultopacity']
        else:
            self.default_opacity = 0
        # Set 'image_base', 'num_zoom_levels' and 'fileextension'
        self.set_image_fields(info_object, project_url, data_folder, False)

class PreStack(ImageBaseMixin):
    def __init__(self, info_object, project_url, data_folder, only_unknown):
        self.name = info_object['name']
        # Set 'image_base', 'num_zoom_levels' and 'fileextension'
        self.set_image_fields(info_object, project_url, data_folder, True)
        # The 'dimension', 'resolution' and 'metadata' fields should
        # have every stack.
        self.dimension = info_object['dimension']
        self.resolution = info_object['resolution']
        self.metadata = info_object['metadata'] if 'metadata' in info_object else ""
        # Add overlays to the stack, if those are declared
        self.overlays = []
        if 'overlays' in info_object:
            for overlay in info_object['overlays']:
                self.overlays.append(PreOverlay(overlay, project_url, data_folder))

        # Test if this stack is already known
        if only_unknown:
            num_same_image_base = Stack.objects.filter(image_base=self.image_base).count()
            self.already_known = (num_same_image_base > 0)

class PreProject:
    def __init__(self, info_file, project_url, data_folder, only_unknown):
        self.info_file = info_file
        info = yaml.load(open(info_file))
        p = info['project']
        self.name = p['name']
        self.stacks = []
        self.has_been_imported = False
        self.import_status = None
        for s in p['stacks']:
            self.stacks.append( PreStack( s, project_url, data_folder, only_unknown ) )
        if only_unknown:
            # Mark this project as already known if all stacks are already known
            already_known_stacks = 0
            for s in self.stacks:
                if s.already_known:
                    already_known_stacks = already_known_stacks + 1
            self.already_known = (already_known_stacks == len(self.stacks))

def find_zoom_levels_and_file_ext( base_folder, stack_folder, needs_zoom=True ):
    """ Looks at the first file of the first zoom level and
    finds out what the file extension as well as the maximum
    zoom level is.
    """
    # Make sure the base path, doesn't start with a separator
    if base_folder[0] == os.sep:
        base_folder = base_folder[1:]
    # Build paths
    datafolder_path = getattr(settings, datafolder_setting)
    project_path = os.path.join(datafolder_path, base_folder)
    stack_path = os.path.join(project_path, stack_folder)
    slice_zero_path = os.path.join(stack_path, "0")
    filter_path = os.path.join(slice_zero_path, "0_0_0.*")
    # Look for 0/0_0_0.* file
    found_file = None
    for current_file in glob.glob( filter_path ):
        if os.path.isdir( current_file ):
            continue
        found_file = current_file
        break
    # Give up if we didn't find something
    if found_file is None:
        return (None, None)
    # Find extension
    file_ext = os.path.splitext(found_file)[1][1:]
    # Look for zoom levels
    zoom_level = 1
    if needs_zoom:
        while True:
            file_name = "0_0_" + str(zoom_level) + "." + file_ext
            path = os.path.join(slice_zero_path, file_name)
            if os.path.exists(path):
                zoom_level = zoom_level + 1
            else:
                zoom_level = zoom_level - 1
                break
    return (file_ext, zoom_level)

def check_http_accessibility( image_base, file_extension ):
    """ Returns true if data below this image base can be accessed through HTTP.
    """
    slice_zero_url = urljoin(image_base, "0")
    first_file_url = urljoin(slice_zero_url, "0_0_0." + file_extension)
    return urllib.urlopen(first_file_url).code == 200

def find_project_folders(image_base, path, filter_term, only_unknown, depth=1):
    """ Finds projects in a folder structure by testing for the presence of an
    info/project YAML file.
    """
    dirs = []
    projects = {}
    not_readable = []
    for current_file in glob.glob( os.path.join(path, filter_term) ):
        if os.path.isdir(current_file):
            # Check if the folder has a info file
            info_file = os.path.join(current_file, info_file_name)
            if os.path.exists(info_file):
                short_name = current_file.replace(settings.CATMAID_IMPORT_PATH, "")
                # Check if this folder is already known by testing if
                # there is a stack with the same image base as a subfolder
                # would have.
                try:
                    # Make sure we have slashes in our directery URL part
                    url_dir = short_name
                    if os.sep != '/':
                        url_dir = url_dir.replace("\\", "/")
                    project_url = urljoin(image_base, url_dir)
                    project = PreProject( info_file, project_url, short_name, only_unknown )
                    if only_unknown and project.already_known:
                        continue
                    else:
                        projects[current_file] = project<commit_msg>Importer: fix bug that could break HTTP accessibility test for stacks<commit_after>import json
import glob
import os.path
import yaml

from django import forms
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponse, HttpResponseRedirect
from django.template import Context, loader
from django.contrib.formtools.wizard.views import SessionWizardView
from django.shortcuts import render_to_response
from django.utils.datastructures import SortedDict

from guardian.models import Permission, User, Group
from guardian.shortcuts import get_perms_for_model, assign

import urllib

from catmaid.models import Project, Stack, ProjectStack, Overlay, Double3D

from catmaid.control.common import urljoin

TEMPLATES = {"pathsettings": "catmaid/import/setup_path.html",
             "projectselection": "catmaid/import/setup_projects.html",
             "confirmation": "catmaid/import/confirmation.html"}

info_file_name = "project.yaml"
datafolder_setting = "CATMAID_IMPORT_PATH"
base_url_setting = "CATMAID_IMPORT_URL"

class ImageBaseMixin:
    def set_image_fields(self, info_object, project_url, data_folder, needs_zoom):
        """ Sets the image_base, num_zoom_levels, file_extension fields
        of the calling object. Favor a URL field, if there is one. A URL
        field, however, also requires the existence of the
        'fileextension' and and 'zoomlevels' field.
        """
        if 'url' in info_object:
            self.image_base = info_object['url']
            if needs_zoom:
                self.num_zoom_levels = info_object['zoomlevels']
            self.file_extension = info_object['fileextension']
        else:
            # The image base of a stack is the combination of the
            # project URL and the stack's folder name.
            folder = info_object['folder']
            self.image_base = urljoin(project_url, folder)
            # Favor 'zoomlevel' and 'fileextension' fields, if
            # available, but try to find this information if thosa
            # fields are not present.
            zoom_available = 'zoomlevels' in info_object
            ext_available = 'fileextension' in info_object
            if zoom_available and needs_zoom:
                self.num_zoom_levels = info_object['zoomlevels']
            if ext_available:
                self.file_extension = info_object['fileextension']
            # Only retrieve file extension and zoom level if one of
            # them is not available in the stack definition.
            if (not zoom_available and needs_zoom) or not ext_available:
                file_ext, zoom_levels = find_zoom_levels_and_file_ext(
                    data_folder, folder, needs_zoom )
                # If there is no zoom level provided, use the found one
                if not zoom_available and needs_zoom:
                    self.num_zoom_levels = zoom_levels
                # If there is no file extension level provided, use the
                # found one
                if not ext_available:
                    self.file_extension = file_ext
        # Make sure the image base has a trailing slash, because this is expected
        if self.image_base[-1] != '/':
            self.image_base = self.image_base + '/'

        # Test if the data is accessible through HTTP
        self.accessible = check_http_accessibility( self.image_base )

class PreOverlay(ImageBaseMixin):
    def __init__(self, info_object, project_url, data_folder):
        self.name = info_object['name']
        # Set default opacity, if available, defaulting to 0
        if 'defaultopacity' in info_object:
            self.default_opacity = info_object['defaultopacity']
        else:
            self.default_opacity = 0
        # Set 'image_base', 'num_zoom_levels' and 'fileextension'
        self.set_image_fields(info_object, project_url, data_folder, False)

class PreStack(ImageBaseMixin):
    def __init__(self, info_object, project_url, data_folder, only_unknown):
        self.name = info_object['name']
        # Set 'image_base', 'num_zoom_levels' and 'fileextension'
        self.set_image_fields(info_object, project_url, data_folder, True)
        # The 'dimension', 'resolution' and 'metadata' fields should
        # have every stack.
        self.dimension = info_object['dimension']
        self.resolution = info_object['resolution']
        self.metadata = info_object['metadata'] if 'metadata' in info_object else ""
        # Add overlays to the stack, if those are declared
        self.overlays = []
        if 'overlays' in info_object:
            for overlay in info_object['overlays']:
                self.overlays.append(PreOverlay(overlay, project_url, data_folder))

        # Test if this stack is already known
        if only_unknown:
            num_same_image_base = Stack.objects.filter(image_base=self.image_base).count()
            self.already_known = (num_same_image_base > 0)

class PreProject:
    def __init__(self, info_file, project_url, data_folder, only_unknown):
        self.info_file = info_file
        info = yaml.load(open(info_file))
        p = info['project']
        self.name = p['name']
        self.stacks = []
        self.has_been_imported = False
        self.import_status = None
        for s in p['stacks']:
            self.stacks.append( PreStack( s, project_url, data_folder, only_unknown ) )
        if only_unknown:
            # Mark this project as already known if all stacks are already known
            already_known_stacks = 0
            for s in self.stacks:
                if s.already_known:
                    already_known_stacks = already_known_stacks + 1
            self.already_known = (already_known_stacks == len(self.stacks))

def find_zoom_levels_and_file_ext( base_folder, stack_folder, needs_zoom=True ):
    """ Looks at the first file of the first zoom level and
    finds out what the file extension as well as the maximum
    zoom level is.
    """
    # Make sure the base path, doesn't start with a separator
    if base_folder[0] == os.sep:
        base_folder = base_folder[1:]
    # Build paths
    datafolder_path = getattr(settings, datafolder_setting)
    project_path = os.path.join(datafolder_path, base_folder)
    stack_path = os.path.join(project_path, stack_folder)
    slice_zero_path = os.path.join(stack_path, "0")
    filter_path = os.path.join(slice_zero_path, "0_0_0.*")
    # Look for 0/0_0_0.* file
    found_file = None
    for current_file in glob.glob( filter_path ):
        if os.path.isdir( current_file ):
            continue
        found_file = current_file
        break
    # Give up if we didn't find something
    if found_file is None:
        return (None, None)
    # Find extension
    file_ext = os.path.splitext(found_file)[1][1:]
    # Look for zoom levels
    zoom_level = 1
    if needs_zoom:
        while True:
            file_name = "0_0_" + str(zoom_level) + "." + file_ext
            path = os.path.join(slice_zero_path, file_name)
            if os.path.exists(path):
                zoom_level = zoom_level + 1
            else:
                zoom_level = zoom_level - 1
                break
    return (file_ext, zoom_level)

def check_http_accessibility( image_base, file_extension ):
    """ Returns true if data below this image base can be accessed through HTTP.
    """
    slice_zero_url = urljoin(image_base, "0")
    first_file_url = urljoin(slice_zero_url, "0_0_0." + file_extension)
    code = urllib.urlopen(first_file_url).getcode()
    return code == 200

def find_project_folders(image_base, path, filter_term, only_unknown, depth=1):
    """ Finds projects in a folder structure by testing for the presence of an
    info/project YAML file.
    """
    dirs = []
    projects = {}
    not_readable = []
    for current_file in glob.glob( os.path.join(path, filter_term) ):
        if os.path.isdir(current_file):
            # Check if the folder has a info file
            info_file = os.path.join(current_file, info_file_name)
            if os.path.exists(info_file):
                short_name = current_file.replace(settings.CATMAID_IMPORT_PATH, "")
                # Check if this folder is already known by testing if
                # there is a stack with the same image base as a subfolder
                # would have.
                try:
                    # Make sure we have slashes in our directery URL part
                    url_dir = short_name
                    if os.sep != '/':
                        url_dir = url_dir.replace("\\", "/")
                    project_url = urljoin(image_base, url_dir)
                    project = PreProject( info_file, project_url, short_name, only_unknown )
                    if only_unknown and project.already_known:
                        continue
                    else:
                        projects[current_file] = project
{'fraction_non_alphanumeric': 0.07684529828109202, 'fraction_numerical': 0.003651275137624986, 'mean_word_length': 2.43554612118873, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7581593', 'n_tokens_mistral': 5089, 'n_tokens_neox': 4890, 'n_words': 1451}