<commit_before># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

import base64
import os
import logging
import shutil
import sys
import tempfile
import uuid

from django.http import Http404, HttpResponse, HttpResponseBadRequest
from django.db import connection

from components import helpers
import components.ingest.helpers as ingest_helpers
import components.filesystem_ajax.helpers as filesystem_ajax_helpers
from main import models

sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import archivematicaFunctions
import databaseFunctions
import elasticSearchFunctions
from archivematicaCreateStructuredDirectory import createStructuredDirectory
import storageService as storage_service

# for unciode sorting support
import locale
locale.setlocale(locale.LC_ALL, '')

logger = logging.getLogger(__name__)
logging.basicConfig(filename="/tmp/archivematicaDashboard.log",
    level=logging.INFO)

SHARED_DIRECTORY_ROOT   = '/var/archivematica/sharedDirectory'
ACTIVE_TRANSFER_DIR     = SHARED_DIRECTORY_ROOT + '/watchedDirectories/activeTransfers'
STANDARD_TRANSFER_DIR   = ACTIVE_TRANSFER_DIR + '/standardTransfer'
ORIGINAL_DIR            = SHARED_DIRECTORY_ROOT + '/www/AIPsStore/transferBacklog/originals'

DEFAULT_ARRANGE_PATH = '/arrange/'


def directory_children_proxy_to_storage_server(request, location_uuid, basePath=False):
    path = ''
    if (basePath):
        path = base64.b64decode(basePath)
    path = path + base64.b64decode(request.GET.get('base_path', ''))
    path = path + base64.b64decode(request.GET.get('path', ''))
    path = base64.b64encode(path)

    response = storage_service.browse_location(location_uuid, path)

    return helpers.json_response(response)

def contents(request):
    path = request.GET.get('path', '/home')
    response = filesystem_ajax_helpers.directory_to_dict(path)
    return helpers.json_response(response)

def arrange_contents(request):
    base_path = request.GET.get('path', DEFAULT_ARRANGE_PATH)

    # Must indicate that base_path is a folder by ending with /
    if not base_path.endswith('/'):
        base_path += '/'

    if not base_path.startswith(DEFAULT_ARRANGE_PATH):
        base_path = DEFAULT_ARRANGE_PATH

    # Query SIP Arrangement for results
    # Get all the paths that are not in SIPs and start with base_path.  We don't
    # need the objects, just the arrange_path
    paths = models.SIPArrange.objects.filter(sip_created=False).filter(arrange_path__startswith=base_path).order_by('arrange_path').values_list('arrange_path', flat=True)

    # Convert the response into an entries [] and directories []
    # 'entries' contains everything (files and directories)
    response = {'entries': [], 'directories': []}
    for path in paths:
        # Stip common prefix
        if path.startswith(base_path):
            path = path[len(base_path):]
        entry = path.split('/', 1)[0]
        # Only insert once
        if entry and entry not in response['entries']:
            response['entries'].append(entry)
            if path.endswith('/'):  # path is a dir
                response['directories'].append(entry)

    return helpers.json_response(response)


def originals_contents(request):
    path = request.GET.get('path', 'originals').lstrip('/')
    # IDEA memoize the backlog location?
    backlog = storage_service.get_location(purpose='BL')[0]

    # TODO need to be able to search on accession ID
    response = storage_service.browse_location(backlog['uuid'], path)

    return helpers.json_response(response)

def delete(request):
    filepath = request.POST.get('filepath', '')
    filepath = os.path.join('/', filepath)
    error = filesystem_ajax_helpers.check_filepath_exists(filepath)

    if error == None:
        filepath = os.path.join(filepath)
        if os.path.isdir(filepath):
            try:
                shutil.rmtree(filepath)
            except:
                error = 'Error attempting to delete directory.'
        else:
            os.remove(filepath)

    response = {}

    # if deleting from originals, delete ES data as well
    if ORIGINAL_DIR in filepath and filepath.index(ORIGINAL_DIR) == 0:
        transfer_uuid = _find_uuid_of_transfer_in_originals_directory_using_path(filepath)
        if transfer_uuid != None:
            elasticSearchFunctions.connect_and_remove_backlog_transfer_files(transfer_uuid)

    if error != None:
      response['message'] = error
      response['error']   = True
    else:
      response['message'] = 'Delete successful.'

    return helpers.json_response(response)

def get_temp_directory(request):
    temp_base_dir = helpers.get_client_config_value('temp_dir')

    response = {}<commit_msg>Update delete from arrange functionality<commit_after># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

import base64
import os
import logging
import shutil
import sys
import tempfile
import uuid

from django.http import Http404, HttpResponse, HttpResponseBadRequest
from django.db import connection

from components import helpers
import components.ingest.helpers as ingest_helpers
import components.filesystem_ajax.helpers as filesystem_ajax_helpers
from main import models

sys.path.append("/usr/lib/archivematica/archivematicaCommon")
import archivematicaFunctions
import databaseFunctions
import elasticSearchFunctions
from archivematicaCreateStructuredDirectory import createStructuredDirectory
import storageService as storage_service

# for unciode sorting support
import locale
locale.setlocale(locale.LC_ALL, '')

logger = logging.getLogger(__name__)
logging.basicConfig(filename="/tmp/archivematicaDashboard.log",
    level=logging.INFO)

SHARED_DIRECTORY_ROOT   = '/var/archivematica/sharedDirectory'
ACTIVE_TRANSFER_DIR     = SHARED_DIRECTORY_ROOT + '/watchedDirectories/activeTransfers'
STANDARD_TRANSFER_DIR   = ACTIVE_TRANSFER_DIR + '/standardTransfer'
ORIGINAL_DIR            = SHARED_DIRECTORY_ROOT + '/www/AIPsStore/transferBacklog/originals'

DEFAULT_ARRANGE_PATH = '/arrange/'


def directory_children_proxy_to_storage_server(request, location_uuid, basePath=False):
    path = ''
    if (basePath):
        path = base64.b64decode(basePath)
    path = path + base64.b64decode(request.GET.get('base_path', ''))
    path = path + base64.b64decode(request.GET.get('path', ''))
    path = base64.b64encode(path)

    response = storage_service.browse_location(location_uuid, path)

    return helpers.json_response(response)

def contents(request):
    path = request.GET.get('path', '/home')
    response = filesystem_ajax_helpers.directory_to_dict(path)
    return helpers.json_response(response)

def arrange_contents(request):
    base_path = request.GET.get('path', DEFAULT_ARRANGE_PATH)

    # Must indicate that base_path is a folder by ending with /
    if not base_path.endswith('/'):
        base_path += '/'

    if not base_path.startswith(DEFAULT_ARRANGE_PATH):
        base_path = DEFAULT_ARRANGE_PATH

    # Query SIP Arrangement for results
    # Get all the paths that are not in SIPs and start with base_path.  We don't
    # need the objects, just the arrange_path
    paths = models.SIPArrange.objects.filter(sip_created=False).filter(arrange_path__startswith=base_path).order_by('arrange_path').values_list('arrange_path', flat=True)

    # Convert the response into an entries [] and directories []
    # 'entries' contains everything (files and directories)
    response = {'entries': [], 'directories': []}
    for path in paths:
        # Stip common prefix
        if path.startswith(base_path):
            path = path[len(base_path):]
        entry = path.split('/', 1)[0]
        # Only insert once
        if entry and entry not in response['entries']:
            response['entries'].append(entry)
            if path.endswith('/'):  # path is a dir
                response['directories'].append(entry)

    return helpers.json_response(response)


def originals_contents(request):
    path = request.GET.get('path', 'originals').lstrip('/')
    # IDEA memoize the backlog location?
    backlog = storage_service.get_location(purpose='BL')[0]

    # TODO need to be able to search on accession ID
    response = storage_service.browse_location(backlog['uuid'], path)

    return helpers.json_response(response)

def delete(request):
    filepath = request.POST.get('filepath', '')
    filepath = os.path.join('/', filepath)
    error = filesystem_ajax_helpers.check_filepath_exists(filepath)

    if error == None:
        if os.path.isdir(filepath):
            try:
                shutil.rmtree(filepath)
            except Exception as e:
                logging.exception('Error deleting directory {}'.format(filepath))
                error = 'Error attempting to delete directory.'
        else:
            os.remove(filepath)
    elif filepath.startswith(DEFAULT_ARRANGE_PATH):
        # Might be for SIP Arrange view
        models.SIPArrange.objects.filter(arrange_path__startswith=filepath).delete()

    # if deleting from originals, delete ES data as well
    if ORIGINAL_DIR in filepath and filepath.index(ORIGINAL_DIR) == 0:
        transfer_uuid = _find_uuid_of_transfer_in_originals_directory_using_path(filepath)
        if transfer_uuid != None:
            elasticSearchFunctions.connect_and_remove_backlog_transfer_files(transfer_uuid)

    if error is not None:
        response = {
            'message': error,
            'error': True,
        }
    else:
        response = {'message': 'Delete successful.'}

    return helpers.json_response(response)

def get_temp_directory(request):
    temp_base_dir = helpers.get_client_config_value('temp_dir')

    response = {}
{'fraction_non_alphanumeric': 0.08984693415451499, 'fraction_numerical': 0.0061588624218820755, 'mean_word_length': 3.877208480565371, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2491185', 'n_tokens_mistral': 3246, 'n_tokens_neox': 3140, 'n_words': 859}