<commit_before>"""
These views handle all actions in Studio related to import and exporting of
courses
"""
import logging
import os
import tarfile
import shutil
import re
from tempfile import mkdtemp
from path import path

from django.conf import settings
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django_future.csrf import ensure_csrf_cookie
from django.core.servers.basehttp import FileWrapper
from django.core.files.temp import NamedTemporaryFile
from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_http_methods, require_GET
from django.utils.translation import ugettext as _

from edxmako.shortcuts import render_to_response

from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.exceptions import SerializationError

from xmodule.modulestore.locator import BlockUsageLocator
from .access import has_course_access

from util.json_request import JsonResponse
from extract_tar import safetar_extractall
from student.roles import CourseInstructorRole, CourseStaffRole
from student import auth


__all__ = ['import_handler', 'import_status_handler', 'export_handler']


log = logging.getLogger(__name__)


# Regex to capture Content-Range header ranges.
CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})")


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
def import_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
    """
    The restful handler for importing a course.

    GET
        html: return html page for import page
        json: not supported
    POST or PUT
        json: import a course via the .tar.gz file specified in request.FILES
    """
    location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block)
    if not has_course_access(request.user, location):
        raise PermissionDenied()

    old_location = loc_mapper().translate_locator_to_location(location)

    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            raise NotImplementedError('coming soon')
        else:
            data_root = path(settings.GITHUB_REPO_ROOT)
            course_subdir = "{0}-{1}-{2}".format(old_location.org, old_location.course, old_location.name)
            course_dir = data_root / course_subdir

            filename = request.FILES['course-data'].name
            if not filename.endswith('.tar.gz'):
                return JsonResponse(
                    {
                        'ErrMsg': _('We only support uploading a .tar.gz file.'),
                        'Stage': 1
                    },
                    status=415
                )
            temp_filepath = course_dir / filename

            if not course_dir.isdir():
                os.mkdir(course_dir)

            logging.debug('importing course to {0}'.format(temp_filepath))

            # Get upload chunks byte ranges
            try:
                matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
                content_range = matches.groupdict()
            except KeyError:    # Single chunk
                # no Content-Range header, so make one that will work
                content_range = {'start': 0, 'stop': 1, 'end': 2}

            # stream out the uploaded files in chunks to disk
            if int(content_range['start']) == 0:
                mode = "wb+"
            else:
                mode = "ab+"
                size = os.path.getsize(temp_filepath)
                # Check to make sure we haven't missed a chunk
                # This shouldn't happen, even if different instances are handling
                # the same session, but it's always better to catch errors earlier.
                if size < int(content_range['start']):
                    log.warning(
                        "Reported range %s does not match size downloaded so far %s",
                        content_range['start'],
                        size
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': _('File upload corrupted. Please try again'),
                            'Stage': 1
                        },
                        status=409
                    )
                # The last request sometimes comes twice. This happens because
                # nginx sends a 499 error code when the response takes too long.
                elif size > int(content_range['stop']) and size == int(content_range['end']):
                    return JsonResponse({'ImportStatus': 1})

            with open(temp_filepath, mode) as temp_file:
                for chunk in request.FILES['course-data'].chunks():
                    temp_file.write(chunk)

            size = os.path.getsize(temp_filepath)

            if int(content_range['stop']) != int(content_range['end']) - 1:
                # More chunks coming
                return JsonResponse({
                    "files": [{
                                  "name": filename,
                                  "size": size,
                                  "deleteUrl": "",
                                  "deleteType": "",
                                  "url": location.url_reverse('import'),
                                  "thumbnailUrl": ""
                              }]
                })

            else:   # This was the last chunk.

                # Use sessions to keep info about import progress
                session_status = request.session.setdefault("import_status", {})
                key = location.package_id + filename
                session_status[key] = 1
                request.session.modified = True

                # Do everything from now on in a try-finally block to make sure
                # everything is properly cleaned up.
                try:

                    tar_file = tarfile.open(temp_filepath)
                    try:
                        safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
                    except SuspiciousOperation as exc:
                        return JsonResponse(
                            {
                                'ErrMsg': 'Unsafe tar file. Aborting import.',
                                'SuspiciousFileOperationMsg': exc.args[0],
                                'Stage': 1
                            },
                            status=400
                        )
                    finally:
                        tar_file.close()

                    session_status[key] = 2
                    request.session.modified = True

                    # find the 'course.xml' file
                    def get_all_files(directory):
                        """
                        For each file in the directory, yield a 2-tuple of (file-name,
                        directory-path)
                        """
                        for dirpath, _dirnames, filenames in os.walk(directory):
                            for filename in filenames:
                                yield (filename, dirpath)

                    def get_dir_for_fname(directory, filename):
                        """
                        Returns the dirpath for the first file found in the directory
                        with the given name.  If there is no file in the directory with
                        the specified name, return None.
                        """
                        for fname, dirpath in get_all_files(directory):
                            if fname == filename:
                                return dirpath
                        return None

                    fname = "course.xml"

                    dirpath = get_dir_for_fname(course_dir, fname)

                    if not dirpath:
                        return JsonResponse(
                            {

                                'ErrMsg': _('Could not find the course.xml file in the package.'),
                                'Stage': 2
                            },
                            status=415
                        )

                    logging.debug('found course.xml at {0}'.format(dirpath))

                    if dirpath != course_dir:
                        for fname in os.listdir(dirpath):
                            shutil.move(dirpath / fname, course_dir)

                    _module_store, course_items = import_from_xml(
                        modulestore('direct'),
                        settings.GITHUB_REPO_ROOT,
                        [course_subdir],
                        load_error_modules=False,
                        static_content_store=contentstore(),
                        target_location_namespace=old_location,
                        draft_store=modulestore()
                    )

                    new_location = course_items[0].location
                    logging.debug('new course at {0}'.format(new_location))

                    session_status[key] = 3
                    request.session.modified = True

                    auth.add_users(request.user, CourseInstructorRole(new_location), request.user)
                    auth.add_users(request.user, CourseStaffRole(new_location), request.user)
                    logging.debug('created all course groups at {0}'.format(new_location))

                # Send errors to client with stage at which error occurred.
                except Exception as exception:   # pylint: disable=W0703
                    return JsonResponse(
                        {
                            'ErrMsg': str(exception),
                            'Stage': session_status[key]<commit_msg>add logging to course import<commit_after>"""
These views handle all actions in Studio related to import and exporting of
courses
"""
import logging
import os
import tarfile
import shutil
import re
from tempfile import mkdtemp
from path import path

from django.conf import settings
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django_future.csrf import ensure_csrf_cookie
from django.core.servers.basehttp import FileWrapper
from django.core.files.temp import NamedTemporaryFile
from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.http import HttpResponseNotFound
from django.views.decorators.http import require_http_methods, require_GET
from django.utils.translation import ugettext as _

from edxmako.shortcuts import render_to_response

from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.exceptions import SerializationError

from xmodule.modulestore.locator import BlockUsageLocator
from .access import has_course_access

from util.json_request import JsonResponse
from extract_tar import safetar_extractall
from student.roles import CourseInstructorRole, CourseStaffRole
from student import auth


__all__ = ['import_handler', 'import_status_handler', 'export_handler']


log = logging.getLogger(__name__)


# Regex to capture Content-Range header ranges.
CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})")


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
def import_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
    """
    The restful handler for importing a course.

    GET
        html: return html page for import page
        json: not supported
    POST or PUT
        json: import a course via the .tar.gz file specified in request.FILES
    """
    location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block)
    if not has_course_access(request.user, location):
        raise PermissionDenied()

    old_location = loc_mapper().translate_locator_to_location(location)

    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            raise NotImplementedError('coming soon')
        else:
            data_root = path(settings.GITHUB_REPO_ROOT)
            course_subdir = "{0}-{1}-{2}".format(old_location.org, old_location.course, old_location.name)
            course_dir = data_root / course_subdir

            filename = request.FILES['course-data'].name
            if not filename.endswith('.tar.gz'):
                return JsonResponse(
                    {
                        'ErrMsg': _('We only support uploading a .tar.gz file.'),
                        'Stage': 1
                    },
                    status=415
                )
            temp_filepath = course_dir / filename

            if not course_dir.isdir():
                os.mkdir(course_dir)

            logging.debug('importing course to {0}'.format(temp_filepath))

            # Get upload chunks byte ranges
            try:
                matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
                content_range = matches.groupdict()
            except KeyError:    # Single chunk
                # no Content-Range header, so make one that will work
                content_range = {'start': 0, 'stop': 1, 'end': 2}

            # stream out the uploaded files in chunks to disk
            if int(content_range['start']) == 0:
                mode = "wb+"
            else:
                mode = "ab+"
                size = os.path.getsize(temp_filepath)
                # Check to make sure we haven't missed a chunk
                # This shouldn't happen, even if different instances are handling
                # the same session, but it's always better to catch errors earlier.
                if size < int(content_range['start']):
                    log.warning(
                        "Reported range %s does not match size downloaded so far %s",
                        content_range['start'],
                        size
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': _('File upload corrupted. Please try again'),
                            'Stage': 1
                        },
                        status=409
                    )
                # The last request sometimes comes twice. This happens because
                # nginx sends a 499 error code when the response takes too long.
                elif size > int(content_range['stop']) and size == int(content_range['end']):
                    return JsonResponse({'ImportStatus': 1})

            with open(temp_filepath, mode) as temp_file:
                for chunk in request.FILES['course-data'].chunks():
                    temp_file.write(chunk)

            size = os.path.getsize(temp_filepath)

            if int(content_range['stop']) != int(content_range['end']) - 1:
                # More chunks coming
                return JsonResponse({
                    "files": [{
                                  "name": filename,
                                  "size": size,
                                  "deleteUrl": "",
                                  "deleteType": "",
                                  "url": location.url_reverse('import'),
                                  "thumbnailUrl": ""
                              }]
                })

            else:   # This was the last chunk.

                # Use sessions to keep info about import progress
                session_status = request.session.setdefault("import_status", {})
                key = location.package_id + filename
                session_status[key] = 1
                request.session.modified = True

                # Do everything from now on in a try-finally block to make sure
                # everything is properly cleaned up.
                try:

                    tar_file = tarfile.open(temp_filepath)
                    try:
                        safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
                    except SuspiciousOperation as exc:
                        return JsonResponse(
                            {
                                'ErrMsg': 'Unsafe tar file. Aborting import.',
                                'SuspiciousFileOperationMsg': exc.args[0],
                                'Stage': 1
                            },
                            status=400
                        )
                    finally:
                        tar_file.close()

                    session_status[key] = 2
                    request.session.modified = True

                    # find the 'course.xml' file
                    def get_all_files(directory):
                        """
                        For each file in the directory, yield a 2-tuple of (file-name,
                        directory-path)
                        """
                        for dirpath, _dirnames, filenames in os.walk(directory):
                            for filename in filenames:
                                yield (filename, dirpath)

                    def get_dir_for_fname(directory, filename):
                        """
                        Returns the dirpath for the first file found in the directory
                        with the given name.  If there is no file in the directory with
                        the specified name, return None.
                        """
                        for fname, dirpath in get_all_files(directory):
                            if fname == filename:
                                return dirpath
                        return None

                    fname = "course.xml"

                    dirpath = get_dir_for_fname(course_dir, fname)

                    if not dirpath:
                        return JsonResponse(
                            {

                                'ErrMsg': _('Could not find the course.xml file in the package.'),
                                'Stage': 2
                            },
                            status=415
                        )

                    logging.debug('found course.xml at {0}'.format(dirpath))

                    if dirpath != course_dir:
                        for fname in os.listdir(dirpath):
                            shutil.move(dirpath / fname, course_dir)

                    _module_store, course_items = import_from_xml(
                        modulestore('direct'),
                        settings.GITHUB_REPO_ROOT,
                        [course_subdir],
                        load_error_modules=False,
                        static_content_store=contentstore(),
                        target_location_namespace=old_location,
                        draft_store=modulestore()
                    )

                    new_location = course_items[0].location
                    logging.debug('new course at {0}'.format(new_location))

                    session_status[key] = 3
                    request.session.modified = True

                    auth.add_users(request.user, CourseInstructorRole(new_location), request.user)
                    auth.add_users(request.user, CourseStaffRole(new_location), request.user)
                    logging.debug('created all course groups at {0}'.format(new_location))

                # Send errors to client with stage at which error occurred.
                except Exception as exception:   # pylint: disable=W0703
                    log.exception(
                        "error importing course at {0}".format(new_location)
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': str(exception),
                            'Stage': session_status[key]
{'fraction_non_alphanumeric': 0.07566875215527859, 'fraction_numerical': 0.005172668604364747, 'mean_word_length': 1.4600096946194863, 'pattern_counts': {'":': 14, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1882314', 'n_tokens_mistral': 5048, 'n_tokens_neox': 4647, 'n_words': 1311}