# Copyright (c) 2016 Red Hat, Inc.
#
# This software is licensed to you under the GNU General Public License,
# version 2 (GPLv2). There is NO WARRANTY for this software, express or
# implied, including the implied warranties of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
# along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
#
# Red Hat trademarks are not licensed under GPLv2. No permission is
# granted to use or replicate Red Hat trademarks that are incorporated
# in this software or its documentation.
#

import sys

import cStringIO
import json
import zipfile
import os
from M2Crypto import X509

from spacewalk.satellite_tools.syncLib import log2
from spacewalk.server.rhnServer.satellite_cert import SatelliteCert

import constants

class Manifest(object):
    """Class containing relevant data from RHSM manifest."""

    SIGNATURE_NAME = "signature"
    INNER_ZIP_NAME = "consumer_export.zip"
    ENTITLEMENTS_PATH = "export/entitlements"
    CERTIFICATE_PATH = "export/extensions"
    PRODUCTS_PATH = "export/products"
    CONSUMER_INFO = "export/consumer.json"
    META_INFO = "export/meta.json"
    UPSTREAM_CONSUMER_PATH = "export/upstream_consumer"

    def __init__(self, zip_path):
        self.all_entitlements = []
        self.manifest_repos = {}
        self.sat5_certificate = None
        self.satellite_version = None
        self.consumer_credentials = None
        self.uuid = None
        self.name = None
        self.ownerid = None
        self.api_url = None
        self.created = None
        # Signature and signed data
        self.signature = None
        self.data = None
        # Open manifest from path
        top_zip = None
        inner_zip = None
        inner_file = None
        try:
            top_zip = zipfile.ZipFile(zip_path, 'r')
            # Fetch inner zip file into memory
            try:
                # inner_file = top_zip.open(zip_path.split('.zip')[0] + '/' + self.INNER_ZIP_NAME)
                inner_file = top_zip.open(self.INNER_ZIP_NAME)
                self.data = inner_file.read()
                inner_file_data = cStringIO.StringIO(self.data)
                signature_file = top_zip.open(self.SIGNATURE_NAME)
                self.signature = signature_file.read()
                # Open the inner zip file
                try:
                    inner_zip = zipfile.ZipFile(inner_file_data)
                    self._load_entitlements(inner_zip)
                    self._extract_certificate(inner_zip)
                    self._extract_consumer_info(inner_zip)
                    self._extract_meta_info(inner_zip)
                    self._extract_consumer_credentials(inner_zip)
                finally:
                    if inner_zip is not None:
                        inner_zip.close()
            finally:
                if inner_file is not None:
                    inner_file.close()
        finally:
            if top_zip is not None:
                top_zip.close()

    def _extract_certificate(self, zip_file):
        files = zip_file.namelist()
        certificates_names = []
        for f in files:
            if f.startswith(self.CERTIFICATE_PATH) and f.endswith(".xml"):
                certificates_names.append(f)
        if len(certificates_names) >= 1:
            # take only first file
            cert_file = zip_file.open(certificates_names[0])  # take only first file
            self.sat5_certificate = cert_file.read().strip()
            cert_file.close()
            # Save version too
            sat5_cert = SatelliteCert()
            sat5_cert.load(self.sat5_certificate)
            self.satellite_version = getattr(sat5_cert, 'satellite-version')
        else:
            raise MissingSatelliteCertificateError("Satellite Certificate was not found in manifest.")

    def _fill_product_repositories(self, zip_file, product):
        product_file = zip_file.open(self.PRODUCTS_PATH + '/' + str(product.get_id()) + '.json')
        product_data = json.load(product_file)
        product_file.close()
        try:
            for content in product_data['productContent']:
                content = content['content']
                product.add_repository(content['label'], content['contentUrl'])
        except KeyError:
            log2(0, 0, "ERROR: Cannot access required field in product '%s'" % product.get_id(), stream=sys.stderr)
            raise

    def _load_entitlements(self, zip_file):
        files = zip_file.namelist()
        entitlements_files = []
        for f in files:
            if f.startswith(self.ENTITLEMENTS_PATH) and f.endswith(".json"):
                entitlements_files.append(f)

        if len(entitlements_files) >= 1:
            self.all_entitlements = []
            for entitlement_file in entitlements_files:
                entitlements = zip_file.open(entitlement_file)
                # try block in try block - this is hack for python 2.4 compatibility
                # to support finally
                try:
                    try:
                        data = json.load(entitlements)

                        # Extract credentials
                        certs = data['certificates']
                        if len(certs) != 1:
                            raise IncorrectEntitlementsFileFormatError(
                                "ERROR: Single certificate in entitlements file is expected, found: %d"
                                % len(certs))
                        cert = certs[0]
                        credentials = Credentials(data['id'], cert['cert'], cert['key'])

                        # Extract product IDs
                        products = []
                        provided_products = data['pool']['providedProducts']
                        for provided_product in provided_products:
                            product = Product(provided_product['productId'])
                            self._fill_product_repositories(zip_file, product)
                            products.append(product)

                        # Skip entitlements not providing any products
                        if products:
                            entitlement = Entitlement(products, credentials)
                            self.all_entitlements.append(entitlement)
                    except KeyError:
                        log2(0, 0, "ERROR: Cannot access required field in file '%s'" % entitlement_file,
                             stream=sys.stderr)
                        raise
                finally:
                    entitlements.close()
        else:
            raise IncorrectEntitlementsFileFormatError(
                "ERROR: There has to be at least one entitlements file")

    def _extract_consumer_info(self, zip_file):
        files = zip_file.namelist()
        found = False
        for f in files:
            if f == self.CONSUMER_INFO:
                found = True
                break
        if found:
            consumer_info = zip_file.open(self.CONSUMER_INFO)
            try:
                try:
                    data = json.load(consumer_info)
                    self.uuid = data['uuid']
                    self.name = data['name']
                    self.ownerid = data['owner']['key']
                    self.api_url = data['urlApi']
                except KeyError:
                    log2(0, 0, "ERROR: Cannot access required field in file '%s'" % self.CONSUMER_INFO,
                         stream=sys.stderr)
                    raise
            finally:
                consumer_info.close()
        else:
            raise MissingConsumerInfoError()

    def _extract_meta_info(self, zip_file):
        files = zip_file.namelist()
        found = False
        for f in files:
            if f == self.META_INFO:
                found = True
                break
        if found:
            meta_info = zip_file.open(self.META_INFO)
            try:
                try:
                    data = json.load(meta_info)
                    self.created = data['created']
                except KeyError:
                    log2(0, 0, "ERROR: Cannot access required field in file '%s'" % self.META_INFO, stream=sys.stderr)
                    raise
            finally:
                meta_info.close()
        else:
            raise MissingMetaInfoError()

    def _extract_consumer_credentials(self, zip_file):
        files = zip_file.namelist()
        consumer_credentials = []
        for f in files:
            if f.startswith(self.UPSTREAM_CONSUMER_PATH) and f.endswith(".json"):
                consumer_credentials.append(f)

        if len(consumer_credentials) == 1:


# Copyright (C) 2014 Sam Parkinson
# Copyright (C) 2015 Martin Abente Lahaye
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from gettext import gettext as _
import math

from gi.repository import GLib
from gi.repository import Gtk
from gi.repository import GObject

from sugar3 import profile
from sugar3.graphics import style
from sugar3.graphics.icon import Icon
from sugar3.graphics.tray import TrayIcon
from sugar3.graphics.palette import Palette
from sugar3.graphics.palettemenu import PaletteMenuBox
from sugar3.graphics.palettemenu import PaletteMenuItem
from sugar3.graphics.palettemenu import PaletteMenuItemSeparator
from sugar3.graphics.xocolor import XoColor

from jarabe.frame.frameinvoker import FrameWidgetInvoker
from jarabe.model import brightness
from jarabe.model.screenshot import take_screenshot
from jarabe import frame

class DeviceView(TrayIcon):

    FRAME_POSITION_RELATIVE = 160

    def __init__(self, label):
        self._color = profile.get_color()
        self._label = label

        TrayIcon.__init__(self, icon_name='brightness-100',
                          xo_color=self._color)

        self.set_palette_invoker(FrameWidgetInvoker(self))
        self.palette_invoker.props.toggle_palette = True

        model = brightness.get_instance()
        if model.get_path():
            self._model = model
            self._model.changed_signal.connect(self.__brightness_changed_cb)
            self._update_output_info()

    def create_palette(self):
        palette = DisplayPalette()
        palette.set_group_id('frame')
        return palette

    def _update_output_info(self, value=None):
        if value is None:
            value = self._model.get_brightness()

        icon_number = math.ceil(
            float(value) * 3 / self._model.get_max_brightness()) * 33

        if icon_number == 99:
            icon_number = 100

        self.icon.props.icon_name = \
            'brightness-{:03d}'.format(int(icon_number))

    def __brightness_changed_cb(self, model, value):
        self._update_output_info(value)

class BrightnessManagerWidget(Gtk.VBox):

    TIMEOUT_DELAY = 100

    def __init__(self, text, icon_name):
        Gtk.VBox.__init__(self)
        self._progress_bar = None
        self._adjustment = None

        icon = Icon(pixel_size=style.SMALL_ICON_SIZE)
        icon.props.icon_name = icon_name
        icon.props.xo_color = XoColor('%s,%s' % (style.COLOR_WHITE.get_svg(),
                                      style.COLOR_BUTTON_GREY.get_svg()))
        icon.show()

        label = Gtk.Label(text)
        label.show()

        grid = Gtk.Grid()
        grid.set_column_spacing(style.DEFAULT_SPACING)
        grid.attach(icon, 0, 0, 1, 1)
        grid.attach(label, 1, 0, 1, 1)
        grid.show()

        alignment = Gtk.Alignment()
        alignment.set(0.5, 0, 0, 0)
        alignment.add(grid)
        alignment.show()
        self.add(alignment)

        alignment = Gtk.Alignment()
        alignment.set(0.5, 0, 0, 0)
        alignment.set_padding(0, 0, style.DEFAULT_SPACING,
                              style.DEFAULT_SPACING)

        self._model = brightness.get_instance()
        self._model_changed_hid = \
            self._model.changed_signal.connect(self.__brightness_changed_cb)

        # if sugar-backlight-helper finds the device
        if self._model.get_path():
            adjustment = Gtk.Adjustment(
                value=self._model.get_brightness(),
                lower=0,
                upper=self._model.get_max_brightness() + 1,
                step_incr=self._model.get_step_amount(),
                page_incr=self._model.get_step_amount(),
                page_size=self._model.get_step_amount())
            self._adjustment = adjustment

            self._adjustment_timeout_id = None
            self._adjustment_hid = \
                self._adjustment.connect('value_changed', self.__adjusted_cb)

            hscale = Gtk.HScale()
            hscale.props.draw_value = False
            hscale.set_adjustment(adjustment)
            hscale.set_digits(0)
            hscale.set_size_request(style.GRID_CELL_SIZE * 4, -1)
            alignment.add(hscale)
            hscale.show()
        else:
            self._progress_bar = Gtk.ProgressBar()
            self._progress_bar.set_size_request(
                style.zoom(style.GRID_CELL_SIZE * 4), -1)
            alignment.props.top_padding = style.DEFAULT_PADDING
            alignment.add(self._progress_bar)
            self._progress_bar.show()

        alignment.show()
        self.add(alignment)

    def __brightness_changed_cb(self, model, value):
        self.update(value)

    def update(self, value=None):
        if value is None:
            value = self._model.get_brightness()

        if self._adjustment:
            self._adjustment.handler_block(self._adjustment_hid)
            self._adjustment.props.value = value
            self._adjustment.handler_unblock(self._adjustment_hid)
        else:
            self._progress_bar.props.fraction = \
                float(value) / self._model.get_max_brightness()

    def __adjusted_cb(self, device, data=None):
        if self._adjustment_timeout_id is not None:
            GLib.source_remove(self._adjustment_timeout_id)
        self._adjustment_timeout_id = GLib.timeout_add(
            self.TIMEOUT_DELAY,  self._adjust_brightness)

    def _adjust_brightness(self):
        self._model.handler_block(self._model_changed_hid)
        self._model.set_brightness(int(self._adjustment.props.value))
        self._model.handler_unblock(self._model_changed_hid)
        self._adjustment_timeout_id = None
        return False

class DisplayPalette(Palette):

    def __init__(self):
        Palette.__init__(self, label=_('My Display'))

        self._screenshot = PaletteMenuItem(_('Take a screenshot'))
        icon = Icon(icon_name='camera-external',
                    pixel_size=style.SMALL_ICON_SIZE)
        self._screenshot.set_image(icon)
        icon.show()
        self._screenshot.connect('activate', self.__screenshot_cb)
        self._screenshot.show()

        self._box = PaletteMenuBox()

        self._brightness_manager = None
        # only add this widget if device available
        if brightness.get_instance().get_path():
            self._add_brightness_manager()

        self._box.append_item(self._screenshot, 0, 0)
        self._box.show()

        self.set_content(self._box)
        self.connect('popup', self.__popup_cb)

    def _add_brightness_manager(self):
        self._brightness_manager = BrightnessManagerWidget(_('Brightness'),
                                                           'brightness-100')
        self._brightness_manager.show()


"""Views for items (modules)."""
from __future__ import absolute_import

import hashlib
import logging
from collections import OrderedDict
from datetime import datetime
from functools import partial
from uuid import uuid4

import dogstats_wrapper as dog_stats_api
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied
from django.http import HttpResponseBadRequest, HttpResponse, Http404
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_http_methods
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryUsageLocator
from pytz import UTC
from xblock.fields import Scope
from xblock.fragment import Fragment

from cms.lib.xblock.authoring_mixin import VISIBILITY_VIEW
from contentstore.utils import (
    find_release_date_source, find_staff_lock_source, is_currently_visible_to_students,
    ancestor_has_staff_lock, has_children_visible_to_specific_content_groups,
    get_user_partition_info,
)
from contentstore.views.helpers import is_unit, xblock_studio_url, xblock_primary_child_category, \
    xblock_type_display_name, get_parent_xblock, create_xblock, usage_key_with_run
from contentstore.views.preview import get_preview_fragment
from contentstore.utils import is_self_paced

from openedx.core.lib.gating import api as gating_api
from edxmako.shortcuts import render_to_string
from models.settings.course_grading import CourseGradingModel
from openedx.core.lib.xblock_utils import wrap_xblock, request_token
from static_replace import replace_static_urls
from student.auth import has_studio_write_access, has_studio_read_access
from util.date_utils import get_default_time_display
from util.json_request import expect_json, JsonResponse
from util.milestones_helpers import is_entrance_exams_enabled
from xmodule.course_module import DEFAULT_START_DATE
from xmodule.modulestore import ModuleStoreEnum, EdxJSONEncoder
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from xmodule.modulestore.inheritance import own_metadata
from xmodule.tabs import CourseTabList
from xmodule.x_module import PREVIEW_VIEWS, STUDIO_VIEW, STUDENT_VIEW, DEPRECATION_VSCOMPAT_EVENT

__all__ = [
    'orphan_handler', 'xblock_handler', 'xblock_view_handler', 'xblock_outline_handler', 'xblock_container_handler'
]

log = logging.getLogger(__name__)

CREATE_IF_NOT_FOUND = ['course_info']

# Useful constants for defining predicates
NEVER = lambda x: False
ALWAYS = lambda x: True

def hash_resource(resource):
    """
    Hash a :class:`xblock.fragment.FragmentResource`.
    """
    md5 = hashlib.md5()
    md5.update(repr(resource))
    return md5.hexdigest()

def _filter_entrance_exam_grader(graders):
    """
    If the entrance exams feature is enabled we need to hide away the grader from
    views/controls like the 'Grade as' dropdown that allows a course author to select
    the grader type for a given section of a course
    """
    if is_entrance_exams_enabled():
        graders = [grader for grader in graders if grader.get('type') != u'Entrance Exam']
    return graders

@require_http_methods(("DELETE", "GET", "PUT", "POST", "PATCH"))
@login_required
@expect_json
def xblock_handler(request, usage_key_string):
    """
    The restful handler for xblock requests.

    DELETE
        json: delete this xblock instance from the course.
    GET
        json: returns representation of the xblock (locator id, data, and metadata).
              if ?fields=graderType, it returns the graderType for the unit instead of the above.
        html: returns HTML for rendering the xblock (which includes both the "preview" view and the "editor" view)
    PUT or POST or PATCH
        json: if xblock locator is specified, update the xblock instance. The json payload can contain
              these fields, all optional:
                :data: the new value for the data.
                :children: the unicode representation of the UsageKeys of children for this xblock.
                :metadata: new values for the metadata fields. Any whose values are None will be deleted not set
                       to None! Absent ones will be left alone.
                :nullout: which metadata fields to set to None
                :graderType: change how this unit is graded
                :isPrereq: Set this xblock as a prerequisite which can be used to limit access to other xblocks
                :prereqUsageKey: Use the xblock identified by this usage key to limit access to this xblock
                :prereqMinScore: The minimum score that needs to be achieved on the prerequisite xblock
                        identifed by prereqUsageKey
                :publish: can be:
                  'make_public': publish the content
                  'republish': publish this item *only* if it was previously published
                  'discard_changes' - reverts to the last published version
                Note: If 'discard_changes', the other fields will not be used; that is, it is not possible
                to update and discard changes in a single operation.
              The JSON representation on the updated xblock (minus children) is returned.

              if usage_key_string is not specified, create a new xblock instance, either by duplicating
              an existing xblock, or creating an entirely new one. The json playload can contain
              these fields:
                :parent_locator: parent for new xblock, required for both duplicate and create new instance
                :duplicate_source_locator: if present, use this as the source for creating a duplicate copy
                :category: type of xblock, required if duplicate_source_locator is not present.
                :display_name: name for new xblock, optional
                :boilerplate: template name for populating fields, optional and only used
                     if duplicate_source_locator is not present
              The locator (unicode representation of a UsageKey) for the created xblock (minus children) is returned.
    """
    if usage_key_string:
        usage_key = usage_key_with_run(usage_key_string)

        access_check = has_studio_read_access if request.method == 'GET' else has_studio_write_access
        if not access_check(request.user, usage_key.course_key):
            raise PermissionDenied()

        if request.method == 'GET':
            accept_header = request.META.get('HTTP_ACCEPT', 'application/json')

            if 'application/json' in accept_header:
                fields = request.GET.get('fields', '').split(',')
                if 'graderType' in fields:
                    # right now can't combine output of this w/ output of _get_module_info, but worthy goal
                    return JsonResponse(CourseGradingModel.get_section_grader_type(usage_key))
                # TODO: pass fields to _get_module_info and only return those
                with modulestore().bulk_operations(usage_key.course_key):
                    response = _get_module_info(_get_xblock(usage_key, request.user))
                return JsonResponse(response)
            else:
                return HttpResponse(status=406)

        elif request.method == 'DELETE':
            _delete_item(usage_key, request.user)
            return JsonResponse()
        else:  # Since we have a usage_key, we are updating an existing xblock.
            return _save_xblock(
                request.user,
                _get_xblock(usage_key, request.user),
                data=request.json.get('data'),
                children_strings=request.json.get('children'),
                metadata=request.json.get('metadata'),
                nullout=request.json.get('nullout'),
                grader_type=request.json.get('graderType'),
                is_prereq=request.json.get('isPrereq'),
                prereq_usage_key=request.json.get('prereqUsageKey'),
                prereq_min_score=request.json.get('prereqMinScore'),
                publish=request.json.get('publish'),
            )
    elif request.method in ('PUT', 'POST'):
        if 'duplicate_source_locator' in request.json:
            parent_usage_key = usage_key_with_run(request.json['parent_locator'])
            duplicate_source_usage_key = usage_key_with_run(request.json['duplicate_source_locator'])

            source_course = duplicate_source_usage_key.course_key
            dest_course = parent_usage_key.course_key
            if (
                    not has_studio_write_access(request.user, dest_course) or
                    not has_studio_read_access(request.user, source_course)
            ):
                raise PermissionDenied()

            dest_usage_key = _duplicate_item(
                parent_usage_key,
                duplicate_source_usage_key,
                request.user,
                request.json.get('display_name'),
            )

            return JsonResponse({"locator": unicode(dest_usage_key), "courseKey": unicode(dest_usage_key.course_key)})
        else:
            return _create_item(request)
    else:
        return HttpResponseBadRequest(
            "Only instance creation is supported without a usage key.",
            content_type="text/plain"
        )

@require_http_methods(("GET"))
@login_required
@expect_json
def xblock_view_handler(request, usage_key_string, view_name):
    """
    The restful handler for requests for rendered xblock views.

    Returns a json object containing two keys:
        html: The rendered html of the view
        resources: A list of tuples where the first element is the resource hash, and
            the second is the resource description
    """
    usage_key = usage_key_with_run(usage_key_string)
    if not has_studio_read_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    accept_header = request.META.get('HTTP_ACCEPT', 'application/json')

    if 'application/json' in accept_header:
        store = modulestore()
        xblock = store.get_item(usage_key)
        container_views = ['container_preview', 'reorderable_container_child_preview', 'container_child_preview']

        # wrap the generated fragment in the xmodule_editor div so that the javascript
        # can bind to it correctly
        xblock.runtime.wrappers.append(partial(
            wrap_xblock,
            'StudioRuntime',
            usage_id_serializer=unicode,
            request_token=request_token(request),
        ))

        if view_name in (STUDIO_VIEW, VISIBILITY_VIEW):
            try:
                fragment = xblock.render(view_name)
            # catch exceptions indiscriminately, since after this point they escape the
            # dungeon and surface as uneditable, unsaveable, and undeletable
            # component-goblins.
            except Exception as exc:                          # pylint: disable=broad-except
                log.debug("Unable to render %s for %r", view_name, xblock, exc_info=True)
                fragment = Fragment(render_to_string('html_error.html', {'message': str(exc)}))

        elif view_name in PREVIEW_VIEWS + container_views:
            is_pages_view = view_name == STUDENT_VIEW   # Only the "Pages" view uses student view in Studio
            can_edit = has_studio_write_access(request.user, usage_key.course_key)

            # Determine the items to be shown as reorderable. Note that the view
            # 'reorderable_container_child_preview' is only rendered for xblocks that
            # are being shown in a reorderable container, so the xblock is automatically
            # added to the list.
            reorderable_items = set()
            if view_name == 'reorderable_container_child_preview':
                reorderable_items.add(xblock.location)

            paging = None
            try:
                if request.GET.get('enable_paging', 'false') == 'true':
                    paging = {
                        'page_number': int(request.GET.get('page_number', 0)),
                        'page_size': int(request.GET.get('page_size', 0)),
                    }
            except ValueError:
                return HttpResponse(
                    content="Couldn't parse paging parameters: enable_paging: "
                            "{0}, page_number: {1}, page_size: {2}".format(
                                request.GET.get('enable_paging', 'false'),
                                request.GET.get('page_number', 0),
                                request.GET.get('page_size', 0)
                            ),
                    status=400,
                    content_type="text/plain",
                )

            force_render = request.GET.get('force_render', None)

            # Set up the context to be passed to each XBlock's render method.
            context = {
                'is_pages_view': is_pages_view,     # This setting disables the recursive wrapping of xblocks
                'is_unit_page': is_unit(xblock),
                'can_edit': can_edit,
                'root_xblock': xblock if (view_name == 'container_preview') else None,
                'reorderable_items': reorderable_items,
                'paging': paging,
                'force_render': force_render,
            }

            fragment = get_preview_fragment(request, xblock, context)

            # Note that the container view recursively adds headers into the preview fragment,
            # so only the "Pages" view requires that this extra wrapper be included.
            if is_pages_view:
                fragment.content = render_to_string('component.html', {
                    'xblock_context': context,
                    'xblock': xblock,
                    'locator': usage_key,
                    'preview': fragment.content,
                    'label': xblock.display_name or xblock.scope_ids.block_type,
                })
        else:
            raise Http404

        hashed_resources = OrderedDict()
        for resource in fragment.resources:
            hashed_resources[hash_resource(resource)] = resource._asdict()

        return JsonResponse({
            'html': fragment.content,
            'resources': hashed_resources.items()
        })

    else:
        return HttpResponse(status=406)

@require_http_methods(("GET"))
@login_required
@expect_json
def xblock_outline_handler(request, usage_key_string):
    """
    The restful handler for requests for XBlock information about the block and its children.
    This is used by the course outline in particular to construct the tree representation of
    a course.
    """
    usage_key = usage_key_with_run(usage_key_string)
    if not has_studio_read_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    response_format = request.GET.get('format', 'html')
    if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        store = modulestore()
        with store.bulk_operations(usage_key.course_key):
            root_xblock = store.get_item(usage_key, depth=None)
            return JsonResponse(create_xblock_info(
                root_xblock,
                include_child_info=True,
                course_outline=True,
                include_children_predicate=lambda xblock: not xblock.category == 'vertical'
            ))
    else:
        return Http404

@require_http_methods(("GET"))
@login_required
@expect_json
def xblock_container_handler(request, usage_key_string):
    """
    The restful handler for requests for XBlock information about the block and its children.
    This is used by the container page in particular to get additional information about publish state
    and ancestor state.
    """
    usage_key = usage_key_with_run(usage_key_string)

    if not has_studio_read_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    response_format = request.GET.get('format', 'html')
    if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        with modulestore().bulk_operations(usage_key.course_key):
            response = _get_module_info(
                _get_xblock(usage_key, request.user), include_ancestor_info=True, include_publishing_info=True
            )
        return JsonResponse(response)
    else:
        return Http404

def _update_with_callback(xblock, user, old_metadata=None, old_content=None):
    """
    Updates the xblock in the modulestore.
    But before doing so, it calls the xblock's editor_saved callback function.
    """
    if callable(getattr(xblock, "editor_saved", None)):
        if old_metadata is None:
            old_metadata = own_metadata(xblock)
        if old_content is None:
            old_content = xblock.get_explicitly_set_fields_by_scope(Scope.content)
        xblock.editor_saved(user, old_metadata, old_content)

    # Update after the callback so any changes made in the callback will get persisted.
    return modulestore().update_item(xblock, user.id)

def _save_xblock(user, xblock, data=None, children_strings=None, metadata=None, nullout=None,
                 grader_type=None, is_prereq=None, prereq_usage_key=None, prereq_min_score=None, publish=None):
    """
    Saves xblock w/ its fields. Has special processing for grader_type, publish, and nullout and Nones in metadata.
    nullout means to truly set the field to None whereas nones in metadata mean to unset them (so they revert
    to default).
    """
    store = modulestore()
    # Perform all xblock changes within a (single-versioned) transaction
    with store.bulk_operations(xblock.location.course_key):

        # Don't allow updating an xblock and discarding changes in a single operation (unsupported by UI).
        if publish == "discard_changes":
            store.revert_to_published(xblock.location, user.id)
            # Returning the same sort of result that we do for other save operations. In the future,
            # we may want to return the full XBlockInfo.
            return JsonResponse({'id': unicode(xblock.location)})

        old_metadata = own_metadata(xblock)
        old_content = xblock.get_explicitly_set_fields_by_scope(Scope.content)

        if data:
            # TODO Allow any scope.content fields not just "data" (exactly like the get below this)
            xblock.data = data
        else:
            data = old_content['data'] if 'data' in old_content else None

        if children_strings is not None:
            children = []
            for child_string in children_strings:
                children.append(usage_key_with_run(child_string))

            # if new children have been added, remove them from their old parents
            new_children = set(children) - set(xblock.children)
            for new_child in new_children:
                old_parent_location = store.get_parent_location(new_child)
                if old_parent_location:
                    old_parent = store.get_item(old_parent_location)
                    old_parent.children.remove(new_child)
                    old_parent = _update_with_callback(old_parent, user)
                else:
                    # the Studio UI currently doesn't present orphaned children, so assume this is an error
                    return JsonResponse({"error": "Invalid data, possibly caused by concurrent authors."}, 400)

            # make sure there are no old children that became orphans
            # In a single-author (no-conflict) scenario, all children in the persisted list on the server should be
            # present in the updated list.  If there are any children that have been dropped as part of this update,
            # then that would be an error.
            #
            # We can be even more restrictive in a multi-author (conflict), by returning an error whenever
            # len(old_children) > 0. However, that conflict can still be "merged" if the dropped child had been
            # re-parented. Hence, the check for the parent in the any statement below.
            #
            # Note that this multi-author conflict error should not occur in modulestores (such as Split) that support
            # atomic write transactions.  In Split, if there was another author who moved one of the "old_children"
            # into another parent, then that child would have been deleted from this parent on the server. However,
            # this is error could occur in modulestores (such as Draft) that do not support atomic write-transactions
            old_children = set(xblock.children) - set(children)
            if any(
                    store.get_parent_location(old_child) == xblock.location
                    for old_child in old_children
            ):
                # since children are moved as part of a single transaction, orphans should not be created
                return JsonResponse({"error": "Invalid data, possibly caused by concurrent authors."}, 400)

            # set the children on the xblock
            xblock.children = children

        # also commit any metadata which might have been passed along
        if nullout is not None or metadata is not None:
            # the postback is not the complete metadata, as there's system metadata which is
            # not presented to the end-user for editing. So let's use the original (existing_item) and
            # 'apply' the submitted metadata, so we don't end up deleting system metadata.
            if nullout is not None:
                for metadata_key in nullout:
                    setattr(xblock, metadata_key, None)

            # update existing metadata with submitted metadata (which can be partial)
            # IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
            # the intent is to make it None, use the nullout field
            if metadata is not None:
                for metadata_key, value in metadata.items():
                    field = xblock.fields[metadata_key]

                    if value is None:
                        field.delete_from(xblock)
                    else:
                        try:
                            value = field.from_json(value)
                        except ValueError as verr:
                            reason = _("Invalid data")
                            if verr.message:
                                reason = _("Invalid data ({details})").format(details=verr.message)
                            return JsonResponse({"error": reason}, 400)

                        field.write_to(xblock, value)

        # update the xblock and call any xblock callbacks
        xblock = _update_with_callback(xblock, user, old_metadata, old_content)

        # for static tabs, their containing course also records their display name
        course = store.get_course(xblock.location.course_key)
        if xblock.location.category == 'static_tab':
            # find the course's reference to this tab and update the name.
            static_tab = CourseTabList.get_tab_by_slug(course.tabs, xblock.location.name)
            # only update if changed
            if static_tab and static_tab['name'] != xblock.display_name:
                static_tab['name'] = xblock.display_name
                store.update_item(course, user.id)

        result = {
            'id': unicode(xblock.location),
            'data': data,
            'metadata': own_metadata(xblock)
        }

        if grader_type is not None:
            result.update(CourseGradingModel.update_section_grader_type(xblock, grader_type, user))

        # Save gating info
        if xblock.category == 'sequential' and course.enable_subsection_gating:
            if is_prereq is not None:
                if is_prereq:
                    gating_api.add_prerequisite(xblock.location.course_key, xblock.location)
                else:
                    gating_api.remove_prerequisite(xblock.location)
                result['is_prereq'] = is_prereq

            if prereq_usage_key is not None:
                gating_api.set_required_content(
                    xblock.location.course_key, xblock.location, prereq_usage_key, prereq_min_score
                )

        # If publish is set to 'republish' and this item is not in direct only categories and has previously been
        # published, then this item should be republished. This is used by staff locking to ensure that changing the
        # draft value of the staff lock will also update the published version, but only at the unit level.
        if publish == 'republish' and xblock.category not in DIRECT_ONLY_CATEGORIES:
            if modulestore().has_published_version(xblock):
                publish = 'make_public'

        # Make public after updating the xblock, in case the caller asked for both an update and a publish.
        # Used by Bok Choy tests and by republishing of staff locks.
        if publish == 'make_public':
            modulestore().publish(xblock.location, user.id)

        # Note that children aren't being returned until we have a use case.
        return JsonResponse(result, encoder=EdxJSONEncoder)

@login_required
@expect_json
def create_item(request):
    """
    Exposes internal helper method without breaking existing bindings/dependencies
    """
    return _create_item(request)

@login_required
@expect_json
def _create_item(request):
    """View for create items."""
    parent_locator = request.json['parent_locator']
    usage_key = usage_key_with_run(parent_locator)
    if not has_studio_write_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    category = request.json['category']
    if isinstance(usage_key, LibraryUsageLocator):
        # Only these categories are supported at this time.
        if category not in ['html', 'problem', 'video']:
            return HttpResponseBadRequest(
                "Category '%s' not supported for Libraries" % category, content_type='text/plain'
            )

    created_block = create_xblock(
        parent_locator=parent_locator,
        user=request.user,
        category=category,
        display_name=request.json.get('display_name'),
        boilerplate=request.json.get('boilerplate')
    )

    return JsonResponse(
        {"locator": unicode(created_block.location), "courseKey": unicode(created_block.location.course_key)}
    )

def _duplicate_item(parent_usage_key, duplicate_source_usage_key, user, display_name=None):
    """
    Duplicate an existing xblock as a child of the supplied parent_usage_key.
    """
    store = modulestore()
    with store.bulk_operations(duplicate_source_usage_key.course_key):
        source_item = store.get_item(duplicate_source_usage_key)
        # Change the blockID to be unique.
        dest_usage_key = source_item.location.replace(name=uuid4().hex)
        category = dest_usage_key.block_type

        # Update the display name to indicate this is a duplicate (unless display name provided).
        # Can't use own_metadata(), b/c it converts data for JSON serialization -
        # not suitable for setting metadata of the new block
        duplicate_metadata = {}
        for field in source_item.fields.values():
            if field.scope == Scope.settings and field.is_set_on(source_item):
                duplicate_metadata[field.name] = field.read_from(source_item)
        if display_name is not None:
            duplicate_metadata['display_name'] = display_name
        else:
            if source_item.display_name is None:
                duplicate_metadata['display_name'] = _("Duplicate of {0}").format(source_item.category)
            else:
                duplicate_metadata['display_name'] = _("Duplicate of '{0}'").format(source_item.display_name)

        asides_to_create = []
        for aside in source_item.runtime.get_asides(source_item):
            for field in aside.fields.values():
                if field.scope in (Scope.settings, Scope.content,) and field.is_set_on(aside):
                    asides_to_create.append(aside)
                    break

        for aside in asides_to_create:
            for field in aside.fields.values():
                if field.scope not in (Scope.settings, Scope.content,):
                    field.delete_from(aside)

        dest_module = store.create_item(
            user.id,
            dest_usage_key.course_key,
            dest_usage_key.block_type,
            block_id=dest_usage_key.block_id,
            definition_data=source_item.get_explicitly_set_fields_by_scope(Scope.content),
            metadata=duplicate_metadata,
            runtime=source_item.runtime,
            asides=asides_to_create
        )

        children_handled = False

        if hasattr(dest_module, 'studio_post_duplicate'):
            # Allow an XBlock to do anything fancy it may need to when duplicated from another block.
            # These blocks may handle their own children or parenting if needed. Let them return booleans to
            # let us know if we need to handle these or not.
            children_handled = dest_module.studio_post_duplicate(store, source_item)

        # Children are not automatically copied over (and not all xblocks have a 'children' attribute).
        # Because DAGs are not fully supported, we need to actually duplicate each child as well.
        if source_item.has_children and not children_handled:
            dest_module.children = dest_module.children or []
            for child in source_item.children:
                dupe = _duplicate_item(dest_module.location, child, user=user)
                if dupe not in dest_module.children:  # _duplicate_item may add the child for us.
                    dest_module.children.append(dupe)
            store.update_item(dest_module, user.id)

        # pylint: disable=protected-access
        if 'detached' not in source_item.runtime.load_block_type(category)._class_tags:
            parent = store.get_item(parent_usage_key)
            # If source was already a child of the parent, add duplicate immediately afterward.
            # Otherwise, add child to end.
            if source_item.location in parent.children:
                source_index = parent.children.index(source_item.location)
                parent.children.insert(source_index + 1, dest_module.location)
            else:
                parent.children.append(dest_module.location)
            store.update_item(parent, user.id)

        return dest_module.location

@login_required
@expect_json
def delete_item(request, usage_key):
    """
    Exposes internal helper method without breaking existing bindings/dependencies
    """
    _delete_item(usage_key, request.user)

def _delete_item(usage_key, user):
    """
    Deletes an existing xblock with the given usage_key.
    If the xblock is a Static Tab, removes it from course.tabs as well.
    """
    store = modulestore()

    with store.bulk_operations(usage_key.course_key):
        # VS[compat] cdodge: This is a hack because static_tabs also have references from the course module, so
        # if we add one then we need to also add it to the policy information (i.e. metadata)
        # we should remove this once we can break this reference from the course to static tabs
        if usage_key.category == 'static_tab':

            dog_stats_api.increment(
                DEPRECATION_VSCOMPAT_EVENT,
                tags=(
                    "location:_delete_item_static_tab",
                    u"course:{}".format(unicode(usage_key.course_key)),
                )
            )

            course = store.get_course(usage_key.course_key)
            existing_tabs = course.tabs or []
            course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != usage_key.name]
            store.update_item(course, user.id)

        store.delete_item(usage_key, user.id)

@login_required
@require_http_methods(("GET", "DELETE"))
def orphan_handler(request, course_key_string):
    """
    View for handling orphan related requests. GET gets all of the current orphans.
    DELETE removes all orphans (requires is_staff access)

    An orphan is a block whose category is not in the DETACHED_CATEGORY list, is not the root, and is not reachable
    from the root via children
    """
    course_usage_key = CourseKey.from_string(course_key_string)
    if request.method == 'GET':
        if has_studio_read_access(request.user, course_usage_key):
            return JsonResponse([unicode(item) for item in modulestore().get_orphans(course_usage_key)])
        else:
            raise PermissionDenied()
    if request.method == 'DELETE':
        if request.user.is_staff:
            deleted_items = _delete_orphans(course_usage_key, request.user.id, commit=True)
            return JsonResponse({'deleted': deleted_items})
        else:
            raise PermissionDenied()

def _delete_orphans(course_usage_key, user_id, commit=False):
    """
    Helper function to delete orphans for a given course.
    If `commit` is False, this function does not actually remove
    the orphans.
    """
    store = modulestore()
    items = store.get_orphans(course_usage_key)
    branch = course_usage_key.branch
    if commit:
        with store.bulk_operations(course_usage_key):
            for itemloc in items:
                revision = ModuleStoreEnum.RevisionOption.all
                # specify branches when deleting orphans
                if branch == ModuleStoreEnum.BranchName.published:
                    revision = ModuleStoreEnum.RevisionOption.published_only
                store.delete_item(itemloc, user_id, revision=revision)
    return [unicode(item) for item in items]

def _get_xblock(usage_key, user):
    """
    Returns the xblock for the specified usage key. Note: if failing to find a key with a category
    in the CREATE_IF_NOT_FOUND list, an xblock will be created and saved automatically.
    """
    store = modulestore()
    with store.bulk_operations(usage_key.course_key):
        try:
            return store.get_item(usage_key, depth=None)
        except ItemNotFoundError:
            if usage_key.category in CREATE_IF_NOT_FOUND:
                # Create a new one for certain categories only. Used for course info handouts.
                return store.create_item(user.id, usage_key.course_key, usage_key.block_type, block_id=usage_key.block_id)
            else:
                raise
        except InvalidLocationError:
            log.error("Can't find item by location.")
            return JsonResponse({"error": "Can't find item by location: " + unicode(usage_key)}, 404)

def _get_module_info(xblock, rewrite_static_links=True, include_ancestor_info=False, include_publishing_info=False):
    """
    metadata, data, id representation of a leaf module fetcher.
    :param usage_key: A UsageKey
    """
    with modulestore().bulk_operations(xblock.location.course_key):
        data = getattr(xblock, 'data', '')
        if rewrite_static_links:
            data = replace_static_urls(
                data,
                None,
                course_id=xblock.location.course_key
            )

        # Pre-cache has changes for the entire course because we'll need it for the ancestor info
        # Except library blocks which don't [yet] use draft/publish


# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import errno
import getopt
import getpass
import imp
import os
import platform
import re
import sha
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import zipfile

# missing in Python 2.4 and before
if not hasattr(os, "SEEK_SET"):
  os.SEEK_SET = 0

class Options(object): pass
OPTIONS = Options()
OPTIONS.search_path = "out/host/linux-x86"
OPTIONS.verbose = False
OPTIONS.tempfiles = []
OPTIONS.device_specific = None
OPTIONS.extras = {}
OPTIONS.info_dict = None

# Values for "certificate" in apkcerts that mean special things.
SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")

class ExternalError(RuntimeError): pass

def Run(args, **kwargs):
  """Create and return a subprocess.Popen object, printing the command
  line on the terminal if -v was specified."""
  if OPTIONS.verbose:
    print "  running: ", " ".join(args)
  return subprocess.Popen(args, **kwargs)

def CloseInheritedPipes():
  """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds
  before doing other work."""
  if platform.system() != "Darwin":
    return
  for d in range(3, 1025):
    try:
      stat = os.fstat(d)
      if stat is not None:
        pipebit = stat[0] & 0x1000
        if pipebit != 0:
          os.close(d)
    except OSError:
      pass

def LoadInfoDict(zip):
  """Read and parse the META/misc_info.txt key/value pairs from the
  input target files and return a dict."""

  d = {}
  try:
    for line in zip.read("META/misc_info.txt").split("\n"):
      line = line.strip()
      if not line or line.startswith("#"): continue
      k, v = line.split("=", 1)
      d[k] = v
  except KeyError:
    # ok if misc_info.txt doesn't exist
    pass

  # backwards compatibility: These values used to be in their own
  # files.  Look for them, in case we're processing an old
  # target_files zip.

  if "mkyaffs2_extra_flags" not in d:
    try:
      d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
    except KeyError:
      # ok if flags don't exist
      pass

  if "recovery_api_version" not in d:
    try:
      d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
    except KeyError:
      raise ValueError("can't find recovery API version in input target-files")

  if "tool_extensions" not in d:
    try:
      d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
    except KeyError:
      # ok if extensions don't exist
      pass

  try:
    data = zip.read("META/imagesizes.txt")
    for line in data.split("\n"):
      if not line: continue
      name, value = line.split(" ", 1)
      if not value: continue
      if name == "blocksize":
        d[name] = value
      else:
        d[name + "_size"] = value
  except KeyError:
    pass

  def makeint(key):
    if key in d:
      d[key] = int(d[key], 0)

  makeint("recovery_api_version")
  makeint("blocksize")
  makeint("system_size")
  makeint("userdata_size")
  makeint("recovery_size")
  makeint("boot_size")

  d["fstab"] = LoadRecoveryFSTab(zip)
  return d

def LoadRecoveryFSTab(zip):
  class Partition(object):
    pass

  try:
    data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
  except KeyError:
    raise ValueError("Could not find RECOVERY/RAMDISK/etc/recovery.fstab")

  d = {}
  for line in data.split("\n"):
    line = line.strip()
    if not line or line.startswith("#"): continue
    pieces = line.split()
    if not (3 <= len(pieces) <= 4):
      raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))

    p = Partition()
    p.mount_point = pieces[0]
    p.fs_type = pieces[1]
    p.device = pieces[2]
    if len(pieces) == 4:
      p.device2 = pieces[3]
    else:
      p.device2 = None

    d[p.mount_point] = p
  return d

def DumpInfoDict(d):
  for k, v in sorted(d.items()):
    print "%-25s = (%s) %s" % (k, type(v).__name__, v)

def BuildAndAddBootableImage(sourcedir, targetname, output_zip, info_dict):
  """Take a kernel, cmdline, and ramdisk directory from the input (in
  'sourcedir'), and turn them into a boot image.  Put the boot image
  into the output zip file under the name 'targetname'.  Returns
  targetname on success or None on failure (if sourcedir does not
  appear to contain files for the requested image)."""

  print "creating %s..." % (targetname,)

  img = BuildBootableImage(sourcedir)
  if img is None:
    return None

  CheckSize(img, targetname, info_dict)
  ZipWriteStr(output_zip, targetname, img)
  return targetname

def BuildBootableImage(sourcedir):
  """Take a kernel, cmdline, and ramdisk directory from the input (in
  'sourcedir'), and turn them into a boot image.  Return the image
  data, or None if sourcedir does not appear to contains files for
  building the requested image."""

  if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
      not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
    return None

  ramdisk_img = tempfile.NamedTemporaryFile()
  img = tempfile.NamedTemporaryFile()

  p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
           stdout=subprocess.PIPE)
  p2 = Run(["minigzip"],
           stdin=p1.stdout, stdout=ramdisk_img.file.fileno())

  p2.wait()
  p1.wait()
  assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
  assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)

  cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]

  fn = os.path.join(sourcedir, "cmdline")
  if os.access(fn, os.F_OK):
    cmd.append("--cmdline")
    cmd.append(open(fn).read().rstrip("\n"))

  fn = os.path.join(sourcedir, "base")
  if os.access(fn, os.F_OK):
    cmd.append("--base")
    cmd.append(open(fn).read().rstrip("\n"))

  fn = os.path.join(sourcedir, "pagesize")
  if os.access(fn, os.F_OK):
    cmd.append("--pagesize")
    cmd.append(open(fn).read().rstrip("\n"))

  cmd.extend(["--ramdisk", ramdisk_img.name,
              "--output", img.name])

  p = Run(cmd, stdout=subprocess.PIPE)
  p.communicate()
  assert p.returncode == 0, "mkbootimg of %s image failed" % (
      os.path.basename(sourcedir),)

  img.seek(os.SEEK_SET, 0)
  data = img.read()

  ramdisk_img.close()
  img.close()

  return data

def AddRecovery(output_zip, info_dict):
  BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"),
                           "recovery.img", output_zip, info_dict)

def AddBoot(output_zip, info_dict):
  BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"),
                           "boot.img", output_zip, info_dict)

def UnzipTemp(filename, pattern=None):
  """Unzip the given archive into a temporary directory and return the name."""

  tmp = tempfile.mkdtemp(prefix="targetfiles-")
  OPTIONS.tempfiles.append(tmp)
  cmd = ["unzip", "-o", "-q", filename, "-d", tmp]
  if pattern is not None:
    cmd.append(pattern)
  p = Run(cmd, stdout=subprocess.PIPE)
  p.communicate()
  if p.returncode != 0:
    raise ExternalError("failed to unzip input target-files \"%s\"" %
                        (filename,))
  return tmp

def GetKeyPasswords(keylist):
  """Given a list of keys, prompt the user to enter passwords for
  those which require them.  Return a {key: password} dict.  password
  will be None if the key has no password."""

  no_passwords = []
  need_passwords = []
  devnull = open("/dev/null", "w+b")
  for k in sorted(keylist):
    # We don't need a password for things that aren't really keys.
    if k in SPECIAL_CERT_STRINGS:
      no_passwords.append(k)
      continue

    p = Run(["openssl", "pkcs8", "-in", k+".pk8",
             "-inform", "DER", "-nocrypt"],
            stdin=devnull.fileno(),
            stdout=devnull.fileno(),
            stderr=subprocess.STDOUT)
    p.communicate()
    if p.returncode == 0:
      no_passwords.append(k)
    else:
      need_passwords.append(k)
  devnull.close()

  key_passwords = PasswordManager().GetPasswords(need_passwords)
  key_passwords.update(dict.fromkeys(no_passwords, None))
  return key_passwords

def SignFile(input_name, output_name, key, password, align=None,
             whole_file=False):
  """Sign the input_name zip/jar/apk, producing output_name.  Use the
  given key and password (the latter may be None if the key does not
  have a password.

  If align is an integer > 1, zipalign is run to align stored files in
  the output zip on 'align'-byte boundaries.

  If whole_file is true, use the "-w" option to SignApk to embed a
  signature that covers the whole file in the archive comment of the
  zip file.
  """

  if align == 0 or align == 1:
    align = None

  if align:
    temp = tempfile.NamedTemporaryFile()
    sign_name = temp.name
  else:
    sign_name = output_name

  cmd = ["java", "-Xmx512m", "-jar",
           os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
  if whole_file:
    cmd.append("-w")
  cmd.extend([key + ".x509.pem", key + ".pk8",
              input_name, sign_name])

  p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
  if password is not None:
    password += "\n"
  p.communicate(password)
  if p.returncode != 0:
    raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))

  if align:
    p = Run(["zipalign", "-f", str(align), sign_name, output_name])
    p.communicate()
    if p.returncode != 0:
      raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
    temp.close()

def CheckSize(data, target, info_dict):
  """Check the data string passed against the max size limit, if
  any, for the given target.  Raise exception if the data is too big.
  Print a warning if the data is nearing the maximum size."""

  if target.endswith(".img"): target = target[:-4]
  mount_point = "/" + target

  if info_dict["fstab"]:
    if mount_point == "/userdata": mount_point = "/data"
    p = info_dict["fstab"][mount_point]
    fs_type = p.fs_type
    limit = info_dict.get(p.device + "_size", None)
  if not fs_type or not limit: return

  if fs_type == "yaffs2":
    # image size should be increased by 1/64th to account for the
    # spare area (64 bytes per 2k page)
    limit = limit / 2048 * (2048+64)

  size = len(data)
  pct = float(size) * 100.0 / limit
  msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
  if pct >= 99.0:
    raise ExternalError(msg)
  elif pct >= 95.0:
    print
    print "  WARNING: ", msg
    print
  elif OPTIONS.verbose:
    print "  ", msg

def ReadApkCerts(tf_zip):
  """Given a target_files ZipFile, parse the META/apkcerts.txt file
  and return a {package: cert} dict."""
  certmap = {}
  for line in tf_zip.read("META/apkcerts.txt").split("\n"):
    line = line.strip()
    if not line: continue
    m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
                 r'private_key="(.*)"$', line)
    if m:
      name, cert, privkey = m.groups()
      if cert in SPECIAL_CERT_STRINGS and not privkey:
        certmap[name] = cert
      elif (cert.endswith(".x509.pem") and
            privkey.endswith(".pk8") and
            cert[:-9] == privkey[:-4]):
        certmap[name] = cert[:-9]
      else:
        raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
  return certmap

COMMON_DOCSTRING = """
  -p  (--path)  <dir>
      Prepend <dir>/bin to the list of places to search for binaries
      run by this script, and expect to find jars in <dir>/framework.

  -s  (--device_specific) <file>
      Path to the python module containing device-specific
      releasetools code.

  -x  (--extra)  <key=value>
      Add a key/value pair to the 'extras' dict, which device-specific
      extension code may look at.

  -v  (--verbose)
      Show command lines being executed.

  -h  (--help)
      Display this usage message and exit.
"""

def Usage(docstring):
  print docstring.rstrip("\n")
  print COMMON_DOCSTRING

def ParseOptions(argv,
                 docstring,
                 extra_opts="", extra_long_opts=(),
                 extra_option_handler=None):
  """Parse the options in argv and return any arguments that aren't
  flags.  docstring is the calling module's docstring, to be displayed
  for errors and -h.  extra_opts and extra_long_opts are for flags
  defined by the caller, which are processed by passing them to
  extra_option_handler."""

  try:
    opts, args = getopt.getopt(
        argv, "hvp:s:x:" + extra_opts,
        ["help", "verbose", "path=", "device_specific=", "extra="] +
          list(extra_long_opts))
  except getopt.GetoptError, err:
    Usage(docstring)
    print "**", str(err), "**"
    sys.exit(2)

  path_specified = False

  for o, a in opts:
    if o in ("-h", "--help"):
      Usage(docstring)
      sys.exit()
    elif o in ("-v", "--verbose"):
      OPTIONS.verbose = True
    elif o in ("-p", "--path"):
      OPTIONS.search_path = a
    elif o in ("-s", "--device_specific"):
      OPTIONS.device_specific = a
    elif o in ("-x", "--extra"):
      key, value = a.split("=", 1)
      OPTIONS.extras[key] = value
    else:
      if extra_option_handler is None or not extra_option_handler(o, a):
        assert False, "unknown option \"%s\"" % (o,)

  os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
                        os.pathsep + os.environ["PATH"])

  return args

def Cleanup():
  for i in OPTIONS.tempfiles:
    if os.path.isdir(i):
      shutil.rmtree(i)
    else:
      os.remove(i)

class PasswordManager(object):
  def __init__(self):
    self.editor = os.getenv("EDITOR", None)
    self.pwfile = os.getenv("ANDROID_PW_FILE", None)

  def GetPasswords(self, items):
    """Get passwords corresponding to each string in 'items',
    returning a dict.  (The dict may have keys in addition to the
    values in 'items'.)

    Uses the passwords in $ANDROID_PW_FILE if available, letting the
    user edit that file to add more needed passwords.  If no editor is
    available, or $ANDROID_PW_FILE isn't define, prompts the user
    interactively in the ordinary way.
    """

    current = self.ReadFile()

    first = True
    while True:
      missing = []
      for i in items:
        if i not in current or not current[i]:
          missing.append(i)
      # Are all the passwords already in the file?
      if not missing: return current

      for i in missing:
        current[i] = ""

      if not first:
        print "key file %s still missing some passwords." % (self.pwfile,)
        answer = raw_input("try to edit again? [y]> ").strip()
        if answer and answer[0] not in 'yY':
          raise RuntimeError("key passwords unavailable")
      first = False

      current = self.UpdateAndReadFile(current)

  def PromptResult(self, current):
    """Prompt the user to enter a value (password) for each key in
    'current' whose value is fales.  Returns a new dict with all the
    values.
    """
    result = {}
    for k, v in sorted(current.iteritems()):
      if v:
        result[k] = v
      else:
      #Alfred test
#        while True:
#          result[k] = getpass.getpass("Enter password for %s key> "
#                                      % (k,)).strip()
#          if result[k]: break
        print ">>>>>PrompResult k %s" % (k,)
        result['build/target/product/security/releasekey']='FIHLX'

    return result

  def UpdateAndReadFile(self, current):
    if not self.editor or not self.pwfile:
      return self.PromptResult(current)

    f = open(self.pwfile, "w")
    os.chmod(self.pwfile, 0600)
    f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
    f.write("# (Additional spaces are harmless.)\n\n")

    first_line = None
    sorted = [(not v, k, v) for (k, v) in current.iteritems()]
    sorted.sort()
    for i, (_, k, v) in enumerate(sorted):
      f.write("[[[  %s  ]]] %s\n" % (v, k))
      if not v and first_line is None:
        # position cursor on first line with no password.
        first_line = i + 4
    f.close()

    p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
    _, _ = p.communicate()

    return self.ReadFile()

  def ReadFile(self):
    result = {}
    if self.pwfile is None: return result
    try:
      f = open(self.pwfile, "r")
      for line in f:
        line = line.strip()
        if not line or line[0] == '#': continue
        m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
        if not m:
          print "failed to parse password file: ", line
        else:
          result[m.group(2)] = m.group(1)
      f.close()
    except IOError, e:
      if e.errno != errno.ENOENT:
        print "error reading password file: ", str(e)
    return result

def ZipWriteStr(zip, filename, data, perms=0644):
  # use a fixed timestamp so the output is repeatable.
  zinfo = zipfile.ZipInfo(filename=filename,
                          date_time=(2009, 1, 1, 0, 0, 0))
  zinfo.compress_type = zip.compression
  zinfo.external_attr = perms << 16
  zip.writestr(zinfo, data)

class DeviceSpecificParams(object):
  module = None
  def __init__(self, **kwargs):
    """Keyword arguments to the constructor become attributes of this
    object, which is passed to all functions in the device-specific
    module."""
    for k, v in kwargs.iteritems():
      setattr(self, k, v)
    self.extras = OPTIONS.extras

    if self.module is None:
      path = OPTIONS.device_specific
      if not path: return
      try:
        if os.path.isdir(path):
          info = imp.find_module("releasetools", [path])
        else:
          d, f = os.path.split(path)
          b, x = os.path.splitext(f)
          if x == ".py":
            f = b
          info = imp.find_module(f, [d])
        self.module = imp.load_module("device_specific", *info)
      except ImportError:
        print "unable to load device-specific module; assuming none"

  def _DoCall(self, function_name, *args, **kwargs):
    """Call the named function in the device-specific module, passing
    the given args and kwargs.  The first argument to the call will be
    the DeviceSpecific object itself.  If there is no module, or the
    module does not define the function, return the value of the
    'default' kwarg (which itself defaults to None)."""
    if self.module is None or not hasattr(self.module, function_name):
      return kwargs.get("default", None)
    return getattr(self.module, function_name)(*((self,) + args), **kwargs)

  def FullOTA_Assertions(self):
    """Called after emitting the block of assertions at the top of a
    full OTA package.  Implementations can add whatever additional
    assertions they like."""
    return self._DoCall("FullOTA_Assertions")

  def FullOTA_InstallEnd(self):
    """Called at the end of full OTA installation; typically this is
    used to install the image for the device's baseband processor."""
    return self._DoCall("FullOTA_InstallEnd")

  def IncrementalOTA_Assertions(self):
    """Called after emitting the block of assertions at the top of an
    incremental OTA package.  Implementations can add whatever
    additional assertions they like."""
    return self._DoCall("IncrementalOTA_Assertions")

  def IncrementalOTA_VerifyEnd(self):
    """Called at the end of the verification phase of incremental OTA
    installation; additional checks can be placed here to abort the
    script before any changes are made."""
    return self._DoCall("IncrementalOTA_VerifyEnd")

  def IncrementalOTA_InstallEnd(self):
    """Called at the end of incremental OTA installation; typically
    this is used to install the image for the device's baseband
    processor."""
    return self._DoCall("IncrementalOTA_InstallEnd")

class File(object):
  def __init__(self, name, data):
    self.name = name
    self.data = data
    self.size = len(data)
    self.sha1 = sha.sha(data).hexdigest()

  def WriteToTemp(self):
    t = tempfile.NamedTemporaryFile()
    t.write(self.data)
    t.flush()
    return t

  def AddToZip(self, z):
    ZipWriteStr(z, self.name, self.data)

DIFF_PROGRAM_BY_EXT = {
    ".gz" : "imgdiff",
    ".zip" : ["imgdiff", "-z"],
    ".jar" : ["imgdiff", "-z"],
    ".apk" : ["imgdiff", "-z"],
    ".img" : "imgdiff",
    }

class Difference(object):
  def __init__(self, tf, sf):
    self.tf = tf
    self.sf = sf
    self.patch = None

  def ComputePatch(self):
    """Compute the patch (as a string of data) needed to turn sf into
    tf.  Returns the same tuple as GetPatch()."""

    tf = self.tf
    sf = self.sf

    ext = os.path.splitext(tf.name)[1]
    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")

    ttemp = tf.WriteToTemp()
    stemp = sf.WriteToTemp()

    ext = os.path.splitext(tf.name)[1]

    try:
      ptemp = tempfile.NamedTemporaryFile()


# -*- coding: utf-8 -*-
"""Calendar handler test."""

from .. import mocks
from ..test_base_handler import TestBaseHandler
from models import User
from models import Institution
from models import Event
from datetime import datetime
from handlers.event_collection_handler import EventCollectionHandler, get_filtered_events
from google.appengine.ext import ndb
import json

from mock import patch

class EventCollectionHandlerTest(TestBaseHandler):
    """Calendar handler test."""

    @classmethod
    def setUp(cls):
        """Provide the base for the tests."""
        super(EventCollectionHandlerTest, cls).setUp()
        app = cls.webapp2.WSGIApplication(
            [("/api/events.*", EventCollectionHandler),
             ], debug=True)
        cls.testapp = cls.webtest.TestApp(app)
        initModels(cls)

    @patch('util.login_service.verify_token', return_value={'email': 'user@gmail.com'})
    def test_post(self, verify_token):
        """Test the calendar_handler's post event method."""

        # Make the request and assign the answer to post
        event = self.testapp.post_json("/api/events", {
            'title': 'new event',
            'institution_key': self.certbio.key.urlsafe(),
            'text': 'testing new event',
            'start_time': '2027-12-24T10:25:48',
            'end_time': '2027-12-25T10:25:48',
            'local': 'local do evento',
            'address': {}
            })
        # Retrieve the entities
        event = json.loads(event._app_iter[0])
        key_event = ndb.Key(urlsafe=event['key'])
        event_obj = key_event.get()
        self.certbio = self.certbio.key.get()
        self.user = self.user.key.get()

        # Check if the event's attributes are the expected
        self.assertEqual(event_obj.title, 'new event',
                         "The title expected was new event")
        self.assertEqual(event_obj.institution_key, self.certbio.key,
                         "The post's institution is not the expected one")
        self.assertEqual(event_obj.text,
                         'testing new event',
                         "The post's text is not the expected one")
        self.assertTrue(self.user.has_permission('remove_post', event_obj.key.urlsafe()))
        self.assertTrue(self.user.has_permission(
            'edit_post', event_obj.key.urlsafe()))

        # Check if raise exception when the end time of event is before the start time
        with self.assertRaises(Exception) as raises_context:
            self.testapp.post_json("/api/events", {
                'title': 'new event',
                'institution_key': self.certbio.key.urlsafe(),
                'text': 'testing new event',
                'start_time': '2027-12-24T10:25:48',
                'end_time': '2027-12-20T10:25:48',
                'local': 'local do evento',
                'address': {}
                })

        message_exception = self.get_message_exception(str(raises_context.exception))

        self.assertEqual(
            message_exception,
            "Error! The end time can not be before the start time",
            "Expected exception message must be equal to " +
            "Error! The end time can not be before the start time")

        # Check if raise exception when the end time of event is before today
        with self.assertRaises(Exception) as raises_context:
            self.testapp.post_json("/api/events", {
                'title': 'new event',
                'institution_key': self.certbio.key.urlsafe(),
                'text': 'testing new event',
                'start_time': '2010-12-24T10:25:48',
                'end_time': '2010-12-25T10:25:48',
                'local': 'local do evento',
                'address': {}
                })

        message_exception = self.get_message_exception(str(raises_context.exception))

        self.assertEqual(
            message_exception,
            "Error! The end time must be after the current time",
            "Excpected exception message must be equal to " +
            "Error! The end time must be after the current time")

    @patch('util.login_service.verify_token', return_value={'email': 'user@gmail.com'})
    def test_get(self, verify_token):
        """Test the calendar_handler's post event method."""

        # Call the get method
        events = self.testapp.get("/api/events?page=0&limit=1")

        # Retrieve the entities
        event = (events.json['events'])[0]
        key_event = ndb.Key(urlsafe=event['key'])
        event_obj = key_event.get()
        self.certbio = self.certbio.key.get()
        self.user = self.user.key.get()

        # Check if the event's attributes are the expected
        self.assertEqual(event_obj.title, 'New Event',
                         "The title expected was new event")
        self.assertEqual(event_obj.institution_key, self.certbio.key,
                         "The post's institution is not the expected one")
        self.assertEqual(event_obj.local,
                         'Event location',
                         "The event's local is not the expected one")

    def test_get_filtered_events_with_date_filters(self):
        """Test the get query of events filtered by date"""

        # test events with filter by current month and year
        filters = [('page', '0'), ('limit', '1'), ('month', str(self.date.month)), ('year', str(self.date.year))]
        query_events = get_filtered_events(filters, self.user).fetch()

        self.assertEqual(len(query_events), 2,
                    "Should return only the events that happens the current month")

        # test events that cross the month and year
        # filter by november 2017
        filters = [('page', '0'), ('limit', '1'), ('month', '11'), ('year', '2017')]
        query_events = get_filtered_events(filters, self.user).fetch()

        self.assertTrue(self.event_through_months in query_events,
            "Should have the event that happens in november and december of 2017")

        self.assertTrue(self.event_through_years in query_events,
            "should have the event that happens on november 2017 to january 2018")

        self.assertTrue(self.event_utc in query_events,
            "should have the event that happens on november 2017")

        # change filter to december 2017
        filters = [('page', '0'), ('limit', '1'), ('month', '12'), ('year', '2017')]
        query_events = get_filtered_events(filters, self.user).fetch()

        self.assertTrue(self.event_through_months in query_events,
            "Should have the event that happens in november and december of 2017")

        self.assertTrue(self.event_through_years in query_events,
            "should have the event that happens on november 2017 to january 2018")

        self.assertEqual(self.event_utc.end_time.strftime("%Y-%m-%d %H:%M:%S"), "2017-12-01 02:55:00",
                "The event ends 'fake' on december because of utc")

        self.assertFalse(self.event_utc in query_events,
                "Should NOT have the event that ends 'fake' on december 2017")

        # change filter to january 2018
        filters = [('page', '0'), ('limit', '1'), ('month', '1'), ('year', '2018')]
        query_events = get_filtered_events(filters, self.user).fetch()

        self.assertFalse(self.event_through_months in query_events,
            "Should NOT have the event that happens in november and december of 2017")

        self.assertTrue(self.event_through_years in query_events,
            "should have the event that happens on november 2017 to january 2018")

        filters = [('page', '0'), ('limit', '1'), ('month', str(self.date.month)), ('year', '2016')]
        query_events = get_filtered_events(filters, self.user).fetch()

        self.assertEqual(len(query_events), 0,
                    "Should return nothing because does not have past events")

    def test_get_filtered_events_without_date_filters(self):
        """Test the get query events not filtered"""

        filters = [('page', '0'), ('limit', '1')]
        query_events = get_filtered_events(filters, self.user).fetch()

        self.assertEqual(len(query_events), 7,
                    "Should return all created events in the initModels")

def setup_title_and_local(event):
    event.title = "New Event"
    event.local = "Event location"
    event.put()

def setup_photo_url(entity):
    entity.photo_url = 'urlphoto'
    entity.put()

def setup_date(event, date):
    one_month_later = datetime(date.year if date.month < 12 else date.year+1, date.month+1 if date.month < 12 else 1, 15)
    event.start_time = one_month_later
    event.end_time = one_month_later
    event.put()

def initModels(cls):
    """Init the models."""
    # new User user
    cls.user = mocks.create_user("user@gmail.com")
    setup_photo_url(cls.user)

    # new Institution CERTBIO


# qgis-ol3 Creates OpenLayers map from QGIS layers
# Copyright (C) 2014 Victor Olaya (volayaf@gmail.com)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import os
import re
from datetime import datetime
from qgis.core import (QgsProject,
                       QgsCoordinateReferenceSystem,
                       QgsCoordinateTransform,
                       QgsRectangle,
                       QgsCsException)
from qgis.PyQt.QtCore import Qt, QObject
from qgis.PyQt.QtGui import QCursor
from qgis.PyQt.QtWidgets import QApplication
from qgis2web.utils import exportLayers, replaceInTemplate
from qgis2web.olFileScripts import (writeFiles,
                                    writeHTMLstart,
                                    writeLayerSearch,
                                    writeScriptIncludes)
from qgis2web.olLayerScripts import writeLayersAndGroups
from qgis2web.olScriptStrings import (measureScript,
                                      measuringScript,
                                      measureControlScript,
                                      measureUnitMetricScript,
                                      measureUnitFeetScript,
                                      measureStyleScript,
                                      layerSearchStyleScript,
                                      geolocation,
                                      geolocateStyle,
                                      geolocationHead,
                                      geocodeLinks,
                                      geocodeJS,
                                      geocodeScript,
                                      getGrid,
                                      getM2px,
                                      getMapUnitLayers)
from qgis2web.olStyleScripts import exportStyles
from qgis2web.writer import (Writer,
                             WriterResult,
                             translator)
from qgis2web.feedbackDialog import Feedback

class OpenLayersWriter(Writer):
    """
    Writer for creation of web maps based on the OpenLayers
    JavaScript library.
    """

    def __init__(self):
        super(OpenLayersWriter, self).__init__()

    @classmethod
    def type(cls):
        return 'openlayers'

    @classmethod
    def name(cls):
        return QObject.tr(translator, 'OpenLayers')

    def write(self, iface, dest_folder, feedback=None):
        if not feedback:
            feedback = Feedback()

        feedback.showFeedback('Creating OpenLayers map...')

        self.preview_file = self.writeOL(iface, feedback,
                                         layers=self.layers,
                                         groups=self.groups,
                                         popup=self.popup,
                                         visible=self.visible,
                                         interactive=self.interactive,
                                         json=self.json,
                                         clustered=self.cluster,
                                         getFeatureInfo=self.getFeatureInfo,
                                         settings=self.params,
                                         folder=dest_folder)
        result = WriterResult()
        result.index_file = self.preview_file
        result.folder = os.path.dirname(self.preview_file)
        for dirpath, dirnames, filenames in os.walk(result.folder):
            result.files.extend([os.path.join(dirpath, f) for f in filenames])
        return result

    @classmethod
    def writeOL(cls, iface, feedback, layers, groups, popup, visible,
                interactive, json, clustered, getFeatureInfo, settings,
                folder):
        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
        mapSettings = iface.mapCanvas().mapSettings()
        controlCount = 0
        stamp = datetime.now().strftime("%Y_%m_%d-%H_%M_%S_%f")
        folder = os.path.join(folder, 'qgis2web_' + stamp)
        restrictToExtent = settings["Scale/Zoom"]["Restrict to extent"]
        matchCRS = settings["Appearance"]["Match project CRS"]
        precision = settings["Data export"]["Precision"]
        optimize = settings["Data export"]["Minify GeoJSON files"]
        extent = settings["Scale/Zoom"]["Extent"]
        mapbounds = bounds(iface, extent == "Canvas extent", layers, matchCRS)
        fullextent = bounds(iface, False, layers, matchCRS)
        geolocateUser = settings["Appearance"]["Geolocate user"]
        maxZoom = int(settings["Scale/Zoom"]["Max zoom level"])
        minZoom = int(settings["Scale/Zoom"]["Min zoom level"])
        popupsOnHover = settings["Appearance"]["Show popups on hover"]
        highlightFeatures = settings["Appearance"]["Highlight on hover"]
        geocode = settings["Appearance"]["Add address search"]
        measureTool = settings["Appearance"]["Measure tool"]
        addLayersList = settings["Appearance"]["Add layers list"]
        htmlTemplate = settings["Appearance"]["Template"]
        layerSearch = settings["Appearance"]["Layer search"]
        searchLayer = settings["Appearance"]["Search layer"]
        widgetAccent = settings["Appearance"]["Widget Icon"]
        widgetBackground = settings["Appearance"]["Widget Background"]

        writeFiles(folder, restrictToExtent, feedback)
        exportLayers(iface, layers, folder, precision, optimize,
                     popup, json, restrictToExtent, extent, feedback, matchCRS)
        mapUnitsLayers = exportStyles(layers, folder, clustered, feedback)
        mapUnitLayers = getMapUnitLayers(mapUnitsLayers)
        osmb = writeLayersAndGroups(layers, groups, visible, interactive,
                                    folder, popup, settings, json, matchCRS,
                                    clustered, getFeatureInfo, iface,
                                    restrictToExtent, extent, mapbounds,
                                    mapSettings.destinationCrs().authid())
        (jsAddress,
         cssAddress, controlCount) = writeHTMLstart(settings, controlCount,
                                                    osmb, feedback)
        (geojsonVars, wfsVars, styleVars) = writeScriptIncludes(layers,
                                                                json, matchCRS)
        popupLayers = "popupLayers = [%s];" % ",".join(
            ['1' for field in popup])
        project = QgsProject.instance()
        controls = getControls(project, measureTool, geolocateUser)
        layersList = getLayersList(addLayersList)
        pageTitle = project.title()
        backgroundColor = getBackground(mapSettings, widgetAccent,
                                        widgetBackground)
        (geolocateCode, controlCount) = geolocateStyle(geolocateUser,
                                                       controlCount)
        backgroundColor += geolocateCode
        mapextent = "extent: %s," % mapbounds if restrictToExtent else ""
        onHover = str(popupsOnHover).lower()
        highlight = str(highlightFeatures).lower()
        highlightFill = mapSettings.selectionColor().name()
        (proj, proj4, view) = getCRSView(mapextent, fullextent, maxZoom,
                                         minZoom, matchCRS, mapSettings)
        (measureControl, measuring, measure, measureUnit, measureStyle,
         controlCount) = getMeasure(measureTool, controlCount)
        geolocateHead = geolocationHead(geolocateUser)
        geolocate = geolocation(geolocateUser)
        geocodingLinks = geocodeLinks(geocode)
        geocodingJS = geocodeJS(geocode)
        geocodingScript = geocodeScript(geocode)
        m2px = getM2px(mapUnitsLayers)
        (extracss, controlCount) = getCSS(geocode, geolocateUser, layerSearch,
                                          controlCount)
        (jsAddress, cssAddress,
         layerSearch, controlCount) = writeLayerSearch(cssAddress, jsAddress,
                                                       controlCount,
                                                       layerSearch,
                                                       searchLayer, feedback)
        ol3layerswitcher = getLayerSwitcher()
        ol3popup = getPopup()
        ol3qgis2webjs = getJS(osmb)
        ol3layers = getLayers()
        mapSize = iface.mapCanvas().size()
        exp_js = getExpJS()
        grid = getGrid(project)
        values = {"@PAGETITLE@": pageTitle,
                  "@CSSADDRESS@": cssAddress,
                  "@EXTRACSS@": extracss,
                  "@JSADDRESS@": jsAddress,
                  "@MAP_WIDTH@": str(mapSize.width()) + "px",
                  "@MAP_HEIGHT@": str(mapSize.height()) + "px",
                  "@OL3_STYLEVARS@": styleVars,
                  "@OL3_BACKGROUNDCOLOR@": backgroundColor,
                  "@OL3_POPUP@": ol3popup,
                  "@OL3_GEOJSONVARS@": geojsonVars,
                  "@OL3_WFSVARS@": wfsVars,
                  "@OL3_PROJ4@": proj4,
                  "@OL3_PROJDEF@": proj,
                  "@OL3_GEOCODINGLINKS@": geocodingLinks,
                  "@OL3_GEOCODINGJS@": geocodingJS,
                  "@QGIS2WEBJS@": ol3qgis2webjs,
                  "@OL3_LAYERSWITCHER@": ol3layerswitcher,
                  "@OL3_LAYERS@": ol3layers,
                  "@OL3_MEASURESTYLE@": measureStyle,
                  "@EXP_JS@": exp_js,
                  "@LEAFLET_ADDRESSCSS@": "",
                  "@LEAFLET_MEASURECSS@": "",
                  "@LEAFLET_LAYERFILTERCSS@": "",
                  "@LEAFLET_EXTRAJS@": "",
                  "@LEAFLET_ADDRESSJS@": "",
                  "@LEAFLET_MEASUREJS@": "",
                  "@LEAFLET_CRSJS@": "",
                  "@LEAFLET_LAYERSEARCHCSS@": "",
                  "@LEAFLET_LAYERSEARCHJS@": "",
                  "@LEAFLET_LAYERFILTERJS@": "",
                  "@LEAFLET_CLUSTERCSS@": "",
                  "@LEAFLET_CLUSTERJS@": "",
                  "@MBGLJS_MEASURE@": "",
                  "@MBGLJS_LOCATE@": ""}
        with open(os.path.join(folder, "index.html"), "w") as f:
            htmlTemplate = htmlTemplate
            if htmlTemplate == "":
                htmlTemplate = "full-screen"
            templateOutput = replaceInTemplate(
                htmlTemplate + ".html", values)
            templateOutput = re.sub(r'\n[\s_]+\n', '\n', templateOutput)
            f.write(templateOutput)
        values = {"@GEOLOCATEHEAD@": geolocateHead,
                  "@BOUNDS@": mapbounds,
                  "@CONTROLS@": ",".join(controls),
                  "@LAYERSLIST@": layersList,
                  "@POPUPLAYERS@": popupLayers,
                  "@VIEW@": view,
                  "@LAYERSEARCH@": layerSearch,
                  "@ONHOVER@": onHover,
                  "@DOHIGHLIGHT@": highlight,
                  "@HIGHLIGHTFILL@": highlightFill,
                  "@GEOLOCATE@": geolocate,
                  "@GEOCODINGSCRIPT@": geocodingScript,
                  "@MEASURECONTROL@": measureControl,
                  "@MEASURING@": measuring,
                  "@MEASURE@": measure,
                  "@MEASUREUNIT@": measureUnit,
                  "@GRID@": grid,
                  "@M2PX@": m2px,
                  "@MAPUNITLAYERS@": mapUnitLayers}
        with open(os.path.join(folder, "resources", "qgis2web.js"),
                  "w") as f:
            out = replaceInScript("qgis2web.js", values)
            f.write(out)
        QApplication.restoreOverrideCursor()
        return os.path.join(folder, "index.html")

def replaceInScript(template, values):
    path = os.path.join(os.path.dirname(__file__), "resources", template)
    with open(path) as f:
        lines = f.readlines()
    s = "".join(lines)
    for name, value in values.items():
        s = s.replace(name, value)
    return s

def bounds(iface, useCanvas, layers, matchCRS):
    if useCanvas:
        canvas = iface.mapCanvas()
        canvasCrs = canvas.mapSettings().destinationCrs()
        if not matchCRS:
            epsg3857 = QgsCoordinateReferenceSystem("EPSG:3857")
            try:
                transform = QgsCoordinateTransform(canvasCrs, epsg3857,
                                                   QgsProject.instance())
            except Exception:
                transform = QgsCoordinateTransform(canvasCrs, epsg3857)

            try:
                extent = transform.transformBoundingBox(canvas.extent())
            except QgsCsException:
                extent = QgsRectangle(-20026376.39, -20048966.10,
                                      20026376.39, 20048966.10)
        else:
            extent = canvas.extent()
    else:
        extent = None
        for layer in layers:
            if not matchCRS: