<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import hashlib
import io
import json
import mimetypes
import os
import zipfile
from xml.etree.ElementTree import SubElement

import html5lib
from django.conf import settings
from django.contrib.staticfiles import finders
from django.http import Http404
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.http.response import FileResponse
from django.http.response import HttpResponseNotModified
from django.template import loader
from django.templatetags.static import static
from django.utils.decorators import method_decorator
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.http import etag
from django.views.generic.base import View

from .api import cache_forever
from .decorators import add_security_headers
from .models import ContentNode
from .utils.paths import get_content_storage_file_path
from kolibri import __version__ as kolibri_version
from kolibri.core.content.errors import InvalidStorageFilenameError
from kolibri.core.content.hooks import ContentNodeDisplayHook

# Do this to prevent import of broken Windows filetype registry that makes guesstype not work.
# https://www.thecodingforums.com/threads/mimetypes-guess_type-broken-in-windows-on-py2-7-and-python-3-x.952693/
mimetypes.init([os.path.join(os.path.dirname(__file__), "constants", "mime.types")])

HASHI_FILENAME = None


def get_hashi_filename():
    global HASHI_FILENAME
    if HASHI_FILENAME is None or getattr(settings, "DEVELOPER_MODE", None):
        with io.open(
            os.path.join(os.path.dirname(__file__), "./build/hashi_filename"),
            mode="r",
            encoding="utf-8",
        ) as f:
            HASHI_FILENAME = f.read().strip()
    return HASHI_FILENAME


def calculate_zip_content_etag(request, *args, **kwargs):
    return hashlib.md5(
        kolibri_version.encode("utf-8") + get_hashi_filename().encode("utf-8")
    ).hexdigest()


def get_path_or_404(zipped_filename):
    try:
        # calculate the local file path to the zip file
        zipped_path = get_content_storage_file_path(zipped_filename)
        # if the zipfile does not exist on disk, return a 404
        if not os.path.exists(zipped_path):
            raise InvalidStorageFilenameError()
        return zipped_path
    except InvalidStorageFilenameError:
        raise Http404(
            '"%(filename)s" is not a valid zip file' % {"filename": zipped_filename}
        )


def recursive_h5p_dependencies(zf, data, prefix=""):

    jsfiles = []
    cssfiles = []

    # load the dependencies, recursively, to extract their JS and CSS paths to include
    for dep in data.get("preloadedDependencies", []):
        packagepath = "{machineName}-{majorVersion}.{minorVersion}/".format(**dep)
        librarypath = packagepath + "library.json"
        info = zf.getinfo(librarypath)
        content = json.loads(zf.open(info).read())
        newjs, newcss = recursive_h5p_dependencies(zf, content, packagepath)
        cssfiles += newcss
        jsfiles += newjs

    # load the JS required for the current package
    for js in data.get("preloadedJs", []):
        path = prefix + js["path"]
        if path not in jsfiles:
            jsfiles.append(path)

    # load the CSS required for the current package
    for css in data.get("preloadedCss", []):
        path = prefix + css["path"]
        if path not in cssfiles:
            cssfiles.append(path)

    return jsfiles, cssfiles


def replace_script(parent, script):
    parent.remove(script)
    template = SubElement(parent, "template", attrib={"hashi-script": "true"})
    if script.get("async") is not None:
        template.set("async", "true")
    template.append(script)


def parse_html(content):
    try:
        document = html5lib.parse(content, namespaceHTMLElements=False)
        if not document:
            # Could not parse
            return content

        for parent in document.findall(".//script/.."):
            for script in parent.findall("script"):
                replace_script(parent, script)
        # Because html5lib parses like a browser, it will
        # always create head and body tags if they are missing.
        head = document.find("head")
        SubElement(
            head,
            "script",
            attrib={
                "src": static(
                    "content/{filename}".format(filename=get_hashi_filename())
                )
            },
        )
        return html5lib.serialize(
            document,
            quote_attr_values="always",
            omit_optional_tags=False,
            minimize_boolean_attributes=False,
            use_trailing_solidus=True,
            space_before_trailing_solidus=False,
        )
    except html5lib.html5parser.ParseError:
        return content


def get_h5p(zf, embedded_filepath):
    if not embedded_filepath:
        # Get the h5p bootloader, and then run it through our hashi templating code.
        # return the H5P bootloader code
        h5pdata = json.loads(zf.open(zf.getinfo("h5p.json")).read())
        jsfiles, cssfiles = recursive_h5p_dependencies(zf, h5pdata)
        contentdata = zf.open(zf.getinfo("content/content.json")).read()
        path_includes_version = (
            "true"
            if "-" in [name for name in zf.namelist() if "/" in name][0]
            else "false"
        )
        template = loader.get_template("content/h5p.html")
        main_library_data = [
            lib
            for lib in h5pdata["preloadedDependencies"]
            if lib["machineName"] == h5pdata["mainLibrary"]
        ][0]
        bootstrap_content = template.render(
            {
                "jsfiles": jsfiles,
                "cssfiles": cssfiles,
                "content": contentdata,
                "library": "{machineName} {majorVersion}.{minorVersion}".format(
                    **main_library_data
                ),
                "path_includes_version": path_includes_version,
            },
            None,
        )
        content = parse_html(bootstrap_content)
        content_type = "text/html"
    elif embedded_filepath.startswith("dist/"):
        # return static H5P dist resources
        path = finders.find("assets/h5p-standalone-" + embedded_filepath)
        with open(path) as f:
            content = f.read()
        # try to guess the MIME type of the embedded file being referenced
        content_type = (
            mimetypes.guess_type(embedded_filepath)[0] or "application/octet-stream"
        )
    response = HttpResponse(content, content_type=content_type)
    response["Content-Length"] = len(content)
    return response


def get_embedded_file(request, zf, zipped_filename, embedded_filepath):
    # if no path, or a directory, is being referenced, look for an index.html file
    if not embedded_filepath or embedded_filepath.endswith("/"):
        embedded_filepath += "index.html"

    # get the details about the embedded file, and ensure it exists
    try:
        info = zf.getinfo(embedded_filepath)
    except KeyError:<commit_msg>Set response content length consistently for h5p rendering.<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import hashlib
import io
import json
import mimetypes
import os
import zipfile
from xml.etree.ElementTree import SubElement

import html5lib
from django.conf import settings
from django.contrib.staticfiles import finders
from django.http import Http404
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.http.response import FileResponse
from django.http.response import HttpResponseNotModified
from django.template import loader
from django.templatetags.static import static
from django.utils.decorators import method_decorator
from django.views.decorators.clickjacking import xframe_options_exempt
from django.views.decorators.http import etag
from django.views.generic.base import View

from .api import cache_forever
from .decorators import add_security_headers
from .models import ContentNode
from .utils.paths import get_content_storage_file_path
from kolibri import __version__ as kolibri_version
from kolibri.core.content.errors import InvalidStorageFilenameError
from kolibri.core.content.hooks import ContentNodeDisplayHook

# Do this to prevent import of broken Windows filetype registry that makes guesstype not work.
# https://www.thecodingforums.com/threads/mimetypes-guess_type-broken-in-windows-on-py2-7-and-python-3-x.952693/
mimetypes.init([os.path.join(os.path.dirname(__file__), "constants", "mime.types")])

HASHI_FILENAME = None


def get_hashi_filename():
    global HASHI_FILENAME
    if HASHI_FILENAME is None or getattr(settings, "DEVELOPER_MODE", None):
        with io.open(
            os.path.join(os.path.dirname(__file__), "./build/hashi_filename"),
            mode="r",
            encoding="utf-8",
        ) as f:
            HASHI_FILENAME = f.read().strip()
    return HASHI_FILENAME


def calculate_zip_content_etag(request, *args, **kwargs):
    return hashlib.md5(
        kolibri_version.encode("utf-8") + get_hashi_filename().encode("utf-8")
    ).hexdigest()


def get_path_or_404(zipped_filename):
    try:
        # calculate the local file path to the zip file
        zipped_path = get_content_storage_file_path(zipped_filename)
        # if the zipfile does not exist on disk, return a 404
        if not os.path.exists(zipped_path):
            raise InvalidStorageFilenameError()
        return zipped_path
    except InvalidStorageFilenameError:
        raise Http404(
            '"%(filename)s" is not a valid zip file' % {"filename": zipped_filename}
        )


def recursive_h5p_dependencies(zf, data, prefix=""):

    jsfiles = []
    cssfiles = []

    # load the dependencies, recursively, to extract their JS and CSS paths to include
    for dep in data.get("preloadedDependencies", []):
        packagepath = "{machineName}-{majorVersion}.{minorVersion}/".format(**dep)
        librarypath = packagepath + "library.json"
        info = zf.getinfo(librarypath)
        content = json.loads(zf.open(info).read())
        newjs, newcss = recursive_h5p_dependencies(zf, content, packagepath)
        cssfiles += newcss
        jsfiles += newjs

    # load the JS required for the current package
    for js in data.get("preloadedJs", []):
        path = prefix + js["path"]
        if path not in jsfiles:
            jsfiles.append(path)

    # load the CSS required for the current package
    for css in data.get("preloadedCss", []):
        path = prefix + css["path"]
        if path not in cssfiles:
            cssfiles.append(path)

    return jsfiles, cssfiles


def replace_script(parent, script):
    parent.remove(script)
    template = SubElement(parent, "template", attrib={"hashi-script": "true"})
    if script.get("async") is not None:
        template.set("async", "true")
    template.append(script)


def parse_html(content):
    try:
        document = html5lib.parse(content, namespaceHTMLElements=False)
        if not document:
            # Could not parse
            return content

        for parent in document.findall(".//script/.."):
            for script in parent.findall("script"):
                replace_script(parent, script)
        # Because html5lib parses like a browser, it will
        # always create head and body tags if they are missing.
        head = document.find("head")
        SubElement(
            head,
            "script",
            attrib={
                "src": static(
                    "content/{filename}".format(filename=get_hashi_filename())
                )
            },
        )
        return html5lib.serialize(
            document,
            quote_attr_values="always",
            omit_optional_tags=False,
            minimize_boolean_attributes=False,
            use_trailing_solidus=True,
            space_before_trailing_solidus=False,
        )
    except html5lib.html5parser.ParseError:
        return content


def get_h5p(zf, embedded_filepath):
    if not embedded_filepath:
        # Get the h5p bootloader, and then run it through our hashi templating code.
        # return the H5P bootloader code
        h5pdata = json.loads(zf.open(zf.getinfo("h5p.json")).read())
        jsfiles, cssfiles = recursive_h5p_dependencies(zf, h5pdata)
        contentdata = zf.open(zf.getinfo("content/content.json")).read()
        path_includes_version = (
            "true"
            if "-" in [name for name in zf.namelist() if "/" in name][0]
            else "false"
        )
        template = loader.get_template("content/h5p.html")
        main_library_data = [
            lib
            for lib in h5pdata["preloadedDependencies"]
            if lib["machineName"] == h5pdata["mainLibrary"]
        ][0]
        bootstrap_content = template.render(
            {
                "jsfiles": jsfiles,
                "cssfiles": cssfiles,
                "content": contentdata,
                "library": "{machineName} {majorVersion}.{minorVersion}".format(
                    **main_library_data
                ),
                "path_includes_version": path_includes_version,
            },
            None,
        )
        content = parse_html(bootstrap_content)
        content_type = "text/html"
    elif embedded_filepath.startswith("dist/"):
        # return static H5P dist resources
        path = finders.find("assets/h5p-standalone-" + embedded_filepath)
        with open(path) as f:
            content = f.read()
        # try to guess the MIME type of the embedded file being referenced
        content_type = (
            mimetypes.guess_type(embedded_filepath)[0] or "application/octet-stream"
        )
    response = HttpResponse(content, content_type=content_type)
    response["Content-Length"] = len(response.content)
    return response


def get_embedded_file(request, zf, zipped_filename, embedded_filepath):
    # if no path, or a directory, is being referenced, look for an index.html file
    if not embedded_filepath or embedded_filepath.endswith("/"):
        embedded_filepath += "index.html"

    # get the details about the embedded file, and ensure it exists
    try:
        info = zf.getinfo(embedded_filepath)
    except KeyError:
{'fraction_non_alphanumeric': 0.09048836084824204, 'fraction_numerical': 0.006700283207846929, 'mean_word_length': 2.7410852713178295, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6586690', 'n_tokens_mistral': 4184, 'n_tokens_neox': 4085, 'n_words': 1076}