<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2019 CERN
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the MIT License; see the
# LICENSE file for more details.

from __future__ import unicode_literals

import base64
import mimetypes
import re
import urlparse
from contextlib import contextmanager

import requests
from flask import current_app, g, request
from flask_webpackext import current_webpack
from flask_webpackext.manifest import JinjaManifestEntry
from pywebpack import Manifest
from werkzeug.urls import url_parse

from indico.core.config import config
from indico.modules.events.layout.models.images import ImageFile
from indico.web.flask.util import endpoint_for_url


_css_url_pattern = r"""url\((['"]?)({}|https?:)?([^)'"]+)\1\)"""
_url_has_extension_re = re.compile(r'.*\.([^/]+)$')
_plugin_url_pattern = r'(?:{})?/static/plugins/([^/]+)/(.*?)(?:__v[0-9a-f]+)?\.([^.]+)$'
_static_url_pattern = r'(?:{})?/(images|dist|fonts)(.*)/(.+?)(?:__v[0-9a-f]+)?\.([^.]+)$'
_custom_url_pattern = r'(?:{})?/static/custom/(.+)$'


def rewrite_static_url(path):
    """Remove __vxxx prefix from static URLs."""
    plugin_pattern = _plugin_url_pattern.format(url_parse(config.BASE_URL).path)
    static_pattern = _static_url_pattern.format(url_parse(config.BASE_URL).path)
    custom_pattern = _custom_url_pattern.format(url_parse(config.BASE_URL).path)
    if re.match(plugin_pattern, path):
        return re.sub(plugin_pattern, r'static/plugins/\1/\2.\3', path)
    elif re.match(static_pattern, path):
        return re.sub(static_pattern, r'static/\1\2/\3.\4', path)
    else:
        return re.sub(custom_pattern, r'static/custom/\1', path)


def _create_data_uri(url, filename):
    """Create a data url that contains the file in question."""
    response = requests.get(url, verify=False)
    if response.status_code != 200:
        # couldn't access the file
        return url
    data = base64.b64encode(response.content)
    content_type = (mimetypes.guess_type(filename)[0] or
                    response.headers.get('Content-Type', 'application/octet-stream'))
    return 'data:{};base64,{}'.format(content_type, data)


def _rewrite_event_asset_url(event, url):
    """Rewrite URLs of assets such as event images.

    Only assets contained within the event will be taken into account
    """
    scheme, netloc, path, qs, anchor = urlparse.urlsplit(url)
    netloc = netloc or current_app.config['SERVER_NAME']
    scheme = scheme or 'https'

    # internal URLs (same server)
    if netloc == current_app.config['SERVER_NAME']:
        # this piece of Flask magic finds the endpoint that corresponds to
        # the URL and checks that it points to an image belonging to this event
        endpoint_info = endpoint_for_url(path)
        if endpoint_info:
            endpoint, data = endpoint_info
            if endpoint == 'event_images.image_display' and int(data['confId']) == event.id:
                image_file = ImageFile.get(data['image_id'])
                if image_file and image_file.event == event:
                    return 'images/{}-{}'.format(image_file.id, image_file.filename), image_file
    # if the URL is not internal or just not an image,
    # we embed the contents using a data URI
    data_uri = _create_data_uri(urlparse.urlunsplit((scheme, netloc, path, qs, '')), urlparse.urlsplit(path)[-1])
    return data_uri, None


def _remove_anchor(url):
    """Remove the anchor from a URL."""
    scheme, netloc, path, qs, anchor = urlparse.urlsplit(url)
    return urlparse.urlunsplit((scheme, netloc, path, qs, ''))


def rewrite_css_urls(event, css):
    """Rewrite CSS in order to handle url(...) properly."""
    # keeping track of used URLs
    used_urls = set()
    used_images = set()

    def _replace_url(m):
        prefix = m.group(2) or ''
        url = m.group(3)
        if url.startswith('/event/') or re.match(r'https?:', prefix):
            rewritten_url, image_file = _rewrite_event_asset_url(event, prefix + url)
            if image_file:
                used_images.add(image_file)
            return 'url({})'.format(rewritten_url)
        else:
            rewritten_url = rewrite_static_url(url)
            used_urls.add(_remove_anchor(rewritten_url))
            if url.startswith('/static/plugins/'):
                return "url('../../../../../{}')".format(rewritten_url)
            else:
                return "url('../../../{}')".format(rewritten_url)

    indico_path = url_parse(config.BASE_URL).path
    return re.sub(_css_url_pattern.format(indico_path), _replace_url, css, flags=re.MULTILINE), used_urls, used_images


def url_to_static_filename(endpoint, url):
    """Handle special endpoint/URLs so that they link to offline content."""
    if re.match('(events)?\.display(_overview)?$', endpoint):
        return 'index.html'
    elif endpoint == 'event_layout.css_display':
        return 'custom.css'
    elif endpoint == 'event_images.logo_display':
        return 'logo.png'

    indico_path = url_parse(config.BASE_URL).path
    if re.match(_static_url_pattern.format(indico_path), url):
        url = rewrite_static_url(url)
    else:<commit_msg>Fix unicode error when building static site<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2019 CERN
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the MIT License; see the
# LICENSE file for more details.

from __future__ import unicode_literals

import base64
import mimetypes
import re
import urlparse
from contextlib import contextmanager

import requests
from flask import current_app, g, request
from flask_webpackext import current_webpack
from flask_webpackext.manifest import JinjaManifestEntry
from pywebpack import Manifest
from werkzeug.urls import url_parse

from indico.core.config import config
from indico.modules.events.layout.models.images import ImageFile
from indico.web.flask.util import endpoint_for_url


_css_url_pattern = r"""url\((['"]?)({}|https?:)?([^)'"]+)\1\)"""
_url_has_extension_re = re.compile(r'.*\.([^/]+)$')
_plugin_url_pattern = r'(?:{})?/static/plugins/([^/]+)/(.*?)(?:__v[0-9a-f]+)?\.([^.]+)$'
_static_url_pattern = r'(?:{})?/(images|dist|fonts)(.*)/(.+?)(?:__v[0-9a-f]+)?\.([^.]+)$'
_custom_url_pattern = r'(?:{})?/static/custom/(.+)$'


def rewrite_static_url(path):
    """Remove __vxxx prefix from static URLs."""
    plugin_pattern = _plugin_url_pattern.format(url_parse(config.BASE_URL).path)
    static_pattern = _static_url_pattern.format(url_parse(config.BASE_URL).path)
    custom_pattern = _custom_url_pattern.format(url_parse(config.BASE_URL).path)
    if re.match(plugin_pattern, path):
        return re.sub(plugin_pattern, r'static/plugins/\1/\2.\3', path)
    elif re.match(static_pattern, path):
        return re.sub(static_pattern, r'static/\1\2/\3.\4', path)
    else:
        return re.sub(custom_pattern, r'static/custom/\1', path)


def _create_data_uri(url, filename):
    """Create a data url that contains the file in question."""
    response = requests.get(url, verify=False)
    if response.status_code != 200:
        # couldn't access the file
        return url
    data = base64.b64encode(response.content)
    content_type = (mimetypes.guess_type(filename)[0] or
                    response.headers.get('Content-Type', 'application/octet-stream'))
    return 'data:{};base64,{}'.format(content_type, data)


def _rewrite_event_asset_url(event, url):
    """Rewrite URLs of assets such as event images.

    Only assets contained within the event will be taken into account
    """
    scheme, netloc, path, qs, anchor = urlparse.urlsplit(url)
    netloc = netloc or current_app.config['SERVER_NAME']
    scheme = scheme or 'https'

    # internal URLs (same server)
    if netloc == current_app.config['SERVER_NAME']:
        # this piece of Flask magic finds the endpoint that corresponds to
        # the URL and checks that it points to an image belonging to this event
        endpoint_info = endpoint_for_url(path)
        if endpoint_info:
            endpoint, data = endpoint_info
            if endpoint == 'event_images.image_display' and int(data['confId']) == event.id:
                image_file = ImageFile.get(data['image_id'])
                if image_file and image_file.event == event:
                    return 'images/{}-{}'.format(image_file.id, image_file.filename), image_file
    # if the URL is not internal or just not an image,
    # we embed the contents using a data URI
    data_uri = _create_data_uri(urlparse.urlunsplit((scheme, netloc, path, qs, '')), urlparse.urlsplit(path)[-1])
    return data_uri, None


def _remove_anchor(url):
    """Remove the anchor from a URL."""
    scheme, netloc, path, qs, anchor = urlparse.urlsplit(url)
    return urlparse.urlunsplit((scheme, netloc, path, qs, ''))


def rewrite_css_urls(event, css):
    """Rewrite CSS in order to handle url(...) properly."""
    # keeping track of used URLs
    used_urls = set()
    used_images = set()

    def _replace_url(m):
        prefix = m.group(2) or ''
        url = m.group(3)
        if url.startswith('/event/') or re.match(r'https?:', prefix):
            rewritten_url, image_file = _rewrite_event_asset_url(event, prefix + url)
            if image_file:
                used_images.add(image_file)
            return 'url({})'.format(rewritten_url)
        else:
            rewritten_url = rewrite_static_url(url)
            used_urls.add(_remove_anchor(rewritten_url))
            if url.startswith('/static/plugins/'):
                return "url('../../../../../{}')".format(rewritten_url)
            else:
                return "url('../../../{}')".format(rewritten_url)

    indico_path = url_parse(config.BASE_URL).path
    new_css = re.sub(_css_url_pattern.format(indico_path), _replace_url, css.decode('utf-8'), flags=re.MULTILINE)
    return new_css.encode('utf-8'), used_urls, used_images


def url_to_static_filename(endpoint, url):
    """Handle special endpoint/URLs so that they link to offline content."""
    if re.match('(events)?\.display(_overview)?$', endpoint):
        return 'index.html'
    elif endpoint == 'event_layout.css_display':
        return 'custom.css'
    elif endpoint == 'event_images.logo_display':
        return 'logo.png'

    indico_path = url_parse(config.BASE_URL).path
    if re.match(_static_url_pattern.format(indico_path), url):
        url = rewrite_static_url(url)
    else:
{'fraction_non_alphanumeric': 0.13481326219512196, 'fraction_numerical': 0.007050304878048781, 'mean_word_length': 3.3994132439228837, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7132848', 'n_tokens_mistral': 3382, 'n_tokens_neox': 3266, 'n_words': 815}