<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import io
import json
import os
import pkg_resources

from flask import current_app, url_for
from flask_cdn import url_for as cdn_url_for

# Store manifests URLs with the following hierarchy
# app > filename (without hash) > URL
_manifests = {}

_registered_manifests = {}  # Here for debug without cache


def has_manifest(app, filename='manifest.json'):
    '''Verify the existance of a JSON assets manifest'''
    try:
        return pkg_resources.resource_exists(app, filename)
    except ImportError:
        return os.path.isabs(filename) and os.path.exists(filename)


def register_manifest(app, filename='manifest.json'):
    '''Register an assets json manifest'''
    if current_app.config.get('TESTING'):
        return  # Do not spend time here when testing
    if not has_manifest(app, filename):
        msg = '{filename} not found for {app}'.format(**locals())
        raise ValueError(msg)
    manifest = _manifests.get(app, {})
    manifest.update(load_manifest(app, filename))
    _manifests[app] = manifest


def load_manifest(app, filename='manifest.json'):
    '''Load an assets json manifest'''
    if os.path.isabs(filename):
        path = filename
    else:
        path = pkg_resources.resource_filename(app, filename)
    with io.open(path, mode='r', encoding='utf8') as stream:
        data = json.load(stream)
    _registered_manifests[app] = path
    return data


def exists_in_manifest(app, filename):
    '''
    Test wether a static file exists in registered manifests or not
    '''
    return app in _manifests and filename in _manifests[app]


def from_manifest(app, filename, **kwargs):
    '''
    Get the path to a static file for a given app entry of a given type
    '''
    cfg = current_app.config

    if current_app.config.get('TESTING'):
        return  # Do not spend time here when testing

    path = _manifests[app][filename]

    if cfg.get('CDN_DOMAIN') and not cfg.get('CDN_DEBUG'):
        prefix = 'https://' if cfg.get('CDN_HTTPS') else '//'
        return ''.join((prefix, cfg['CDN_DOMAIN'], path))
    return path


def manifests_paths():
    return _registered_manifests.values()


def cdn_for(endpoint, **kwargs):
    '''
    Get a CDN URL for a static assets.

    Do not use a replacement for all flask.url_for calls
    as it is only meant for CDN assets URLS.
    (There is some extra round trip which cost is justified
    by the CDN assets prformance improvements)
    '''
    if current_app.config['CDN_DOMAIN']:
        if not current_app.config.get('CDN_DEBUG'):
            kwargs.pop('_external', None)  # Avoid the _external parameter in URL
        return cdn_url_for(endpoint, **kwargs)
    return url_for(endpoint, **kwargs)
<commit_msg>Fixes assets.from_manifest and add raw parameter<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import io
import json
import os
import pkg_resources

from flask import current_app, url_for
from flask_cdn import url_for as cdn_url_for

# Store manifests URLs with the following hierarchy
# app > filename (without hash) > URL
_manifests = {}

_registered_manifests = {}  # Here for debug without cache


def has_manifest(app, filename='manifest.json'):
    '''Verify the existance of a JSON assets manifest'''
    try:
        return pkg_resources.resource_exists(app, filename)
    except ImportError:
        return os.path.isabs(filename) and os.path.exists(filename)


def register_manifest(app, filename='manifest.json'):
    '''Register an assets json manifest'''
    if current_app.config.get('TESTING'):
        return  # Do not spend time here when testing
    if not has_manifest(app, filename):
        msg = '{filename} not found for {app}'.format(**locals())
        raise ValueError(msg)
    manifest = _manifests.get(app, {})
    manifest.update(load_manifest(app, filename))
    _manifests[app] = manifest


def load_manifest(app, filename='manifest.json'):
    '''Load an assets json manifest'''
    if os.path.isabs(filename):
        path = filename
    else:
        path = pkg_resources.resource_filename(app, filename)
    with io.open(path, mode='r', encoding='utf8') as stream:
        data = json.load(stream)
    _registered_manifests[app] = path
    return data


def exists_in_manifest(app, filename):
    '''
    Test wether a static file exists in registered manifests or not
    '''
    return app in _manifests and filename in _manifests[app]


def from_manifest(app, filename, raw=False, **kwargs):
    '''
    Get the path to a static file for a given app entry of a given type.

    :param str app: The application key to which is tied this manifest
    :param str filename: the original filename (without hash)
    :param bool raw: if True, doesn't add prefix to the manifest
    :return: the resolved file path from manifest
    :rtype: str
    '''
    cfg = current_app.config

    if current_app.config.get('TESTING'):
        return  # Do not spend time here when testing

    path = _manifests[app][filename]

    if not raw and cfg.get('CDN_DOMAIN') and not cfg.get('CDN_DEBUG'):
        prefix = 'https://' if cfg.get('CDN_HTTPS') else '//'
        if not path.startswith('/'):
            path = '/' + path
        return ''.join((prefix, cfg['CDN_DOMAIN'], path))
    return path


def manifests_paths():
    return _registered_manifests.values()


def cdn_for(endpoint, **kwargs):
    '''
    Get a CDN URL for a static assets.

    Do not use a replacement for all flask.url_for calls
    as it is only meant for CDN assets URLS.
    (There is some extra round trip which cost is justified
    by the CDN assets prformance improvements)
    '''
    if current_app.config['CDN_DOMAIN']:
        if not current_app.config.get('CDN_DEBUG'):
            kwargs.pop('_external', None)  # Avoid the _external parameter in URL
        return cdn_url_for(endpoint, **kwargs)
    return url_for(endpoint, **kwargs)

{'fraction_non_alphanumeric': 0.10175732217573222, 'fraction_numerical': 0.0006694560669456067, 'mean_word_length': 3.3556851311953353, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3776874', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1722, 'n_words': 594}