<commit_before>/**
 * This file is part of the Gribbit Web Framework.
 * 
 *     https://github.com/lukehutch/gribbit
 * 
 * @author <NAME>
 * 
 * --
 * 
 * @license Apache 2.0 
 * 
 * Copyright 2015 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gribbit.util;

import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_ENCODING;
import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
import gribbit.server.GribbitServer;
import gribbit.server.config.GribbitProperties;
import gribbit.util.thirdparty.UTF8;
import io.netty.handler.codec.http.HttpHeaders;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.regex.Pattern;

public class WebUtils {

    /**
     * Pattern for valid id or name attribute values. NB '.' and ':' are also technically allowed in the standard, but
     * they cause problems with jQuery, so they are disallowed here. Also note that both character cases are allowed,
     * but browsers may not handle case sensitivity correctly in all related contexts. See
     * http://stackoverflow.com/questions/70579/what-are-valid-values-for-the-id-attribute-in-html .
     */
    public static final Pattern VALID_HTML_NAME_OR_ID = Pattern.compile("[a-zA-Z][\\w\\-]*");

    /**
     * Pattern for CSS class name (includes ' ', because class attributes can list multiple classes).
     * 
     * NB CSS class names can start with '-' or '_', but this is technically reserved for vendor-specific extensions, so
     * we don't allow that here.
     */
    public static final Pattern VALID_CSS_ID = Pattern
            .compile("\\s*[a-zA-Z][_a-zA-Z0-9\\-]*(\\s+[a-zA-Z][_a-zA-Z0-9\\-]*)*\\s*");

    /**
     * Pattern for valid email address: from http://www.regular-expressions.info/email.html .
     * 
     * N.B. this is different than the validation performed by Chrome's type="email" form field validation (it pretty
     * much only requires an '@' character somewhere), so it's possible to submit form data like "x@y" that passes
     * Chrome's validation but fails serverside validation.
     */
    public static final Pattern VALID_EMAIL_ADDRESS = Pattern
            .compile("[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*"
                    + "@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?");

    // -----------------------------------------------------------------------------------------------------

    /** HTML tags that should not be closed. http://www.w3.org/TR/html-markup/syntax.html#void-element */
    public static final HashSet<String> VOID_ELEMENTS = toSet(new String[] { "area", "base", "br", "col", "command",
            "embed", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr", "!doctype",
            "!DOCTYPE" });

    /** HTML inline elements. https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elemente */
    public static final HashSet<String> INLINE_ELEMENTS = toSet(new String[] { "a", "abbr", "acronym", "b", "bdo",
            "big", "br", "button", "cite", "code", "dfn", "em", "i", "img", "input", "kbd", "label", "map", "object",
            "q", "samp", "select", "small", "span", "strong", "sub", "sup", "textarea", "title", "tt", "var" });

    // -----------------------------------------------------------------------------------------------------

    /**
     * HTML5 attributes that can take a URL: http://stackoverflow.com/questions/2725156/complete-list-of-html
     * -tag-attributes-which-have-a-url-value
     * 
     * (Applet and object tags are rejected during template loading, so those tags' attributes are not listed here, but
     * they also take URL params.)
     */
    private static final HashMap<String, HashSet<String>> URL_ELT_ATTRS = toMap(new String[] { "a.href", "area.href",
            "base.href", "blockquote.cite", "body.background", "del.cite", "form.action", "frame.longdesc",
            "frame.src", "head.profile", "iframe.longdesc", "iframe.src", "img.longdesc", "img.src", "img.usemap",
            "input.src", "input.usemap", "ins.cite", "link.href", "q.cite", "script.src", "audio.src",
            "button.formaction", "command.icon", "embed.src", "html.manifest", "input.formaction", "source.src",
            "video.poster", "video.src" }, ".");

    /** Return true if the given HTML attribute takes a URL as a value */
    public static boolean isURLAttr(String tagName, String attrName) {
        HashSet<String> attrs = URL_ELT_ATTRS.get(tagName.toLowerCase());
        return attrs == null ? false : attrs.contains(attrName.toLowerCase());
    }

    /**
     * Add a URL-typed attribute (used for adding attributes from custom elements or Polymer elements that take a URL
     * parameter, and therefore should be checked for URI validity).
     */
    public static void addCustomURLAttr(String customElementName, String attrName) {
        HashSet<String> set = URL_ELT_ATTRS.get(customElementName);
        if (set == null) {
            URL_ELT_ATTRS.put(customElementName, set = new HashSet<>());
        }
        set.add(attrName);
    }

    /**
     * Whitelisted attributes that can't be exploited in an XSS attack -- source: https://www.owasp.org/index.
     * php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet#XSS_Prevention_Rules_Summary
     */
    public static final HashSet<String> XSS_SAFE_ATTRS = toSet(new String[] { "align", "alink", "alt", "bgcolor",
            "border", "cellpadding", "cellspacing", "class", "color", "cols", "colspan", "coords", "dir", "face",
            "height", "hspace", "ismap", "lang", "marginheight", "marginwidth", "multiple", "nohref", "noresize",
            "noshade", "nowrap", "ref", "rel", "rev", "rows", "rowspan", "scrolling", "shape", "span", "summary",
            "tabindex", "title", "usemap", "valign", "value", "vlink", "vspace", "width" });

    // -----------------------------------------------------------------------------------------------------

    private static final Map<String, String> EXTENSION_TO_MIMETYPE = toMap(new String[][] {
            // See https://github.com/h5bp/server-configs-nginx/blob/master/mime.types for more
            { "txt", "text/plain" }, //
            { "htm", "text/html" }, //
            { "html", "text/html" }, //
            { "js", "application/javascript" }, //
            { "json", "application/json" }, //
            { "css", "text/css" }, //
            { "xml", "application/xml" }, //
            { "ico", "image/x-icon" }, //
            { "png", "image/png" }, //
            { "webp", "image/webp" }, //
            { "gif", "image/gif" }, //
            { "mng", "image/mng" }, //
            { "jpg", "image/jpeg" }, //
            { "jpeg", "image/jpeg" }, //
            { "svg", "image/svg+xml" }, //
            { "svgz", "image/svg+xml" }, //
            { "woff", "application/font-woff" }, //
            { "woff2", "application/font-woff2" }, //
            { "ttf", "application/x-font-ttf" }, //
            { "ttc", "application/x-font-ttf" }, //
            { "otf", "font/opentype" }, //
            { "gz", "application/x-gzip" }, //
            { "bz2", "application/x-bzip2" }, //
            { "zip", "application/zip" }, //
            { "pdf", "application/pdf" }, //
            { "ogg", "audio/ogg" }, //
            { "mp3", "audio/mpeg" }, //
            { "wav", "audio/x-wav" }, //
            { "csv", "text/comma-separated-values" }, //
    });

    public static void setContentTypeHeaders(HttpHeaders headers, String path) {
        String contentType = "application/octet-stream";
        int dotIdx = path.lastIndexOf('.'), slashIdx = path.lastIndexOf(File.separatorChar);
        if (dotIdx > 0 && slashIdx < dotIdx) {
            String ext = path.substring(dotIdx + 1).toLowerCase();
            String mimeType = WebUtils.EXTENSION_TO_MIMETYPE.get(ext);
            if (mimeType != null) {
                contentType = mimeType;
            }
            // .svgz files need an additional header -- see http://kaioa.com/node/45
            if (ext.equals("svgz")) {
                headers.set(CONTENT_ENCODING, "gzip");
            }
        }
        headers.set(CONTENT_TYPE, contentType);
    }<commit_msg>Allow cross-domain serving of fonts<commit_after>/**
 * This file is part of the Gribbit Web Framework.
 * 
 *     https://github.com/lukehutch/gribbit
 * 
 * @author <NAME>
 * 
 * --
 * 
 * @license Apache 2.0 
 * 
 * Copyright 2015 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gribbit.util;

import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_ENCODING;
import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN;
import gribbit.server.GribbitServer;
import gribbit.server.config.GribbitProperties;
import gribbit.util.thirdparty.UTF8;
import io.netty.handler.codec.http.HttpHeaders;

import java.io.File;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.regex.Pattern;

public class WebUtils {

    /**
     * Pattern for valid id or name attribute values. NB '.' and ':' are also technically allowed in the standard, but
     * they cause problems with jQuery, so they are disallowed here. Also note that both character cases are allowed,
     * but browsers may not handle case sensitivity correctly in all related contexts. See
     * http://stackoverflow.com/questions/70579/what-are-valid-values-for-the-id-attribute-in-html .
     */
    public static final Pattern VALID_HTML_NAME_OR_ID = Pattern.compile("[a-zA-Z][\\w\\-]*");

    /**
     * Pattern for CSS class name (includes ' ', because class attributes can list multiple classes).
     * 
     * NB CSS class names can start with '-' or '_', but this is technically reserved for vendor-specific extensions, so
     * we don't allow that here.
     */
    public static final Pattern VALID_CSS_ID = Pattern
            .compile("\\s*[a-zA-Z][_a-zA-Z0-9\\-]*(\\s+[a-zA-Z][_a-zA-Z0-9\\-]*)*\\s*");

    /**
     * Pattern for valid email address: from http://www.regular-expressions.info/email.html .
     * 
     * N.B. this is different than the validation performed by Chrome's type="email" form field validation (it pretty
     * much only requires an '@' character somewhere), so it's possible to submit form data like "x@y" that passes
     * Chrome's validation but fails serverside validation.
     */
    public static final Pattern VALID_EMAIL_ADDRESS = Pattern
            .compile("[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*"
                    + "@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?");

    // -----------------------------------------------------------------------------------------------------

    /** HTML tags that should not be closed. http://www.w3.org/TR/html-markup/syntax.html#void-element */
    public static final HashSet<String> VOID_ELEMENTS = toSet(new String[] { "area", "base", "br", "col", "command",
            "embed", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr", "!doctype",
            "!DOCTYPE" });

    /** HTML inline elements. https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elemente */
    public static final HashSet<String> INLINE_ELEMENTS = toSet(new String[] { "a", "abbr", "acronym", "b", "bdo",
            "big", "br", "button", "cite", "code", "dfn", "em", "i", "img", "input", "kbd", "label", "map", "object",
            "q", "samp", "select", "small", "span", "strong", "sub", "sup", "textarea", "title", "tt", "var" });

    // -----------------------------------------------------------------------------------------------------

    /**
     * HTML5 attributes that can take a URL: http://stackoverflow.com/questions/2725156/complete-list-of-html
     * -tag-attributes-which-have-a-url-value
     * 
     * (Applet and object tags are rejected during template loading, so those tags' attributes are not listed here, but
     * they also take URL params.)
     */
    private static final HashMap<String, HashSet<String>> URL_ELT_ATTRS = toMap(new String[] { "a.href", "area.href",
            "base.href", "blockquote.cite", "body.background", "del.cite", "form.action", "frame.longdesc",
            "frame.src", "head.profile", "iframe.longdesc", "iframe.src", "img.longdesc", "img.src", "img.usemap",
            "input.src", "input.usemap", "ins.cite", "link.href", "q.cite", "script.src", "audio.src",
            "button.formaction", "command.icon", "embed.src", "html.manifest", "input.formaction", "source.src",
            "video.poster", "video.src" }, ".");

    /** Return true if the given HTML attribute takes a URL as a value */
    public static boolean isURLAttr(String tagName, String attrName) {
        HashSet<String> attrs = URL_ELT_ATTRS.get(tagName.toLowerCase());
        return attrs == null ? false : attrs.contains(attrName.toLowerCase());
    }

    /**
     * Add a URL-typed attribute (used for adding attributes from custom elements or Polymer elements that take a URL
     * parameter, and therefore should be checked for URI validity).
     */
    public static void addCustomURLAttr(String customElementName, String attrName) {
        HashSet<String> set = URL_ELT_ATTRS.get(customElementName);
        if (set == null) {
            URL_ELT_ATTRS.put(customElementName, set = new HashSet<>());
        }
        set.add(attrName);
    }

    /**
     * Whitelisted attributes that can't be exploited in an XSS attack -- source: https://www.owasp.org/index.
     * php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet#XSS_Prevention_Rules_Summary
     */
    public static final HashSet<String> XSS_SAFE_ATTRS = toSet(new String[] { "align", "alink", "alt", "bgcolor",
            "border", "cellpadding", "cellspacing", "class", "color", "cols", "colspan", "coords", "dir", "face",
            "height", "hspace", "ismap", "lang", "marginheight", "marginwidth", "multiple", "nohref", "noresize",
            "noshade", "nowrap", "ref", "rel", "rev", "rows", "rowspan", "scrolling", "shape", "span", "summary",
            "tabindex", "title", "usemap", "valign", "value", "vlink", "vspace", "width" });

    // -----------------------------------------------------------------------------------------------------

    private static final Map<String, String> EXTENSION_TO_MIMETYPE = toMap(new String[][] {
            // See https://github.com/h5bp/server-configs-nginx/blob/master/mime.types for more
            { "txt", "text/plain" }, //
            { "htm", "text/html" }, //
            { "html", "text/html" }, //
            { "js", "application/javascript" }, //
            { "json", "application/json" }, //
            { "css", "text/css" }, //
            { "xml", "application/xml" }, //
            { "ico", "image/x-icon" }, //
            { "png", "image/png" }, //
            { "webp", "image/webp" }, //
            { "gif", "image/gif" }, //
            { "mng", "image/mng" }, //
            { "jpg", "image/jpeg" }, //
            { "jpeg", "image/jpeg" }, //
            { "svg", "image/svg+xml" }, //
            { "svgz", "image/svg+xml" }, //
            { "woff", "application/font-woff" }, //
            { "woff2", "application/font-woff2" }, //
            { "ttf", "application/x-font-ttf" }, //
            { "ttc", "application/x-font-ttf" }, //
            { "otf", "font/opentype" }, //
            { "gz", "application/x-gzip" }, //
            { "bz2", "application/x-bzip2" }, //
            { "zip", "application/zip" }, //
            { "pdf", "application/pdf" }, //
            { "ogg", "audio/ogg" }, //
            { "mp3", "audio/mpeg" }, //
            { "wav", "audio/x-wav" }, //
            { "csv", "text/comma-separated-values" }, //
    });

    private static final HashSet<String> FONT_EXTENSION = toSet(new String[] { "eot", "otf", "ttf", "ttc", "woff" });

    public static void setContentTypeHeaders(HttpHeaders headers, String path) {
        String contentType = "application/octet-stream";
        int dotIdx = path.lastIndexOf('.'), slashIdx = path.lastIndexOf(File.separatorChar);
        if (dotIdx > 0 && slashIdx < dotIdx) {
            String leaf = path.substring(slashIdx + 1).toLowerCase();
            String ext = path.substring(dotIdx + 1).toLowerCase();
            String mimeType = WebUtils.EXTENSION_TO_MIMETYPE.get(ext);
            if (mimeType != null) {
                contentType = mimeType;
            }
            // .svgz files need an additional header -- see http://kaioa.com/node/45
            if (ext.equals("svgz")) {
                headers.set(CONTENT_ENCODING, "gzip");
            }
            // Fonts need a CORS header if served across domains, to work in Firefox and IE (and according to spec)
            // -- see http://davidwalsh.name/cdn-fonts
            if (FONT_EXTENSION.contains(ext) || leaf.equals("font.css") || leaf.equals("fonts.css")) {
                headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
            }
        }
        headers.set(CONTENT_TYPE, contentType);
    }
{'fraction_non_alphanumeric': 0.19597989949748743, 'fraction_numerical': 0.005997730588426001, 'mean_word_length': 2.320416218155723, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 8, 'lorem ipsum': 0, 'www.': 8, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5875901', 'n_tokens_mistral': 5775, 'n_tokens_neox': 5615, 'n_words': 1413}