<commit_before>/*
 * Copyright 2008-2010 Digital Enterprise Research Institute (DERI)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.deri.any23.servlet;

import org.deri.any23.http.HTTPClient;
import org.deri.any23.servlet.conneg.Any23Negotiator;
import org.deri.any23.servlet.conneg.MediaRangeSpec;
import org.deri.any23.source.ByteArrayDocumentSource;
import org.deri.any23.source.DocumentSource;
import org.deri.any23.source.HTTPDocumentSource;
import org.deri.any23.source.StringDocumentSource;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.regex.Pattern;

/**
 * A <i>Servlet</i> that fetches a client-specified <i>URI</i>,
 * RDFizes the content, and returns it in a format chosen by the client.
 *
 * @author <NAME>
 * @author <NAME> (<EMAIL>)
 */
public class Servlet extends HttpServlet {

    public static final String DEFAULT_BASE_URI = "http://any23.org/tmp";

    private static final long serialVersionUID = 8207685628715421336L;

    // RFC 3986: scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
    private final static Pattern schemeRegex =
            Pattern.compile("^[a-zA-Z][a-zA-Z0-9.+-]*:");

    @Override
    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException {
        // Show /resources/form.html for GET requests to the app's root
        if (("/".equals(req.getPathInfo()) && req.getQueryString() == null)) {
            getServletContext().getRequestDispatcher("/resources/form.html").forward(req, resp);
            return;
        }
        // forward requests to /resources/* to the default servlet, this is
        // where we can put static files
        if (req.getPathInfo().startsWith("/resources/")) {
            getServletContext().getNamedDispatcher("default").forward(req, resp);
            return;
        }
        WebResponder responder = new WebResponder(this, resp);
        String format = getFormatFromRequestOrNegotiation(req);
        if (format == null) {
            responder.sendError(406, "Client accept header does not include a supported output format");
            return;
        }
        String uri = getInputURIFromRequest(req);
        if (uri == null) {
            responder.sendError(404, "Missing URI in GET request. Try /format/http://example.com/myfile");
            return;
        }
        responder.runExtraction(createHTTPDocumentSource(responder, uri), format);
    }

    @Override
    protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        WebResponder responder = new WebResponder(this, resp);
        if (req.getContentType() == null) {
            responder.sendError(400, "Invalid POST request, no Content-Type for the message body specified");
            return;
        }
        String uri = getInputURIFromRequest(req);
        String format = getFormatFromRequestOrNegotiation(req);
        if (format == null) {
            responder.sendError(406, "Client accept header does not include a supported output format");
            return;
        }
        if ("application/x-www-form-urlencoded".equals(getContentTypeHeader(req))) {
            if (uri != null) {
                log("Attempting conversion to '" + format + "' from URI <" + uri + ">");
                responder.runExtraction(createHTTPDocumentSource(responder, uri), format);
                return;
            }
            if (req.getParameter("body") == null) {
                responder.sendError(400, "Invalid POST request, parameter 'uri' or 'body' required");
                return;
            }
            String type = null;
            if (req.getParameter("type") != null && !"".equals(req.getParameter("type"))) {
                type = req.getParameter("type");
            }
            log("Attempting conversion to '" + format + "' from body parameter");
            responder.runExtraction(
                    new StringDocumentSource(req.getParameter("body"), Servlet.DEFAULT_BASE_URI, type),
                    format);
            return;
        }
        log("Attempting conversion to '" + format + "' from POST body");
        responder.runExtraction(
                new ByteArrayDocumentSource(req.getInputStream(), Servlet.DEFAULT_BASE_URI, getContentTypeHeader(req)),
                format);
    }

    private String getFormatFromRequestOrNegotiation(HttpServletRequest request) {
        String fromRequest = getFormatFromRequest(request);
        if (fromRequest != null && !"".equals(fromRequest) && !"best".equals(fromRequest)) {
            return fromRequest;
        }
        MediaRangeSpec result = Any23Negotiator.getNegotiator().getBestMatch(request.getHeader("Accept"));
        if (result == null) {
            return null;
        }
        if ("text/turtle".equals(result.getMediaType())) {
            return "turtle";
        }
        if ("text/rdf+n3".equals(result.getMediaType())) {
            return "n3";
        }
        if ("application/rdf+xml".equals(result.getMediaType())) {
            return "rdf";
        }
        if ("text/plain".equals(result.getMediaType())) {
            return "nt";
        }
        return "turtle";    // shouldn't happen
    }

    private String getFormatFromRequest(HttpServletRequest request) {
        if (request.getPathInfo() == null) return "best";
        String[] args = request.getPathInfo().split("/", 3);
        if (args.length < 2 || "".equals(args[1])) {
            if (request.getParameter("format") == null) {
                return "best";
            } else {
                return request.getParameter("format");
            }
        }
        return args[1];
    }

    private String getInputURIFromRequest(HttpServletRequest request) {
        if (request.getPathInfo() == null) return null;
        String[] args = request.getPathInfo().split("/", 3);
        if (args.length < 3) {
            if (request.getParameter("uri") != null) {
                return request.getParameter("uri");
            }
            if (request.getParameter("url") != null) {
                return request.getParameter("url");
            }
            return null;
        }
        String uri = args[2];
        if (request.getQueryString() != null) {
            uri = uri + "?" + request.getQueryString();
        }
        if (!hasScheme(uri)) {
            uri = "http://" + uri;
        } else if (hasOnlySingleSlashAfterScheme(uri)) {
            // This is to work around an issue where Tomcat 6.0.18 is
            // too smart for us. Tomcat normalizes double-slashes in
            // the path, and thus turns "http://" into "http:/" if it
            // occurs in the path. So we restore the double slash.
            uri = uri.replaceFirst(":/", "://");
        }
        return uri;
    }


    private boolean hasScheme(String uri) {
        return schemeRegex.matcher(uri).find();
    }

    private final static Pattern schemeAndSingleSlashRegex =
            Pattern.compile("^[a-zA-Z][a-zA-Z0-9.+-]*:/[^/]");

    private boolean hasOnlySingleSlashAfterScheme(String uri) {
        return schemeAndSingleSlashRegex.matcher(uri).find();
    }

    private String getContentTypeHeader(HttpServletRequest req) {
        if (req.getHeader("Content-Type") == null) return null;
        if ("".equals(req.getHeader("Content-Type"))) return null;<commit_msg>Added tolerance to spaces in URI form parameter.<commit_after>/*
 * Copyright 2008-2010 Digital Enterprise Research Institute (DERI)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.deri.any23.servlet;

import org.deri.any23.http.HTTPClient;
import org.deri.any23.servlet.conneg.Any23Negotiator;
import org.deri.any23.servlet.conneg.MediaRangeSpec;
import org.deri.any23.source.ByteArrayDocumentSource;
import org.deri.any23.source.DocumentSource;
import org.deri.any23.source.HTTPDocumentSource;
import org.deri.any23.source.StringDocumentSource;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.regex.Pattern;

/**
 * A <i>Servlet</i> that fetches a client-specified <i>URI</i>,
 * RDFizes the content, and returns it in a format chosen by the client.
 *
 * @author <NAME>
 * @author <NAME> (<EMAIL>)
 */
public class Servlet extends HttpServlet {

    public static final String DEFAULT_BASE_URI = "http://any23.org/tmp";

    private static final long serialVersionUID = 8207685628715421336L;

    // RFC 3986: scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
    private final static Pattern schemeRegex =
            Pattern.compile("^[a-zA-Z][a-zA-Z0-9.+-]*:");

    @Override
    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException {
        // Show /resources/form.html for GET requests to the app's root
        if (("/".equals(req.getPathInfo()) && req.getQueryString() == null)) {
            getServletContext().getRequestDispatcher("/resources/form.html").forward(req, resp);
            return;
        }
        // forward requests to /resources/* to the default servlet, this is
        // where we can put static files
        if (req.getPathInfo().startsWith("/resources/")) {
            getServletContext().getNamedDispatcher("default").forward(req, resp);
            return;
        }
        WebResponder responder = new WebResponder(this, resp);
        String format = getFormatFromRequestOrNegotiation(req);
        if (format == null) {
            responder.sendError(406, "Client accept header does not include a supported output format");
            return;
        }
        String uri = getInputURIFromRequest(req);
        if (uri == null) {
            responder.sendError(404, "Missing URI in GET request. Try /format/http://example.com/myfile");
            return;
        }
        responder.runExtraction(createHTTPDocumentSource(responder, uri), format);
    }

    @Override
    protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException {
        WebResponder responder = new WebResponder(this, resp);
        if (req.getContentType() == null) {
            responder.sendError(400, "Invalid POST request, no Content-Type for the message body specified");
            return;
        }
        String uri = getInputURIFromRequest(req);
        String format = getFormatFromRequestOrNegotiation(req);
        if (format == null) {
            responder.sendError(406, "Client accept header does not include a supported output format");
            return;
        }
        if ("application/x-www-form-urlencoded".equals(getContentTypeHeader(req))) {
            if (uri != null) {
                log("Attempting conversion to '" + format + "' from URI <" + uri + ">");
                responder.runExtraction(createHTTPDocumentSource(responder, uri), format);
                return;
            }
            if (req.getParameter("body") == null) {
                responder.sendError(400, "Invalid POST request, parameter 'uri' or 'body' required");
                return;
            }
            String type = null;
            if (req.getParameter("type") != null && !"".equals(req.getParameter("type"))) {
                type = req.getParameter("type");
            }
            log("Attempting conversion to '" + format + "' from body parameter");
            responder.runExtraction(
                    new StringDocumentSource(req.getParameter("body"), Servlet.DEFAULT_BASE_URI, type),
                    format);
            return;
        }
        log("Attempting conversion to '" + format + "' from POST body");
        responder.runExtraction(
                new ByteArrayDocumentSource(req.getInputStream(), Servlet.DEFAULT_BASE_URI, getContentTypeHeader(req)),
                format);
    }

    private String getFormatFromRequestOrNegotiation(HttpServletRequest request) {
        String fromRequest = getFormatFromRequest(request);
        if (fromRequest != null && !"".equals(fromRequest) && !"best".equals(fromRequest)) {
            return fromRequest;
        }
        MediaRangeSpec result = Any23Negotiator.getNegotiator().getBestMatch(request.getHeader("Accept"));
        if (result == null) {
            return null;
        }
        if ("text/turtle".equals(result.getMediaType())) {
            return "turtle";
        }
        if ("text/rdf+n3".equals(result.getMediaType())) {
            return "n3";
        }
        if ("application/rdf+xml".equals(result.getMediaType())) {
            return "rdf";
        }
        if ("text/plain".equals(result.getMediaType())) {
            return "nt";
        }
        return "turtle";    // shouldn't happen
    }

    private String getFormatFromRequest(HttpServletRequest request) {
        if (request.getPathInfo() == null) return "best";
        String[] args = request.getPathInfo().split("/", 3);
        if (args.length < 2 || "".equals(args[1])) {
            if (request.getParameter("format") == null) {
                return "best";
            } else {
                return request.getParameter("format");
            }
        }
        return args[1];
    }

    private String getInputURIFromRequest(HttpServletRequest request) {
        if (request.getPathInfo() == null) return null;
        String[] args = request.getPathInfo().split("/", 3);
        if (args.length < 3) {
            if (request.getParameter("uri") != null) {
                return request.getParameter("uri").trim();
            }
            if (request.getParameter("url") != null) {
                return request.getParameter("url").trim();
            }
            return null;
        }
        String uri = args[2];
        if (request.getQueryString() != null) {
            uri = uri + "?" + request.getQueryString();
        }
        if (!hasScheme(uri)) {
            uri = "http://" + uri;
        } else if (hasOnlySingleSlashAfterScheme(uri)) {
            // This is to work around an issue where Tomcat 6.0.18 is
            // too smart for us. Tomcat normalizes double-slashes in
            // the path, and thus turns "http://" into "http:/" if it
            // occurs in the path. So we restore the double slash.
            uri = uri.replaceFirst(":/", "://");
        }
        return uri.trim();
    }


    private boolean hasScheme(String uri) {
        return schemeRegex.matcher(uri).find();
    }

    private final static Pattern schemeAndSingleSlashRegex =
            Pattern.compile("^[a-zA-Z][a-zA-Z0-9.+-]*:/[^/]");

    private boolean hasOnlySingleSlashAfterScheme(String uri) {
        return schemeAndSingleSlashRegex.matcher(uri).find();
    }

    private String getContentTypeHeader(HttpServletRequest req) {
        if (req.getHeader("Content-Type") == null) return null;
        if ("".equals(req.getHeader("Content-Type"))) return null;
{'fraction_non_alphanumeric': 0.1205115592720118, 'fraction_numerical': 0.010944417117560256, 'mean_word_length': 2.4415996614473126, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6645438', 'n_tokens_mistral': 4540, 'n_tokens_neox': 4342, 'n_words': 1132}