<commit_before>/** See the file "LICENSE" for the full license governing this code. */
package au.org.ands.vocabs.toolkit.restlet;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.nio.file.Paths;
import java.util.Hashtable;

import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.container.AsyncResponse;
import javax.ws.rs.container.Suspended;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import au.org.ands.vocabs.toolkit.db.AccessPointsUtils;
import au.org.ands.vocabs.toolkit.db.VersionsUtils;
import au.org.ands.vocabs.toolkit.db.VocabulariesUtils;
import au.org.ands.vocabs.toolkit.db.model.AccessPoints;
import au.org.ands.vocabs.toolkit.db.model.Versions;
import au.org.ands.vocabs.toolkit.db.model.Vocabularies;
import au.org.ands.vocabs.toolkit.utils.ToolkitFileUtils;

/** Restlet for downloading a vocabulary. */
@Path("download")
public class Download {

    /** Logger for this class. */
    private Logger logger = LoggerFactory.getLogger(
            MethodHandles.lookup().lookupClass());


    /** Mapping of Sesame Download formats to MIME types. */
    public static final Hashtable<String, String>
    SESAME_FORMAT_TO_MIMETYPE_MAP =
    new Hashtable<String, String>();

    // List taken from:
    // http://rdf4j.org/sesame/2.8/docs/system.docbook?view#content-types
    static {
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("rdf", "application/rdf+xml");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("nt", "text/plain");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("ttl", "text/turtle");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("n3", "text/rdf+n3");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("nq", "text/x-nquads");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("json", "application/rdf+json");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("trix", "application/trix");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("trig", "application/x-trig");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("bin", "application/x-binary-rdf");
    }

    /** Mapping of file formats to MIME types. */
    public static final Hashtable<String, String>
    FILE_FORMAT_TO_MIMETYPE_MAP =
    new Hashtable<String, String>();

    // The keys should match those in:
    // ANDS-Registry-Core/applications/portal/vocabs/assets/js/versionCtrl.js
    static {
        FILE_FORMAT_TO_MIMETYPE_MAP.put("RDF/XML", "application/rdf+xml");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TTL", "text/turtle");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("N-Triples", "text/plain");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("JSON", "application/json");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TriG", "application/x-trig");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TriX", "application/trix");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("N3", "text/rdf+n3");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("CSV", "text/csv");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TSV", "text/csv");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("XLS", "application/vnd.ms-excel");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("XLSX",
                "application/vnd.openxmlformats-officedocument."
                + "spreadsheetml.sheet");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("BinaryRDF",
                "application/x-binary-rdf");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("ODS",
                "application/vnd.oasis.opendocument.spreadsheet");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("ZIP", "application/zip");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("XML", "application/xml");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TXT", "text/plain");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("ODT",
                "application/vnd.oasis.opendocument.text");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TEXT", "text/plain");
//        FILE_FORMAT_TO_MIMETYPE_MAP.put("", "");

//        FILE_FORMAT_TO_MIMETYPE_MAP.put("nq", "text/x-nquads");
//        FILE_FORMAT_TO_MIMETYPE_MAP.put("json", "application/rdf+json");
    }



    /** Get the download for an access point. The Path for this
     * method does not include a filename, but allows for a format
     * query parameter.
     * @param response Asynchronous response for this request
     * @param accessPointId Access point id
     * @param downloadFormat The download format. This may be
     * ignored, depending (for example) on the access point type. */
    @Path("{access_point_id}")
    @GET
    public final void download(
            @Suspended final AsyncResponse response,
            @PathParam("access_point_id")
            final int accessPointId,
            @DefaultValue("rdf")
            @QueryParam("format")
            final String downloadFormat) {
        logger.info("Called download: " + accessPointId
                + ", download format: " + downloadFormat);
        // Have a look at the downloadFormat, irrespective of
        // whether it is supported -- which is currently only
        // is for sesameDownload. In future, we _may_ support
        // transforms for file downloads.
        final String mimeType =
                SESAME_FORMAT_TO_MIMETYPE_MAP.get(downloadFormat);
        if (mimeType == null) {
            response.resume(Response.status(Status.NOT_FOUND).
                    entity("Not found: no such format").build());
            return;
        }
        AccessPoints ap = AccessPointsUtils.getAccessPointById(accessPointId);
        if (ap == null) {
            response.resume(Response.status(Status.NOT_FOUND).
                    entity("Not found: no such access point").build());
            return;
        }

        switch (ap.getType()) {
        case "file":
            fileDownload(response, accessPointId, ap);
            break;
        case "sesameDownload":
            sesameDownload(response, accessPointId, ap,
                    downloadFormat, mimeType);
            return;
        default:
            logger.error("download: invalid type for access point: "
                    + ap.getType());
            response.resume(Response.status(Status.NOT_FOUND).
                    entity("Invalid access point type").build());
            return;
        }
    }

    /** Get the download for an access point. The Path for this
     * method includes a filename with an extension.
     * The use of this method enables the URL to contain a path component
     * with a filename. This enables the use of e.g., curl or wget
     * to save the download to the "correct" filename.
     * @param response Asynchronous response for this request
     * @param accessPointId Access point id.
     * @param filename The filename specified in the URL. This may
     * be ignored in constructing the response headers.
     * @param extension The download format. This may be
     * ignored, depending (for example) on the access point type. */
    @Path("{access_point_id}/{filename}.{extension}")
    @GET<commit_msg>Download: don't check downloadFormat for file access points<commit_after>/** See the file "LICENSE" for the full license governing this code. */
package au.org.ands.vocabs.toolkit.restlet;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.nio.file.Paths;
import java.util.Hashtable;

import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.container.AsyncResponse;
import javax.ws.rs.container.Suspended;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import au.org.ands.vocabs.toolkit.db.AccessPointsUtils;
import au.org.ands.vocabs.toolkit.db.VersionsUtils;
import au.org.ands.vocabs.toolkit.db.VocabulariesUtils;
import au.org.ands.vocabs.toolkit.db.model.AccessPoints;
import au.org.ands.vocabs.toolkit.db.model.Versions;
import au.org.ands.vocabs.toolkit.db.model.Vocabularies;
import au.org.ands.vocabs.toolkit.utils.ToolkitFileUtils;

/** Restlet for downloading a vocabulary. */
@Path("download")
public class Download {

    /** Logger for this class. */
    private Logger logger = LoggerFactory.getLogger(
            MethodHandles.lookup().lookupClass());


    /** Mapping of Sesame Download formats to MIME types. */
    public static final Hashtable<String, String>
    SESAME_FORMAT_TO_MIMETYPE_MAP =
    new Hashtable<String, String>();

    // List taken from:
    // http://rdf4j.org/sesame/2.8/docs/system.docbook?view#content-types
    static {
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("rdf", "application/rdf+xml");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("nt", "text/plain");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("ttl", "text/turtle");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("n3", "text/rdf+n3");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("nq", "text/x-nquads");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("json", "application/rdf+json");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("trix", "application/trix");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("trig", "application/x-trig");
        SESAME_FORMAT_TO_MIMETYPE_MAP.put("bin", "application/x-binary-rdf");
    }

    /** Mapping of file formats to MIME types. */
    public static final Hashtable<String, String>
    FILE_FORMAT_TO_MIMETYPE_MAP =
    new Hashtable<String, String>();

    // The keys should match those in:
    // ANDS-Registry-Core/applications/portal/vocabs/assets/js/versionCtrl.js
    static {
        FILE_FORMAT_TO_MIMETYPE_MAP.put("RDF/XML", "application/rdf+xml");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TTL", "text/turtle");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("N-Triples", "text/plain");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("JSON", "application/json");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TriG", "application/x-trig");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TriX", "application/trix");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("N3", "text/rdf+n3");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("CSV", "text/csv");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TSV", "text/csv");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("XLS", "application/vnd.ms-excel");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("XLSX",
                "application/vnd.openxmlformats-officedocument."
                + "spreadsheetml.sheet");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("BinaryRDF",
                "application/x-binary-rdf");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("ODS",
                "application/vnd.oasis.opendocument.spreadsheet");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("ZIP", "application/zip");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("XML", "application/xml");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TXT", "text/plain");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("ODT",
                "application/vnd.oasis.opendocument.text");
        FILE_FORMAT_TO_MIMETYPE_MAP.put("TEXT", "text/plain");
//        FILE_FORMAT_TO_MIMETYPE_MAP.put("", "");

//        FILE_FORMAT_TO_MIMETYPE_MAP.put("nq", "text/x-nquads");
//        FILE_FORMAT_TO_MIMETYPE_MAP.put("json", "application/rdf+json");
    }



    /** Get the download for an access point. The Path for this
     * method does not include a filename, but allows for a format
     * query parameter.
     * @param response Asynchronous response for this request
     * @param accessPointId Access point id
     * @param downloadFormat The download format. This may be
     * ignored, depending (for example) on the access point type. */
    @Path("{access_point_id}")
    @GET
    public final void download(
            @Suspended final AsyncResponse response,
            @PathParam("access_point_id")
            final int accessPointId,
            @DefaultValue("rdf")
            @QueryParam("format")
            final String downloadFormat) {
        logger.info("Called download: " + accessPointId
                + ", download format: " + downloadFormat);
        AccessPoints ap = AccessPointsUtils.getAccessPointById(accessPointId);
        if (ap == null) {
            response.resume(Response.status(Status.NOT_FOUND).
                    entity("Not found: no such access point").build());
            return;
        }

        switch (ap.getType()) {
        case "file":
            // For now, transforms for file access points are not supported,
            // so we don't look at downloadFormat. In future, we _may_ support
            // transforms for file access points. If that happens, note that
            // downloadWithFilename() and
            // downloadWithFilenameWithoutExtension() delegate to this method,
            // and the downloadFormat parameter may need some care
            // (i.e., it may be wrong, but the wrongness may or may not be
            // a problem).
            fileDownload(response, accessPointId, ap);
            break;
        case "sesameDownload":
            // Have a look at the downloadFormat before proceeding.
            final String mimeType =
                    SESAME_FORMAT_TO_MIMETYPE_MAP.get(downloadFormat);
            if (mimeType == null) {
                response.resume(Response.status(Status.NOT_FOUND).
                        entity("Not found: no such format").build());
                return;
            }
            sesameDownload(response, accessPointId, ap,
                    downloadFormat, mimeType);
            return;
        default:
            logger.error("download: invalid type for access point: "
                    + ap.getType());
            response.resume(Response.status(Status.NOT_FOUND).
                    entity("Invalid access point type").build());
            return;
        }
    }

    /** Get the download for an access point. The Path for this
     * method includes a filename with an extension.
     * The use of this method enables the URL to contain a path component
     * with a filename. This enables the use of e.g., curl or wget
     * to save the download to the "correct" filename.
     * @param response Asynchronous response for this request
     * @param accessPointId Access point id.
     * @param filename The filename specified in the URL. This may
     * be ignored in constructing the response headers.
     * @param extension The download format. This may be
     * ignored, depending (for example) on the access point type. */
    @Path("{access_point_id}/{filename}.{extension}")
    @GET
{'fraction_non_alphanumeric': 0.11396351575456053, 'fraction_numerical': 0.0011940298507462687, 'mean_word_length': 2.9209362808842654, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2141888', 'n_tokens_mistral': 4681, 'n_tokens_neox': 4677, 'n_words': 973}