<commit_before>/*
 * Copyright 2015-2020 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.opencb.cellbase.server.rest;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.factories.SchemaFactoryWrapper;
import com.google.common.base.Splitter;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.opencb.cellbase.core.CellBaseDataResponse;
import org.opencb.cellbase.core.config.CellBaseConfiguration;
import org.opencb.cellbase.core.exception.CellbaseException;
import org.opencb.cellbase.core.result.CellBaseDataResult;
import org.opencb.cellbase.lib.managers.CellBaseManagerFactory;
import org.opencb.cellbase.lib.monitor.Monitor;
import org.opencb.cellbase.server.exception.VersionException;
import org.opencb.commons.datastore.core.Event;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.ws.rs.core.Response.ResponseBuilder;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;

@Path("/{version}/{species}")
@Produces("text/plain")
public class GenericRestWSServer implements IWSServer {

    protected String version;
    protected String species;

    @ApiParam(name = "assembly", value = "Set the reference genome assembly, e.g. grch38. For a full list of"
            + "potentially available assemblies, please refer to: "
            + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species")
    @DefaultValue("")
    @QueryParam("assembly")
    protected String assembly;

    protected Query query;
    protected QueryOptions queryOptions;
    protected ObjectMap params;
    protected Map<String, String> uriParams;
    protected UriInfo uriInfo;
    protected HttpServletRequest httpServletRequest;
    protected ObjectMapper jsonObjectMapper;
    protected static ObjectWriter jsonObjectWriter;
    protected String SERVICE_START_DATE;
    protected StopWatch WATCH;
    protected AtomicBoolean initialized;
    protected long startTime;
    protected Logger logger;

    /**
     * Loading properties file just one time to be more efficient. All methods
     * will check parameters so to avoid extra operations this config can load
     * versions and species
     */
    protected CellBaseConfiguration cellBaseConfiguration;
    protected CellBaseManagerFactory cellBaseManagerFactory;
    protected Monitor monitor;
    private static final String ERROR = "error";
    private static final String OK = "ok";
    // this webservice has no species, do not validate
    private static final String DONT_CHECK_SPECIES = "do not validate species";

    public GenericRestWSServer(@PathParam("version") String version, @Context UriInfo uriInfo, @Context HttpServletRequest hsr)
            throws VersionException, IOException, CellbaseException {
        this(version, DONT_CHECK_SPECIES, uriInfo, hsr);
    }

    public GenericRestWSServer(@PathParam("version") String version, @PathParam("species") String species, @Context UriInfo uriInfo,
                               @Context HttpServletRequest hsr)
            throws VersionException, IOException, CellbaseException {

        this.version = version;
        this.uriInfo = uriInfo;
        this.httpServletRequest = hsr;
        this.species = species;

        init();
        initQuery();
    }

    private void init() throws IOException, CellbaseException {
        // we need to make sure we only init one single time
        if (initialized == null || initialized.compareAndSet(false, true)) {
            initialized = new AtomicBoolean(true);

            SERVICE_START_DATE = new SimpleDateFormat("yyyyMMdd_HHmmss").format(Calendar.getInstance().getTime());
            WATCH = new StopWatch();
            WATCH.start();

            jsonObjectMapper = new ObjectMapper();
            jsonObjectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
            jsonObjectMapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true);
            jsonObjectWriter = jsonObjectMapper.writer();

            logger = LoggerFactory.getLogger(this.getClass());

            // We must load the configuration file from CELLBASE_HOME, this must happen only the first time!
            String cellbaseHome = System.getenv("CELLBASE_HOME");
            if (StringUtils.isEmpty(cellbaseHome)) {
                // ENV variable isn't set, try the servlet context instead
                ServletContext context = httpServletRequest.getServletContext();
                if (StringUtils.isNotEmpty(context.getInitParameter("CELLBASE_HOME"))) {
                    cellbaseHome = context.getInitParameter("CELLBASE_HOME");
                } else {
                    logger.error("No valid configuration directory provided!");
                    throw new CellbaseException("No CELLBASE_HOME found");
                }
            }

            logger.debug("CELLBASE_HOME set to: {}", cellbaseHome);

            cellBaseConfiguration = CellBaseConfiguration.load(Paths.get(cellbaseHome).resolve("conf").resolve("configuration.yml"));
            cellBaseManagerFactory = new CellBaseManagerFactory(cellBaseConfiguration);

            // Initialize Monitor
            monitor = new Monitor(cellBaseManagerFactory.getMetaManager());
        }
    }

    private void initQuery() throws VersionException {
        startTime = System.currentTimeMillis();
//        query = new Query();
//        // This needs to be an ArrayList since it may be added some extra fields later
        // TODO move to manager
//        queryOptions = new QueryOptions("exclude", new ArrayList<>(Arrays.asList("_id", "_chunkIds")));
//        params = new ObjectMap();
        uriParams = convertMultiToMap(uriInfo.getQueryParameters());
        checkVersion();
    }

    private void checkVersion() throws VersionException {
        if (version == null) {
            throw new VersionException("Version not valid: '" + version + "'");
        }

        if (!cellBaseConfiguration.getVersion().equalsIgnoreCase(version)) {
            logger.error("Version '{}' does not match configuration '{}'", this.version, cellBaseConfiguration.getVersion());
            throw new VersionException("Version not valid: '" + version + "'");
        }
    }

    public Map<String, String> convertMultiToMap(MultivaluedMap<String, String> m) {
        Map<String, String> map = new HashMap<String, String>();
        if (m == null) {
            return map;
        }
        for (Map.Entry<String, List<String>> entry : m.entrySet()) {
            map.put(entry.getKey(), String.join(",", entry.getValue()));
        }
        return map;
    }

    public void parseQueryParams() {
        MultivaluedMap<String, String> multivaluedMap = uriInfo.getQueryParameters();

        queryOptions.put("metadata", multivaluedMap.get("metadata") == null || multivaluedMap.get("metadata").get(0).equals("true"));
<commit_msg>transform dot notation to camelcased parameter names. need to do the reverse right before the mongo query is run<commit_after>/*
 * Copyright 2015-2020 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.opencb.cellbase.server.rest;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.module.jsonSchema.JsonSchema;
import com.fasterxml.jackson.module.jsonSchema.factories.SchemaFactoryWrapper;
import com.google.common.base.Splitter;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.opencb.cellbase.core.CellBaseDataResponse;
import org.opencb.cellbase.core.config.CellBaseConfiguration;
import org.opencb.cellbase.core.exception.CellbaseException;
import org.opencb.cellbase.core.result.CellBaseDataResult;
import org.opencb.cellbase.lib.managers.CellBaseManagerFactory;
import org.opencb.cellbase.lib.monitor.Monitor;
import org.opencb.cellbase.server.exception.VersionException;
import org.opencb.commons.datastore.core.Event;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.*;
import javax.ws.rs.core.Response.ResponseBuilder;
import java.io.IOException;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;

@Path("/{version}/{species}")
@Produces("text/plain")
public class GenericRestWSServer implements IWSServer {

    protected String version;
    protected String species;

    @ApiParam(name = "assembly", value = "Set the reference genome assembly, e.g. grch38. For a full list of"
            + "potentially available assemblies, please refer to: "
            + "https://bioinfo.hpc.cam.ac.uk/cellbase/webservices/rest/v4/meta/species")
    @DefaultValue("")
    @QueryParam("assembly")
    protected String assembly;

    protected Query query;
    protected QueryOptions queryOptions;
    protected ObjectMap params;
    protected Map<String, String> uriParams;
    protected UriInfo uriInfo;
    protected HttpServletRequest httpServletRequest;
    protected ObjectMapper jsonObjectMapper;
    protected static ObjectWriter jsonObjectWriter;
    protected String SERVICE_START_DATE;
    protected StopWatch WATCH;
    protected AtomicBoolean initialized;
    protected long startTime;
    protected Logger logger;

    /**
     * Loading properties file just one time to be more efficient. All methods
     * will check parameters so to avoid extra operations this config can load
     * versions and species
     */
    protected CellBaseConfiguration cellBaseConfiguration;
    protected CellBaseManagerFactory cellBaseManagerFactory;
    protected Monitor monitor;
    private static final String ERROR = "error";
    private static final String OK = "ok";
    // this webservice has no species, do not validate
    private static final String DONT_CHECK_SPECIES = "do not validate species";

    public GenericRestWSServer(@PathParam("version") String version, @Context UriInfo uriInfo, @Context HttpServletRequest hsr)
            throws VersionException, IOException, CellbaseException {
        this(version, DONT_CHECK_SPECIES, uriInfo, hsr);
    }

    public GenericRestWSServer(@PathParam("version") String version, @PathParam("species") String species, @Context UriInfo uriInfo,
                               @Context HttpServletRequest hsr)
            throws VersionException, IOException, CellbaseException {

        this.version = version;
        this.uriInfo = uriInfo;
        this.httpServletRequest = hsr;
        this.species = species;

        init();
        initQuery();
    }

    private void init() throws IOException, CellbaseException {
        // we need to make sure we only init one single time
        if (initialized == null || initialized.compareAndSet(false, true)) {
            initialized = new AtomicBoolean(true);

            SERVICE_START_DATE = new SimpleDateFormat("yyyyMMdd_HHmmss").format(Calendar.getInstance().getTime());
            WATCH = new StopWatch();
            WATCH.start();

            jsonObjectMapper = new ObjectMapper();
            jsonObjectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
            jsonObjectMapper.configure(MapperFeature.REQUIRE_SETTERS_FOR_GETTERS, true);
            jsonObjectWriter = jsonObjectMapper.writer();

            logger = LoggerFactory.getLogger(this.getClass());

            // We must load the configuration file from CELLBASE_HOME, this must happen only the first time!
            String cellbaseHome = System.getenv("CELLBASE_HOME");
            if (StringUtils.isEmpty(cellbaseHome)) {
                // ENV variable isn't set, try the servlet context instead
                ServletContext context = httpServletRequest.getServletContext();
                if (StringUtils.isNotEmpty(context.getInitParameter("CELLBASE_HOME"))) {
                    cellbaseHome = context.getInitParameter("CELLBASE_HOME");
                } else {
                    logger.error("No valid configuration directory provided!");
                    throw new CellbaseException("No CELLBASE_HOME found");
                }
            }

            logger.debug("CELLBASE_HOME set to: {}", cellbaseHome);

            cellBaseConfiguration = CellBaseConfiguration.load(Paths.get(cellbaseHome).resolve("conf").resolve("configuration.yml"));
            cellBaseManagerFactory = new CellBaseManagerFactory(cellBaseConfiguration);

            // Initialize Monitor
            monitor = new Monitor(cellBaseManagerFactory.getMetaManager());
        }
    }

    private void initQuery() throws VersionException {
        startTime = System.currentTimeMillis();
//        query = new Query();
//        // This needs to be an ArrayList since it may be added some extra fields later
        // TODO move to manager
//        queryOptions = new QueryOptions("exclude", new ArrayList<>(Arrays.asList("_id", "_chunkIds")));
//        params = new ObjectMap();
        uriParams = convertMultiToMap(uriInfo.getQueryParameters());
        checkVersion();
    }

    private void checkVersion() throws VersionException {
        if (version == null) {
            throw new VersionException("Version not valid: '" + version + "'");
        }

        if (!cellBaseConfiguration.getVersion().equalsIgnoreCase(version)) {
            logger.error("Version '{}' does not match configuration '{}'", this.version, cellBaseConfiguration.getVersion());
            throw new VersionException("Version not valid: '" + version + "'");
        }
    }

    public Map<String, String> convertMultiToMap(MultivaluedMap<String, String> m) {
        Map<String, String> map = new HashMap<String, String>();
        if (m == null) {
            return map;
        }
        for (Map.Entry<String, List<String>> entry : m.entrySet()) {
            String camelCased = convertDotToCamelCase(entry.getKey());
            map.put(camelCased, String.join(",", entry.getValue()));
        }
        return map;
    }

    /**
     * Converts to canelcase, e.g. transcripts.biotype --> transcriptsBiotype. It's translated back to the full path right before the
     * mongo query is run.
     *
     * @param dotPath e.g. transcripts.biotype
     * @return the string camelcased
     */
    private String convertDotToCamelCase(String dotPath) {
        String[] paths = dotPath.split("\\.");
        StringBuilder sb = new StringBuilder();
        for (String path  : paths) {
            if (sb.length() > 0) {
                path = StringUtils.capitalize(path);
            }
            sb.append(path);
        }
        return sb.toString();
    }

    public void parseQueryParams() {
        MultivaluedMap<String, String> multivaluedMap = uriInfo.getQueryParameters();

        queryOptions.put("metadata", multivaluedMap.get("metadata") == null || multivaluedMap.get("metadata").get(0).equals("true"));

{'fraction_non_alphanumeric': 0.0967985590610656, 'fraction_numerical': 0.0023821974318749637, 'mean_word_length': 3.2987012987012987, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5221343', 'n_tokens_mistral': 4564, 'n_tokens_neox': 4405, 'n_words': 1184}