<commit_before>/*
 * Copyright 2015-2016 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.opencb.opencga.storage.core.search.solr;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
import org.apache.solr.client.solrj.response.*;
import org.apache.solr.common.util.NamedList;
import org.opencb.biodata.formats.variant.io.VariantReader;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.models.variant.VariantSource;
import org.opencb.biodata.models.variant.annotation.ConsequenceTypeMappings;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.datastore.core.result.FacetedQueryResult;
import org.opencb.commons.datastore.core.result.FacetedQueryResultItem;
import org.opencb.commons.utils.FileUtils;
import org.opencb.opencga.core.results.VariantQueryResult;
import org.opencb.opencga.storage.core.config.StorageConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.exceptions.VariantSearchException;
import org.opencb.opencga.storage.core.metadata.StudyConfigurationManager;
import org.opencb.opencga.storage.core.search.VariantIterator;
import org.opencb.opencga.storage.core.search.VariantSearchIterator;
import org.opencb.opencga.storage.core.search.VariantSearchModel;
import org.opencb.opencga.storage.core.search.VariantSearchToVariantConverter;
import org.opencb.opencga.storage.core.utils.CellBaseUtils;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;
import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * Created by wasim on 09/11/16.
 */
public class VariantSearchManager {

    private SolrClient solrClient;
    private StorageConfiguration storageConfiguration;
    private VariantSearchToVariantConverter variantSearchToVariantConverter;
    private SolrQueryParser solrQueryParser;

    private Logger logger;

    public static final String SUMMARY = "summary";
    private static final int DEFAULT_INSERT_SIZE = 10000;

    @Deprecated
    public VariantSearchManager(String host, String collection) {
        this.solrClient = new HttpSolrClient.Builder(host + collection).build();
//        this.solrClient.setRequestWriter(new BinaryRequestWriter());
        variantSearchToVariantConverter = new VariantSearchToVariantConverter();
    }

    public VariantSearchManager(StudyConfigurationManager studyConfigurationManager, CellBaseUtils cellbaseUtils,
                                StorageConfiguration storageConfiguration) {
        this.storageConfiguration = storageConfiguration;

        logger = LoggerFactory.getLogger(VariantSearchManager.class);

        this.variantSearchToVariantConverter = new VariantSearchToVariantConverter();
        this.solrQueryParser = new SolrQueryParser(studyConfigurationManager, cellbaseUtils);

        init();
    }

//    public VariantSearchManager(String collection, StorageConfiguration storageConfiguration) {
////        this.host = storageConfiguration.getSearch().getHost();
////        this.collection = collection;
//        this.storageConfiguration = storageConfiguration;
//
//        this.solrClient = new HttpSolrClient.Builder(storageConfiguration.getSearch().getHost() + collection).build();
//        this.solrClient.setRequestWriter(new BinaryRequestWriter());
//
//        variantSearchToVariantConverter = new VariantSearchToVariantConverter();
//
//        logger = LoggerFactory.getLogger(VariantSearchManager.class);
//    }

    private void init() {
        this.solrClient = new HttpSolrClient.Builder(storageConfiguration.getSearch().getHost()).build();

        // Default implementation is HttpSolrClient and we can set up some parameters
        ((HttpSolrClient)this.solrClient).setRequestWriter(new BinaryRequestWriter());
        ((HttpSolrClient)this.solrClient).setSoTimeout(storageConfiguration.getSearch().getTimeout());
    }

    public boolean isAlive() throws IOException {
        try {
            return solrClient.ping().getResponse().get("status").equals("OK");
        } catch (SolrServerException e) {
            e.printStackTrace();
        }
        return false;
    }

    @Deprecated
    public boolean isAlive(String collection) {
        try {
            // init collection
//            init(collection);

            return solrClient.ping().getResponse().get("status").equals("OK");
        } catch (SolrServerException | IOException e) {
            return false;
        }
    }

    /**
     * Create a Solr core from a configuration set directory. By default, the configuration set directory is located
     * inside the folder server/solr/configsets.
     *
     * @param coreName      Core name
     * @param configSet     Configuration set name
     */
    public void createCore(String coreName, String configSet) {
        try {
            logger.debug("Creating core={}, core={}, configSet={}", storageConfiguration.getSearch().getHost(), coreName, configSet);
//            HttpSolrClient solrClient = new HttpSolrClient.Builder(storageConfiguration.getSearch().getHost()).build();
            CoreAdminRequest.Create request = new CoreAdminRequest.Create();
            request.setCoreName(coreName);
            request.setConfigSet(configSet);
            request.process(solrClient);
        } catch (Exception e) {
            e.printStackTrace();<commit_msg>storage: update isAlive function (VariantSearchManager) to take into account the collection parameter, #639<commit_after>/*
 * Copyright 2015-2016 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.opencb.opencga.storage.core.search.solr;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
import org.apache.solr.client.solrj.request.SolrPing;
import org.apache.solr.client.solrj.response.*;
import org.apache.solr.common.util.NamedList;
import org.opencb.biodata.formats.variant.io.VariantReader;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.models.variant.VariantSource;
import org.opencb.biodata.models.variant.annotation.ConsequenceTypeMappings;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.datastore.core.result.FacetedQueryResult;
import org.opencb.commons.datastore.core.result.FacetedQueryResultItem;
import org.opencb.commons.utils.FileUtils;
import org.opencb.opencga.core.results.VariantQueryResult;
import org.opencb.opencga.storage.core.config.StorageConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.exceptions.VariantSearchException;
import org.opencb.opencga.storage.core.metadata.StudyConfigurationManager;
import org.opencb.opencga.storage.core.search.VariantIterator;
import org.opencb.opencga.storage.core.search.VariantSearchIterator;
import org.opencb.opencga.storage.core.search.VariantSearchModel;
import org.opencb.opencga.storage.core.search.VariantSearchToVariantConverter;
import org.opencb.opencga.storage.core.utils.CellBaseUtils;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;
import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.TimeUnit;

/**
 * Created by wasim on 09/11/16.
 */
public class VariantSearchManager {

    private SolrClient solrClient;
    private StorageConfiguration storageConfiguration;
    private VariantSearchToVariantConverter variantSearchToVariantConverter;
    private SolrQueryParser solrQueryParser;

    private Logger logger;

    public static final String SUMMARY = "summary";
    private static final int DEFAULT_INSERT_SIZE = 10000;

    @Deprecated
    public VariantSearchManager(String host, String collection) {
        this.solrClient = new HttpSolrClient.Builder(host + collection).build();
//        this.solrClient.setRequestWriter(new BinaryRequestWriter());
        variantSearchToVariantConverter = new VariantSearchToVariantConverter();
    }

    public VariantSearchManager(StudyConfigurationManager studyConfigurationManager, CellBaseUtils cellbaseUtils,
                                StorageConfiguration storageConfiguration) {
        this.storageConfiguration = storageConfiguration;

        logger = LoggerFactory.getLogger(VariantSearchManager.class);

        this.variantSearchToVariantConverter = new VariantSearchToVariantConverter();
        this.solrQueryParser = new SolrQueryParser(studyConfigurationManager, cellbaseUtils);

        init();
    }

//    public VariantSearchManager(String collection, StorageConfiguration storageConfiguration) {
////        this.host = storageConfiguration.getSearch().getHost();
////        this.collection = collection;
//        this.storageConfiguration = storageConfiguration;
//
//        this.solrClient = new HttpSolrClient.Builder(storageConfiguration.getSearch().getHost() + collection).build();
//        this.solrClient.setRequestWriter(new BinaryRequestWriter());
//
//        variantSearchToVariantConverter = new VariantSearchToVariantConverter();
//
//        logger = LoggerFactory.getLogger(VariantSearchManager.class);
//    }

    private void init() {
        this.solrClient = new HttpSolrClient.Builder(storageConfiguration.getSearch().getHost()).build();

        // Default implementation is HttpSolrClient and we can set up some parameters
        ((HttpSolrClient)this.solrClient).setRequestWriter(new BinaryRequestWriter());
        ((HttpSolrClient)this.solrClient).setSoTimeout(storageConfiguration.getSearch().getTimeout());
    }

    public boolean isAlive(String collection) {
        try {
            SolrPing solrPing = new SolrPing();
            SolrPingResponse response = solrPing.process(solrClient, collection);
            return response.getResponse().get("status").equals("OK");
        } catch (SolrServerException | IOException e) {
            return false;
        }
    }

    /**
     * Create a Solr core from a configuration set directory. By default, the configuration set directory is located
     * inside the folder server/solr/configsets.
     *
     * @param coreName      Core name
     * @param configSet     Configuration set name
     */
    public void createCore(String coreName, String configSet) {
        try {
            logger.debug("Creating core={}, core={}, configSet={}", storageConfiguration.getSearch().getHost(), coreName, configSet);
//            HttpSolrClient solrClient = new HttpSolrClient.Builder(storageConfiguration.getSearch().getHost()).build();
            CoreAdminRequest.Create request = new CoreAdminRequest.Create();
            request.setCoreName(coreName);
            request.setConfigSet(configSet);
            request.process(solrClient);
        } catch (Exception e) {
            e.printStackTrace();
{'fraction_non_alphanumeric': 0.09828427747374649, 'fraction_numerical': 0.004215352758467682, 'mean_word_length': 4.0895747083176515, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5457870', 'n_tokens_mistral': 3822, 'n_tokens_neox': 3692, 'n_words': 745}