<commit_before>package org.opencb.opencga.storage.core.search;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.opencb.biodata.formats.variant.io.VariantReader;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.models.variant.VariantSource;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.utils.FileUtils;
import org.opencb.opencga.storage.core.config.SearchConfiguration;
import org.opencb.opencga.storage.core.config.StorageConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.search.solr.ParseSolrQuery;
import org.opencb.opencga.storage.core.search.solr.SolrVariantSearchIterator;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;
import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by wasim on 09/11/16.
 */
public class VariantSearchManager {

    private String hostName;
    private String collectionName;

    private SearchConfiguration searchConfiguration;
    private HttpSolrClient solrClient;
    private static VariantSearchToVariantConverter variantSearchToVariantConverter;

    public VariantSearchManager(String host, String collection) {
        this.hostName = host;
        this.collectionName = collection;

        this.solrClient = new HttpSolrClient.Builder(host + collection).build();
        this.solrClient.setRequestWriter(new BinaryRequestWriter());
        variantSearchToVariantConverter = new VariantSearchToVariantConverter();
    }

    public VariantSearchManager(StorageConfiguration storageConfiguration) {
        this(storageConfiguration.getSearch().getHost(), storageConfiguration.getSearch().getCollection());
        this.searchConfiguration = storageConfiguration.getSearch();
    }

    /**
     * Create a Solr core from a configuration set directory. By default, the configuration set directory is located
     * inside the folder server/solr/configsets.
     *
     * @param coreName      Core name
     * @param configSet     Configuration set name
     */
    public void createCore(String coreName, String configSet) {
        try {
            System.out.println("Creating core: " + hostName + ", core=" + coreName + ", configSet=" + configSet);
            HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
            CoreAdminRequest.Create request = new CoreAdminRequest.Create();
            request.setCoreName(coreName);
            request.setConfigSet(configSet);
            request.process(solrClient);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * Check if a given core exists.
     *
     * @param coreName          Core name
     * @return                  True or false
     * @throws Exception        Exception
     */
    public boolean existCore(String coreName) throws Exception {
        HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
        CoreStatus status = CoreAdminRequest.getCoreStatus(coreName, solrClient);
        try {
            // if the status.response is null, catch the exception
            status.getInstanceDirectory();
        } catch (Exception e) {
            return false;
        }
        return true;
    }

    /**
     * Create a Solr collection from a configuration directory. The configuration has to be uploaded to the zookeeper,
     * $ ./bin/solr zk upconfig -n <config name> -d <path to the config dir> -z <host:port zookeeper>.
     * For Solr, collection name, configuration name and number of shards are mandatory in order to create a collection.
     * Number of replicas is optional.
     *
     * @param collectionName     Collection name
     * @param config             Configuration name
     * @param numShards          Number of shards
     * @param numReplicas        Number of replicas
     */
    public void createCollection(String collectionName, String config, int numShards, int numReplicas) {
        System.out.println("Creating core: " + hostName + ", collection=" + collectionName + ", config=" + config
                + ", numShards=" + numShards + ", numReplicas=" + numReplicas);
        try {
            HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
            CollectionAdminRequest request = CollectionAdminRequest.createCollection(collectionName, config,
                    numShards, numReplicas);
            request.process(solrClient);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * Check if a given collection exists.
     *
     * @param collectionName    Collection name
     * @return                  True or false
     * @throws Exception        Exception
     */
    public boolean existCollection(String collectionName) throws Exception {

//        HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
//
//        List<String> collections = CollectionAdminRequest.listCollections(solrClient);
//        for (String collection: collections) {
//            System.out.println(collection);
//        }
//        for (String collection: collections) {
//            if (collection.equals(collectionName)) {
//                return true;<commit_msg>opencga-storage: update collection creation collection (search command line)<commit_after>package org.opencb.opencga.storage.core.search;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.opencb.biodata.formats.variant.io.VariantReader;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.models.variant.VariantSource;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.utils.FileUtils;
import org.opencb.opencga.storage.core.config.SearchConfiguration;
import org.opencb.opencga.storage.core.config.StorageConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
import org.opencb.opencga.storage.core.search.solr.ParseSolrQuery;
import org.opencb.opencga.storage.core.search.solr.SolrVariantSearchIterator;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;
import org.opencb.opencga.storage.core.variant.io.VariantReaderUtils;

import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by wasim on 09/11/16.
 */
public class VariantSearchManager {

    private String hostName;
    private String collectionName;

    private SearchConfiguration searchConfiguration;
    private HttpSolrClient solrClient;
    private static VariantSearchToVariantConverter variantSearchToVariantConverter;

    public VariantSearchManager(String host, String collection) {
        this.hostName = host;
        this.collectionName = collection;

        this.solrClient = new HttpSolrClient.Builder(host + collection).build();
        this.solrClient.setRequestWriter(new BinaryRequestWriter());
        variantSearchToVariantConverter = new VariantSearchToVariantConverter();
    }

    public VariantSearchManager(StorageConfiguration storageConfiguration) {
        this(storageConfiguration.getSearch().getHost(), storageConfiguration.getSearch().getCollection());
        this.searchConfiguration = storageConfiguration.getSearch();
    }

    /**
     * Create a Solr core from a configuration set directory. By default, the configuration set directory is located
     * inside the folder server/solr/configsets.
     *
     * @param coreName      Core name
     * @param configSet     Configuration set name
     */
    public void createCore(String coreName, String configSet) {
        try {
            System.out.println("Creating core: " + hostName + ", core=" + coreName + ", configSet=" + configSet);
            HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
            CoreAdminRequest.Create request = new CoreAdminRequest.Create();
            request.setCoreName(coreName);
            request.setConfigSet(configSet);
            request.process(solrClient);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * Check if a given core exists.
     *
     * @param coreName          Core name
     * @return                  True or false
     * @throws Exception        Exception
     */
    public boolean existCore(String coreName) throws Exception {
        HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
        CoreStatus status = CoreAdminRequest.getCoreStatus(coreName, solrClient);
        try {
            // if the status.response is null, catch the exception
            status.getInstanceDirectory();
        } catch (Exception e) {
            return false;
        }
        return true;
    }

    /**
     * Create a Solr collection from a configuration directory. The configuration has to be uploaded to the zookeeper,
     * $ ./bin/solr zk upconfig -n <config name> -d <path to the config dir> -z <host:port zookeeper>.
     * For Solr, collection name, configuration name and number of shards are mandatory in order to create a collection.
     * Number of replicas is optional.
     *
     * @param collectionName     Collection name
     * @param config             Configuration name
     * @param numShards          Number of shards
     * @param numReplicas        Number of replicas
     */
    public void createCollection(String collectionName, String config, int numShards, int numReplicas) {
        System.out.println("Creating collection: " + hostName + ", collection=" + collectionName + ", config=" + config
                + ", numShards=" + numShards + ", numReplicas=" + numReplicas);
        try {
            HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
            CollectionAdminRequest request = CollectionAdminRequest.createCollection(collectionName, config,
                    numShards, numReplicas);
            request.process(solrClient);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * Check if a given collection exists.
     *
     * @param collectionName    Collection name
     * @return                  True or false
     * @throws Exception        Exception
     */
    public boolean existCollection(String collectionName) throws Exception {

//        HttpSolrClient solrClient = new HttpSolrClient.Builder(hostName).build();
//
//        List<String> collections = CollectionAdminRequest.listCollections(solrClient);
//        for (String collection: collections) {
//            System.out.println(collection);
//        }
//        for (String collection: collections) {
//            if (collection.equals(collectionName)) {
//                return true;
{'fraction_non_alphanumeric': 0.09270007357148696, 'fraction_numerical': 0.0009809531594866345, 'mean_word_length': 3.0875375877046443, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4251607', 'n_tokens_mistral': 3365, 'n_tokens_neox': 3266, 'n_words': 744}