<commit_before>/*
 * Copyright 2008-2013 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.atlas.solr.query;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.SortedSetMultimap;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import uk.ac.ebi.atlas.commands.GenesNotFoundException;
import uk.ac.ebi.atlas.solr.BioentityProperty;
import uk.ac.ebi.atlas.web.controllers.ResourceNotFoundException;

import javax.inject.Inject;
import javax.inject.Named;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.google.common.base.Preconditions.checkArgument;

@Named
@Scope("prototype") //can't be singleton because RestTemplate is not thread safe
public class SolrClient {
    private static final Logger LOGGER = Logger.getLogger(SolrClient.class);

    private static final Pattern NON_WORD_CHARACTERS_PATTERN = Pattern.compile("[^\\w ]|_");

    private static final String SOLR_AUTOCOMPLETE_PROPERTIES_TEMPLATE = "suggest_properties?q=\"{0}\" AND species:\"{1}\"&wt=json&omitHeader=true&rows=0&json.nl=arrarr";

    private static final String SOLR_AUTOCOMPLETE_PROPERTIES_TEMPLATE_NO_SPECIES = "suggest_properties?q=\"{0}\"&wt=json&omitHeader=true&rows=0&json.nl=arrarr";

    private BioentityPropertyValueTokenizer bioentityPropertyValueTokenizer;

    @Value("#{configuration['index.server.url']}")
    private String serverURL;

    @Value("#{configuration['index.types.tooltip']}")
    private String tooltipPropertyTypes;

    private RestTemplate restTemplate;

    private final SolrQueryService solrQueryService;

    @Inject
    public SolrClient(RestTemplate restTemplate, SolrQueryService solrQueryService, BioentityPropertyValueTokenizer bioentityPropertyValueTokenizer) {
        this.restTemplate = restTemplate;
        this.solrQueryService = solrQueryService;
        this.bioentityPropertyValueTokenizer = bioentityPropertyValueTokenizer;
    }

    void setTooltipPropertyTypes(String tooltipPropertyTypes) {
        this.tooltipPropertyTypes = tooltipPropertyTypes;
    }

    public SortedSetMultimap<String, String> fetchTooltipProperties(String identifier) {

        List<String> propertyTypes = Arrays.asList(tooltipPropertyTypes.trim().split(","));
        return fetchProperties(identifier, propertyTypes);

    }

    public SortedSetMultimap<String, String> fetchGenePageProperties(String identifier, List<String> propertyTypes) {
        SortedSetMultimap<String, String> propertiesByType = fetchProperties(identifier, propertyTypes);
        if (propertiesByType.isEmpty()) {
            throw new ResultNotFoundException("Gene/protein with accession : " + identifier + " is not found!");
        }
        return propertiesByType;
    }

    SortedSetMultimap<String, String> fetchProperties(String identifier, List<String> propertyTypes) {

        return solrQueryService.fetchProperties(identifier, propertyTypes);

    }

    public Set<String> fetchGeneIdentifiersFromSolr(String queryString, String bioentityType, String... propertyNames) {
        StringBuilder sb = new StringBuilder();
        sb.append(SolrQueryService.PROPERTY_LOWER_FIELD);
        sb.append(":\"");
        sb.append(queryString);
        sb.append("\" AND (");
        for (String propertyName : propertyNames) {
            sb.append(SolrQueryService.PROPERTY_NAME_FIELD);
            sb.append(":\"");
            sb.append(propertyName);
            sb.append("\" OR ");
        }
        sb.delete(sb.length() - 3, sb.length());
        sb.append(") AND ");
        sb.append(SolrQueryService.BIOENTITY_TYPE_FIELD);
        sb.append(":");
        sb.append(bioentityType);

        return solrQueryService.fetchGeneIdentifiersFromSolr(sb.toString());
    }

    public String findSpeciesForBioentityId(String identifier) {

        return getSpeciesForPropertyValue(identifier, SolrQueryService.BIOENTITY_IDENTIFIER_FIELD);

    }

    public List<String> findPropertyValuesForGeneId(String identifier, String propertyType) {

        return solrQueryService.getPropertyValuesForIdentifier(identifier, propertyType);

    }

    public String getSpeciesForPropertyValue(String propertyValue) {
        return getSpeciesForPropertyValue(propertyValue, "");
    }

    public String getSpeciesForPropertyValue(String propertyValue, String propertyName) {
        List<String> propertyValueTokens = bioentityPropertyValueTokenizer.split(propertyValue);
        Set<String> species = Sets.newHashSet();
        for (String propertyValueToken : propertyValueTokens) {
            species.addAll(solrQueryService.getSpeciesForPropertyValue(propertyValueToken, propertyName));
        }
        if (species.size() != 1) {
            throw new ResourceNotFoundException("Exact species can't be determined for propertyValue: " + propertyValue + " and propertyName: " + propertyName);
        }
        return species.iterator().next();
    }

    public GeneQueryResponse findGeneIdsOrSets(String geneQuery, boolean exactMatch, String species, boolean tokenizeQuery) throws GenesNotFoundException {

        checkArgument(StringUtils.isNotBlank(geneQuery));

        species = limitSpeciesNameToTwoWords(species);

        GeneQueryResponse geneQueryResponse = new GeneQueryResponse();

        if (tokenizeQuery) {
            for (String queryToken : bioentityPropertyValueTokenizer.split(geneQuery)) {
                Set<String> geneIds = solrQueryService.getGeneIds(queryToken, exactMatch, species);
                geneQueryResponse.addGeneIds(queryToken, geneIds);
            }
        } else {
            Set<String> geneIds = solrQueryService.getGeneIds(geneQuery, exactMatch, species);
            geneQueryResponse.addGeneIds(geneQuery, geneIds);
        }
        return geneQueryResponse;

    }

    String limitSpeciesNameToTwoWords(String species) {

        String[] words = StringUtils.split(species);
        if (ArrayUtils.getLength(words) > 2){
            return words[0].concat(" ").concat(words[1]);
        }
        return species;
    }

    public Set<String> findGeneIdsForSpecies(String species) {

        checkArgument(StringUtils.isNotBlank(species));

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdsForSpecies(species);
    }

    public List<String> findGeneIdSuggestionsInName(String geneName, String species) {

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdSuggestionsInName(geneName, species);
    }

    public List<String> findGeneIdSuggestionsInSynonym(String geneName, String species) {

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdSuggestionsInSynonym(geneName, species);
    }

    public List<String> findGeneIdSuggestionsInIdentifier(String geneName, String species) {

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdSuggestionsInIdentifier(geneName, species);

    }

    public List<String> findGenePropertySuggestions(String multiTermToken, String species) {

        species = limitSpeciesNameToTwoWords(species);

        Matcher notSpellCheckableMatcher = NON_WORD_CHARACTERS_PATTERN.matcher(multiTermToken);

        if (notSpellCheckableMatcher.find()) {<commit_msg>removed solr query method that isn't used anymore<commit_after>/*
 * Copyright 2008-2013 Microarray Informatics Team, EMBL-European Bioinformatics Institute
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *
 * For further details of the Gene Expression Atlas project, including source code,
 * downloads and documentation, please see:
 *
 * http://gxa.github.com/gxa
 */

package uk.ac.ebi.atlas.solr.query;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.SortedSetMultimap;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import uk.ac.ebi.atlas.commands.GenesNotFoundException;
import uk.ac.ebi.atlas.solr.BioentityProperty;
import uk.ac.ebi.atlas.web.controllers.ResourceNotFoundException;

import javax.inject.Inject;
import javax.inject.Named;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.google.common.base.Preconditions.checkArgument;

@Named
@Scope("prototype") //can't be singleton because RestTemplate is not thread safe
public class SolrClient {
    private static final Logger LOGGER = Logger.getLogger(SolrClient.class);

    private static final Pattern NON_WORD_CHARACTERS_PATTERN = Pattern.compile("[^\\w ]|_");

    private static final String SOLR_AUTOCOMPLETE_PROPERTIES_TEMPLATE = "suggest_properties?q=\"{0}\" AND species:\"{1}\"&wt=json&omitHeader=true&rows=0&json.nl=arrarr";

    private static final String SOLR_AUTOCOMPLETE_PROPERTIES_TEMPLATE_NO_SPECIES = "suggest_properties?q=\"{0}\"&wt=json&omitHeader=true&rows=0&json.nl=arrarr";

    private BioentityPropertyValueTokenizer bioentityPropertyValueTokenizer;

    @Value("#{configuration['index.server.url']}")
    private String serverURL;

    @Value("#{configuration['index.types.tooltip']}")
    private String tooltipPropertyTypes;

    private RestTemplate restTemplate;

    private final SolrQueryService solrQueryService;

    @Inject
    public SolrClient(RestTemplate restTemplate, SolrQueryService solrQueryService, BioentityPropertyValueTokenizer bioentityPropertyValueTokenizer) {
        this.restTemplate = restTemplate;
        this.solrQueryService = solrQueryService;
        this.bioentityPropertyValueTokenizer = bioentityPropertyValueTokenizer;
    }

    void setTooltipPropertyTypes(String tooltipPropertyTypes) {
        this.tooltipPropertyTypes = tooltipPropertyTypes;
    }

    public SortedSetMultimap<String, String> fetchTooltipProperties(String identifier) {

        List<String> propertyTypes = Arrays.asList(tooltipPropertyTypes.trim().split(","));
        return fetchProperties(identifier, propertyTypes);

    }

    public SortedSetMultimap<String, String> fetchGenePageProperties(String identifier, List<String> propertyTypes) {
        SortedSetMultimap<String, String> propertiesByType = fetchProperties(identifier, propertyTypes);
        if (propertiesByType.isEmpty()) {
            throw new ResultNotFoundException("Gene/protein with accession : " + identifier + " is not found!");
        }
        return propertiesByType;
    }

    SortedSetMultimap<String, String> fetchProperties(String identifier, List<String> propertyTypes) {

        return solrQueryService.fetchProperties(identifier, propertyTypes);

    }

    public Set<String> fetchGeneIdentifiersFromSolr(String queryString, String bioentityType, String... propertyNames) {
        StringBuilder sb = new StringBuilder();
        sb.append(SolrQueryService.PROPERTY_LOWER_FIELD);
        sb.append(":\"");
        sb.append(queryString);
        sb.append("\" AND (");
        for (String propertyName : propertyNames) {
            sb.append(SolrQueryService.PROPERTY_NAME_FIELD);
            sb.append(":\"");
            sb.append(propertyName);
            sb.append("\" OR ");
        }
        sb.delete(sb.length() - 3, sb.length());
        sb.append(") AND ");
        sb.append(SolrQueryService.BIOENTITY_TYPE_FIELD);
        sb.append(":");
        sb.append(bioentityType);

        return solrQueryService.fetchGeneIdentifiersFromSolr(sb.toString());
    }

    public String findSpeciesForBioentityId(String identifier) {

        return getSpeciesForPropertyValue(identifier, SolrQueryService.BIOENTITY_IDENTIFIER_FIELD);

    }

    public List<String> findPropertyValuesForGeneId(String identifier, String propertyType) {

        return solrQueryService.getPropertyValuesForIdentifier(identifier, propertyType);

    }

    public String getSpeciesForPropertyValue(String propertyValue) {
        return getSpeciesForPropertyValue(propertyValue, "");
    }

    public String getSpeciesForPropertyValue(String propertyValue, String propertyName) {
        List<String> propertyValueTokens = bioentityPropertyValueTokenizer.split(propertyValue);
        Set<String> species = Sets.newHashSet();
        for (String propertyValueToken : propertyValueTokens) {
            species.addAll(solrQueryService.getSpeciesForPropertyValue(propertyValueToken, propertyName));
        }
        if (species.size() != 1) {
            throw new ResourceNotFoundException("Exact species can't be determined for propertyValue: " + propertyValue + " and propertyName: " + propertyName);
        }
        return species.iterator().next();
    }

    public GeneQueryResponse findGeneIdsOrSets(String geneQuery, boolean exactMatch, String species, boolean tokenizeQuery) throws GenesNotFoundException {

        checkArgument(StringUtils.isNotBlank(geneQuery));

        species = limitSpeciesNameToTwoWords(species);

        GeneQueryResponse geneQueryResponse = new GeneQueryResponse();

        if (tokenizeQuery) {
            for (String queryToken : bioentityPropertyValueTokenizer.split(geneQuery)) {
                Set<String> geneIds = solrQueryService.getGeneIds(queryToken, exactMatch, species);
                geneQueryResponse.addGeneIds(queryToken, geneIds);
            }
        } else {
            Set<String> geneIds = solrQueryService.getGeneIds(geneQuery, exactMatch, species);
            geneQueryResponse.addGeneIds(geneQuery, geneIds);
        }
        return geneQueryResponse;

    }

    String limitSpeciesNameToTwoWords(String species) {

        String[] words = StringUtils.split(species);
        if (ArrayUtils.getLength(words) > 2){
            return words[0].concat(" ").concat(words[1]);
        }
        return species;
    }

    public List<String> findGeneIdSuggestionsInName(String geneName, String species) {

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdSuggestionsInName(geneName, species);
    }

    public List<String> findGeneIdSuggestionsInSynonym(String geneName, String species) {

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdSuggestionsInSynonym(geneName, species);
    }

    public List<String> findGeneIdSuggestionsInIdentifier(String geneName, String species) {

        species = limitSpeciesNameToTwoWords(species);

        return solrQueryService.getGeneIdSuggestionsInIdentifier(geneName, species);

    }

    public List<String> findGenePropertySuggestions(String multiTermToken, String species) {

        species = limitSpeciesNameToTwoWords(species);

        Matcher notSpellCheckableMatcher = NON_WORD_CHARACTERS_PATTERN.matcher(multiTermToken);

        if (notSpellCheckableMatcher.find()) {
{'fraction_non_alphanumeric': 0.09197628690497153, 'fraction_numerical': 0.0027000058695779774, 'mean_word_length': 4.098144823459006, 'pattern_counts': {'":': 6, '<': 38, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5449943', 'n_tokens_mistral': 4788, 'n_tokens_neox': 4535, 'n_words': 972}