<commit_before>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see http://www.gnu.org/licenses/
 */
package org.phenotips.ncbieutils.internal;

import org.phenotips.ncbieutils.NCBIEUtilsService;

import java.io.BufferedInputStream;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import javax.inject.Inject;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

/**
 * Base implementation for {@link NCBIEUtilsService}.
 *
 * @version $Id$
 */
public abstract class AbstractSpecializedNCBIEUtilsAccessService implements NCBIEUtilsService
{
    @Inject
    private Logger logger;

    protected static final String SERVER_URL = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/";

    protected static final String TERM_SEARCH_QUERY_SCRIPT = "esearch.fcgi";

    protected static final String TERM_SEARCH_PARAM_NAME = "term";

    protected static final String TERM_SUMMARY_QUERY_SCRIPT = "esummary.fcgi";

    protected static final String TERM_SUMMARY_PARAM_NAME = "id";

    protected static final String SPELL_CHECK_QUERY_SCRIPT = "espell.fcgi";

    protected static final String SPELL_CHECK_PARAM_NAME = "term";

    protected static final String DB_PARAM_NAME = "db";

    protected abstract String getDatabaseName();

    @Override
    public List<Map<String, Object>> getSuggestions(final String query)
    {
        return getSuggestions(query, 10, 0);
    }

    @Override
    public List<Map<String, Object>> getSuggestions(final String query, final int rows, final int start)
    {
        // Step 1: get spelling suggestions for query
        String correctedQuery = getCorrectedQuery(query);
        // Step 2: get ID list matching the corrected query
        List<String> idList = getMatches(correctedQuery, rows, start);
        // step 3: get Summaries for ID list
        List<Map<String, Object>> result = getSummaries(idList);
        return result;
    }

    @Override
    public String getSuggestionsXML(final String query)
    {
        return getSuggestionsXML(query, 10, 0);
    }

    @Override
    public String getSuggestionsXML(final String query, final int rows, final int start)
    {
        // Step 1: get spelling suggestions for query
        String correctedQuery = getCorrectedQuery(query);
        // Step 2: get ID list matching the corrected query
        List<String> idList = getMatches(correctedQuery, rows, start);
        // step 3: get Summaries for ID list
        String result = getSummariesXML(idList);
        return result;
    }

    @Override
    public String getName(String id)
    {
        String url = composeURL(TERM_SUMMARY_QUERY_SCRIPT, TERM_SUMMARY_PARAM_NAME, id);
        String result = id;
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("DocSum");
            if (nodes.getLength() > 0) {
                Map<String, String> idToName = this.getNameForId(nodes.item(0));
                result = idToName.get(id);
                if (result != null && !id.equals(result)) {
                    return result;
                }
            }
            this.logger.warn("Name not found for OMIM id " + id);
            return id;
        } catch (Exception ex) {
            this.logger.error("Error while trying to retrieve name for " + getDatabaseName() + " id " + id + " "
                + ex.getClass().getName() + " " + ex.getMessage(), ex);
        }
        return id;
    }

    @Override
    public Map<String, String> getNames(List<String> idList)
    {
        Map<String, String> result = new HashMap<String, String>();
        String queryList = getSerializedList(idList);
        String url = composeURL(TERM_SUMMARY_QUERY_SCRIPT, TERM_SUMMARY_PARAM_NAME, queryList);
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("DocSum");
            for (int i = 0; i < nodes.getLength(); ++i) {
                Map<String, String> idToName = this.getNameForId(nodes.item(i));
                for (String id : idToName.keySet()) {
                    if (idList.contains(id)) {
                        result.put(id, idToName.get(id));
                    } else {
                        this.logger.warn("Unrequested " + getDatabaseName() + " id " + id);
                    }
                }
            }
        } catch (Exception ex) {
            this.logger.error("Error while trying to retrieve name for " + getDatabaseName() + " ids " + queryList
                + " " + ex.getClass().getName() + " " + ex.getMessage(), ex);
        }
        return result;
    }

    @Override
    public String getCorrectedQuery(String query)
    {
        // response example at http://eutils.ncbi.nlm.nih.gov/entrez/eutils/espell.fcgi?db=omim&term=atention+sindrom
        // response type: XML
        // get corrected query from /eSpellResult/CorrectedQuery (single element)
        // use original query if this element is empty
        String url = composeURL(SPELL_CHECK_QUERY_SCRIPT, SPELL_CHECK_PARAM_NAME, query);
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("CorrectedQuery");
            if (nodes.getLength() > 0) {
                String result = nodes.item(0).getTextContent();
                if (result == null) {
                    result = "";
                } else {
                    result.trim();
                }
                return (!"".equals(result)) ? result : query;
            }

        } catch (Exception ex) {
            this.logger.error("Error while trying to retrieve corrected query for " + query + " "
                + ex.getClass().getName() + " " + ex.getMessage(), ex);
        }
        return query;
    }

    @Override
    public List<String> getMatches(final String query, final int rows, final int start)
    {
        // response example at http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=omim&term=down
        // response type: XML
        // get corrected query from /eSearchResult/IdList/Id (multiple elements)
        String url = composeURL(TERM_SEARCH_QUERY_SCRIPT, TERM_SEARCH_PARAM_NAME, query, rows, start);
        List<String> result = new ArrayList<String>();
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("IdList");
            if (nodes.getLength() > 0) {
                nodes = nodes.item(0).getChildNodes();
                for (int i = 0; i < nodes.getLength(); ++i) {
                    Node n = nodes.item(i);
                    if (n.getNodeType() == Node.ELEMENT_NODE && n.getNodeName().equals("Id")) {
                        result.add(n.getTextContent());
                    }
                }
            }

        } catch (Exception ex) {<commit_msg>PT-2318: AbstractSpecializedNCBIEUtilsAccessService calls String.trim() without saving the result Fixed<commit_after>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see http://www.gnu.org/licenses/
 */
package org.phenotips.ncbieutils.internal;

import org.phenotips.ncbieutils.NCBIEUtilsService;

import java.io.BufferedInputStream;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import javax.inject.Inject;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

/**
 * Base implementation for {@link NCBIEUtilsService}.
 *
 * @version $Id$
 */
public abstract class AbstractSpecializedNCBIEUtilsAccessService implements NCBIEUtilsService
{
    @Inject
    private Logger logger;

    protected static final String SERVER_URL = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/";

    protected static final String TERM_SEARCH_QUERY_SCRIPT = "esearch.fcgi";

    protected static final String TERM_SEARCH_PARAM_NAME = "term";

    protected static final String TERM_SUMMARY_QUERY_SCRIPT = "esummary.fcgi";

    protected static final String TERM_SUMMARY_PARAM_NAME = "id";

    protected static final String SPELL_CHECK_QUERY_SCRIPT = "espell.fcgi";

    protected static final String SPELL_CHECK_PARAM_NAME = "term";

    protected static final String DB_PARAM_NAME = "db";

    protected abstract String getDatabaseName();

    @Override
    public List<Map<String, Object>> getSuggestions(final String query)
    {
        return getSuggestions(query, 10, 0);
    }

    @Override
    public List<Map<String, Object>> getSuggestions(final String query, final int rows, final int start)
    {
        // Step 1: get spelling suggestions for query
        String correctedQuery = getCorrectedQuery(query);
        // Step 2: get ID list matching the corrected query
        List<String> idList = getMatches(correctedQuery, rows, start);
        // step 3: get Summaries for ID list
        List<Map<String, Object>> result = getSummaries(idList);
        return result;
    }

    @Override
    public String getSuggestionsXML(final String query)
    {
        return getSuggestionsXML(query, 10, 0);
    }

    @Override
    public String getSuggestionsXML(final String query, final int rows, final int start)
    {
        // Step 1: get spelling suggestions for query
        String correctedQuery = getCorrectedQuery(query);
        // Step 2: get ID list matching the corrected query
        List<String> idList = getMatches(correctedQuery, rows, start);
        // step 3: get Summaries for ID list
        String result = getSummariesXML(idList);
        return result;
    }

    @Override
    public String getName(String id)
    {
        String url = composeURL(TERM_SUMMARY_QUERY_SCRIPT, TERM_SUMMARY_PARAM_NAME, id);
        String result = id;
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("DocSum");
            if (nodes.getLength() > 0) {
                Map<String, String> idToName = this.getNameForId(nodes.item(0));
                result = idToName.get(id);
                if (result != null && !id.equals(result)) {
                    return result;
                }
            }
            this.logger.warn("Name not found for OMIM id " + id);
            return id;
        } catch (Exception ex) {
            this.logger.error("Error while trying to retrieve name for " + getDatabaseName() + " id " + id + " "
                + ex.getClass().getName() + " " + ex.getMessage(), ex);
        }
        return id;
    }

    @Override
    public Map<String, String> getNames(List<String> idList)
    {
        Map<String, String> result = new HashMap<String, String>();
        String queryList = getSerializedList(idList);
        String url = composeURL(TERM_SUMMARY_QUERY_SCRIPT, TERM_SUMMARY_PARAM_NAME, queryList);
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("DocSum");
            for (int i = 0; i < nodes.getLength(); ++i) {
                Map<String, String> idToName = this.getNameForId(nodes.item(i));
                for (String id : idToName.keySet()) {
                    if (idList.contains(id)) {
                        result.put(id, idToName.get(id));
                    } else {
                        this.logger.warn("Unrequested " + getDatabaseName() + " id " + id);
                    }
                }
            }
        } catch (Exception ex) {
            this.logger.error("Error while trying to retrieve name for " + getDatabaseName() + " ids " + queryList
                + " " + ex.getClass().getName() + " " + ex.getMessage(), ex);
        }
        return result;
    }

    @Override
    public String getCorrectedQuery(String query)
    {
        // response example at http://eutils.ncbi.nlm.nih.gov/entrez/eutils/espell.fcgi?db=omim&term=atention+sindrom
        // response type: XML
        // get corrected query from /eSpellResult/CorrectedQuery (single element)
        // use original query if this element is empty
        String url = composeURL(SPELL_CHECK_QUERY_SCRIPT, SPELL_CHECK_PARAM_NAME, query);
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("CorrectedQuery");
            if (nodes.getLength() > 0) {
                String result = nodes.item(0).getTextContent();
                result = StringUtils.trim(result);
                return StringUtils.isNotEmpty(result) ? result : query;
            }
        } catch (Exception ex) {
            this.logger.error("Error while trying to retrieve corrected query for " + query + " "
                + ex.getClass().getName() + " " + ex.getMessage(), ex);
        }
        return query;
    }

    @Override
    public List<String> getMatches(final String query, final int rows, final int start)
    {
        // response example at http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=omim&term=down
        // response type: XML
        // get corrected query from /eSearchResult/IdList/Id (multiple elements)
        String url = composeURL(TERM_SEARCH_QUERY_SCRIPT, TERM_SEARCH_PARAM_NAME, query, rows, start);
        List<String> result = new ArrayList<String>();
        try {
            Document response = readXML(url);
            NodeList nodes = response.getElementsByTagName("IdList");
            if (nodes.getLength() > 0) {
                nodes = nodes.item(0).getChildNodes();
                for (int i = 0; i < nodes.getLength(); ++i) {
                    Node n = nodes.item(i);
                    if (n.getNodeType() == Node.ELEMENT_NODE && n.getNodeName().equals("Id")) {
                        result.add(n.getTextContent());
                    }
                }
            }

        } catch (Exception ex) {
{'fraction_non_alphanumeric': 0.09914498817536838, 'fraction_numerical': 0.0036383481899217754, 'mean_word_length': 2.497031382527566, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6039824', 'n_tokens_mistral': 4753, 'n_tokens_neox': 4518, 'n_words': 1270}