<commit_before>/*******************************************************************************
 * Copyright (c) 2009, 2017, The University of Manchester
 *
 * Licensed under the New BSD License.
 * Please see LICENSE file that is distributed with the source code
 *  
 *******************************************************************************/
package uk.ac.manchester.cs.owl.semspreadsheets.repository.bioportal;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;

import org.apache.log4j.Logger;

import uk.ac.manchester.cs.owl.semspreadsheets.repository.Repository;
import uk.ac.manchester.cs.owl.semspreadsheets.repository.RepositoryAccessor;
import uk.ac.manchester.cs.owl.semspreadsheets.repository.RepositoryItem;
import uk.ac.manchester.cs.owl.semspreadsheets.ui.ErrorHandler;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

/**
 * @author <NAME>
 * @author <NAME>
 */
public class BioPortalRepositoryAccessor implements RepositoryAccessor {

	private static Logger logger = Logger.getLogger(BioPortalRepositoryAccessor.class);
	
	private final int CONNECT_TIMEOUT = 120000;
	
	private final boolean SAVE_PROPERTIES_CACHE = false;
	
    private BioPortalRepository repository;
    
    

    public String getRepositoryName() {
        return BioPortalRepository.NAME;
    }

    public synchronized Repository getRepository() {
        if(repository == null) {
            repository = new BioPortalRepository();
        }
        return repository;
    }    
       
    public String fetchOntologyFormat(String ontologyAcronym) {
    	logger.debug("Fetching format for "+ontologyAcronym);
    	BioPortalCache cache = BioPortalCache.getInstance();
    	String format = cache.getFormat(ontologyAcronym);
    	if (format!=null) {
    		if (logger.isDebugEnabled()) {
    			logger.debug("The ontology format for "+ontologyAcronym+" was found in the cache as "+format);
    		}
    	}
    	else{
    		logger.info("The format for "+ontologyAcronym+" not found in cache, fetching");
    		try {
        		URL url = new URL(BioPortalRepository.ONTOLOGY_LIST+"/"+ontologyAcronym+"/"+BioPortalRepository.LATEST_SUBMISSION +"?format=json&apikey=" + BioPortalRepository.readAPIKey());
        		logger.info("About to fetch more ontology information from "+url.toExternalForm());
        		HttpURLConnection connection = (HttpURLConnection)url.openConnection();
        		connection.setConnectTimeout(CONNECT_TIMEOUT);
                connection.setReadTimeout(CONNECT_TIMEOUT);
                int responseCode = connection.getResponseCode();
                logger.info("BioPortal http response: " + responseCode);
                
                if (responseCode == 400 || responseCode == 403) {
                	throw new BioPortalAccessDeniedException();
                }
                
                JsonParser parser = new JsonFactory().createParser(connection.getInputStream());
                while (format==null && parser.nextToken()!=null) {
                	String name = parser.getCurrentName();            	
                	if ("hasOntologyLanguage".equals(name)) {
                		parser.nextToken();
                		format = parser.getText();                		                	
                	}
                } 
                if (format==null) {
                	format="unknown";
                }
                cache.storeFormat(ontologyAcronym, format);
        	}
        	catch (UnknownHostException e) {
                ErrorHandler.getErrorHandler().handleError(e);
            }
            catch (MalformedURLException e) {
                logger.error("Error with URL for BioPortal rest API",e);
            }
            catch (SocketTimeoutException e) {
            	logger.error("Timeout connecting to BioPortal",e);
            	ErrorHandler.getErrorHandler().handleError(e);
            }
            catch (IOException e) {        	
                logger.error("Error communiciating with BioPortal rest API",e);                    	
            }
            catch (BioPortalAccessDeniedException e) {
            	ErrorHandler.getErrorHandler().handleError(e);
            }
    	}
    	 
    	return format;
    }     
    
    public Collection<RepositoryItem> getOntologies() {
        final Collection<RepositoryItem> items = new ArrayList<RepositoryItem>();
        URL url = null;
        try {
        	        	                    
            url = new URL(BioPortalRepository.ONTOLOGY_LIST + "?format=json&apikey=" + BioPortalRepository.readAPIKey());        	
            
            logger.debug("Contacting BioPortal REST API at: "+url.toExternalForm());
                                   
            HttpURLConnection connection = (HttpURLConnection)url.openConnection();            
            connection.setConnectTimeout(CONNECT_TIMEOUT);
            connection.setReadTimeout(CONNECT_TIMEOUT);
            int responseCode = connection.getResponseCode();
            logger.info("BioPortal http response: " + responseCode);
            
            if (responseCode == 400 || responseCode == 403) {
            	throw new BioPortalAccessDeniedException();
            }            
            ObjectMapper mapper = new ObjectMapper();            
            
            JsonNode node = mapper.readTree(connection.getInputStream());
            for (final JsonNode item : node) {
            	String name = item.get("name").asText();
            	String id = item.get("acronym").asText();
            	            	
        		BioPortalRepositoryItem repositoryItem = new BioPortalRepositoryItem(id,name,this);        		
    			if (repositoryItem.isCompatible()) {
    				items.add(repositoryItem);
    			}        	<commit_msg>added 401 detection<commit_after>/*******************************************************************************
 * Copyright (c) 2009, 2017, The University of Manchester
 *
 * Licensed under the New BSD License.
 * Please see LICENSE file that is distributed with the source code
 *  
 *******************************************************************************/
package uk.ac.manchester.cs.owl.semspreadsheets.repository.bioportal;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;

import org.apache.log4j.Logger;

import uk.ac.manchester.cs.owl.semspreadsheets.repository.Repository;
import uk.ac.manchester.cs.owl.semspreadsheets.repository.RepositoryAccessor;
import uk.ac.manchester.cs.owl.semspreadsheets.repository.RepositoryItem;
import uk.ac.manchester.cs.owl.semspreadsheets.ui.ErrorHandler;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;

/**
 * @author <NAME>
 * @author <NAME>
 */
public class BioPortalRepositoryAccessor implements RepositoryAccessor {

	private static Logger logger = Logger.getLogger(BioPortalRepositoryAccessor.class);
	
	private final int CONNECT_TIMEOUT = 120000;
	
	private final boolean SAVE_PROPERTIES_CACHE = false;
	
    private BioPortalRepository repository;
    
    

    public String getRepositoryName() {
        return BioPortalRepository.NAME;
    }

    public synchronized Repository getRepository() {
        if(repository == null) {
            repository = new BioPortalRepository();
        }
        return repository;
    }    
       
    public String fetchOntologyFormat(String ontologyAcronym) {
    	logger.debug("Fetching format for "+ontologyAcronym);
    	BioPortalCache cache = BioPortalCache.getInstance();
    	String format = cache.getFormat(ontologyAcronym);
    	if (format!=null) {
    		if (logger.isDebugEnabled()) {
    			logger.debug("The ontology format for "+ontologyAcronym+" was found in the cache as "+format);
    		}
    	}
    	else{
    		logger.info("The format for "+ontologyAcronym+" not found in cache, fetching");
    		try {
        		URL url = new URL(BioPortalRepository.ONTOLOGY_LIST+"/"+ontologyAcronym+"/"+BioPortalRepository.LATEST_SUBMISSION +"?format=json&apikey=" + BioPortalRepository.readAPIKey());
        		logger.info("About to fetch more ontology information from "+url.toExternalForm());
        		HttpURLConnection connection = (HttpURLConnection)url.openConnection();
        		connection.setConnectTimeout(CONNECT_TIMEOUT);
                connection.setReadTimeout(CONNECT_TIMEOUT);
                int responseCode = connection.getResponseCode();
                logger.info("BioPortal http response: " + responseCode);
                
                if (responseCode == 400 || responseCode == 401 || responseCode == 403) {
                	throw new BioPortalAccessDeniedException();
                }
                
                JsonParser parser = new JsonFactory().createParser(connection.getInputStream());
                while (format==null && parser.nextToken()!=null) {
                	String name = parser.getCurrentName();            	
                	if ("hasOntologyLanguage".equals(name)) {
                		parser.nextToken();
                		format = parser.getText();                		                	
                	}
                } 
                if (format==null) {
                	format="unknown";
                }
                cache.storeFormat(ontologyAcronym, format);
        	}
        	catch (UnknownHostException e) {
                ErrorHandler.getErrorHandler().handleError(e);
            }
            catch (MalformedURLException e) {
                logger.error("Error with URL for BioPortal rest API",e);
            }
            catch (SocketTimeoutException e) {
            	logger.error("Timeout connecting to BioPortal",e);
            	ErrorHandler.getErrorHandler().handleError(e);
            }
            catch (IOException e) {        	
                logger.error("Error communiciating with BioPortal rest API",e);                    	
            }
            catch (BioPortalAccessDeniedException e) {
            	ErrorHandler.getErrorHandler().handleError(e);
            }
    	}
    	 
    	return format;
    }     
    
    public Collection<RepositoryItem> getOntologies() {
        final Collection<RepositoryItem> items = new ArrayList<RepositoryItem>();
        URL url = null;
        try {
        	        	                    
            url = new URL(BioPortalRepository.ONTOLOGY_LIST + "?format=json&apikey=" + BioPortalRepository.readAPIKey());        	
            
            logger.debug("Contacting BioPortal REST API at: "+url.toExternalForm());
                                   
            HttpURLConnection connection = (HttpURLConnection)url.openConnection();            
            connection.setConnectTimeout(CONNECT_TIMEOUT);
            connection.setReadTimeout(CONNECT_TIMEOUT);
            int responseCode = connection.getResponseCode();
            logger.info("BioPortal http response: " + responseCode);
            
           if (responseCode == 400 || responseCode == 401 || responseCode == 403) {
        	   logger.error("We are here");
            	throw new BioPortalAccessDeniedException();
            }            
            ObjectMapper mapper = new ObjectMapper();            
            
            JsonNode node = mapper.readTree(connection.getInputStream());
            for (final JsonNode item : node) {
            	String name = item.get("name").asText();
            	String id = item.get("acronym").asText();
            	            	
        		BioPortalRepositoryItem repositoryItem = new BioPortalRepositoryItem(id,name,this);        		
    			if (repositoryItem.isCompatible()) {
    				items.add(repositoryItem);
    			}        	
{'fraction_non_alphanumeric': 0.11659852820932134, 'fraction_numerical': 0.005151267375306623, 'mean_word_length': 2.2932148626817446, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4644036', 'n_tokens_mistral': 3266, 'n_tokens_neox': 3070, 'n_words': 619}