<commit_before>package com.assaydepot;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonProcessingException;
import org.codehaus.jackson.map.ObjectMapper;

import com.assaydepot.conf.Configuration;
import com.assaydepot.result.Provider;
import com.assaydepot.result.ProviderRef;
import com.assaydepot.result.Results;
import com.assaydepot.result.Ware;
import com.assaydepot.result.WareRef;

public class AssayDepotTreeImpl implements AssayDepot {

	private static final String BASE_PROVIDER_REF_QUERY_URL = "https://www.assaydepot.com/api/providers.json";
	private static final String BASE_WARE_REF_QUERY_URL = "https://www.assaydepot.com/api/wares.json";
	private static final String BASE_PROVIDER_URL = "https://www.assaydepot.com/api/providers/";
	private static final String BASE_WARE_URL = "https://www.assaydepot.com/api/wares/";
	
	private static final Logger log = Logger.getLogger( AssayDepot.class );
	private Configuration conf;
	
	AssayDepotTreeImpl( Configuration conf ) {
		this.conf = conf;
	}
	
	public Results getProviderRefsByFacets( List <String> facetNames, List<String> facetValues, String query ) {
		StringBuilder urlBuilder = new StringBuilder( BASE_PROVIDER_REF_QUERY_URL );
		if( conf.getApiToken() != null ) {
			urlBuilder.append( "?access_token=" ).append( conf.getApiToken() );
		}
		urlBuilder.append( buildFacetString( facetNames, facetValues ));
		if( query != null ) {
			urlBuilder.append( "?q=" ).append( query );
		}
		return getProviderRefsByURL( urlBuilder.toString() );
	}
	
	public Results getWareRefsByFacets( List<String> facetNames, List<String> facetValues, String query ) {
		StringBuilder urlBuilder = new StringBuilder( BASE_WARE_REF_QUERY_URL );
		if( conf.getApiToken() != null ) {
			urlBuilder.append( "?access_token=" ).append( conf.getApiToken() );
		}
		urlBuilder.append( buildFacetString( facetNames, facetValues ));
		if( query != null ) {
			urlBuilder.append( "?q=" ).append( query );
		}
		return getWareRefsByURL( urlBuilder.toString() );
	}
	

	public Provider getProvider( String id )  {
		StringBuilder urlBuilder = new StringBuilder( BASE_PROVIDER_URL );
		if( id != null ) {
			urlBuilder
				.append( id )
				.append( ".json?access_token=" )
				.append( conf.getApiToken() );
			
		}
		
		JsonNode rootNode = doParseURL( urlBuilder.toString() );
		JsonNode pNode = rootNode.path( "provider" );
		Provider provider = new Provider();
		provider.setServiceAreas( new ArrayList<String>() );
		provider.setLaboratories( new ArrayList<String>() );
		provider.setKeywords( new ArrayList<String>() );
		provider.setCertifications( new ArrayList<String>() );
		provider.setProfessionalAssociations( new ArrayList<String>() );
		
		provider.setId( pNode.path( "id" ).getTextValue() );
		provider.setSlug( pNode.path( "slug" ).getTextValue() );
		provider.setName( pNode.path( "name" ).getTextValue() );
		provider.setPhoneNumber( pNode.path( "phone_number" ).getTextValue() );
		provider.setWebsite( pNode.path( "website" ).getTextValue() );
		provider.setServiceAreas( doArray( pNode.path( "service_areas" )));
		provider.setHeadquarters( pNode.path( "headquarters" ).getTextValue() );
		provider.setLaboratories( doArray( pNode.path( "laboratories" )));
		provider.setYearEstablished( pNode.path( "year_established" ).getIntValue() );
		provider.setNumEmployees( pNode.path( "number_of_employees" ).asText() );
		provider.setDescription( pNode.path( "description" ).getTextValue() );
		provider.setKeywords( doArray( pNode.path( "keywords" )));
		provider.setCertifications( doArray( pNode.path( "certifications" )));
		provider.setProfessionalAssociations( doArray( pNode.path( "professional_associations" )));
		provider.setPermission( pNode.path( "permission" ).getTextValue() );
		provider.setOrigin( pNode.path( "origin" ).getTextValue() );
		provider.setGreen( pNode.path( "green" ).asBoolean() );
		provider.setGreenExplanation( pNode.path( "green_explaination" ).getTextValue() );
		provider.setDiversity( pNode.path( "diversity" ).asBoolean() );
		provider.setDiversityExplanation( pNode.path( "diversity_explaination" ).getTextValue() );
		provider.setCreatedAt( pNode.path( "created_at" ).asText() );
		provider.setUpdatedAt( pNode.path( "updated_at" ).asText() );
		provider.setUrls( doStringMap( pNode.path( "urls") ));
		
		return provider;
	}
	
	private List<String> doArray( JsonNode arrayNode ) {
		List<String> list = new ArrayList<String>();
		for( int i=0; i < arrayNode.size(); i++ ) {
			list.add( arrayNode.get( i ).getTextValue() );
		}
		return list;
	}
	
	
	public Results getProviderRefs(String query) {
		StringBuilder urlBuilder = new StringBuilder( BASE_PROVIDER_REF_QUERY_URL );
		if( query != null ) {
			urlBuilder.append( "?q=" ).append( query );
		}
		if( conf.getApiToken() != null ) {
			urlBuilder.append( "&access_token=" ).append( conf.getApiToken() );
		}

		return getProviderRefsByURL( urlBuilder.toString() );
	}
	
	private Results getProviderRefsByURL( String url ) {
		JsonNode rootNode = doParseURL( url );		

		Results results = new Results();

		results.setTotal( rootNode.path( "total" ).getIntValue() );
		results.setPage( rootNode.path( "page" ).getIntValue() );
		results.setPerPage( rootNode.path( "per_page" ).getIntValue() );
		results.setQueryTime( rootNode.path( "query_time" ).getDoubleValue() );
		results.setFacets( doFacets( rootNode.path( "facets" )));
		results.setProviderRefs( doProviderRefs( rootNode.path( "provider_refs" )));
		
		return results;
		
	}
	private JsonNode doParseURL( String urlString ) {
		ObjectMapper mapper = new ObjectMapper();
		JsonNode rootNode = null;
		URL url = null;
		try {
			url = new URL( urlString );
			JsonFactory f = new JsonFactory();
			JsonParser jp = f.createJsonParser( url.openStream() );
			rootNode = mapper.readTree( jp );
		} catch (JsonProcessingException jpex ) {
			log.error("Problem accessing json are you getting json back on this url ["+urlString+"] ?", jpex);
		} catch (MalformedURLException malex) {
			log.error( "Problem accessing url ["+urlString+"]", malex );
		} catch (IOException ioex ) {
			if( ioex.getMessage().contains( "500" ) && ioex.getMessage().contains("HTTP response")) {
				log.error( "It's possible the facet you are requesting does not exist for this type or the server is busy.", ioex );
			} else {
				log.error("Most likely problem with url or problem accessing internet", ioex );
			}
		}
		return rootNode;
	}
	
	private List<ProviderRef> doProviderRefs( JsonNode pRefNode ) {
		ProviderRef newRef = null;
		int numRefs = pRefNode.size();
		JsonNode node = null;
		List<ProviderRef> providerRefs = new ArrayList<ProviderRef>();
		for( int i=0; i < numRefs; i++ ) {
			node = pRefNode.get( i );
			newRef = new ProviderRef();
			newRef.setId( node.path("id").getTextValue() );
			newRef.setSlug( node.path("slug").getTextValue() );
			newRef.setName( node.path("name").getTextValue() );
			newRef.setSnippet( node.path("snippet").getTextValue() );
			newRef.setPermission( node.path("permission").getTextValue() );
			newRef.setScore( node.path("score").getDoubleValue() );<commit_msg>updated message the url is not always added by default<commit_after>package com.assaydepot;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonProcessingException;
import org.codehaus.jackson.map.ObjectMapper;

import com.assaydepot.conf.Configuration;
import com.assaydepot.result.Provider;
import com.assaydepot.result.ProviderRef;
import com.assaydepot.result.Results;
import com.assaydepot.result.Ware;
import com.assaydepot.result.WareRef;

public class AssayDepotTreeImpl implements AssayDepot {

	private static final String BASE_PROVIDER_REF_QUERY_URL = "https://www.assaydepot.com/api/providers.json";
	private static final String BASE_WARE_REF_QUERY_URL = "https://www.assaydepot.com/api/wares.json";
	private static final String BASE_PROVIDER_URL = "https://www.assaydepot.com/api/providers/";
	private static final String BASE_WARE_URL = "https://www.assaydepot.com/api/wares/";
	
	private static final Logger log = Logger.getLogger( AssayDepot.class );
	private Configuration conf;
	
	AssayDepotTreeImpl( Configuration conf ) {
		this.conf = conf;
	}
	
	public Results getProviderRefsByFacets( List <String> facetNames, List<String> facetValues, String query ) {
		StringBuilder urlBuilder = new StringBuilder( BASE_PROVIDER_REF_QUERY_URL );
		if( conf.getApiToken() != null ) {
			urlBuilder.append( "?access_token=" ).append( conf.getApiToken() );
		}
		urlBuilder.append( buildFacetString( facetNames, facetValues ));
		if( query != null ) {
			urlBuilder.append( "?q=" ).append( query );
		}
		return getProviderRefsByURL( urlBuilder.toString() );
	}
	
	public Results getWareRefsByFacets( List<String> facetNames, List<String> facetValues, String query ) {
		StringBuilder urlBuilder = new StringBuilder( BASE_WARE_REF_QUERY_URL );
		if( conf.getApiToken() != null ) {
			urlBuilder.append( "?access_token=" ).append( conf.getApiToken() );
		}
		urlBuilder.append( buildFacetString( facetNames, facetValues ));
		if( query != null ) {
			urlBuilder.append( "?q=" ).append( query );
		}
		return getWareRefsByURL( urlBuilder.toString() );
	}
	

	public Provider getProvider( String id )  {
		StringBuilder urlBuilder = new StringBuilder( BASE_PROVIDER_URL );
		if( id != null ) {
			urlBuilder
				.append( id )
				.append( ".json?access_token=" )
				.append( conf.getApiToken() );
			
		}
		
		JsonNode rootNode = doParseURL( urlBuilder.toString() );
		JsonNode pNode = rootNode.path( "provider" );
		Provider provider = new Provider();
		provider.setServiceAreas( new ArrayList<String>() );
		provider.setLaboratories( new ArrayList<String>() );
		provider.setKeywords( new ArrayList<String>() );
		provider.setCertifications( new ArrayList<String>() );
		provider.setProfessionalAssociations( new ArrayList<String>() );
		
		provider.setId( pNode.path( "id" ).getTextValue() );
		provider.setSlug( pNode.path( "slug" ).getTextValue() );
		provider.setName( pNode.path( "name" ).getTextValue() );
		provider.setPhoneNumber( pNode.path( "phone_number" ).getTextValue() );
		provider.setWebsite( pNode.path( "website" ).getTextValue() );
		provider.setServiceAreas( doArray( pNode.path( "service_areas" )));
		provider.setHeadquarters( pNode.path( "headquarters" ).getTextValue() );
		provider.setLaboratories( doArray( pNode.path( "laboratories" )));
		provider.setYearEstablished( pNode.path( "year_established" ).getIntValue() );
		provider.setNumEmployees( pNode.path( "number_of_employees" ).asText() );
		provider.setDescription( pNode.path( "description" ).getTextValue() );
		provider.setKeywords( doArray( pNode.path( "keywords" )));
		provider.setCertifications( doArray( pNode.path( "certifications" )));
		provider.setProfessionalAssociations( doArray( pNode.path( "professional_associations" )));
		provider.setPermission( pNode.path( "permission" ).getTextValue() );
		provider.setOrigin( pNode.path( "origin" ).getTextValue() );
		provider.setGreen( pNode.path( "green" ).asBoolean() );
		provider.setGreenExplanation( pNode.path( "green_explaination" ).getTextValue() );
		provider.setDiversity( pNode.path( "diversity" ).asBoolean() );
		provider.setDiversityExplanation( pNode.path( "diversity_explaination" ).getTextValue() );
		provider.setCreatedAt( pNode.path( "created_at" ).asText() );
		provider.setUpdatedAt( pNode.path( "updated_at" ).asText() );
		provider.setUrls( doStringMap( pNode.path( "urls") ));
		
		return provider;
	}
	
	private List<String> doArray( JsonNode arrayNode ) {
		List<String> list = new ArrayList<String>();
		for( int i=0; i < arrayNode.size(); i++ ) {
			list.add( arrayNode.get( i ).getTextValue() );
		}
		return list;
	}
	
	
	public Results getProviderRefs(String query) {
		StringBuilder urlBuilder = new StringBuilder( BASE_PROVIDER_REF_QUERY_URL );
		if( query != null ) {
			urlBuilder.append( "?q=" ).append( query );
		}
		if( conf.getApiToken() != null ) {
			urlBuilder.append( "&access_token=" ).append( conf.getApiToken() );
		}

		return getProviderRefsByURL( urlBuilder.toString() );
	}
	
	private Results getProviderRefsByURL( String url ) {
		JsonNode rootNode = doParseURL( url );		

		Results results = new Results();

		results.setTotal( rootNode.path( "total" ).getIntValue() );
		results.setPage( rootNode.path( "page" ).getIntValue() );
		results.setPerPage( rootNode.path( "per_page" ).getIntValue() );
		results.setQueryTime( rootNode.path( "query_time" ).getDoubleValue() );
		results.setFacets( doFacets( rootNode.path( "facets" )));
		results.setProviderRefs( doProviderRefs( rootNode.path( "provider_refs" )));
		
		return results;
		
	}
	private JsonNode doParseURL( String urlString ) {
		ObjectMapper mapper = new ObjectMapper();
		JsonNode rootNode = null;
		URL url = null;
		try {
			url = new URL( urlString );
			JsonFactory f = new JsonFactory();
			JsonParser jp = f.createJsonParser( url.openStream() );
			rootNode = mapper.readTree( jp );
		} catch (JsonProcessingException jpex ) {
			log.error("Problem accessing json are you getting json back on this url ["+urlString+"] ?", jpex);
		} catch (MalformedURLException malex) {
			log.error( "Problem accessing url ["+urlString+"]", malex );
		} catch (IOException ioex ) {
			if( ioex.getMessage().contains( "500" ) && ioex.getMessage().contains("HTTP response")) {
				log.error( "It's possible the facet you are requesting does not exist for this type or the server is busy.", ioex );
			} else {
				log.error("Most likely problem with url or problem accessing internet, url ["+urlString+"]", ioex );
			}
		}
		return rootNode;
	}
	
	private List<ProviderRef> doProviderRefs( JsonNode pRefNode ) {
		ProviderRef newRef = null;
		int numRefs = pRefNode.size();
		JsonNode node = null;
		List<ProviderRef> providerRefs = new ArrayList<ProviderRef>();
		for( int i=0; i < numRefs; i++ ) {
			node = pRefNode.get( i );
			newRef = new ProviderRef();
			newRef.setId( node.path("id").getTextValue() );
			newRef.setSlug( node.path("slug").getTextValue() );
			newRef.setName( node.path("name").getTextValue() );
			newRef.setSnippet( node.path("snippet").getTextValue() );
			newRef.setPermission( node.path("permission").getTextValue() );
			newRef.setScore( node.path("score").getDoubleValue() );
{'fraction_non_alphanumeric': 0.13858839050131927, 'fraction_numerical': 0.0007915567282321899, 'mean_word_length': 4.89005439005439, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 33, 'https://': 8, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6434850', 'n_tokens_mistral': 5304, 'n_tokens_neox': 4809, 'n_words': 1069}