<commit_before>/**
 * Copyright © 2011-2014 EMBL - European Bioinformatics Institute
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License.  
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package uk.ac.ebi.generic.util;

import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import net.sf.json.JSONSerializer;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.springframework.stereotype.Service;

import uk.ac.ebi.phenotype.web.util.DrupalHttpProxy;
import uk.ac.ebi.phenotype.web.util.HttpProxy;

@Service
public class SolrIndex {

	private Logger log = Logger.getLogger(this.getClass().getCanonicalName());

	@Resource(name = "globalConfiguration")
	private Map<String, String> config;

	/**
	 * Return the number of documents found for a specified solr query on a
	 * specified core.
	 * 
	 * @param query
	 *            the query
	 * @param core
	 *            which solr core to query
	 * @param mode
	 *            which configuration mode to operate in
	 * @param solrParams
	 *            the default solr parameters to also restrict the query
	 * @return integer count of the number of matching documents
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public Integer getNumFound(String query, String core, String mode,
			String solrParams) throws IOException, URISyntaxException {
		JSONObject json = getResults(composeSolrUrl(core, mode, query,
				solrParams, 0, 0, false));
		return json.getJSONObject("response").getInt("numFound");
	}

	/**
	 * Gets the json string representation for the query.
	 * 
	 * @param query
	 *            the query for which documents
	 * @param core
	 *            the solr core to query
	 * @param gridSolrParams
	 *            the default solr parameters to append to the query
	 * @param mode
	 *            the configuration mode to operate in
	 * @param start
	 *            where to start the offset
	 * @param length
	 *            how many documents to return
	 * @param showImgView
	 *            is this query showing the annotation view of the images
	 *            (true/false)
	 * @return
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public JSONObject getDataTableJson(String query, String core,
			String gridSolrParams, String mode, int start, int length,
			boolean showImgView) throws IOException, URISyntaxException {

		if (gridSolrParams.equals("")) {
			gridSolrParams = "qf=auto_suggest&defType=edismax&wt=json&q=*:*";
		}

		return getResults(composeSolrUrl(core, mode, query, gridSolrParams,
				start, length, showImgView));

	}

	/**
	 * Get rows for saving to an external file.
	 * 
	 * @param core
	 *            the solr core to query
	 * @param gridSolrParams
	 *            the default parameters to use in the query
	 * @param start
	 *            where to start the query
	 * @param gridFields
	 *            the default solr parameters to append to the query
	 * @param length
	 *            how many documents to return
	 * @return json representation of the results of the solr query
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public JSONObject getDataTableExportRows(String core,
			String gridSolrParams, String gridFields, int start, int length)
			throws IOException, URISyntaxException {

		System.out.println("GRID SOLR PARAMS : " + gridSolrParams);
		
		if (core.equals("gene")) {			
			//gridFields += ",imits_report_phenotyping_complete_date,imits_report_genotype_confirmed_date,imits_report_mi_plan_status,escell,ikmc_project,imits_phenotype_started,imits_phenotype_complete,imits_phenotype_status";
		}

		//String newgridSolrParams = gridSolrParams + "&rows=" + length
		gridSolrParams = gridSolrParams.replace("rows=10", "rows="+length);
		String newgridSolrParams = gridSolrParams 		
				+ "&start=" + start + "&fl=" + gridFields;

		String url = composeSolrUrl(core, "", "", newgridSolrParams, start,
				length, false);
		
		log.debug("Export data URL: " + url);
		System.out.println("Export data URL: " + url);
		return getResults(url);
	}

	/**
	 * Prepare a url for querying the solr indexes based on the passed in
	 * arguments.
	 * 
	 * @param core
	 *            which solr core to query
	 * @param mode
	 *            which configuration mode to operate in
	 * @param query
	 *            what to query
	 * @param gridSolrParams
	 *            default parameters to add to the solr query
	 * @param iDisplayStart
	 *            starting point of the query
	 * @param iDisplayLength
	 *            length of the query
	 * @param showImgView
	 *            which image mode to operate in
	 * @return the constructed url including all parameters
	 */
	private String composeSolrUrl(String core, String mode, String query,
			String gridSolrParams, Integer iDisplayStart,
			Integer iDisplayLength, boolean showImgView) {
		String internalSolrUrl = config.get("internalSolrUrl");
		
		String url = internalSolrUrl + "/" + core + "/select?";
						
		System.out.println(("BASEURL: " + url));
		if (mode.equals("mpPage")) {
			url += "q=" + query;
			url += "&start=0&rows=0&wt=json&qf=auto_suggest&defType=edismax";
		} else if (mode.equals("geneGrid")) {			
			url += gridSolrParams + "&start=" + iDisplayStart + "&rows="
					+ iDisplayLength;
			System.out.println("GENE PARAMS: " + url);
		} else if (mode.equals("pipelineGrid")) {
			url += gridSolrParams + "&start=" + iDisplayStart + "&rows="
					+ iDisplayLength;
//			System.out.println("PROTOCOL PARAMS: " + url);
		} else if (mode.equals("imagesGrid")) {
			url += gridSolrParams + "&start=" + iDisplayStart + "&rows="
					+ iDisplayLength;
			if (!showImgView) {
				url += "&facet=on&facet.field=symbol_gene&facet.field=expName_exp&facet.field=maTermName&facet.field=mpTermName&facet.mincount=1&facet.limit=-1";
			}
			System.out.println("IMG PARAMS: " + url);
		} else if (mode.equals("mpGrid")) {
			url += gridSolrParams.replaceAll(" ", "%20") + "&start="
					+ iDisplayStart + "&rows=" + iDisplayLength;
			System.out.println("MP PARAMS: " + url);
		} else if (mode.equals("maGrid")) {
			url += gridSolrParams.replaceAll(" ", "%20") + "&start="
					+ iDisplayStart + "&rows=" + iDisplayLength;
//			System.out.println("MA PARAMS: " + url);
		} else if ( mode.equals("diseaseGrid") ){			
			url += gridSolrParams.replaceAll(" ", "%20") + "&start="
					+ iDisplayStart + "&rows=" + iDisplayLength;
//			System.out.println("DISEASE PARAMS: " + url);
		} else if (mode.equals("ikmcAlleleGrid")) {
			url += "q=" + query;
			url += "&start=0&rows=0&wt=json";
//			System.out.println("IKMC ALLELE PARAMS: " + url);
		} else if (mode.equals("all") || mode.equals("page") || mode.equals("")) { // download search page result
			url += gridSolrParams;
			if (core.equals("images") && !showImgView) {				
				url += "&facet=on&facet.field=symbol_gene&facet.field=expName_exp&facet.field=maTermName&facet.field=mpTermName&facet.mincount=1&facet.limit=-1";
			}
//			System.out.println("GRID DUMP PARAMS - " + core + ": " + url);
		}
		// OTHER solrCoreNames to be added here
		
		return url;
	}
	/**
	 * Get the simplified production status of ES cells/mice for a document.
	 * 
	 * @param doc
	 *            represents a gene with imits status fields
	 * @return the latest status at the gene level for both ES cells and alleles
	 */
	public String deriveLatestProductionStatusForEsCellAndMice(JSONObject doc, HttpServletRequest request, boolean toExport, String geneLink){		
		
		String esCellStatus = fetchEsCellStatus(doc, request, toExport);		
		String miceStatus = "";		
		final List<String> exportMiceStatus = new ArrayList<String>();
				
		Map<String, String> sh = new HashMap<String, String>();
				
		try {		
						
			// mice production status			
			// Mice: blue tm1/tm1a/tm1e... mice (depending on how many allele docs) 
			if ( doc.containsKey("mouse_status") ){
				
				JSONArray alleleNames = doc.getJSONArray("allele_name");				
				JSONArray mouseStatus = doc.getJSONArray("mouse_status");
				
				for ( int i=0; i< mouseStatus.size(); i++ ) {		
					String mouseStatusStr = mouseStatus.get(i).toString();					
					sh.put(mouseStatusStr, "yes");
				}		
								
				// if no mice status found but there is already allele produced, mark it as "mice produced planned"				
				for ( int j=0; j< alleleNames.size(); j++ ) {
					String alleleName = alleleNames.get(j).toString();
					//System.out.println("allele: " + alleleName);
					if ( !alleleName.equals("") && !alleleName.equals("None") && !alleleName.contains("tm1e") && mouseStatus.get(j).toString().equals("") ){	
						sh.put("mice production planned", "yes");						
					}				
				}
				
				if ( sh.containsKey("Mice Produced") ){
					miceStatus = "<a class='status done' oldtitle='Mice Produced' title='' href='" + geneLink + "#order'>"
							   +  "<span>Mice</span>"
							   +  "</a>";
						
					exportMiceStatus.add("mice produced");
				}
				else if ( sh.containsKey("Assigned for Mouse Production and Phenotyping") ){
					miceStatus = "<a class='status inprogress' oldtitle='Mice production in progress' title=''>"
							   +  "<span>Mice</span>"
							   +  "</a>";
					exportMiceStatus.add("mice production in progress");
				}
				else if ( sh.containsKey("mice production planned") ){
					miceStatus = "<a class='status none' oldtitle='Mice production planned' title=''>"
							   +  "<span>Mice</span>"
							   +  "</a>";
					exportMiceStatus.add("mice production in progress");
				}				
			}
		} 
		catch (Exception e) {
			log.error("Error getting ES cell/Mice status");
			log.error(e.getLocalizedMessage());
		}
		
		if ( toExport ){
			exportMiceStatus.add(0, esCellStatus); // want to keep this at front
			return StringUtils.join(exportMiceStatus, ", ");
		}
		return esCellStatus + miceStatus;
		
	}
	/**
	 * Get the production status of ES cells/mice for a document.
	 * 
	 * @param doc
	 *            represents a gene with imits status fields
	 * @return the latest status at the gene level for ES cells and all statuses at the allele level for mice as a comma separated string
	 */
	public String deriveProductionStatusForEsCellAndMice(JSONObject doc, HttpServletRequest request, boolean toExport){		
		
		String esCellStatus = fetchEsCellStatus(doc, request, toExport);		
		String miceStatus = "";		
		final List<String> exportMiceStatus = new ArrayList<String>();
		
		String patternStr = "(tm.*)\\(.+\\).+"; // allele name pattern
		Pattern pattern = Pattern.compile(patternStr);
		
		try {		
						
			// mice production status
			
			// Mice: blue tm1/tm1a/tm1e... mice (depending on how many allele docs) 
			if ( doc.containsKey("mouse_status") ){
				
				JSONArray alleleNames = doc.getJSONArray("allele_name");
				JSONArray mouseStatus = doc.getJSONArray("mouse_status");
				
				for ( int i=0; i< mouseStatus.size(); i++ ) {		
					String mouseStatusStr = mouseStatus.get(i).toString();	
					
					if ( mouseStatusStr.equals("Mice Produced") ){
						String alleleName = alleleNames.getString(i).toString();						
						Matcher matcher = pattern.matcher(alleleName);
						//System.out.println(matcher.toString());
							
						if (matcher.find()) {
							String alleleType = matcher.group(1);						
							miceStatus += "<span class='status done' oldtitle='" + mouseStatusStr + "' title=''>"
									+  "<span>Mice<br>" + alleleType + "</span>"
									+  "</span>";
							
							exportMiceStatus.add(alleleType + " mice produced");
						}
					}
					else if (mouseStatusStr.equals("Assigned for Mouse Production and Phenotyping") ){
						String alleleName = alleleNames.getString(i).toString();						
						Matcher matcher = pattern.matcher(alleleName);
						//System.out.println(matcher.toString());
							
						if (matcher.find()) {
							String alleleType = matcher.group(1);						
							miceStatus += "<span class='status inprogress' oldtitle='Mice production in progress' title=''>"
									+  "<span>Mice<br>" + alleleType + "</span>"
									+  "</span>";
							exportMiceStatus.add(alleleType + " mice production in progress");
						}						
					}					
				}	
				// if no mice status found but there is already allele produced, mark it as "mice produced planned"				
				for ( int j=0; j< alleleNames.size(); j++ ) {
					String alleleName = alleleNames.get(j).toString();
					if ( !alleleName.equals("") && !alleleName.equals("None") && mouseStatus.get(j).toString().equals("") ){	
						Matcher matcher = pattern.matcher(alleleName);
						//System.out.println(matcher.toString());
							
						if (matcher.find()) {
							String alleleType = matcher.group(1);						
							miceStatus += "<span class='status none' oldtitle='Mice production planned' title=''>"
									+  "<span>Mice<br>" + alleleType + "</span>"
									+  "</span>";
							
							exportMiceStatus.add(alleleType + " mice production planned");
						}	
					}						
				}
			}
		} 
		catch (Exception e) {
			log.error("Error getting ES cell/Mice status");
			log.error(e.getLocalizedMessage());
		}
		
		if ( toExport ){
			exportMiceStatus.add(0, esCellStatus); // want to keep this at front
			return StringUtils.join(exportMiceStatus, ", ");
		}
		return esCellStatus + miceStatus;
		
	}
	/**
	 * Get the latest production status of ES cells for a document.
	 * 
	 * @param doc
	 *            represents a gene with imits status fields
	 * @return the latest status at the gene level for ES cells as a string
	 */
	public String fetchEsCellStatus(JSONObject doc, HttpServletRequest request, boolean toExport){
		
		String mgiId = doc.getString("mgi_accession_id");
		String geneUrl = request.getAttribute("baseUrl") + "/genes/" + mgiId;
				
		String esCellStatus = "";	
		String exportEsCellStatus = "";	
		try {	
			final String field = "latest_es_cell_status"; 
			// ES cell production status
			if ( doc.containsKey(field)  ){
				// blue es cell status				
				esCellStatus = doc.getString(field);
				if ( esCellStatus.equals("ES Cell Targeting Confirmed") ){
						esCellStatus = "<a class='status done' href='" + geneUrl + "' oldtitle='ES Cells produced' title=''>"
									 + " <span>ES cells</span>"
									 + "</a>";
						
						exportEsCellStatus += "Es cells produced";
				}
				else if ( esCellStatus.equals("ES Cell Production in Progress") ){
						esCellStatus = "<span class='status inprogress' oldtitle='ES cells production in progress' title=''>"
						   	 		 +  "	<span>ES Cells</span>"
						   	 		 +  "</span>";
						
						exportEsCellStatus += "Es cells production in progress";
				}
				else {<commit_msg>ES cells button was not showing anymore due to the status being changed at the Sanger. Now updated.<commit_after>/**
 * Copyright © 2011-2014 EMBL - European Bioinformatics Institute
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License.  
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package uk.ac.ebi.generic.util;

import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import net.sf.json.JSONSerializer;

import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.springframework.stereotype.Service;

import uk.ac.ebi.phenotype.web.util.DrupalHttpProxy;
import uk.ac.ebi.phenotype.web.util.HttpProxy;

@Service
public class SolrIndex {

	private Logger log = Logger.getLogger(this.getClass().getCanonicalName());

	@Resource(name = "globalConfiguration")
	private Map<String, String> config;

	/**
	 * Return the number of documents found for a specified solr query on a
	 * specified core.
	 * 
	 * @param query
	 *            the query
	 * @param core
	 *            which solr core to query
	 * @param mode
	 *            which configuration mode to operate in
	 * @param solrParams
	 *            the default solr parameters to also restrict the query
	 * @return integer count of the number of matching documents
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public Integer getNumFound(String query, String core, String mode,
			String solrParams) throws IOException, URISyntaxException {
		JSONObject json = getResults(composeSolrUrl(core, mode, query,
				solrParams, 0, 0, false));
		return json.getJSONObject("response").getInt("numFound");
	}

	/**
	 * Gets the json string representation for the query.
	 * 
	 * @param query
	 *            the query for which documents
	 * @param core
	 *            the solr core to query
	 * @param gridSolrParams
	 *            the default solr parameters to append to the query
	 * @param mode
	 *            the configuration mode to operate in
	 * @param start
	 *            where to start the offset
	 * @param length
	 *            how many documents to return
	 * @param showImgView
	 *            is this query showing the annotation view of the images
	 *            (true/false)
	 * @return
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public JSONObject getDataTableJson(String query, String core,
			String gridSolrParams, String mode, int start, int length,
			boolean showImgView) throws IOException, URISyntaxException {

		if (gridSolrParams.equals("")) {
			gridSolrParams = "qf=auto_suggest&defType=edismax&wt=json&q=*:*";
		}

		return getResults(composeSolrUrl(core, mode, query, gridSolrParams,
				start, length, showImgView));

	}

	/**
	 * Get rows for saving to an external file.
	 * 
	 * @param core
	 *            the solr core to query
	 * @param gridSolrParams
	 *            the default parameters to use in the query
	 * @param start
	 *            where to start the query
	 * @param gridFields
	 *            the default solr parameters to append to the query
	 * @param length
	 *            how many documents to return
	 * @return json representation of the results of the solr query
	 * @throws IOException
	 * @throws URISyntaxException
	 */
	public JSONObject getDataTableExportRows(String core,
			String gridSolrParams, String gridFields, int start, int length)
			throws IOException, URISyntaxException {

		System.out.println("GRID SOLR PARAMS : " + gridSolrParams);
		
		if (core.equals("gene")) {			
			//gridFields += ",imits_report_phenotyping_complete_date,imits_report_genotype_confirmed_date,imits_report_mi_plan_status,escell,ikmc_project,imits_phenotype_started,imits_phenotype_complete,imits_phenotype_status";
		}

		//String newgridSolrParams = gridSolrParams + "&rows=" + length
		gridSolrParams = gridSolrParams.replace("rows=10", "rows="+length);
		String newgridSolrParams = gridSolrParams 		
				+ "&start=" + start + "&fl=" + gridFields;

		String url = composeSolrUrl(core, "", "", newgridSolrParams, start,
				length, false);
		
		log.debug("Export data URL: " + url);
		System.out.println("Export data URL: " + url);
		return getResults(url);
	}

	/**
	 * Prepare a url for querying the solr indexes based on the passed in
	 * arguments.
	 * 
	 * @param core
	 *            which solr core to query
	 * @param mode
	 *            which configuration mode to operate in
	 * @param query
	 *            what to query
	 * @param gridSolrParams
	 *            default parameters to add to the solr query
	 * @param iDisplayStart
	 *            starting point of the query
	 * @param iDisplayLength
	 *            length of the query
	 * @param showImgView
	 *            which image mode to operate in
	 * @return the constructed url including all parameters
	 */
	private String composeSolrUrl(String core, String mode, String query,
			String gridSolrParams, Integer iDisplayStart,
			Integer iDisplayLength, boolean showImgView) {
		String internalSolrUrl = config.get("internalSolrUrl");
		
		String url = internalSolrUrl + "/" + core + "/select?";
						
		System.out.println(("BASEURL: " + url));
		if (mode.equals("mpPage")) {
			url += "q=" + query;
			url += "&start=0&rows=0&wt=json&qf=auto_suggest&defType=edismax";
		} else if (mode.equals("geneGrid")) {			
			url += gridSolrParams + "&start=" + iDisplayStart + "&rows="
					+ iDisplayLength;
			System.out.println("GENE PARAMS: " + url);
		} else if (mode.equals("pipelineGrid")) {
			url += gridSolrParams + "&start=" + iDisplayStart + "&rows="
					+ iDisplayLength;
//			System.out.println("PROTOCOL PARAMS: " + url);
		} else if (mode.equals("imagesGrid")) {
			url += gridSolrParams + "&start=" + iDisplayStart + "&rows="
					+ iDisplayLength;
			if (!showImgView) {
				url += "&facet=on&facet.field=symbol_gene&facet.field=expName_exp&facet.field=maTermName&facet.field=mpTermName&facet.mincount=1&facet.limit=-1";
			}
			System.out.println("IMG PARAMS: " + url);
		} else if (mode.equals("mpGrid")) {
			url += gridSolrParams.replaceAll(" ", "%20") + "&start="
					+ iDisplayStart + "&rows=" + iDisplayLength;
			System.out.println("MP PARAMS: " + url);
		} else if (mode.equals("maGrid")) {
			url += gridSolrParams.replaceAll(" ", "%20") + "&start="
					+ iDisplayStart + "&rows=" + iDisplayLength;
//			System.out.println("MA PARAMS: " + url);
		} else if ( mode.equals("diseaseGrid") ){			
			url += gridSolrParams.replaceAll(" ", "%20") + "&start="
					+ iDisplayStart + "&rows=" + iDisplayLength;
//			System.out.println("DISEASE PARAMS: " + url);
		} else if (mode.equals("ikmcAlleleGrid")) {
			url += "q=" + query;
			url += "&start=0&rows=0&wt=json";
//			System.out.println("IKMC ALLELE PARAMS: " + url);
		} else if (mode.equals("all") || mode.equals("page") || mode.equals("")) { // download search page result
			url += gridSolrParams;
			if (core.equals("images") && !showImgView) {				
				url += "&facet=on&facet.field=symbol_gene&facet.field=expName_exp&facet.field=maTermName&facet.field=mpTermName&facet.mincount=1&facet.limit=-1";
			}
//			System.out.println("GRID DUMP PARAMS - " + core + ": " + url);
		}
		// OTHER solrCoreNames to be added here
		
		return url;
	}
	/**
	 * Get the simplified production status of ES cells/mice for a document.
	 * 
	 * @param doc
	 *            represents a gene with imits status fields
	 * @return the latest status at the gene level for both ES cells and alleles
	 */
	public String deriveLatestProductionStatusForEsCellAndMice(JSONObject doc, HttpServletRequest request, boolean toExport, String geneLink){		
		
		String esCellStatus = fetchEsCellStatus(doc, request, toExport);		
		String miceStatus = "";		
		final List<String> exportMiceStatus = new ArrayList<String>();
				
		Map<String, String> sh = new HashMap<String, String>();
				
		try {		
						
			// mice production status			
			// Mice: blue tm1/tm1a/tm1e... mice (depending on how many allele docs) 
			if ( doc.containsKey("mouse_status") ){
				
				JSONArray alleleNames = doc.getJSONArray("allele_name");				
				JSONArray mouseStatus = doc.getJSONArray("mouse_status");
				
				for ( int i=0; i< mouseStatus.size(); i++ ) {		
					String mouseStatusStr = mouseStatus.get(i).toString();					
					sh.put(mouseStatusStr, "yes");
				}		
								
				// if no mice status found but there is already allele produced, mark it as "mice produced planned"				
				for ( int j=0; j< alleleNames.size(); j++ ) {
					String alleleName = alleleNames.get(j).toString();
					//System.out.println("allele: " + alleleName);
					if ( !alleleName.equals("") && !alleleName.equals("None") && !alleleName.contains("tm1e") && mouseStatus.get(j).toString().equals("") ){	
						sh.put("mice production planned", "yes");						
					}				
				}
				
				if ( sh.containsKey("Mice Produced") ){
					miceStatus = "<a class='status done' oldtitle='Mice Produced' title='' href='" + geneLink + "#order'>"
							   +  "<span>Mice</span>"
							   +  "</a>";
						
					exportMiceStatus.add("mice produced");
				}
				else if ( sh.containsKey("Assigned for Mouse Production and Phenotyping") ){
					miceStatus = "<a class='status inprogress' oldtitle='Mice production in progress' title=''>"
							   +  "<span>Mice</span>"
							   +  "</a>";
					exportMiceStatus.add("mice production in progress");
				}
				else if ( sh.containsKey("mice production planned") ){
					miceStatus = "<a class='status none' oldtitle='Mice production planned' title=''>"
							   +  "<span>Mice</span>"
							   +  "</a>";
					exportMiceStatus.add("mice production in progress");
				}				
			}
		} 
		catch (Exception e) {
			log.error("Error getting ES cell/Mice status");
			log.error(e.getLocalizedMessage());
		}
		
		if ( toExport ){
			exportMiceStatus.add(0, esCellStatus); // want to keep this at front
			return StringUtils.join(exportMiceStatus, ", ");
		}
		return esCellStatus + miceStatus;
		
	}
	/**
	 * Get the production status of ES cells/mice for a document.
	 * 
	 * @param doc
	 *            represents a gene with imits status fields
	 * @return the latest status at the gene level for ES cells and all statuses at the allele level for mice as a comma separated string
	 */
	public String deriveProductionStatusForEsCellAndMice(JSONObject doc, HttpServletRequest request, boolean toExport){		
		
		String esCellStatus = fetchEsCellStatus(doc, request, toExport);		
		String miceStatus = "";		
		final List<String> exportMiceStatus = new ArrayList<String>();
		
		String patternStr = "(tm.*)\\(.+\\).+"; // allele name pattern
		Pattern pattern = Pattern.compile(patternStr);
		
		try {		
						
			// mice production status
			
			// Mice: blue tm1/tm1a/tm1e... mice (depending on how many allele docs) 
			if ( doc.containsKey("mouse_status") ){
				
				JSONArray alleleNames = doc.getJSONArray("allele_name");
				JSONArray mouseStatus = doc.getJSONArray("mouse_status");
				
				for ( int i=0; i< mouseStatus.size(); i++ ) {		
					String mouseStatusStr = mouseStatus.get(i).toString();	
					
					if ( mouseStatusStr.equals("Mice Produced") ){
						String alleleName = alleleNames.getString(i).toString();						
						Matcher matcher = pattern.matcher(alleleName);
						//System.out.println(matcher.toString());
							
						if (matcher.find()) {
							String alleleType = matcher.group(1);						
							miceStatus += "<span class='status done' oldtitle='" + mouseStatusStr + "' title=''>"
									+  "<span>Mice<br>" + alleleType + "</span>"
									+  "</span>";
							
							exportMiceStatus.add(alleleType + " mice produced");
						}
					}
					else if (mouseStatusStr.equals("Assigned for Mouse Production and Phenotyping") ){
						String alleleName = alleleNames.getString(i).toString();						
						Matcher matcher = pattern.matcher(alleleName);
						//System.out.println(matcher.toString());
							
						if (matcher.find()) {
							String alleleType = matcher.group(1);						
							miceStatus += "<span class='status inprogress' oldtitle='Mice production in progress' title=''>"
									+  "<span>Mice<br>" + alleleType + "</span>"
									+  "</span>";
							exportMiceStatus.add(alleleType + " mice production in progress");
						}						
					}					
				}	
				// if no mice status found but there is already allele produced, mark it as "mice produced planned"				
				for ( int j=0; j< alleleNames.size(); j++ ) {
					String alleleName = alleleNames.get(j).toString();
					if ( !alleleName.equals("") && !alleleName.equals("None") && mouseStatus.get(j).toString().equals("") ){	
						Matcher matcher = pattern.matcher(alleleName);
						//System.out.println(matcher.toString());
							
						if (matcher.find()) {
							String alleleType = matcher.group(1);						
							miceStatus += "<span class='status none' oldtitle='Mice production planned' title=''>"
									+  "<span>Mice<br>" + alleleType + "</span>"
									+  "</span>";
							
							exportMiceStatus.add(alleleType + " mice production planned");
						}	
					}						
				}
			}
		} 
		catch (Exception e) {
			log.error("Error getting ES cell/Mice status");
			log.error(e.getLocalizedMessage());
		}
		
		if ( toExport ){
			exportMiceStatus.add(0, esCellStatus); // want to keep this at front
			return StringUtils.join(exportMiceStatus, ", ");
		}
		return esCellStatus + miceStatus;
		
	}
	/**
	 * Get the latest production status of ES cells for a document.
	 * 
	 * @param doc
	 *            represents a gene with imits status fields
	 * @return the latest status at the gene level for ES cells as a string
	 */
	public String fetchEsCellStatus(JSONObject doc, HttpServletRequest request, boolean toExport){
		String mgiId = doc.getString("mgi_accession_id");
		String geneUrl = request.getAttribute("baseUrl") + "/genes/" + mgiId;
				
		String esCellStatus = "";	
		String exportEsCellStatus = "";	
		try {	
			final String field = "latest_es_cell_status"; 
			// ES cell production status
			if ( doc.containsKey(field)  ){
				// blue es cell status				
				esCellStatus = doc.getString(field);
				if ( esCellStatus.equals("ES Cells Produced") ){
						esCellStatus = "<a class='status done' href='" + geneUrl + "' oldtitle='ES Cells produced' title=''>"
									 + " <span>ES Cells</span>"
									 + "</a>";
						
						exportEsCellStatus += "ES cells produced";
				}
				else if ( esCellStatus.equals("Assigned for ES Cell Production") ){
						esCellStatus = "<span class='status inprogress' oldtitle='ES cells production in progress' title=''>"
						   	 		 +  "	<span>ES Cells</span>"
						   	 		 +  "</span>";
						
						exportEsCellStatus += "ES cells production in progress";
				}
				else {
{'fraction_non_alphanumeric': 0.1251208040719026, 'fraction_numerical': 0.0030281554023580954, 'mean_word_length': 3.096463446819741, 'pattern_counts': {'":': 2, '<': 95, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3516122', 'n_tokens_mistral': 11402, 'n_tokens_neox': 9158, 'n_words': 2538}