<commit_before>package com.gallatinsystems.surveyal.app.web;

import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.servlet.http.HttpServletRequest;

import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto.QuestionType;
import org.waterforpeople.mapping.dao.SurveyInstanceDAO;
import org.waterforpeople.mapping.domain.QuestionAnswerStore;
import org.waterforpeople.mapping.domain.SurveyInstance;

import com.gallatinsystems.common.util.PropertyUtil;
import com.gallatinsystems.framework.rest.AbstractRestApiServlet;
import com.gallatinsystems.framework.rest.RestRequest;
import com.gallatinsystems.framework.rest.RestResponse;
import com.gallatinsystems.gis.geography.dao.CountryDao;
import com.gallatinsystems.gis.geography.domain.Country;
import com.gallatinsystems.gis.location.GeoLocationServiceGeonamesImpl;
import com.gallatinsystems.gis.location.GeoPlace;
import com.gallatinsystems.gis.map.domain.OGRFeature;
import com.gallatinsystems.metric.dao.MetricDao;
import com.gallatinsystems.metric.dao.SurveyMetricMappingDao;
import com.gallatinsystems.metric.domain.Metric;
import com.gallatinsystems.metric.domain.SurveyMetricMapping;
import com.gallatinsystems.survey.dao.QuestionDao;
import com.gallatinsystems.survey.dao.SurveyDAO;
import com.gallatinsystems.survey.domain.Question;
import com.gallatinsystems.survey.domain.Survey;
import com.gallatinsystems.surveyal.dao.SurveyedLocaleDao;
import com.gallatinsystems.surveyal.domain.SurveyalValue;
import com.gallatinsystems.surveyal.domain.SurveyedLocale;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;

/**
 * RESTFul servlet that can handle handle operations on SurveyedLocale and
 * related domain objects.
 * 
 * TODO: consider storing survey question list, metrics and mappings in a
 * Soft-Reference map to speed up processing.
 * 
 * @author <NAME>
 * 
 */
public class SurveyalRestServlet extends AbstractRestApiServlet {
	private static final long serialVersionUID = 5923399458369692813L;
	private static final String COMMUNITY_METRIC_NAME = "Community";
	private static final double TOLERANCE = 0.01;
	private static final double UNSET_VAL = -9999.9;
	private static final String DEFAULT = "DEFAULT";
	private static final String DEFAULT_ORG_PROP = "defaultOrg";
	private static final Logger log = Logger
			.getLogger(SurveyalRestServlet.class.getName());

	private SurveyInstanceDAO surveyInstanceDao;
	private SurveyedLocaleDao surveyedLocaleDao;
	private QuestionDao qDao;
	private CountryDao countryDao;
	private SurveyMetricMappingDao metricMappingDao;
	private MetricDao metricDao;
	private boolean useConfigStatusScore = false;
	private String statusFragment;
	private Map<String, String> scoredVals;
	private boolean mergeNearby;

	/**
	 * initializes the servlet by instantiating all needed Dao classes and
	 * loading properties from the configuration.
	 * 
	 * 
	 */
	public SurveyalRestServlet() {
		surveyInstanceDao = new SurveyInstanceDAO();
		surveyedLocaleDao = new SurveyedLocaleDao();
		qDao = new QuestionDao();
		countryDao = new CountryDao();
		metricDao = new MetricDao();
		metricMappingDao = new SurveyMetricMappingDao();
		mergeNearby = true;
		String mergeProp = PropertyUtil.getProperty("mergeNearbyLocales");
		if (mergeProp != null && "false".equalsIgnoreCase(mergeProp.trim())) {
			mergeNearby = false;
		}
		// TODO: once the appropriate metric types are defined and reliably
		// assigned, consider removing this in favor of metrics
		statusFragment = PropertyUtil.getProperty("statusQuestionText");
		if (statusFragment != null && statusFragment.trim().length() > 0) {
			useConfigStatusScore = true;
			String[] fields = statusFragment.split(";");
			statusFragment = fields[0].toLowerCase();
			scoredVals = new HashMap<String, String>();
			if (fields.length > 1) {
				for (int i = 1; i < fields.length; i++) {
					if (fields[i].contains("=")) {
						String[] kvp = fields[i].split("=");
						scoredVals.put(kvp[0], kvp[1]);
					}
				}
			}
		}
	}

	@Override
	protected RestRequest convertRequest() throws Exception {
		HttpServletRequest req = getRequest();
		RestRequest restRequest = new SurveyalRestRequest();
		restRequest.populateFromHttpRequest(req);
		return restRequest;
	}

	@Override
	protected RestResponse handleRequest(RestRequest req) throws Exception {
		RestResponse resp = new RestResponse();
		SurveyalRestRequest sReq = (SurveyalRestRequest) req;
		if (SurveyalRestRequest.INGEST_INSTANCE_ACTION.equalsIgnoreCase(req
				.getAction())) {
			try {
				ingestSurveyInstance(sReq.getSurveyInstanceId());
			} catch (RuntimeException e) {
				log.log(Level.SEVERE,
						"Could not process instance: "
								+ sReq.getSurveyInstanceId() + ": "
								+ e.getMessage());
			}
		} else if (SurveyalRestRequest.RERUN_ACTION.equalsIgnoreCase(req
				.getAction())) {
			rerunForSurvey(sReq.getSurveyId());
		} else if (SurveyalRestRequest.REINGEST_INSTANCE_ACTION
				.equalsIgnoreCase(req.getAction())) {
			log.log(Level.INFO,
					"Reprocessing SurveyInstanceId: "
							+ sReq.getSurveyInstanceId());
			ingestSurveyInstance(sReq.getSurveyInstanceId());
		}
		return resp;
	}

	/**
	 * reruns the locale hydration for a survey
	 * 
	 * @param surveyId
	 */
	private void rerunForSurvey(Long surveyId) {
		if (surveyId != null) {
			Queue queue = QueueFactory.getDefaultQueue();
			Iterable<Entity> siList = surveyInstanceDao
					.listSurveyInstanceKeysBySurveyId(surveyId);
			if (siList != null) {
				int i = 0;
				for (Entity inst : siList) {
					if (inst != null && inst.getKey() != null) {
						String item = inst.getKey().toString();
						Integer startPos = item.indexOf("(");
						Integer endPos = item.indexOf(")");
						String surveyInstanceIdString = item.substring(
								startPos + 1, endPos);
						if (surveyInstanceIdString != null
								&& !surveyInstanceIdString.trim()
										.equalsIgnoreCase("")) {
							TaskOptions to = TaskOptions.Builder
									.withUrl("/app_worker/surveyalservlet")
									.param(SurveyalRestRequest.ACTION_PARAM,
											SurveyalRestRequest.REINGEST_INSTANCE_ACTION)
									.param(SurveyalRestRequest.SURVEY_INSTANCE_PARAM,
											surveyInstanceIdString);
							queue.add(to);

							i++;
						}
					} else {
						String instString = null;
						if (inst != null)
							instString = inst.toString();
						log.log(Level.INFO,
								"Inside rerunForSurvey in the null or empty instanceid branch: "
										+ instString);
					}
				}
				log.log(Level.INFO, "Submitted: " + i
						+ " SurveyInstances for remapping");
			}

		}
	}

	private void ingestSurveyInstance(Long surveyInstanceId) {
		SurveyInstance instance = surveyInstanceDao.getByKey(surveyInstanceId);
		if (instance != null) {
			ingestSurveyInstance(instance);
		} else
			log.log(Level.INFO,
					"Got to ingestSurveyInstance, but instance is null for surveyInstanceId: "
							+ surveyInstanceId);

	}

	/**
	 * looks up a surveyInstance by key and creates (or updates) a
	 * surveyedLocale based on the data contained therein. This method is
	 * unlikely to run in under 1 minute (based on datastore latency) so it is
	 * best invoked via a task queue
	 * 
	 * @param surveyInstanceId
	 */
	private void ingestSurveyInstance(SurveyInstance instance) {

		SurveyedLocale locale = null;
		if (instance != null) {
			List<QuestionAnswerStore> answers = surveyInstanceDao
					.listQuestionAnswerStore(instance.getKey().getId(), null);
			QuestionAnswerStore geoQ = null;
			SurveyDAO surveyDao = new SurveyDAO();
			Survey survey = surveyDao.getByKey(instance.getSurveyId());
			String pointType = null;
			if (survey != null) {
				pointType = survey.getPointType();
			}
			if (answers != null) {
				for (QuestionAnswerStore q : answers) {
					if (QuestionType.GEO.toString().equals(q.getType())) {
						geoQ = q;
						break;
					}
				}
			}

			if (instance.getSurveyedLocaleId() != null) {
				locale = surveyedLocaleDao.getByKey(instance
						.getSurveyedLocaleId());
			}
			GeoPlace geoPlace = null;

			// only create a "locale" if we have a geographic question
			if (geoQ != null && geoQ.getValue() != null) {
				double lat = UNSET_VAL;
				double lon = UNSET_VAL;
				boolean ambiguousFlag = false;
				String code = null;
				String[] tokens = geoQ.getValue().split("\\|");
				if (tokens.length >= 2) {
					lat = Double.parseDouble(tokens[0]);
					lon = Double.parseDouble(tokens[1]);
					if (tokens.length >= 4) {
						code = tokens[tokens.length - 1];
					}
				}
				if (code == null) {
					Long codeNum = Long
							.parseLong((int) ((Math.abs(lat) * 10000d)) + ""
									+ (int) ((Math.abs(lon) * 10000d)));
					code = Long.toString(codeNum, 36);
				}
				if (lat == UNSET_VAL || lon == UNSET_VAL) {
					throw new RuntimeException(
							"Could not parse lat/lon from Geo Question "
									+ geoQ.getQuestionID());
				} else if (mergeNearby && locale == null) {
					// if we have a geo question but no locale id, see if we can
					// find one based on lat/lon
					List<SurveyedLocale> candidates = surveyedLocaleDao
							.listLocalesByCoordinates(pointType, lat, lon,
									TOLERANCE);
					if (candidates != null && candidates.size() == 1) {
						locale = candidates.get(0);
					} else if (candidates != null && candidates.size() > 1) {
						log.log(Level.WARNING,
								"Geo based lookup of surveyed locale returned more than one candidate so we are creating a new one");
						ambiguousFlag = true;
					}
				}
				geoPlace = getGeoPlace(lat, lon);
				if (locale == null) {
					locale = new SurveyedLocale();
					locale.setAmbiguous(ambiguousFlag);
					locale.setLatitude(lat);
					locale.setLongitude(lon);
					setGeoData(geoPlace, locale);
					if (survey != null) {
						locale.setLocaleType(survey.getPointType());
					}
					locale.setIdentifier(code);
					// TODO: for multi-org instances, set org on Survey and pull
					// from there
					if (locale.getOrganization() == null) {
						locale.setOrganization(PropertyUtil
								.getProperty(DEFAULT_ORG_PROP));
					}
					locale = surveyedLocaleDao.save(locale);
				} else {
					if (survey.getPointType() != null
							&& !survey.getPointType().equals(
									locale.getLocaleType())) {
						locale.setLocaleType(survey.getPointType());
					}
				}
			}
			if (instance != null && geoPlace != null) {
				instance.setCountryCode(geoPlace.getCountryCode());
				instance.setSublevel1(geoPlace.getSub1());
				instance.setSublevel2(geoPlace.getSub2());
				instance.setSublevel3(geoPlace.getSub3());
				instance.setSublevel4(geoPlace.getSub4());
				instance.setSublevel5(geoPlace.getSub5());
				instance.setSublevel6(geoPlace.getSub6());
				if (answers != null) {
					// look for the Community metric
					List<Metric> metrics = metricDao.listMetrics(
							COMMUNITY_METRIC_NAME, null, null, null, null);
					if (metrics != null) {
						List<SurveyMetricMapping> mappings = new ArrayList<SurveyMetricMapping>();
						for (Metric m : metrics) {
							List<SurveyMetricMapping> temp = metricMappingDao
									.listMetricsBySurveyAndMetric(m.getKey()
											.getId(), instance.getSurveyId());
							if (temp != null) {
								mappings.addAll(temp);
							}
						}
						for (SurveyMetricMapping mapping : mappings) {
							for (QuestionAnswerStore ans : answers) {
								if (ans.getQuestionID().equals(
										mapping.getSurveyQuestionId())) {
									instance.setCommunity(ans.getValue());
									break;
								}
							}
						}
					}
				}
			}

			if (locale != null && locale.getKey() != null && answers != null) {
				locale.setLastSurveyedDate(instance.getCollectionDate());
				locale.setLastSurveyalInstanceId(instance.getKey().getId());
				instance.setSurveyedLocaleId(locale.getKey().getId());
				List<SurveyalValue> values = constructValues(locale, answers);
				if (values != null) {
					surveyedLocaleDao.save(values);
					if (!useConfigStatusScore) {
						// now check the values to see if we have a status to
						// update
						// check the metrics first
						boolean found = false;
						for (SurveyalValue val : values) {
							if (isStatus(val.getMetricName())
									&& val.getStringValue() != null) {
								found = true;
								locale.setCurrentStatus(val.getStringValue());
								break;
							}
						}
						// if no luck, check the question text
						if (!found) {
							for (SurveyalValue val : values) {
								if (isStatus(val.getQuestionText())
										&& val.getStringValue() != null) {
									found = true;
									locale.setCurrentStatus(val
											.getStringValue());
									break;
								}
							}
						}
					} else {
						for (SurveyalValue val : values) {
							if (val.getQuestionText() != null
									&& val.getQuestionText().toLowerCase()
											.contains(statusFragment)) {
								String scoredField = scoredVals.get(val
										.getStringValue());
								if (scoredField == null) {
									scoredField = scoredVals.get(DEFAULT);
								}
								locale.setCurrentStatus(scoredField);
							}<commit_msg>fixes for testing missing files<commit_after>package com.gallatinsystems.surveyal.app.web;

import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.servlet.http.HttpServletRequest;

import org.waterforpeople.mapping.app.gwt.client.survey.QuestionDto.QuestionType;
import org.waterforpeople.mapping.dao.SurveyInstanceDAO;
import org.waterforpeople.mapping.domain.QuestionAnswerStore;
import org.waterforpeople.mapping.domain.SurveyInstance;

import com.gallatinsystems.common.util.PropertyUtil;
import com.gallatinsystems.framework.rest.AbstractRestApiServlet;
import com.gallatinsystems.framework.rest.RestRequest;
import com.gallatinsystems.framework.rest.RestResponse;
import com.gallatinsystems.gis.geography.dao.CountryDao;
import com.gallatinsystems.gis.geography.domain.Country;
import com.gallatinsystems.gis.location.GeoLocationServiceGeonamesImpl;
import com.gallatinsystems.gis.location.GeoPlace;
import com.gallatinsystems.gis.map.domain.OGRFeature;
import com.gallatinsystems.metric.dao.MetricDao;
import com.gallatinsystems.metric.dao.SurveyMetricMappingDao;
import com.gallatinsystems.metric.domain.Metric;
import com.gallatinsystems.metric.domain.SurveyMetricMapping;
import com.gallatinsystems.survey.dao.QuestionDao;
import com.gallatinsystems.survey.dao.SurveyDAO;
import com.gallatinsystems.survey.domain.Question;
import com.gallatinsystems.survey.domain.Survey;
import com.gallatinsystems.surveyal.dao.SurveyedLocaleDao;
import com.gallatinsystems.surveyal.domain.SurveyalValue;
import com.gallatinsystems.surveyal.domain.SurveyedLocale;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;

/**
 * RESTFul servlet that can handle handle operations on SurveyedLocale and
 * related domain objects.
 * 
 * TODO: consider storing survey question list, metrics and mappings in a
 * Soft-Reference map to speed up processing.
 * 
 * @author <NAME>
 * 
 */
public class SurveyalRestServlet extends AbstractRestApiServlet {
	private static final long serialVersionUID = 5923399458369692813L;
	private static final String COMMUNITY_METRIC_NAME = "Community";
	private static final double TOLERANCE = 0.01;
	private static final double UNSET_VAL = -9999.9;
	private static final String DEFAULT = "DEFAULT";
	private static final String DEFAULT_ORG_PROP = "defaultOrg";
	private static final Logger log = Logger
			.getLogger(SurveyalRestServlet.class.getName());

	private SurveyInstanceDAO surveyInstanceDao;
	private SurveyedLocaleDao surveyedLocaleDao;
	private QuestionDao qDao;
	private CountryDao countryDao;
	private SurveyMetricMappingDao metricMappingDao;
	private MetricDao metricDao;
	private boolean useConfigStatusScore = false;
	private boolean useDynamicScoring = false;
	private String statusFragment;
	private Map<String, String> scoredVals;
	private boolean mergeNearby;

	/**
	 * initializes the servlet by instantiating all needed Dao classes and
	 * loading properties from the configuration.
	 * 
	 * 
	 */
	public SurveyalRestServlet() {
		surveyInstanceDao = new SurveyInstanceDAO();
		surveyedLocaleDao = new SurveyedLocaleDao();
		qDao = new QuestionDao();
		countryDao = new CountryDao();
		metricDao = new MetricDao();
		metricMappingDao = new SurveyMetricMappingDao();
		mergeNearby = true;
		String mergeProp = PropertyUtil.getProperty("mergeNearbyLocales");
		useDynamicScoring = Boolean.parseBoolean(PropertyUtil.getProperty("scoreLocaleDynmaic"));
		if (mergeProp != null && "false".equalsIgnoreCase(mergeProp.trim())) {
			mergeNearby = false;
		}
		// TODO: once the appropriate metric types are defined and reliably
		// assigned, consider removing this in favor of metrics
		statusFragment = PropertyUtil.getProperty("statusQuestionText");
		if (statusFragment != null && statusFragment.trim().length() > 0) {
			useConfigStatusScore = true;
			String[] fields = statusFragment.split(";");
			statusFragment = fields[0].toLowerCase();
			scoredVals = new HashMap<String, String>();
			if (fields.length > 1) {
				for (int i = 1; i < fields.length; i++) {
					if (fields[i].contains("=")) {
						String[] kvp = fields[i].split("=");
						scoredVals.put(kvp[0], kvp[1]);
					}
				}
			}
		}
	}

	@Override
	protected RestRequest convertRequest() throws Exception {
		HttpServletRequest req = getRequest();
		RestRequest restRequest = new SurveyalRestRequest();
		restRequest.populateFromHttpRequest(req);
		return restRequest;
	}

	@Override
	protected RestResponse handleRequest(RestRequest req) throws Exception {
		RestResponse resp = new RestResponse();
		SurveyalRestRequest sReq = (SurveyalRestRequest) req;
		if (SurveyalRestRequest.INGEST_INSTANCE_ACTION.equalsIgnoreCase(req
				.getAction())) {
			try {
				ingestSurveyInstance(sReq.getSurveyInstanceId());
			} catch (RuntimeException e) {
				log.log(Level.SEVERE,
						"Could not process instance: "
								+ sReq.getSurveyInstanceId() + ": "
								+ e.getMessage());
			}
		} else if (SurveyalRestRequest.RERUN_ACTION.equalsIgnoreCase(req
				.getAction())) {
			rerunForSurvey(sReq.getSurveyId());
		} else if (SurveyalRestRequest.REINGEST_INSTANCE_ACTION
				.equalsIgnoreCase(req.getAction())) {
			log.log(Level.INFO,
					"Reprocessing SurveyInstanceId: "
							+ sReq.getSurveyInstanceId());
			ingestSurveyInstance(sReq.getSurveyInstanceId());
		}
		return resp;
	}

	/**
	 * reruns the locale hydration for a survey
	 * 
	 * @param surveyId
	 */
	private void rerunForSurvey(Long surveyId) {
		if (surveyId != null) {
			Queue queue = QueueFactory.getDefaultQueue();
			Iterable<Entity> siList = surveyInstanceDao
					.listSurveyInstanceKeysBySurveyId(surveyId);
			if (siList != null) {
				int i = 0;
				for (Entity inst : siList) {
					if (inst != null && inst.getKey() != null) {
						String item = inst.getKey().toString();
						Integer startPos = item.indexOf("(");
						Integer endPos = item.indexOf(")");
						String surveyInstanceIdString = item.substring(
								startPos + 1, endPos);
						if (surveyInstanceIdString != null
								&& !surveyInstanceIdString.trim()
										.equalsIgnoreCase("")) {
							TaskOptions to = TaskOptions.Builder
									.withUrl("/app_worker/surveyalservlet")
									.param(SurveyalRestRequest.ACTION_PARAM,
											SurveyalRestRequest.REINGEST_INSTANCE_ACTION)
									.param(SurveyalRestRequest.SURVEY_INSTANCE_PARAM,
											surveyInstanceIdString);
							queue.add(to);

							i++;
						}
					} else {
						String instString = null;
						if (inst != null)
							instString = inst.toString();
						log.log(Level.INFO,
								"Inside rerunForSurvey in the null or empty instanceid branch: "
										+ instString);
					}
				}
				log.log(Level.INFO, "Submitted: " + i
						+ " SurveyInstances for remapping");
			}

		}
	}

	private void ingestSurveyInstance(Long surveyInstanceId) {
		SurveyInstance instance = surveyInstanceDao.getByKey(surveyInstanceId);
		if (instance != null) {
			ingestSurveyInstance(instance);
		} else
			log.log(Level.INFO,
					"Got to ingestSurveyInstance, but instance is null for surveyInstanceId: "
							+ surveyInstanceId);

	}

	/**
	 * looks up a surveyInstance by key and creates (or updates) a
	 * surveyedLocale based on the data contained therein. This method is
	 * unlikely to run in under 1 minute (based on datastore latency) so it is
	 * best invoked via a task queue
	 * 
	 * @param surveyInstanceId
	 */
	private void ingestSurveyInstance(SurveyInstance instance) {

		SurveyedLocale locale = null;
		if (instance != null) {
			List<QuestionAnswerStore> answers = surveyInstanceDao
					.listQuestionAnswerStore(instance.getKey().getId(), null);
			QuestionAnswerStore geoQ = null;
			SurveyDAO surveyDao = new SurveyDAO();
			Survey survey = surveyDao.getByKey(instance.getSurveyId());
			String pointType = null;
			if (survey != null) {
				pointType = survey.getPointType();
			}
			if (answers != null) {
				for (QuestionAnswerStore q : answers) {
					if (QuestionType.GEO.toString().equals(q.getType())) {
						geoQ = q;
						break;
					}
				}
			}

			if (instance.getSurveyedLocaleId() != null) {
				locale = surveyedLocaleDao.getByKey(instance
						.getSurveyedLocaleId());
			}
			GeoPlace geoPlace = null;

			// only create a "locale" if we have a geographic question
			if (geoQ != null && geoQ.getValue() != null) {
				double lat = UNSET_VAL;
				double lon = UNSET_VAL;
				boolean ambiguousFlag = false;
				String code = null;
				String[] tokens = geoQ.getValue().split("\\|");
				if (tokens.length >= 2) {
					lat = Double.parseDouble(tokens[0]);
					lon = Double.parseDouble(tokens[1]);
					if (tokens.length >= 4) {
						code = tokens[tokens.length - 1];
					}
				}
				if (code == null) {
					Long codeNum = Long
							.parseLong((int) ((Math.abs(lat) * 10000d)) + ""
									+ (int) ((Math.abs(lon) * 10000d)));
					code = Long.toString(codeNum, 36);
				}
				if (lat == UNSET_VAL || lon == UNSET_VAL) {
					throw new RuntimeException(
							"Could not parse lat/lon from Geo Question "
									+ geoQ.getQuestionID());
				} else if (mergeNearby && locale == null) {
					// if we have a geo question but no locale id, see if we can
					// find one based on lat/lon
					List<SurveyedLocale> candidates = surveyedLocaleDao
							.listLocalesByCoordinates(pointType, lat, lon,
									TOLERANCE);
					if (candidates != null && candidates.size() == 1) {
						locale = candidates.get(0);
					} else if (candidates != null && candidates.size() > 1) {
						log.log(Level.WARNING,
								"Geo based lookup of surveyed locale returned more than one candidate so we are creating a new one");
						ambiguousFlag = true;
					}
				}
				geoPlace = getGeoPlace(lat, lon);
				if (locale == null) {
					locale = new SurveyedLocale();
					locale.setAmbiguous(ambiguousFlag);
					locale.setLatitude(lat);
					locale.setLongitude(lon);
					setGeoData(geoPlace, locale);
					if (survey != null) {
						locale.setLocaleType(survey.getPointType());
					}
					locale.setIdentifier(code);
					// TODO: for multi-org instances, set org on Survey and pull
					// from there
					if (locale.getOrganization() == null) {
						locale.setOrganization(PropertyUtil
								.getProperty(DEFAULT_ORG_PROP));
					}
					locale = surveyedLocaleDao.save(locale);
				} else {
					if (survey.getPointType() != null
							&& !survey.getPointType().equals(
									locale.getLocaleType())) {
						locale.setLocaleType(survey.getPointType());
					}
				}
			}
			if (instance != null && geoPlace != null) {
				instance.setCountryCode(geoPlace.getCountryCode());
				instance.setSublevel1(geoPlace.getSub1());
				instance.setSublevel2(geoPlace.getSub2());
				instance.setSublevel3(geoPlace.getSub3());
				instance.setSublevel4(geoPlace.getSub4());
				instance.setSublevel5(geoPlace.getSub5());
				instance.setSublevel6(geoPlace.getSub6());
				if (answers != null) {
					// look for the Community metric
					List<Metric> metrics = metricDao.listMetrics(
							COMMUNITY_METRIC_NAME, null, null, null, null);
					if (metrics != null) {
						List<SurveyMetricMapping> mappings = new ArrayList<SurveyMetricMapping>();
						for (Metric m : metrics) {
							List<SurveyMetricMapping> temp = metricMappingDao
									.listMetricsBySurveyAndMetric(m.getKey()
											.getId(), instance.getSurveyId());
							if (temp != null) {
								mappings.addAll(temp);
							}
						}
						for (SurveyMetricMapping mapping : mappings) {
							for (QuestionAnswerStore ans : answers) {
								if (ans.getQuestionID().equals(
										mapping.getSurveyQuestionId())) {
									instance.setCommunity(ans.getValue());
									break;
								}
							}
						}
					}
				}
			}

			if (locale != null && locale.getKey() != null && answers != null) {
				locale.setLastSurveyedDate(instance.getCollectionDate());
				locale.setLastSurveyalInstanceId(instance.getKey().getId());
				instance.setSurveyedLocaleId(locale.getKey().getId());
				List<SurveyalValue> values = constructValues(locale, answers);
				if (values != null) {
					surveyedLocaleDao.save(values);
					if (!useConfigStatusScore) {
						// now check the values to see if we have a status to
						// update
						// check the metrics first
						boolean found = false;
						for (SurveyalValue val : values) {
							if (isStatus(val.getMetricName())
									&& val.getStringValue() != null) {
								found = true;
								locale.setCurrentStatus(val.getStringValue());
								break;
							}
						}
						// if no luck, check the question text
						if (!found) {
							for (SurveyalValue val : values) {
								if (isStatus(val.getQuestionText())
										&& val.getStringValue() != null) {
									found = true;
									locale.setCurrentStatus(val
											.getStringValue());
									break;
								}
							}
						}
					} else if (useDynamicScoring) {
						
					} else {
						for (SurveyalValue val : values) {
							if (val.getQuestionText() != null
									&& val.getQuestionText().toLowerCase()
											.contains(statusFragment)) {
								String scoredField = scoredVals.get(val
										.getStringValue());
								if (scoredField == null) {
									scoredField = scoredVals.get(DEFAULT);
								}
								locale.setCurrentStatus(scoredField);
							}
{'fraction_non_alphanumeric': 0.112488928255093, 'fraction_numerical': 0.005019191024505462, 'mean_word_length': 3.3924461014751177, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5396844', 'n_tokens_mistral': 10595, 'n_tokens_neox': 8199, 'n_words': 1823}