<commit_before>/*
 *  Copyright (C) 2010-2012 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.waterforpeople.mapping.app.gwt.server.surveyinstance;

import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.waterforpeople.mapping.app.gwt.client.surveyinstance.QuestionAnswerStoreDto;
import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceDto;
import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceService;
import org.waterforpeople.mapping.app.util.DtoMarshaller;
import org.waterforpeople.mapping.dao.SurveyAttributeMappingDao;
import org.waterforpeople.mapping.dao.SurveyInstanceDAO;
import org.waterforpeople.mapping.domain.QuestionAnswerStore;
import org.waterforpeople.mapping.domain.SurveyAttributeMapping;
import org.waterforpeople.mapping.domain.SurveyInstance;
import org.waterforpeople.mapping.helper.SurveyEventHelper;

import com.gallatinsystems.framework.analytics.summarization.DataSummarizationRequest;
import com.gallatinsystems.framework.domain.DataChangeRecord;
import com.gallatinsystems.framework.gwt.dto.client.ResponseDto;
import com.gallatinsystems.survey.dao.QuestionDao;
import com.gallatinsystems.survey.dao.SurveyDAO;
import com.gallatinsystems.survey.domain.Question;
import com.gallatinsystems.survey.domain.Survey;
import com.gallatinsystems.surveyal.app.web.SurveyalRestRequest;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.gwt.user.server.rpc.RemoteServiceServlet;

public class SurveyInstanceServiceImpl extends RemoteServiceServlet implements
		SurveyInstanceService {

	private static final long serialVersionUID = -9175237700587455358L;
	private static final Logger log = Logger
			.getLogger(SurveyInstanceServiceImpl.class);

	@Override
	public ResponseDto<ArrayList<SurveyInstanceDto>> listSurveyInstance(
			Date beginDate, Date toDate, boolean unapprovedOnlyFlag,
			Long surveyId, String source, String cursorString) {
		SurveyInstanceDAO dao = new SurveyInstanceDAO();
		SurveyDAO surveyDao = new SurveyDAO();
		List<Survey> surveyList = surveyDao.list("all");
		HashMap<Long, String> surveyMap = new HashMap<Long, String>();
		for (Survey s : surveyList) {
			surveyMap.put(s.getKey().getId(), s.getPath() + "/" + s.getCode());
		}
		List<SurveyInstance> siList = null;
		if (beginDate == null && toDate == null) {
			Calendar c = Calendar.getInstance();
			c.add(Calendar.YEAR, -90);
			beginDate = c.getTime();
		}
		siList = dao.listByDateRange(beginDate, toDate, unapprovedOnlyFlag,
				surveyId, source, cursorString);
		String newCursor = SurveyInstanceDAO.getCursor(siList);

		ArrayList<SurveyInstanceDto> siDtoList = new ArrayList<SurveyInstanceDto>();
		for (SurveyInstance siItem : siList) {
			String code = surveyMap.get(siItem.getSurveyId());
			SurveyInstanceDto siDto = marshalToDto(siItem);
			if (code != null)
				siDto.setSurveyCode(code);
			siDtoList.add(siDto);

		}
		ResponseDto<ArrayList<SurveyInstanceDto>> response = new ResponseDto<ArrayList<SurveyInstanceDto>>();
		response.setCursorString(newCursor);
		response.setPayload(siDtoList);
		return response;
	}

	public List<QuestionAnswerStoreDto> listQuestionsByInstance(Long instanceId) {
		List<QuestionAnswerStoreDto> questionDtos = new ArrayList<QuestionAnswerStoreDto>();
		SurveyInstanceDAO dao = new SurveyInstanceDAO();
		List<QuestionAnswerStore> questions = dao.listQuestionAnswerStore(
				instanceId, null);
		QuestionDao qDao = new QuestionDao();

		if (questions != null && questions.size() > 0) {
			List<Question> qList = qDao.listQuestionInOrder(questions.get(0)
					.getSurveyId());
			Map<Integer, QuestionAnswerStoreDto> orderedResults = new TreeMap<Integer, QuestionAnswerStoreDto>();
			int notFoundCount = 0;
			if (qList != null) {
				for (QuestionAnswerStore qas : questions) {
					QuestionAnswerStoreDto qasDto = new QuestionAnswerStoreDto();
					DtoMarshaller.copyToDto(qas, qasDto);
					int idx = -1 - notFoundCount;
					for (int i = 0; i < qList.size(); i++) {
						if (Long.parseLong(qas.getQuestionID()) == qList.get(i)
								.getKey().getId()) {
							qasDto.setQuestionText(qList.get(i).getText());
							idx = i;
							break;
						}
					}
					if (idx < 0) {
						// do this to prevent collisions on the -1 key if there
						// is more than one questionAnswerStore item that isn't
						// in the question list
						notFoundCount++;
					}
					orderedResults.put(idx, qasDto);
				}
			}
			questionDtos = new ArrayList<QuestionAnswerStoreDto>(
					orderedResults.values());
		}
		return questionDtos;
	}

	private SurveyInstanceDto marshalToDto(SurveyInstance si) {
		SurveyInstanceDto siDto = new SurveyInstanceDto();
		DtoMarshaller.copyToDto(si, siDto);
		siDto.setQuestionAnswersStore(null);
		if (si.getQuestionAnswersStore() != null) {
			for (QuestionAnswerStore qas : si.getQuestionAnswersStore()) {
				siDto.addQuestionAnswerStore(marshalToDto(qas));
			}
		}
		return siDto;
	}

	private QuestionAnswerStoreDto marshalToDto(QuestionAnswerStore qas) {
		QuestionAnswerStoreDto qasDto = new QuestionAnswerStoreDto();
		DtoMarshaller.copyToDto(qas, qasDto);
		return qasDto;
	}

	/**
	 * updates the list of QAS dto objects passed in and fires summarization
	 * messages to the async queues
	 */
	@Override
	public List<QuestionAnswerStoreDto> updateQuestions(
			List<QuestionAnswerStoreDto> dtoList, boolean isApproved) {
		return updateQuestions(dtoList, isApproved, true);
	}

	public List<QuestionAnswerStoreDto> updateQuestions(
			List<QuestionAnswerStoreDto> dtoList, boolean isApproved,
			boolean processSummaries) {
		List<QuestionAnswerStore> domainList = new ArrayList<QuestionAnswerStore>();
		for (QuestionAnswerStoreDto dto : dtoList) {
			QuestionAnswerStore answer = new QuestionAnswerStore();
			DtoMarshaller.copyToCanonical(answer, dto);
			if (answer.getValue() != null) {
				answer.setValue(answer.getValue().replaceAll("\t", ""));
			}
			domainList.add(answer);
		}
		SurveyInstanceDAO dao = new SurveyInstanceDAO();
		
		dao.save(domainList);
		if (isApproved && processSummaries) {
			SurveyAttributeMappingDao mappingDao = new SurveyAttributeMappingDao();
			// now send a change message for each item
			Queue queue = QueueFactory.getQueue("dataUpdate");
			for (QuestionAnswerStoreDto item : dtoList) {
				DataChangeRecord value = new DataChangeRecord(
						QuestionAnswerStore.class.getName(),
						item.getQuestionID(), item.getOldValue(),
						item.getValue());
				queue.add(TaskOptions.Builder
						.withUrl("/app_worker/dataupdate")
						.param(DataSummarizationRequest.OBJECT_KEY,
								item.getQuestionID())
						.param(DataSummarizationRequest.OBJECT_TYPE,
								"QuestionDataChange")
						.param(DataSummarizationRequest.VALUE_KEY,
								value.packString()));
				// see if the question is mapped. And if it is, send an Access
				// Point
				// change message<commit_msg>Issue #185 - add comment to surveyInstanceServiceImpl<commit_after>/*
 *  Copyright (C) 2010-2012 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.waterforpeople.mapping.app.gwt.server.surveyinstance;

import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.waterforpeople.mapping.app.gwt.client.surveyinstance.QuestionAnswerStoreDto;
import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceDto;
import org.waterforpeople.mapping.app.gwt.client.surveyinstance.SurveyInstanceService;
import org.waterforpeople.mapping.app.util.DtoMarshaller;
import org.waterforpeople.mapping.dao.SurveyAttributeMappingDao;
import org.waterforpeople.mapping.dao.SurveyInstanceDAO;
import org.waterforpeople.mapping.domain.QuestionAnswerStore;
import org.waterforpeople.mapping.domain.SurveyAttributeMapping;
import org.waterforpeople.mapping.domain.SurveyInstance;
import org.waterforpeople.mapping.helper.SurveyEventHelper;

import com.gallatinsystems.framework.analytics.summarization.DataSummarizationRequest;
import com.gallatinsystems.framework.domain.DataChangeRecord;
import com.gallatinsystems.framework.gwt.dto.client.ResponseDto;
import com.gallatinsystems.survey.dao.QuestionDao;
import com.gallatinsystems.survey.dao.SurveyDAO;
import com.gallatinsystems.survey.domain.Question;
import com.gallatinsystems.survey.domain.Survey;
import com.gallatinsystems.surveyal.app.web.SurveyalRestRequest;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;
import com.google.gwt.user.server.rpc.RemoteServiceServlet;

public class SurveyInstanceServiceImpl extends RemoteServiceServlet implements
		SurveyInstanceService {

	private static final long serialVersionUID = -9175237700587455358L;
	private static final Logger log = Logger
			.getLogger(SurveyInstanceServiceImpl.class);

	@Override
	public ResponseDto<ArrayList<SurveyInstanceDto>> listSurveyInstance(
			Date beginDate, Date toDate, boolean unapprovedOnlyFlag,
			Long surveyId, String source, String cursorString) {
		SurveyInstanceDAO dao = new SurveyInstanceDAO();
		SurveyDAO surveyDao = new SurveyDAO();
		List<Survey> surveyList = surveyDao.list("all");
		HashMap<Long, String> surveyMap = new HashMap<Long, String>();
		for (Survey s : surveyList) {
			surveyMap.put(s.getKey().getId(), s.getPath() + "/" + s.getCode());
		}
		List<SurveyInstance> siList = null;
		if (beginDate == null && toDate == null) {
			Calendar c = Calendar.getInstance();
			c.add(Calendar.YEAR, -90);
			beginDate = c.getTime();
		}
		siList = dao.listByDateRange(beginDate, toDate, unapprovedOnlyFlag,
				surveyId, source, cursorString);
		String newCursor = SurveyInstanceDAO.getCursor(siList);

		ArrayList<SurveyInstanceDto> siDtoList = new ArrayList<SurveyInstanceDto>();
		for (SurveyInstance siItem : siList) {
			String code = surveyMap.get(siItem.getSurveyId());
			SurveyInstanceDto siDto = marshalToDto(siItem);
			if (code != null)
				siDto.setSurveyCode(code);
			siDtoList.add(siDto);

		}
		ResponseDto<ArrayList<SurveyInstanceDto>> response = new ResponseDto<ArrayList<SurveyInstanceDto>>();
		response.setCursorString(newCursor);
		response.setPayload(siDtoList);
		return response;
	}

	public List<QuestionAnswerStoreDto> listQuestionsByInstance(Long instanceId) {
		List<QuestionAnswerStoreDto> questionDtos = new ArrayList<QuestionAnswerStoreDto>();
		SurveyInstanceDAO dao = new SurveyInstanceDAO();
		List<QuestionAnswerStore> questions = dao.listQuestionAnswerStore(
				instanceId, null);
		QuestionDao qDao = new QuestionDao();

		if (questions != null && questions.size() > 0) {
			List<Question> qList = qDao.listQuestionInOrder(questions.get(0)
					.getSurveyId());
			Map<Integer, QuestionAnswerStoreDto> orderedResults = new TreeMap<Integer, QuestionAnswerStoreDto>();
			int notFoundCount = 0;
			if (qList != null) {
				for (QuestionAnswerStore qas : questions) {
					QuestionAnswerStoreDto qasDto = new QuestionAnswerStoreDto();
					DtoMarshaller.copyToDto(qas, qasDto);
					int idx = -1 - notFoundCount;
					for (int i = 0; i < qList.size(); i++) {
						if (Long.parseLong(qas.getQuestionID()) == qList.get(i)
								.getKey().getId()) {
							qasDto.setQuestionText(qList.get(i).getText());
							idx = i;
							break;
						}
					}
					if (idx < 0) {
						// do this to prevent collisions on the -1 key if there
						// is more than one questionAnswerStore item that isn't
						// in the question list
						notFoundCount++;
					}
					orderedResults.put(idx, qasDto);
				}
			}
			questionDtos = new ArrayList<QuestionAnswerStoreDto>(
					orderedResults.values());
		}
		return questionDtos;
	}

	private SurveyInstanceDto marshalToDto(SurveyInstance si) {
		SurveyInstanceDto siDto = new SurveyInstanceDto();
		DtoMarshaller.copyToDto(si, siDto);
		siDto.setQuestionAnswersStore(null);
		if (si.getQuestionAnswersStore() != null) {
			for (QuestionAnswerStore qas : si.getQuestionAnswersStore()) {
				siDto.addQuestionAnswerStore(marshalToDto(qas));
			}
		}
		return siDto;
	}

	private QuestionAnswerStoreDto marshalToDto(QuestionAnswerStore qas) {
		QuestionAnswerStoreDto qasDto = new QuestionAnswerStoreDto();
		DtoMarshaller.copyToDto(qas, qasDto);
		return qasDto;
	}

	/**
	 * updates the list of QAS dto objects passed in and fires summarization
	 * messages to the async queues
	 */
	@Override
	public List<QuestionAnswerStoreDto> updateQuestions(
			List<QuestionAnswerStoreDto> dtoList, boolean isApproved) {
		return updateQuestions(dtoList, isApproved, true);
	}

	public List<QuestionAnswerStoreDto> updateQuestions(
			List<QuestionAnswerStoreDto> dtoList, boolean isApproved,
			boolean processSummaries) {
		List<QuestionAnswerStore> domainList = new ArrayList<QuestionAnswerStore>();
		for (QuestionAnswerStoreDto dto : dtoList) {
			QuestionAnswerStore answer = new QuestionAnswerStore();
			DtoMarshaller.copyToCanonical(answer, dto);
			if (answer.getValue() != null) {
				answer.setValue(answer.getValue().replaceAll("\t", ""));
			}
			domainList.add(answer);
		}
		SurveyInstanceDAO dao = new SurveyInstanceDAO();
		dao.save(domainList);
		
		// this is not active - questionAnswerSummary objects are updated in bulk 
		// after the import  in RawDataSpreadsheetImporter
		if (isApproved && processSummaries) {
			SurveyAttributeMappingDao mappingDao = new SurveyAttributeMappingDao();
			// now send a change message for each item
			Queue queue = QueueFactory.getQueue("dataUpdate");
			for (QuestionAnswerStoreDto item : dtoList) {
				DataChangeRecord value = new DataChangeRecord(
						QuestionAnswerStore.class.getName(),
						item.getQuestionID(), item.getOldValue(),
						item.getValue());
				queue.add(TaskOptions.Builder
						.withUrl("/app_worker/dataupdate")
						.param(DataSummarizationRequest.OBJECT_KEY,
								item.getQuestionID())
						.param(DataSummarizationRequest.OBJECT_TYPE,
								"QuestionDataChange")
						.param(DataSummarizationRequest.VALUE_KEY,
								value.packString()));
				// see if the question is mapped. And if it is, send an Access
				// Point
				// change message
{'fraction_non_alphanumeric': 0.10085999747059568, 'fraction_numerical': 0.005122043758694827, 'mean_word_length': 4.694994598487576, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '835565', 'n_tokens_mistral': 5497, 'n_tokens_neox': 4887, 'n_words': 1091}