<commit_before>package ca.corefacility.bioinformatics.irida.ria.web.projects;

import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.security.Principal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;

import javax.servlet.http.HttpSession;

import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;

import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException;
import ca.corefacility.bioinformatics.irida.exceptions.MetadataImportFileTypeNotSupportedError;
import ca.corefacility.bioinformatics.irida.model.project.Project;
import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplate;
import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField;
import ca.corefacility.bioinformatics.irida.model.sample.Sample;
import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry;
import ca.corefacility.bioinformatics.irida.ria.utilities.SampleMetadataStorage;
import ca.corefacility.bioinformatics.irida.service.ProjectService;
import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService;
import ca.corefacility.bioinformatics.irida.service.sample.SampleService;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;

/**
 * This class is designed to be used for bulk actions on {@link MetadataEntry}
 * within a {@link Project}.
 */
@Controller
@RequestMapping("/projects/{projectId}/sample-metadata")
public class ProjectSampleMetadataController {
	private static final Logger logger = LoggerFactory.getLogger(ProjectSampleMetadataController.class);
	private final MessageSource messageSource;
	private final ProjectControllerUtils projectControllerUtils;
	private final ProjectService projectService;
	private final SampleService sampleService;
	private final MetadataTemplateService metadataTemplateService;

	@Autowired
	public ProjectSampleMetadataController(MessageSource messageSource, ProjectControllerUtils projectControllerUtils,
			ProjectService projectService, SampleService sampleService, MetadataTemplateService metadataTemplateService) {
		this.messageSource = messageSource;
		this.projectControllerUtils = projectControllerUtils;
		this.projectService = projectService;
		this.sampleService = sampleService;
		this.metadataTemplateService = metadataTemplateService;
	}

	/**
	 * Get the page to create a new {@link MetadataTemplate}
	 *
	 * @param projectId
	 * 		the {@link Long} identifier for the current project
	 * @param model
	 * 		{@link Model}
	 * @param principal
	 * 		{@link Principal} currently logged in user
	 * @param locale
	 * 		{@link Locale} of the logged in user.
	 *
	 * @return {@link String} path to the page
	 */
	@RequestMapping("/template")
	public String getCreateNewSampleMetadataTemplatePage(@PathVariable Long projectId,
			@RequestParam(required = false) Long templateId,
			Model model, Principal principal,
			Locale locale) {
		// Set up the template information
		Project project = projectService.read(projectId);
		projectControllerUtils.getProjectTemplateDetails(model, principal, project);
		return "projects/project_samples_metadata_template";
	}

	/**
	 * Search all Metadata keys available for adding to a template.
	 * 
	 * @param query
	 *            the query to search for
	 * @return a list of keys matching the query
	 */
	@RequestMapping("/fields")
	@ResponseBody
	public List<String> getMetadataKeysForProject(@RequestParam String query) {
		return metadataTemplateService.getAllMetadataFieldsByQueryString(query).stream()
				.map(MetadataTemplateField::getLabel).collect(Collectors.toList());
	}

	/**
	 * Handle the page request to upload {@link Sample} metadata
	 *
	 * @param model
	 * 		{@link Model}
	 * @param projectId
	 * 		{@link Long} identifier for the current {@link Project}
	 *
	 * @return {@link String} the path to the metadata import page
	 */
	@RequestMapping(value = "upload", method = RequestMethod.GET)
	public String getProjectSamplesMetadataUploadPage(final Model model, @PathVariable long projectId) {
		model.addAttribute("project", projectService.read(projectId));
		return "projects/project_samples_metadata_upload";
	}

	/**
	 * Upload Excel file containing sample metadata and extract the headers.  The file is stored in the session until
	 * the column that corresponds to a {@link Sample} identifier has been sent.
	 *
	 * @param session
	 * 		{@link HttpSession}
	 * @param projectId
	 * 		{@link Long} identifier for the current {@link Project}
	 * @param file
	 * 		{@link MultipartFile} The excel file containing the metadata.
	 *
	 * @return {@link Map} of headers and rows from the excel file for the user to select the header corresponding the
	 * {@link Sample} identifier.
	 */
	@RequestMapping(value = "/upload/file", method = RequestMethod.POST)
	@ResponseBody
	public SampleMetadataStorage createProjectSampleMetadata(HttpSession session, @PathVariable long projectId,
			@RequestParam("file") MultipartFile file) throws MetadataImportFileTypeNotSupportedError {
		// We want to return a list of the table headers back to the UI.
		SampleMetadataStorage storage = new SampleMetadataStorage();
		try {
			// Need an input stream
			String filename = file.getOriginalFilename();
			byte[] byteArr = file.getBytes();
			InputStream fis = new ByteArrayInputStream(byteArr);

			Workbook workbook;
			String extension = Files.getFileExtension(filename);

			// Check the type of workbook
			switch (extension) {
			case "xlsx":
				workbook = new XSSFWorkbook(fis);
				break;
			case "xls":
				workbook = new HSSFWorkbook(fis);
				break;
			default:
				// Should never reach here as the uploader limits to .xlsx and .xlx files.
				throw new MetadataImportFileTypeNotSupportedError(extension);
			}

			// Only look at the first sheet in the workbook as this should be the file we want.
			Sheet sheet = workbook.getSheetAt(0);
			Iterator<Row> rowIterator = sheet.iterator();

			List<String> headers = getWorkbookHeaders(rowIterator.next());
			storage.saveHeaders(headers);

			// Get the metadata out of the table.
			List<Map<String, String>> rows = new ArrayList<>();
			while (rowIterator.hasNext()) {
				int headerCounter = 0;
				Map<String, String> rowMap = new HashMap<>();
				Row row = rowIterator.next();
				Iterator<Cell> cellIterator = row.cellIterator();
				while (cellIterator.hasNext() && headerCounter < headers.size()) {
					Cell cell = cellIterator.next();
					String cellValue = cell.getStringCellValue().trim();
					String header = headers.get(headerCounter).trim();
					rowMap.put(header, cellValue);
					headerCounter += 1;
				}
				rows.add(rowMap);
			}
			storage.saveRows(rows);

			fis.close();
		} catch (FileNotFoundException e) {
			logger.debug("No file found for uploading an excel file of metadata.");
		} catch (IOException e) {
			logger.error("Error opening file" + file.getOriginalFilename());
		}

		session.setAttribute("pm-" + projectId, storage);
		return storage;
	}

	/**
	 * Extract the headers from an excel file.
	 *
	 * @param row
	 * 		{@link Row} First row from the excel file.
	 *
	 * @return {@link List} of {@link String} header values.
	 */
	private List<String> getWorkbookHeaders(Row row) {
		// We want to return a list of the table headers back to the UI.<commit_msg>set cell type to string on metadata upload<commit_after>package ca.corefacility.bioinformatics.irida.ria.web.projects;

import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.security.Principal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;

import javax.servlet.http.HttpSession;

import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;

import ca.corefacility.bioinformatics.irida.exceptions.EntityNotFoundException;
import ca.corefacility.bioinformatics.irida.exceptions.MetadataImportFileTypeNotSupportedError;
import ca.corefacility.bioinformatics.irida.model.project.Project;
import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplate;
import ca.corefacility.bioinformatics.irida.model.sample.MetadataTemplateField;
import ca.corefacility.bioinformatics.irida.model.sample.Sample;
import ca.corefacility.bioinformatics.irida.model.sample.metadata.MetadataEntry;
import ca.corefacility.bioinformatics.irida.ria.utilities.SampleMetadataStorage;
import ca.corefacility.bioinformatics.irida.service.ProjectService;
import ca.corefacility.bioinformatics.irida.service.sample.MetadataTemplateService;
import ca.corefacility.bioinformatics.irida.service.sample.SampleService;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;

/**
 * This class is designed to be used for bulk actions on {@link MetadataEntry}
 * within a {@link Project}.
 */
@Controller
@RequestMapping("/projects/{projectId}/sample-metadata")
public class ProjectSampleMetadataController {
	private static final Logger logger = LoggerFactory.getLogger(ProjectSampleMetadataController.class);
	private final MessageSource messageSource;
	private final ProjectControllerUtils projectControllerUtils;
	private final ProjectService projectService;
	private final SampleService sampleService;
	private final MetadataTemplateService metadataTemplateService;

	@Autowired
	public ProjectSampleMetadataController(MessageSource messageSource, ProjectControllerUtils projectControllerUtils,
			ProjectService projectService, SampleService sampleService, MetadataTemplateService metadataTemplateService) {
		this.messageSource = messageSource;
		this.projectControllerUtils = projectControllerUtils;
		this.projectService = projectService;
		this.sampleService = sampleService;
		this.metadataTemplateService = metadataTemplateService;
	}

	/**
	 * Get the page to create a new {@link MetadataTemplate}
	 *
	 * @param projectId
	 * 		the {@link Long} identifier for the current project
	 * @param model
	 * 		{@link Model}
	 * @param principal
	 * 		{@link Principal} currently logged in user
	 * @param locale
	 * 		{@link Locale} of the logged in user.
	 *
	 * @return {@link String} path to the page
	 */
	@RequestMapping("/template")
	public String getCreateNewSampleMetadataTemplatePage(@PathVariable Long projectId,
			@RequestParam(required = false) Long templateId,
			Model model, Principal principal,
			Locale locale) {
		// Set up the template information
		Project project = projectService.read(projectId);
		projectControllerUtils.getProjectTemplateDetails(model, principal, project);
		return "projects/project_samples_metadata_template";
	}

	/**
	 * Search all Metadata keys available for adding to a template.
	 * 
	 * @param query
	 *            the query to search for
	 * @return a list of keys matching the query
	 */
	@RequestMapping("/fields")
	@ResponseBody
	public List<String> getMetadataKeysForProject(@RequestParam String query) {
		return metadataTemplateService.getAllMetadataFieldsByQueryString(query).stream()
				.map(MetadataTemplateField::getLabel).collect(Collectors.toList());
	}

	/**
	 * Handle the page request to upload {@link Sample} metadata
	 *
	 * @param model
	 * 		{@link Model}
	 * @param projectId
	 * 		{@link Long} identifier for the current {@link Project}
	 *
	 * @return {@link String} the path to the metadata import page
	 */
	@RequestMapping(value = "upload", method = RequestMethod.GET)
	public String getProjectSamplesMetadataUploadPage(final Model model, @PathVariable long projectId) {
		model.addAttribute("project", projectService.read(projectId));
		return "projects/project_samples_metadata_upload";
	}

	/**
	 * Upload Excel file containing sample metadata and extract the headers.  The file is stored in the session until
	 * the column that corresponds to a {@link Sample} identifier has been sent.
	 *
	 * @param session
	 * 		{@link HttpSession}
	 * @param projectId
	 * 		{@link Long} identifier for the current {@link Project}
	 * @param file
	 * 		{@link MultipartFile} The excel file containing the metadata.
	 *
	 * @return {@link Map} of headers and rows from the excel file for the user to select the header corresponding the
	 * {@link Sample} identifier.
	 */
	@RequestMapping(value = "/upload/file", method = RequestMethod.POST)
	@ResponseBody
	public SampleMetadataStorage createProjectSampleMetadata(HttpSession session, @PathVariable long projectId,
			@RequestParam("file") MultipartFile file) throws MetadataImportFileTypeNotSupportedError {
		// We want to return a list of the table headers back to the UI.
		SampleMetadataStorage storage = new SampleMetadataStorage();
		try {
			// Need an input stream
			String filename = file.getOriginalFilename();
			byte[] byteArr = file.getBytes();
			InputStream fis = new ByteArrayInputStream(byteArr);

			Workbook workbook;
			String extension = Files.getFileExtension(filename);

			// Check the type of workbook
			switch (extension) {
			case "xlsx":
				workbook = new XSSFWorkbook(fis);
				break;
			case "xls":
				workbook = new HSSFWorkbook(fis);
				break;
			default:
				// Should never reach here as the uploader limits to .xlsx and .xlx files.
				throw new MetadataImportFileTypeNotSupportedError(extension);
			}

			// Only look at the first sheet in the workbook as this should be the file we want.
			Sheet sheet = workbook.getSheetAt(0);
			Iterator<Row> rowIterator = sheet.iterator();

			List<String> headers = getWorkbookHeaders(rowIterator.next());
			storage.saveHeaders(headers);

			// Get the metadata out of the table.
			List<Map<String, String>> rows = new ArrayList<>();
			while (rowIterator.hasNext()) {
				int headerCounter = 0;
				Map<String, String> rowMap = new HashMap<>();
				Row row = rowIterator.next();
				Iterator<Cell> cellIterator = row.cellIterator();
				while (cellIterator.hasNext() && headerCounter < headers.size()) {
					Cell cell = cellIterator.next();
					cell.setCellType(Cell.CELL_TYPE_STRING);
					String cellValue = cell.getStringCellValue().trim();
					String header = headers.get(headerCounter).trim();
					rowMap.put(header, cellValue);
					headerCounter += 1;
				}
				rows.add(rowMap);
			}
			storage.saveRows(rows);

			fis.close();
		} catch (FileNotFoundException e) {
			logger.debug("No file found for uploading an excel file of metadata.");
		} catch (IOException e) {
			logger.error("Error opening file" + file.getOriginalFilename());
		}

		session.setAttribute("pm-" + projectId, storage);
		return storage;
	}

	/**
	 * Extract the headers from an excel file.
	 *
	 * @param row
	 * 		{@link Row} First row from the excel file.
	 *
	 * @return {@link List} of {@link String} header values.
	 */
	private List<String> getWorkbookHeaders(Row row) {
		// We want to return a list of the table headers back to the UI.
{'fraction_non_alphanumeric': 0.09657911898974328, 'fraction_numerical': 0.0005928736586233473, 'mean_word_length': 4.820565907522429, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1122207', 'n_tokens_mistral': 5110, 'n_tokens_neox': 4591, 'n_words': 1277}