<commit_before>package fi.csc.chipster.comp;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;

import org.springframework.util.StringUtils;

import fi.csc.chipster.sessiondb.RestException;
import fi.csc.chipster.sessiondb.SessionDbClient;
import fi.csc.chipster.sessiondb.model.Dataset;
import fi.csc.chipster.sessiondb.model.Input;
import fi.csc.chipster.sessiondb.model.MetadataEntry;
import fi.csc.microarray.messaging.message.GenericResultMessage;
import fi.csc.microarray.util.ToolUtils;

public class RestPhenodataUtils {

	public static final String FILE_PHENODATA2 = "phenodata2.tsv";
	public static final String FILE_PHENODATA = "phenodata.tsv";
	
	public static final String HEADER_SAMPLE = "sample";
	public static final String HEADER_COLUMN = "column";
	public static final String HEADER_DATASET = "dataset";
	
	public static final String PREFIX_CHIP = "chip.";

	public static void writePhenodata(File jobWorkDir, HashMap<String, List<MetadataEntry>> metadata, boolean columnPhenodata, boolean fullPhenodata) throws IOException {
		
		
		// safety checks
		for (String inputId : metadata.keySet()) {
			List<MetadataEntry> datasetMetadatas = metadata.get(inputId);
			for (MetadataEntry entry : datasetMetadatas) {
				
				if (!entry.getColumn().matches(ToolUtils.NAME_PATTERN)) {
					throw new IllegalArgumentException("Dataset column " + entry.getColumn() + " contains illegal characters.");
				}
				
				if (!entry.getKey().matches(ToolUtils.NAME_PATTERN)) {
					throw new IllegalArgumentException("Phenodata column " + entry.getKey() + " contains illegal characters.");
				}
				
				if (!entry.getValue().matches(ToolUtils.NAME_PATTERN)) {
					throw new IllegalArgumentException("Phenodata value " + entry.getValue() + " contains illegal characters.");
				}
			}
		}
		
		ArrayList<String> headers = getHeaders(metadata);
		ArrayList<PhenodataRow> array = getArray(metadata, headers);	
		
		HashSet<String> uniqueInputs = new HashSet<>();
		for (PhenodataRow row : array) {
			uniqueInputs.add(row.getInputId());
		}
		
		// write column phenodata (microarray)
		if (columnPhenodata) {
			// the legacy format doesn't support multiple datasets
			if (uniqueInputs.size() > 1) {
				throw new IllegalArgumentException(
						"The tool requires phenodata in the old format, which doesn't support "
								+ "multipele datasets, but more than one input dataset contains phenodata. "
								+ "Please remove the phenodata form other input datasets to indicate which "
								+ "one to use.");
			}
			try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(jobWorkDir, FILE_PHENODATA)))) {
<commit_msg>Allow null columns and values in the phenodata<commit_after>package fi.csc.chipster.comp;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;

import org.springframework.util.StringUtils;

import fi.csc.chipster.sessiondb.RestException;
import fi.csc.chipster.sessiondb.SessionDbClient;
import fi.csc.chipster.sessiondb.model.Dataset;
import fi.csc.chipster.sessiondb.model.Input;
import fi.csc.chipster.sessiondb.model.MetadataEntry;
import fi.csc.microarray.messaging.message.GenericResultMessage;
import fi.csc.microarray.util.ToolUtils;

public class RestPhenodataUtils {

	public static final String FILE_PHENODATA2 = "phenodata2.tsv";
	public static final String FILE_PHENODATA = "phenodata.tsv";
	
	public static final String HEADER_SAMPLE = "sample";
	public static final String HEADER_COLUMN = "column";
	public static final String HEADER_DATASET = "dataset";
	
	public static final String PREFIX_CHIP = "chip.";

	public static void writePhenodata(File jobWorkDir, HashMap<String, List<MetadataEntry>> metadata, boolean columnPhenodata, boolean fullPhenodata) throws IOException {
		
		
		// safety checks
		for (String inputId : metadata.keySet()) {
			List<MetadataEntry> datasetMetadatas = metadata.get(inputId);
			for (MetadataEntry entry : datasetMetadatas) {
				
				if (entry.getColumn() != null && !entry.getColumn().matches(ToolUtils.NAME_PATTERN)) {
					throw new IllegalArgumentException("Dataset column " + entry.getColumn() + " contains illegal characters.");
				}
				
				if (!entry.getKey().matches(ToolUtils.NAME_PATTERN)) {
					throw new IllegalArgumentException("Phenodata column " + entry.getKey() + " contains illegal characters.");
				}
				
				if (entry.getValue() != null && !entry.getValue().matches(ToolUtils.NAME_PATTERN)) {
					throw new IllegalArgumentException("Phenodata value " + entry.getValue() + " contains illegal characters.");
				}
			}
		}
		
		ArrayList<String> headers = getHeaders(metadata);
		ArrayList<PhenodataRow> array = getArray(metadata, headers);	
		
		HashSet<String> uniqueInputs = new HashSet<>();
		for (PhenodataRow row : array) {
			uniqueInputs.add(row.getInputId());
		}
		
		// write column phenodata (microarray)
		if (columnPhenodata) {
			// the legacy format doesn't support multiple datasets
			if (uniqueInputs.size() > 1) {
				throw new IllegalArgumentException(
						"The tool requires phenodata in the old format, which doesn't support "
								+ "multipele datasets, but more than one input dataset contains phenodata. "
								+ "Please remove the phenodata form other input datasets to indicate which "
								+ "one to use.");
			}
			try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(jobWorkDir, FILE_PHENODATA)))) {

{'fraction_non_alphanumeric': 0.1043338683788122, 'fraction_numerical': 0.0009630818619582664, 'mean_word_length': 4.593357271095153, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4821224', 'n_tokens_mistral': 2075, 'n_tokens_neox': 1834, 'n_words': 463}