<commit_before>package org.cytoscape.io.internal.read.datatable;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.cytoscape.equations.Equation;
import org.cytoscape.equations.EquationCompiler;
import org.cytoscape.equations.EquationUtil;
import org.cytoscape.io.read.CyTableReader;
import org.cytoscape.model.CyRow;
import org.cytoscape.model.CyTable;
import org.cytoscape.model.CyTableFactory;
import org.cytoscape.service.util.CyServiceRegistrar;
import org.cytoscape.work.TaskMonitor;

import au.com.bytecode.opencsv.CSVReader;

/*
 * #%L
 * Cytoscape IO Impl (io-impl)
 * $Id:$
 * $HeadURL:$
 * %%
 * Copyright (C) 2006 - 2016 The Cytoscape Consortium
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as 
 * published by the Free Software Foundation, either version 2.1 of the 
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Lesser Public License for more details.
 * 
 * You should have received a copy of the GNU General Lesser Public 
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
 * #L%
 */

public class CSVCyReader implements CyTableReader {
	
	private final static Pattern classPattern = Pattern.compile("([^<>]+)(<(.*?)>)?");

	private final InputStream stream;
	private final boolean readSchema;
	private final boolean handleEquations;
	private final String encoding;

	private boolean isCanceled;
	private CyTable table;
	
	private final CyServiceRegistrar serviceRegistrar;

	public CSVCyReader(
			final InputStream stream,
			final boolean readSchema,
			final boolean handleEquations, 
			final String encoding,
			final CyServiceRegistrar serviceRegistrar
	) {
		this.stream = stream;
		this.readSchema = readSchema;
		this.handleEquations = handleEquations;
		this.encoding = encoding;
		this.serviceRegistrar = serviceRegistrar;
	}

	@Override
	public void cancel() {
		isCanceled = true;
	}

	@Override
	public void run(TaskMonitor taskMonitor) throws Exception {
		taskMonitor.setProgress(0.0);

		CSVReader reader = new CSVReader(new InputStreamReader(stream, encoding), ',', '"', '\0');
		taskMonitor.setProgress(0.2);

		TableInfo info = readHeader(reader);
		table = createTable(reader, info);
		taskMonitor.setProgress(1.0);
	}

	CyTable createTable(CSVReader reader, TableInfo info) throws IOException, SecurityException {
		final ColumnInfo[] columns = info.getColumns();
		final CyTableFactory tableFactory = serviceRegistrar.getService(CyTableFactory.class);
		
		final CyTable table = tableFactory.createTable(info.getTitle(), columns[0].getName(),
		                                               columns[0].getType(), info.isPublic(),
		                                               true);

		final Map<String, Class<?>> variableNameToTypeMap = new HashMap<>();
		
		for (final ColumnInfo colInfo : columns)
			variableNameToTypeMap.put(colInfo.getName(), colInfo.getType() == Integer.class ? Long.class : colInfo.getType());

		for (int i = 1; i < columns.length; i++) {
			ColumnInfo column = columns[i];
			Class<?> type = column.getType();
			
			if (type.equals(List.class)) {
				table.createListColumn(column.getName(), column.getListElementType(), !column.isMutable());
			} else {
				table.createColumn(column.getName(), type, !column.isMutable());
			}
		}
		
		final EquationCompiler compiler = serviceRegistrar.getService(EquationCompiler.class);
		String[] values = reader.readNext();
		
		while (values != null) {
			if (isCanceled)
				return null;

			Object key = parseValue(columns[0].getType(), null, values[0]);
			CyRow row = table.getRow(key);
			
			for (int i = 1; i < values.length; i++) {
				ColumnInfo column = columns[i];
				String name = column.getName();
				final Class<?> columnType = column.getType();
				final Class<?> columnListElementType = column.getListElementType();
				
				if (handleEquations && values[i].startsWith("=")) {
					final Class<?> expectedType = variableNameToTypeMap.remove(name);
					
					try {
						final Equation equation;
						final Class<?> eqnType;
						
						if (compiler.compile(values[i], variableNameToTypeMap)) {
							eqnType = compiler.getEquation().getType();
							if(EquationUtil.eqnTypeIsCompatible(columnType, columnListElementType, eqnType))
								equation = compiler.getEquation();
							else {
								final String errorMsg = "Equation result type is "
									+ EquationUtil.getUnqualifiedName(eqnType) + ", column type is "
									+ EquationUtil.getUnqualifiedName(columnType) + ".";
								equation = compiler.getErrorEquation(values[i], expectedType, errorMsg);
							}
						} else {
							equation = compiler.getErrorEquation(values[i], expectedType, compiler.getLastErrorMsg());
						}
						
						row.set(name, equation);
					} catch (final Exception e) {
						throw new IOException(e.getMessage(), e.getCause());
					}
					
					variableNameToTypeMap.put(name, expectedType);
				} else {
					Object value = parseValue(columnType, columnListElementType, values[i]);
					
					if (value != null)
						row.set(name, value);
				}
			}
			
			values = reader.readNext();
		}
		return table;
	}

	Object parseValue(Class<?> type, Class<?> listElementType, String value) {
		if (type.equals(List.class)) {
			List<Object> list = new ArrayList<>();
			String[] values = value.split("\n");
			
			for (String item : values) {
				list.add(parseValue(listElementType, null, item));
			}

			if (list.size() == 1 && list.get(0) == null) 
				return null;

			return list;
		} else if (type.equals(String.class))
			return value;
		else {
			try {
				Method method = type.getMethod("valueOf", String.class);
				return method.invoke(null, value);
			} catch (Exception e) {
				return null;
			}
		}
	}

	TableInfo readHeader(CSVReader reader) throws IOException, ClassNotFoundException {
		String[] values = reader.readNext();
		int schemaVersion;
		if (values.length == 2 && "CyCSV-Version".equals(values[0])) {
			schemaVersion = Integer.parseInt(values[1]);
			values = reader.readNext();
		} else {
			schemaVersion = 0;
		}
		
		TableInfo table = new TableInfo();
		ColumnInfo[] columns = new ColumnInfo[values.length];
		for (int i = 0; i < values.length; i++) {
			ColumnInfo column = new ColumnInfo();
			column.setName(values[i]);
			columns[i] = column;
		}
		table.setColumns(columns);
		if (!readSchema) {<commit_msg>Improve performance of table parsing during session loading. Refs #4218 <commit_after>package org.cytoscape.io.internal.read.datatable;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.cytoscape.equations.Equation;
import org.cytoscape.equations.EquationCompiler;
import org.cytoscape.equations.EquationUtil;
import org.cytoscape.io.read.CyTableReader;
import org.cytoscape.model.CyRow;
import org.cytoscape.model.CyTable;
import org.cytoscape.model.CyTableFactory;
import org.cytoscape.service.util.CyServiceRegistrar;
import org.cytoscape.work.TaskMonitor;

import au.com.bytecode.opencsv.CSVReader;

/*
 * #%L
 * Cytoscape IO Impl (io-impl)
 * $Id:$
 * $HeadURL:$
 * %%
 * Copyright (C) 2006 - 2016 The Cytoscape Consortium
 * %%
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as 
 * published by the Free Software Foundation, either version 2.1 of the 
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Lesser Public License for more details.
 * 
 * You should have received a copy of the GNU General Lesser Public 
 * License along with this program.  If not, see
 * <http://www.gnu.org/licenses/lgpl-2.1.html>.
 * #L%
 */

public class CSVCyReader implements CyTableReader {
	
	private final static Pattern classPattern = Pattern.compile("([^<>]+)(<(.*?)>)?");

	private final InputStream stream;
	private final boolean readSchema;
	private final boolean handleEquations;
	private final String encoding;

	private boolean isCanceled;
	private CyTable table;
	
	private final CyServiceRegistrar serviceRegistrar;

	public CSVCyReader(
			final InputStream stream,
			final boolean readSchema,
			final boolean handleEquations, 
			final String encoding,
			final CyServiceRegistrar serviceRegistrar
	) {
		this.stream = stream;
		this.readSchema = readSchema;
		this.handleEquations = handleEquations;
		this.encoding = encoding;
		this.serviceRegistrar = serviceRegistrar;
	}

	@Override
	public void cancel() {
		isCanceled = true;
	}

	@Override
	public void run(TaskMonitor taskMonitor) throws Exception {
		taskMonitor.setProgress(0.0);

		CSVReader reader = new CSVReader(new InputStreamReader(stream, encoding), ',', '"', '\0');
		taskMonitor.setProgress(0.2);

		TableInfo info = readHeader(reader);
		table = createTable(reader, info);
		taskMonitor.setProgress(1.0);
	}

	CyTable createTable(CSVReader reader, TableInfo info) throws IOException, SecurityException {
		final ColumnInfo[] columns = info.getColumns();
		final CyTableFactory tableFactory = serviceRegistrar.getService(CyTableFactory.class);
		
		final CyTable table = tableFactory.createTable(info.getTitle(), columns[0].getName(),
		                                               columns[0].getType(), info.isPublic(),
		                                               true);

		final Map<String, Class<?>> variableNameToTypeMap = new HashMap<>();
		
		for (final ColumnInfo colInfo : columns)
			variableNameToTypeMap.put(colInfo.getName(), colInfo.getType() == Integer.class ? Long.class : colInfo.getType());

		for (int i = 1; i < columns.length; i++) {
			ColumnInfo column = columns[i];
			Class<?> type = column.getType();
			
			if (type.equals(List.class)) {
				table.createListColumn(column.getName(), column.getListElementType(), !column.isMutable());
			} else {
				table.createColumn(column.getName(), type, !column.isMutable());
			}
		}
		
		final EquationCompiler compiler = serviceRegistrar.getService(EquationCompiler.class);
		String[] values = reader.readNext();
		
		while (values != null) {
			if (isCanceled)
				return null;

			Object key = parseValue(columns[0].getType(), null, values[0]);
			CyRow row = table.getRow(key);
			
			for (int i = 1; i < values.length; i++) {
				ColumnInfo column = columns[i];
				String name = column.getName();
				final Class<?> columnType = column.getType();
				final Class<?> columnListElementType = column.getListElementType();
				
				if (handleEquations && values[i].startsWith("=")) {
					final Class<?> expectedType = variableNameToTypeMap.remove(name);
					
					try {
						final Equation equation;
						final Class<?> eqnType;
						
						if (compiler.compile(values[i], variableNameToTypeMap)) {
							eqnType = compiler.getEquation().getType();
							if(EquationUtil.eqnTypeIsCompatible(columnType, columnListElementType, eqnType))
								equation = compiler.getEquation();
							else {
								final String errorMsg = "Equation result type is "
									+ EquationUtil.getUnqualifiedName(eqnType) + ", column type is "
									+ EquationUtil.getUnqualifiedName(columnType) + ".";
								equation = compiler.getErrorEquation(values[i], expectedType, errorMsg);
							}
						} else {
							equation = compiler.getErrorEquation(values[i], expectedType, compiler.getLastErrorMsg());
						}
						
						row.set(name, equation);
					} catch (final Exception e) {
						throw new IOException(e.getMessage(), e.getCause());
					}
					
					variableNameToTypeMap.put(name, expectedType);
				} else {
					Object value = parseValue(columnType, columnListElementType, values[i]);
					
					if (value != null)
						row.set(name, value);
				}
			}
			
			values = reader.readNext();
		}
		return table;
	}

	Object parseValue(Class<?> type, Class<?> listElementType, String value) {
		if (type.equals(List.class)) {
			List<Object> list = new ArrayList<>();
			String[] values = value.split("\n");
			for (String item : values) {
				list.add(parseValue(listElementType, null, item));
			}
			if (list.size() == 1 && list.get(0) == null) 
				return null;
			return list;
		} else if (type.equals(String.class)) {
			return value;
		} else if(value.isEmpty()) {
			return null;
		} else {
			try {
				if (type.equals(Long.class)) {
					return Long.valueOf(value);
				} else if (type.equals(Boolean.class)) {
					return Boolean.valueOf(value);
				} else if (type.equals(Double.class)) {
					return Double.valueOf(value);
				} else if (type.equals(Integer.class)) {
					return Integer.valueOf(value); 
				} else {
					Method method = type.getMethod("valueOf", String.class);
					return method.invoke(null, value);
				}
			} catch (Exception e) {
				return null;
			}
		}
	}

	TableInfo readHeader(CSVReader reader) throws IOException, ClassNotFoundException {
		String[] values = reader.readNext();
		int schemaVersion;
		if (values.length == 2 && "CyCSV-Version".equals(values[0])) {
			schemaVersion = Integer.parseInt(values[1]);
			values = reader.readNext();
		} else {
			schemaVersion = 0;
		}
		
		TableInfo table = new TableInfo();
		ColumnInfo[] columns = new ColumnInfo[values.length];
		for (int i = 0; i < values.length; i++) {
			ColumnInfo column = new ColumnInfo();
			column.setName(values[i]);
			columns[i] = column;
		}
		table.setColumns(columns);
		if (!readSchema) {
{'fraction_non_alphanumeric': 0.12032932235592148, 'fraction_numerical': 0.0047850256843290405, 'mean_word_length': 3.6038224813735016, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2147850', 'n_tokens_mistral': 5007, 'n_tokens_neox': 4141, 'n_words': 1090}