<commit_before>/*
 * Copyright (c) 2017 Data and Web Science Group, University of Mannheim, Germany (http://dws.informatik.uni-mannheim.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */
package de.uni_mannheim.informatik.dws.winter.webtables;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import de.uni_mannheim.informatik.dws.winter.model.Pair;
import de.uni_mannheim.informatik.dws.winter.preprocessing.datatypes.ColumnType;
import de.uni_mannheim.informatik.dws.winter.preprocessing.datatypes.DataType;
import de.uni_mannheim.informatik.dws.winter.preprocessing.datatypes.TypeConverter;
import de.uni_mannheim.informatik.dws.winter.utils.MapUtils;
import de.uni_mannheim.informatik.dws.winter.utils.StringUtils;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Consumer;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Parallel;
import de.uni_mannheim.informatik.dws.winter.utils.query.Func;
import de.uni_mannheim.informatik.dws.winter.utils.query.Q;
import de.uni_mannheim.informatik.dws.winter.webtables.detectors.TableKeyIdentification;
import de.uni_mannheim.informatik.dws.winter.webtables.detectors.TypeDetector;
import de.uni_mannheim.informatik.dws.winter.webtables.detectors.TypeGuesser;

/**
 * Represents a Web Table.
 * 
 * @author <NAME> (<EMAIL>)
 *
 */
public class Table implements Serializable {

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;

	public static class TablePathComparator implements Comparator<Table> {

		/*
		 * (non-Javadoc)
		 * 
		 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
		 */
		@Override
		public int compare(Table o1, Table o2) {
			return o1.getPath().compareTo(o2.getPath());
		}

	}

	public static class TableIdComparator implements Comparator<Table> {

		/*
		 * (non-Javadoc)
		 * 
		 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
		 */
		@Override
		public int compare(Table o1, Table o2) {
			return Integer.compare(o1.getTableId(), o2.getTableId());
		}

	}

	public static class TablePathProjection implements Func<String, Table> {

		/*
		 * (non-Javadoc)
		 * 
		 * @see
		 * de.uni_mannheim.informatik.dws.t2k.utils.query.Func#invoke(java.lang.
		 * Object)
		 */
		@Override
		public String invoke(Table in) {
			return in.getPath();
		}

	}

	private ArrayList<TableRow> rows = new ArrayList<>();
	private String path;
	TableSchema schema = new TableSchema();
	private int subjectColumnIndex = -1;
	private int tableId = 0;
	private TableMapping mapping;
	private TableContext context;

	public Table() {
	}

	public TableSchema getSchema() {
		return schema;
	}

	public int getSubjectColumnIndex() {
		return subjectColumnIndex;
	}

	public void setSubjectColumnIndex(int keyIndex) {
		this.subjectColumnIndex = keyIndex;
	}

	public boolean hasSubjectColumn() {
		return subjectColumnIndex >= 0;
	}

	public TableColumn getSubjectColumn() {
		if (hasSubjectColumn()) {
			return schema.get(getSubjectColumnIndex());
		} else {
			return null;
		}
	}

	public String getPath() {
		return path;
	}

	public void setPath(String path) {
		this.path = path;

		// changing the path will change the identifiers of all rows and columns of this table
		// so we have to update all HashMaps which contain rows or columns
		getSchema().updateIdentifiers();
	}

	public Collection<TableColumn> getColumns() {
		return schema.getRecords();
	}

	public int getSize() {
		return rows.size();
	}

	public void addColumn(TableColumn c) {
		schema.addColumn(c);
	}

	public void addRow(TableRow r) {
		rows.add(r);
	}

	/**
	 * Removes all rows from the table
	 */
	public void clear() {
		rows.clear();
		rows.trimToSize();
	}

	/**
	 * Should be called after the loading is done, so unneeded memory overhead
	 * can be freed
	 */
	public void endLoad() {
		rows.trimToSize();
	}

	public List<TableRow> getRows() {
		return rows;
	}

	public void setRows(ArrayList<TableRow> rows) {
		this.rows = rows;
	}
	
	public TableRow get(int rowIndex) {
		if (rows.size() > rowIndex) {
			return rows.get(rowIndex);
		} else {
			return null;
		}
	}

	/**
	 * @return the tableId
	 */
	public int getTableId() {
		return tableId;
	}

	/**
	 * @param tableId
	 *            the tableId to set
	 */
	public void setTableId(int tableId) {
		this.tableId = tableId;
	}

	/**
	 * @return the mapping
	 */
	public TableMapping getMapping() {
		if (mapping == null) {
			mapping = new TableMapping();
		}
		return mapping;
	}

	/**
	 * @param mapping
	 *            the mapping to set
	 */
	public void setMapping(TableMapping mapping) {
		this.mapping = mapping;
	}

	/**
	 * @return the context
	 */
	public TableContext getContext() {
		return context;
	}

	/**
	 * @param context
	 *            the context to set
	 */
	public void setContext(TableContext context) {
		this.context = context;
	}

	@Override
	public String toString() {
		return path;
	}

	/**
	 * Adds a new column to the schema and updates all rows with the new
	 * indices.
	 * 
	 * @param index
	 * @param c
	 */
	public void insertColumn(int index, TableColumn c) {

		if (subjectColumnIndex != -1 && index <= subjectColumnIndex) {
			subjectColumnIndex++;
		}

		getSchema().insertColumn(index, c);

		for (TableRow r : getRows()) {
			Object[] oldValues = r.getValueArray();
			Object[] newValues = new Object[oldValues.length + 1];

			for (int i = 0; i < oldValues.length; i++) {
				if (i < index) {
					newValues[i] = oldValues[i];
				} else if (i >= index) {
					newValues[i + 1] = oldValues[i];
				}
			}

			r.set(newValues);
		}
	}

	/**
	 * Removes a column from the schema and updates all rows with the new
	 * indices
	 * 
	 * @param c
	 */
	public void removeColumn(TableColumn c) {
		if (subjectColumnIndex != -1 && c.getColumnIndex() <= subjectColumnIndex) {
			subjectColumnIndex--;
		} else if (c.getColumnIndex() == subjectColumnIndex) {
			subjectColumnIndex = -1;
		}

		getSchema().removeColumn(c);

		for (TableRow r : getRows()) {
			Object[] oldValues = r.getValueArray();
			Object[] newValues = new Object[oldValues.length - 1];

			for (int i = 0; i < newValues.length; i++) {
				if (i < c.getColumnIndex()) {
					newValues[i] = oldValues[i];
				} else if (i >= c.getColumnIndex()) {
					newValues[i] = oldValues[i + 1];
				}
			}

			r.set(newValues);
		}
	}

	/**
	 * 
	 * Removes a row from the table by its row number. Call reorganiseRowNumbers() afterwards to make sure that row numbers reflect the row's position in the table.
	 * 
	 * @param rowNumber
	 * 				The row number of the row that should be removed
	 * @return
	 * 				Returns the row that was removed
	 */
	public TableRow removeRow(int rowNumber) {
		return getRows().remove(rowNumber);
	}
	
	/**
	 * Detects data types and units for all columns with data type 'unknown' and
	 * converts their values to the corresponding types
	 */
	public void inferSchemaAndConvertValues() {
		// type & unit detection

		inferSchema();
		convertValues();
	}

	/**
	 * Detects data types and units for all columns with data type 'unknown' and
	 * converts their values to the corresponding types
	 */
	public void inferSchemaAndConvertValues(TypeDetector td) {
		// type & unit detection
		inferSchema(td);
		convertValues();
	}

	/**
	 * Detects data types and units for all columns with data type 'unknown'
	 */
	public void inferSchema() {
		final TypeGuesser tg = new TypeGuesser();

		inferSchema(tg);
	}

	/**
	 * Detects data types and units for all columns with data type 'unknown'
	 */
	public void inferSchema(final TypeDetector td) { 
		try {
			Parallel.forLoop(0, getSchema().getSize(), new Consumer<Integer>() {

				@Override
				public void execute(Integer i) {
					String attributeLabel = getSchema().get(i).getHeader();
					String[] column = new String[getSize() + 1];

					if (getSchema().get(i).getDataType() == DataType.unknown) {

						// detect types and units per value
						int rowCounter = 0;
						column[rowCounter] = getSchema().get(i).getHeader();
						rowCounter++;

						boolean nullValues = true;
						for (TableRow r : getRows()) {

							if (nullValues == true && r.get(i) != null)
								nullValues = false;
							if(ListHandler.isArray(r.get(i))) {
								column[rowCounter] = (String) ((Object[])r.get(i))[0];
							} else {
								column[rowCounter] = (String) r.get(i);
							}
							rowCounter++;
						}

						ColumnType columnType = null;
						if (!nullValues) {
							columnType = td.detectTypeForColumn(column, attributeLabel);

							if (columnType == null || columnType.getType() == null)
								columnType = new ColumnType(DataType.string, null);
	
						} else {
							columnType = new ColumnType(DataType.string, null);
						}
						
						if (columnType.getType() == DataType.unit) {
							getSchema().get(i).setDataType(DataType.numeric);
							getSchema().get(i).setUnit(columnType.getUnit());
						} else {
							getSchema().get(i).setDataType(columnType.getType());
						}
						
					}
				}
			});
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	/**
	 * Converts all values to the type corresponding to the respective column's
	 * data type (no unit detection/conversion)
	 */
	public void convertValues() {
		TypeConverter tc = new TypeConverter();

		for (TableRow r : getRows()) {

			for (int i = 0; i < getSchema().getSize(); i++) {

				Object typedValue = null;
				
				if(ListHandler.isArray(r.get(i))) {
					
					Object[] values = (Object[])r.get(i);
					Object[] typedValues = new Object[values.length];
					
					for(int j = 0; j < values.length; j++) {
						typedValues[j] =tc.typeValue((String) values[j], getSchema().get(i).getDataType(), getSchema().get(i).getUnit()); 
					}
					
					typedValue = typedValues;
					
				} else {
					typedValue = tc.typeValue((String) r.get(i), getSchema().get(i).getDataType(), getSchema().get(i).getUnit());
				}
				
				r.set(i, typedValue);

			}

		}
	}

	/**
	 * Adds the current data types of all columns to this tables mapping
	 */
	public void addDataTypesToMapping() {
		for(TableColumn c : getColumns()) {
			getMapping().setDataType(c.getColumnIndex(), c.getDataType());
		}
	}

	/**
	 * Detects and sets the key column for this table
	 */
	public void identifySubjectColumn() {
		identifySubjectColumn(0.3);
	}

	/**
	 * Detects and sets the key column for this table
	 */
	public void identifySubjectColumn(double uniquenessThreshold) {
		identifySubjectColumn(uniquenessThreshold, false);
	}

	public void identifySubjectColumn(double uniquenessThreshold, boolean verbose) {
		if (hasSubjectColumn()) {
			// reset existing subject column
			setSubjectColumnIndex(-1);
		}

		TableKeyIdentification tki = new TableKeyIdentification();

		tki.setKeyUniquenessThreshold(uniquenessThreshold);
		tki.setVerbose(verbose);

		tki.identifyKeys(this);
	}

	/**
	 * Appends the rows of the given table to this table's rows. Both tables
	 * must have the same schema!
	 * 
	 * @param t
	 */
	public void append(Table t) {
		for (TableRow r : t.getRows()) {
			TableRow r2 = new TableRow(getRows().size(), this);
			r2.set(r.getValueArray());
			r2.addProvenanceForRow(r);
			addRow(r2);
		}
	}

	/**
	 * removes all provenance information from the table
	 */
	public void clearProvenance() {
		for (TableRow r : getRows()) {
			r.getProvenance().clear();
		}
		for (TableColumn c : getColumns()) {
			c.getProvenance().clear();
		}
	}

	/**
	 * re-assigns the row numbers to rows. After rows are removed from the
	 * table, the row numbers no longer reflect the position of the row in the
	 * table, which can be corrected by calling this method.
	 */
	public void reorganiseRowNumbers() {
		int number = 0;
		for (TableRow r : getRows()) {
			r.setRowNumber(number++);
		}
	}

	/**
	 * Projects the table and returns a map that translates the column indices to the projected column indices
	 * @param projectedColumns	the columns to project
	 * @return	a map that translates the column indices to the projected column indices
	 */
	public Map<Integer, Integer> projectColumnIndices(Collection<TableColumn> projectedColumns) {
		Map<Integer, Integer> columnIndexProjection = new HashMap<>();

		// project the table schema
		int idx = 0;
		for (int i = 0; i < getColumns().size(); i++) {
			TableColumn c = getSchema().get(i);

			if (projectedColumns.contains(c)) {
				columnIndexProjection.put(i, idx++);
			}
		}
		
		return columnIndexProjection;
	}
	
	public Table project(Collection<TableColumn> projectedColumns) throws Exception {
		return project(projectedColumns, true);
	}
	
	public Table project(Collection<TableColumn> projectedColumns, boolean addProvenance) throws Exception {
		Table result = new Table();

		Map<Integer, Integer> columnIndexProjection = new HashMap<>();

		// project the table schema
		int idx = 0;
		for (int i = 0; i < getColumns().size(); i++) {
			TableColumn c = getSchema().get(i);

			if (projectedColumns.contains(c)) {
				columnIndexProjection.put(i, idx);
				result.addColumn(c.copy(result, idx++, addProvenance));
			}
		}

		// copy path and key index
		if (columnIndexProjection.containsKey(getSubjectColumnIndex())) {
			result.setSubjectColumnIndex(columnIndexProjection.get(getSubjectColumnIndex()));
		}
		result.setPath(getPath());

		// copy functional dependencies
		for (Set<TableColumn> det : getSchema().getFunctionalDependencies().keySet()) {

			Set<TableColumn> dep = getSchema().getFunctionalDependencies().get(det);
			Set<TableColumn> depIntersection = Q.intersection(projectedColumns,dep);
			if (det!=null && dep!=null && projectedColumns.containsAll(det) && depIntersection.size()>0) {
				Set<TableColumn> newDet = new HashSet<>();

				for (TableColumn c : det) {
					newDet.add(result.getSchema().get(columnIndexProjection.get(c.getColumnIndex())));
				}

				Set<TableColumn> newDep = new HashSet<>();
				for (TableColumn c : depIntersection) {
					newDep.add(result.getSchema().get(columnIndexProjection.get(c.getColumnIndex())));
				}

				result.getSchema().getFunctionalDependencies().put(newDet, newDep);
			}
		}

		// copy candidate keys
		for (Set<TableColumn> key : getSchema().getCandidateKeys()) {
			if (projectedColumns.containsAll(key)) {
				Set<TableColumn> newKey = new HashSet<>();
				for (TableColumn c : key) {
					newKey.add(result.getSchema().get(columnIndexProjection.get(c.getColumnIndex())));
				}
				result.getSchema().getCandidateKeys().add(newKey);
			}
		}<commit_msg>updated copying of FDs in Table.project<commit_after>/*
 * Copyright (c) 2017 Data and Web Science Group, University of Mannheim, Germany (http://dws.informatik.uni-mannheim.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */
package de.uni_mannheim.informatik.dws.winter.webtables;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import de.uni_mannheim.informatik.dws.winter.model.Pair;
import de.uni_mannheim.informatik.dws.winter.preprocessing.datatypes.ColumnType;
import de.uni_mannheim.informatik.dws.winter.preprocessing.datatypes.DataType;
import de.uni_mannheim.informatik.dws.winter.preprocessing.datatypes.TypeConverter;
import de.uni_mannheim.informatik.dws.winter.utils.MapUtils;
import de.uni_mannheim.informatik.dws.winter.utils.StringUtils;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Consumer;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Parallel;
import de.uni_mannheim.informatik.dws.winter.utils.query.Func;
import de.uni_mannheim.informatik.dws.winter.utils.query.Q;
import de.uni_mannheim.informatik.dws.winter.webtables.detectors.TableKeyIdentification;
import de.uni_mannheim.informatik.dws.winter.webtables.detectors.TypeDetector;
import de.uni_mannheim.informatik.dws.winter.webtables.detectors.TypeGuesser;

/**
 * Represents a Web Table.
 * 
 * @author <NAME> (<EMAIL>)
 *
 */
public class Table implements Serializable {

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;

	public static class TablePathComparator implements Comparator<Table> {

		/*
		 * (non-Javadoc)
		 * 
		 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
		 */
		@Override
		public int compare(Table o1, Table o2) {
			return o1.getPath().compareTo(o2.getPath());
		}

	}

	public static class TableIdComparator implements Comparator<Table> {

		/*
		 * (non-Javadoc)
		 * 
		 * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
		 */
		@Override
		public int compare(Table o1, Table o2) {
			return Integer.compare(o1.getTableId(), o2.getTableId());
		}

	}

	public static class TablePathProjection implements Func<String, Table> {

		/*
		 * (non-Javadoc)
		 * 
		 * @see
		 * de.uni_mannheim.informatik.dws.t2k.utils.query.Func#invoke(java.lang.
		 * Object)
		 */
		@Override
		public String invoke(Table in) {
			return in.getPath();
		}

	}

	private ArrayList<TableRow> rows = new ArrayList<>();
	private String path;
	TableSchema schema = new TableSchema();
	private int subjectColumnIndex = -1;
	private int tableId = 0;
	private TableMapping mapping;
	private TableContext context;

	public Table() {
	}

	public TableSchema getSchema() {
		return schema;
	}

	public int getSubjectColumnIndex() {
		return subjectColumnIndex;
	}

	public void setSubjectColumnIndex(int keyIndex) {
		this.subjectColumnIndex = keyIndex;
	}

	public boolean hasSubjectColumn() {
		return subjectColumnIndex >= 0;
	}

	public TableColumn getSubjectColumn() {
		if (hasSubjectColumn()) {
			return schema.get(getSubjectColumnIndex());
		} else {
			return null;
		}
	}

	public String getPath() {
		return path;
	}

	public void setPath(String path) {
		this.path = path;

		// changing the path will change the identifiers of all rows and columns of this table
		// so we have to update all HashMaps which contain rows or columns
		getSchema().updateIdentifiers();
	}

	public Collection<TableColumn> getColumns() {
		return schema.getRecords();
	}

	public int getSize() {
		return rows.size();
	}

	public void addColumn(TableColumn c) {
		schema.addColumn(c);
	}

	public void addRow(TableRow r) {
		rows.add(r);
	}

	/**
	 * Removes all rows from the table
	 */
	public void clear() {
		rows.clear();
		rows.trimToSize();
	}

	/**
	 * Should be called after the loading is done, so unneeded memory overhead
	 * can be freed
	 */
	public void endLoad() {
		rows.trimToSize();
	}

	public List<TableRow> getRows() {
		return rows;
	}

	public void setRows(ArrayList<TableRow> rows) {
		this.rows = rows;
	}
	
	public TableRow get(int rowIndex) {
		if (rows.size() > rowIndex) {
			return rows.get(rowIndex);
		} else {
			return null;
		}
	}

	/**
	 * @return the tableId
	 */
	public int getTableId() {
		return tableId;
	}

	/**
	 * @param tableId
	 *            the tableId to set
	 */
	public void setTableId(int tableId) {
		this.tableId = tableId;
	}

	/**
	 * @return the mapping
	 */
	public TableMapping getMapping() {
		if (mapping == null) {
			mapping = new TableMapping();
		}
		return mapping;
	}

	/**
	 * @param mapping
	 *            the mapping to set
	 */
	public void setMapping(TableMapping mapping) {
		this.mapping = mapping;
	}

	/**
	 * @return the context
	 */
	public TableContext getContext() {
		return context;
	}

	/**
	 * @param context
	 *            the context to set
	 */
	public void setContext(TableContext context) {
		this.context = context;
	}

	@Override
	public String toString() {
		return path;
	}

	/**
	 * Adds a new column to the schema and updates all rows with the new
	 * indices.
	 * 
	 * @param index
	 * @param c
	 */
	public void insertColumn(int index, TableColumn c) {

		if (subjectColumnIndex != -1 && index <= subjectColumnIndex) {
			subjectColumnIndex++;
		}

		getSchema().insertColumn(index, c);

		for (TableRow r : getRows()) {
			Object[] oldValues = r.getValueArray();
			Object[] newValues = new Object[oldValues.length + 1];

			for (int i = 0; i < oldValues.length; i++) {
				if (i < index) {
					newValues[i] = oldValues[i];
				} else if (i >= index) {
					newValues[i + 1] = oldValues[i];
				}
			}

			r.set(newValues);
		}
	}

	/**
	 * Removes a column from the schema and updates all rows with the new
	 * indices
	 * 
	 * @param c
	 */
	public void removeColumn(TableColumn c) {
		if (subjectColumnIndex != -1 && c.getColumnIndex() <= subjectColumnIndex) {
			subjectColumnIndex--;
		} else if (c.getColumnIndex() == subjectColumnIndex) {
			subjectColumnIndex = -1;
		}

		getSchema().removeColumn(c);

		for (TableRow r : getRows()) {
			Object[] oldValues = r.getValueArray();
			Object[] newValues = new Object[oldValues.length - 1];

			for (int i = 0; i < newValues.length; i++) {
				if (i < c.getColumnIndex()) {
					newValues[i] = oldValues[i];
				} else if (i >= c.getColumnIndex()) {
					newValues[i] = oldValues[i + 1];
				}
			}

			r.set(newValues);
		}
	}

	/**
	 * 
	 * Removes a row from the table by its row number. Call reorganiseRowNumbers() afterwards to make sure that row numbers reflect the row's position in the table.
	 * 
	 * @param rowNumber
	 * 				The row number of the row that should be removed
	 * @return
	 * 				Returns the row that was removed
	 */
	public TableRow removeRow(int rowNumber) {
		return getRows().remove(rowNumber);
	}
	
	/**
	 * Detects data types and units for all columns with data type 'unknown' and
	 * converts their values to the corresponding types
	 */
	public void inferSchemaAndConvertValues() {
		// type & unit detection

		inferSchema();
		convertValues();
	}

	/**
	 * Detects data types and units for all columns with data type 'unknown' and
	 * converts their values to the corresponding types
	 */
	public void inferSchemaAndConvertValues(TypeDetector td) {
		// type & unit detection
		inferSchema(td);
		convertValues();
	}

	/**
	 * Detects data types and units for all columns with data type 'unknown'
	 */
	public void inferSchema() {
		final TypeGuesser tg = new TypeGuesser();

		inferSchema(tg);
	}

	/**
	 * Detects data types and units for all columns with data type 'unknown'
	 */
	public void inferSchema(final TypeDetector td) { 
		try {
			Parallel.forLoop(0, getSchema().getSize(), new Consumer<Integer>() {

				@Override
				public void execute(Integer i) {
					String attributeLabel = getSchema().get(i).getHeader();
					String[] column = new String[getSize() + 1];

					if (getSchema().get(i).getDataType() == DataType.unknown) {

						// detect types and units per value
						int rowCounter = 0;
						column[rowCounter] = getSchema().get(i).getHeader();
						rowCounter++;

						boolean nullValues = true;
						for (TableRow r : getRows()) {

							if (nullValues == true && r.get(i) != null)
								nullValues = false;
							if(ListHandler.isArray(r.get(i))) {
								column[rowCounter] = (String) ((Object[])r.get(i))[0];
							} else {
								column[rowCounter] = (String) r.get(i);
							}
							rowCounter++;
						}

						ColumnType columnType = null;
						if (!nullValues) {
							columnType = td.detectTypeForColumn(column, attributeLabel);

							if (columnType == null || columnType.getType() == null)
								columnType = new ColumnType(DataType.string, null);
	
						} else {
							columnType = new ColumnType(DataType.string, null);
						}
						
						if (columnType.getType() == DataType.unit) {
							getSchema().get(i).setDataType(DataType.numeric);
							getSchema().get(i).setUnit(columnType.getUnit());
						} else {
							getSchema().get(i).setDataType(columnType.getType());
						}
						
					}
				}
			});
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	/**
	 * Converts all values to the type corresponding to the respective column's
	 * data type (no unit detection/conversion)
	 */
	public void convertValues() {
		TypeConverter tc = new TypeConverter();

		for (TableRow r : getRows()) {

			for (int i = 0; i < getSchema().getSize(); i++) {

				Object typedValue = null;
				
				if(ListHandler.isArray(r.get(i))) {
					
					Object[] values = (Object[])r.get(i);
					Object[] typedValues = new Object[values.length];
					
					for(int j = 0; j < values.length; j++) {
						typedValues[j] =tc.typeValue((String) values[j], getSchema().get(i).getDataType(), getSchema().get(i).getUnit()); 
					}
					
					typedValue = typedValues;
					
				} else {
					typedValue = tc.typeValue((String) r.get(i), getSchema().get(i).getDataType(), getSchema().get(i).getUnit());
				}
				
				r.set(i, typedValue);

			}

		}
	}

	/**
	 * Adds the current data types of all columns to this tables mapping
	 */
	public void addDataTypesToMapping() {
		for(TableColumn c : getColumns()) {
			getMapping().setDataType(c.getColumnIndex(), c.getDataType());
		}
	}

	/**
	 * Detects and sets the key column for this table
	 */
	public void identifySubjectColumn() {
		identifySubjectColumn(0.3);
	}

	/**
	 * Detects and sets the key column for this table
	 */
	public void identifySubjectColumn(double uniquenessThreshold) {
		identifySubjectColumn(uniquenessThreshold, false);
	}

	public void identifySubjectColumn(double uniquenessThreshold, boolean verbose) {
		if (hasSubjectColumn()) {
			// reset existing subject column
			setSubjectColumnIndex(-1);
		}

		TableKeyIdentification tki = new TableKeyIdentification();

		tki.setKeyUniquenessThreshold(uniquenessThreshold);
		tki.setVerbose(verbose);

		tki.identifyKeys(this);
	}

	/**
	 * Appends the rows of the given table to this table's rows. Both tables
	 * must have the same schema!
	 * 
	 * @param t
	 */
	public void append(Table t) {
		for (TableRow r : t.getRows()) {
			TableRow r2 = new TableRow(getRows().size(), this);
			r2.set(r.getValueArray());
			r2.addProvenanceForRow(r);
			addRow(r2);
		}
	}

	/**
	 * removes all provenance information from the table
	 */
	public void clearProvenance() {
		for (TableRow r : getRows()) {
			r.getProvenance().clear();
		}
		for (TableColumn c : getColumns()) {
			c.getProvenance().clear();
		}
	}

	/**
	 * re-assigns the row numbers to rows. After rows are removed from the
	 * table, the row numbers no longer reflect the position of the row in the
	 * table, which can be corrected by calling this method.
	 */
	public void reorganiseRowNumbers() {
		int number = 0;
		for (TableRow r : getRows()) {
			r.setRowNumber(number++);
		}
	}

	/**
	 * Projects the table and returns a map that translates the column indices to the projected column indices
	 * @param projectedColumns	the columns to project
	 * @return	a map that translates the column indices to the projected column indices
	 */
	public Map<Integer, Integer> projectColumnIndices(Collection<TableColumn> projectedColumns) {
		Map<Integer, Integer> columnIndexProjection = new HashMap<>();

		// project the table schema
		int idx = 0;
		for (int i = 0; i < getColumns().size(); i++) {
			TableColumn c = getSchema().get(i);

			if (projectedColumns.contains(c)) {
				columnIndexProjection.put(i, idx++);
			}
		}
		
		return columnIndexProjection;
	}
	
	public Table project(Collection<TableColumn> projectedColumns) throws Exception {
		return project(projectedColumns, true);
	}
	
	public Table project(Collection<TableColumn> projectedColumns, boolean addProvenance) throws Exception {
		Table result = new Table();

		Map<Integer, Integer> columnIndexProjection = new HashMap<>();

		// project the table schema
		int idx = 0;
		for (int i = 0; i < getColumns().size(); i++) {
			TableColumn c = getSchema().get(i);

			if (projectedColumns.contains(c)) {
				columnIndexProjection.put(i, idx);
				result.addColumn(c.copy(result, idx++, addProvenance));
			}
		}

		// copy path and key index
		if (columnIndexProjection.containsKey(getSubjectColumnIndex())) {
			result.setSubjectColumnIndex(columnIndexProjection.get(getSubjectColumnIndex()));
		}
		result.setPath(getPath());

		// copy functional dependencies
		for(Pair<Set<TableColumn>,Set<TableColumn>> fd : Pair.fromMap(getSchema().getFunctionalDependencies())) {
			Set<TableColumn> det = fd.getFirst();

			Set<TableColumn> dep = fd.getSecond();
			Set<TableColumn> depIntersection = Q.intersection(projectedColumns,dep);
			if (projectedColumns.containsAll(det) && depIntersection.size()>0) {
				Set<TableColumn> newDet = new HashSet<>();

				for (TableColumn c : det) {
					newDet.add(result.getSchema().get(columnIndexProjection.get(c.getColumnIndex())));
				}

				Set<TableColumn> newDep = new HashSet<>();
				for (TableColumn c : depIntersection) {
					newDep.add(result.getSchema().get(columnIndexProjection.get(c.getColumnIndex())));
				}

				result.getSchema().getFunctionalDependencies().put(newDet, newDep);
			}
		}

		// copy candidate keys
		for (Set<TableColumn> key : getSchema().getCandidateKeys()) {
			if (projectedColumns.containsAll(key)) {
				Set<TableColumn> newKey = new HashSet<>();
				for (TableColumn c : key) {
					newKey.add(result.getSchema().get(columnIndexProjection.get(c.getColumnIndex())));
				}
				result.getSchema().getCandidateKeys().add(newKey);
			}
		}
{'fraction_non_alphanumeric': 0.12505344867282833, 'fraction_numerical': 0.0032233661151860014, 'mean_word_length': 3.457410936812784, 'pattern_counts': {'":': 0, '<': 84, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3407668', 'n_tokens_mistral': 11012, 'n_tokens_neox': 9410, 'n_words': 2499}