<commit_before>/*******************************************************************************
 * Caleydo - Visualization for Molecular Biology - http://caleydo.org
 * Copyright (c) The Caleydo Team. All rights reserved.
 * Licensed under the new BSD license, available at http://caleydo.org/license
 ******************************************************************************/
package generic;

import generic.ImportSpec.ColumnSpec;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import org.caleydo.core.util.collection.Pair;
import org.caleydo.core.util.logging.Logger;
import org.caleydo.vis.lineup.model.ARankColumnModel;
import org.caleydo.vis.lineup.model.RankRankColumnModel;
import org.caleydo.vis.lineup.model.RankTableModel;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.jface.dialogs.ErrorDialog;

import demo.RankTableDemo.IModelBuilder;

/**
 * @author <NAME>
 *
 */
public class GenericModelBuilder implements IModelBuilder {
	private static final Logger log = Logger.create(GenericModelBuilder.class);
	private ImportSpec spec;

	/**
	 *
	 */
	public GenericModelBuilder(ImportSpec spec) {
		this.spec = spec;
	}

	@Override
	public void apply(RankTableModel table) throws Exception {
		if (spec == null)
			return;
		Pair<List<GenericRow>, String[]> r = readData();
		table.addData(r.getFirst());

		String[] headers = r.getSecond();
		table.add(new RankRankColumnModel());
		if (headers != null) {
			for (ColumnSpec col : spec.getColumns()) {
				table.add(col.create(headers));
			}
		}
	}

	/**
	 * @return
	 */
	private Pair<List<GenericRow>, String[]> readData() {
		List<ColumnSpec> cols = spec.getColumns();
		StringBuilder report = new StringBuilder();
		try (BufferedReader r = Files.newBufferedReader(new File(spec.getDataSourcePath()).toPath(),
				Charset.defaultCharset())) {
			String header = r.readLine();
			String[] columns = header.split(spec.getDelimiter());
			String line;
			List<GenericRow> result = new ArrayList<>();
			for(int line_i = 0; ((line = r.readLine()) != null); ++line_i) {
				String[] vals = line.split(spec.getDelimiter());
				Object[] data = new Object[spec.getColumns().size()];
				for (int i = 0; i < cols.size(); ++i) {
					try {
						data[i] = cols.get(i).parse(vals);<commit_msg>fix charset<commit_after>/*******************************************************************************
 * Caleydo - Visualization for Molecular Biology - http://caleydo.org
 * Copyright (c) The Caleydo Team. All rights reserved.
 * Licensed under the new BSD license, available at http://caleydo.org/license
 ******************************************************************************/
package generic;

import generic.ImportSpec.ColumnSpec;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import org.caleydo.core.util.collection.Pair;
import org.caleydo.core.util.logging.Logger;
import org.caleydo.vis.lineup.model.ARankColumnModel;
import org.caleydo.vis.lineup.model.RankRankColumnModel;
import org.caleydo.vis.lineup.model.RankTableModel;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.jface.dialogs.ErrorDialog;

import demo.RankTableDemo.IModelBuilder;

/**
 * @author <NAME>
 *
 */
public class GenericModelBuilder implements IModelBuilder {
	private static final Logger log = Logger.create(GenericModelBuilder.class);
	private ImportSpec spec;

	/**
	 *
	 */
	public GenericModelBuilder(ImportSpec spec) {
		this.spec = spec;
	}

	@Override
	public void apply(RankTableModel table) throws Exception {
		if (spec == null)
			return;
		Pair<List<GenericRow>, String[]> r = readData();
		table.addData(r.getFirst());

		String[] headers = r.getSecond();
		table.add(new RankRankColumnModel());
		if (headers != null) {
			for (ColumnSpec col : spec.getColumns()) {
				table.add(col.create(headers));
			}
		}
	}

	/**
	 * @return
	 */
	private Pair<List<GenericRow>, String[]> readData() {
		List<ColumnSpec> cols = spec.getColumns();
		StringBuilder report = new StringBuilder();
		try (BufferedReader r = Files.newBufferedReader(new File(spec.getDataSourcePath()).toPath(),
				Charset.forName("UTF-8"))) {
			String header = r.readLine();
			String[] columns = header.split(spec.getDelimiter());
			String line;
			List<GenericRow> result = new ArrayList<>();
			for(int line_i = 0; ((line = r.readLine()) != null); ++line_i) {
				String[] vals = line.split(spec.getDelimiter());
				Object[] data = new Object[spec.getColumns().size()];
				for (int i = 0; i < cols.size(); ++i) {
					try {
						data[i] = cols.get(i).parse(vals);
{'fraction_non_alphanumeric': 0.19549621363092864, 'fraction_numerical': 0.0009964129135113591, 'mean_word_length': 4.74914089347079, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3129949', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1511, 'n_words': 336}