<commit_before>/**
 * 
 */
package org.openforis.calc.collect;

import java.sql.Connection;
import java.util.List;

import javax.sql.DataSource;

import org.jooq.Configuration;
import org.jooq.DataType;
import org.jooq.Field;
import org.jooq.Record;
import org.jooq.SelectQuery;
import org.jooq.TableField;
import org.jooq.impl.DataSourceConnectionProvider;
import org.jooq.impl.SchemaImpl;
import org.jooq.impl.TableImpl;
import org.openforis.calc.engine.Task;
import org.openforis.calc.engine.Workspace;
import org.openforis.calc.metadata.Entity;
import org.openforis.calc.metadata.Variable;
import org.openforis.calc.psql.Psql;
import org.openforis.calc.schema.AbstractTable;
import org.openforis.calc.schema.InputSchema;
import org.openforis.calc.schema.InputTable;
import org.openforis.calc.schema.Schemas;
import org.openforis.collect.relational.CollectRdbException;
import org.openforis.collect.relational.RelationalSchemaCreator;
import org.openforis.collect.relational.liquibase.LiquibaseRelationalSchemaCreator;
import org.openforis.collect.relational.model.RelationalSchema;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.datasource.DataSourceUtils;

/**
 * @author <NAME>
 *
 */
public class CollectInputSchemaCreatorTask extends Task {

	private static final String VIEW_SUFFIX = "_view";
	
	@Autowired
	private Configuration config;
	
	@Override
	public String getName() {
		return "Create relational schema";
	}
	
	@Override
	protected void execute() throws Throwable {
		dropInputSchema();
		
		createInputSchema();
		
		addUserDefinedVariableColumns();
		
		addViews();
	}

	private void dropInputSchema() {
		String inputSchemaName = getWorkspace().getInputSchema();
		psql()
			.dropSchemaIfExists(new SchemaImpl(inputSchemaName))
			.cascade()
			.execute();
	}

	private RelationalSchema createInputSchema() throws CollectRdbException {
		String inputSchemaName = getWorkspace().getInputSchema();
		psql()
			.createSchema(new SchemaImpl(inputSchemaName))
			.execute();
		DataSourceConnectionProvider connectionProvider = (DataSourceConnectionProvider) config.connectionProvider();
		DataSource dataSource = connectionProvider.dataSource();
		RelationalSchema schema = ((CollectJob) getJob()).createInputRelationalSchema();
		RelationalSchemaCreator relationalSchemaCreator = new LiquibaseRelationalSchemaCreator();
		Connection connection = DataSourceUtils.getConnection(dataSource);
		relationalSchemaCreator.createRelationalSchema(schema, connection);
		return schema;
	}
	
	private void addUserDefinedVariableColumns() {
		Workspace ws = getWorkspace();
		for (Variable<?> v : ws.getUserDefinedVariables()) {
			Entity entity = v.getEntity();
			
			InputSchemaTable table = new InputSchemaTable(entity.getDataTable(), ws.getInputSchema());
			TableField<Record, ?> field = table.getOrCreateField(v.getInputValueColumn(), Psql.DOUBLE_PRECISION);
			
			psql()
				.alterTable(table)
				.addColumn(field)
				.execute();
		}
	}

	private void addViews() {
		Workspace ws = getWorkspace();
		List<Entity> entities = ws.getEntities();
		for (Entity entity : entities) {
			if ( entity.getParent() != null ) {
				addView(entity);
			}
		}
	}

	private void addView(Entity entity) {
		Workspace ws = getWorkspace();
		
		Schemas schemas = new Schemas(ws);
		InputSchema inputSchema = schemas.getInputSchema();
		
		InputTable table = inputSchema.getDataTable(entity);
		
		//create inner select
		SelectQuery<Record> select = psql().selectQuery();
		select.addFrom(table);
		select.addSelect(table.fields());
		
		//for every ancestor, add join condition and select fields<commit_msg>CALC-187 - Create views for each InputTable in order to facilitate queries<commit_after>/**
 * 
 */
package org.openforis.calc.collect;

import java.sql.Connection;
import java.util.List;

import javax.sql.DataSource;

import org.jooq.Configuration;
import org.jooq.DataType;
import org.jooq.Field;
import org.jooq.Record;
import org.jooq.SelectQuery;
import org.jooq.TableField;
import org.jooq.impl.DataSourceConnectionProvider;
import org.jooq.impl.SchemaImpl;
import org.jooq.impl.TableImpl;
import org.openforis.calc.engine.Task;
import org.openforis.calc.engine.Workspace;
import org.openforis.calc.metadata.Entity;
import org.openforis.calc.metadata.Variable;
import org.openforis.calc.psql.Psql;
import org.openforis.calc.schema.AbstractTable;
import org.openforis.calc.schema.InputSchema;
import org.openforis.calc.schema.InputTable;
import org.openforis.calc.schema.Schemas;
import org.openforis.collect.relational.CollectRdbException;
import org.openforis.collect.relational.RelationalSchemaCreator;
import org.openforis.collect.relational.liquibase.LiquibaseRelationalSchemaCreator;
import org.openforis.collect.relational.model.RelationalSchema;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.datasource.DataSourceUtils;

/**
 * @author <NAME>
 *
 */
public class CollectInputSchemaCreatorTask extends Task {

	private static final String VIEW_SUFFIX = "_view";
	
	@Autowired
	private Configuration config;
	
	@Override
	public String getName() {
		return "Create relational schema";
	}
	
	@Override
	protected long countTotalItems() {
		return 4;
	}
	
	@Override
	protected void execute() throws Throwable {
		dropInputSchema();
		
		createInputSchema();
		
		addUserDefinedVariableColumns();
		
		addViews();
	}

	private void dropInputSchema() {
		String inputSchemaName = getWorkspace().getInputSchema();
		psql()
			.dropSchemaIfExists(new SchemaImpl(inputSchemaName))
			.cascade()
			.execute();
		incrementItemsProcessed();
	}

	private RelationalSchema createInputSchema() throws CollectRdbException {
		String inputSchemaName = getWorkspace().getInputSchema();
		psql()
			.createSchema(new SchemaImpl(inputSchemaName))
			.execute();
		DataSourceConnectionProvider connectionProvider = (DataSourceConnectionProvider) config.connectionProvider();
		DataSource dataSource = connectionProvider.dataSource();
		RelationalSchema schema = ((CollectJob) getJob()).createInputRelationalSchema();
		RelationalSchemaCreator relationalSchemaCreator = new LiquibaseRelationalSchemaCreator();
		Connection connection = DataSourceUtils.getConnection(dataSource);
		relationalSchemaCreator.createRelationalSchema(schema, connection);
		incrementItemsProcessed();
		return schema;
	}
	
	private void addUserDefinedVariableColumns() {
		Workspace ws = getWorkspace();
		for (Variable<?> v : ws.getUserDefinedVariables()) {
			Entity entity = v.getEntity();
			
			InputSchemaTable table = new InputSchemaTable(entity.getDataTable(), ws.getInputSchema());
			TableField<Record, ?> field = table.getOrCreateField(v.getInputValueColumn(), Psql.DOUBLE_PRECISION);
			
			psql()
				.alterTable(table)
				.addColumn(field)
				.execute();
		}
		incrementItemsProcessed();
	}

	private void addViews() {
		Workspace ws = getWorkspace();
		List<Entity> entities = ws.getEntities();
		for (Entity entity : entities) {
			addView(entity);
		}
		incrementItemsProcessed();
	}

	private void addView(Entity entity) {
		Workspace ws = getWorkspace();
		
		Schemas schemas = new Schemas(ws);
		InputSchema inputSchema = schemas.getInputSchema();
		
		InputTable table = inputSchema.getDataTable(entity);
		
		//create inner select
		SelectQuery<Record> select = psql().selectQuery();
		select.addFrom(table);
		select.addSelect(table.fields());
		
		//for every ancestor, add join condition and select fields
{'fraction_non_alphanumeric': 0.10698108180122569, 'fraction_numerical': 0.0005329070077271516, 'mean_word_length': 5.356477561388654, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3755065', 'n_tokens_mistral': 2416, 'n_tokens_neox': 2202, 'n_words': 444}