<commit_before>/*
 * Copyright (C) 2004-2005  exedio GmbH (www.exedio.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package com.exedio.cope;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import com.exedio.cope.search.Condition;
import com.exedio.cope.util.Day;
import com.exedio.dsmf.Driver;
import com.exedio.dsmf.SQLRuntimeException;
import com.exedio.dsmf.Schema;

abstract class Database
{
	private final List tables = new ArrayList();
	private final HashMap uniqueConstraintsByID = new HashMap();
	private final HashMap itemColumnsByIntegrityConstraintName = new HashMap();
	private boolean buildStage = true;
	final Driver driver;
	private final boolean prepare;
	private final boolean useDefineColumnTypes;
	private final boolean log;
	private final boolean butterflyPkSource;
	final ConnectionPool connectionPool;
	private final java.util.Properties forcedNames;
	final java.util.Properties tableOptions;
	final int limitSupport;
	private List expectedCalls = null;
	
	protected Database(final Driver driver, final Properties properties)
	{
		this.driver = driver;
		this.prepare = !properties.getDatabaseDontSupportPreparedStatements();
		this.useDefineColumnTypes = this instanceof DatabaseColumnTypesDefinable;
		this.log = properties.getDatabaseLog();
		this.butterflyPkSource = properties.getPkSourceButterfly();
		this.connectionPool = new ConnectionPool(properties);
		this.forcedNames = properties.getDatabaseForcedNames();
		this.tableOptions = properties.getDatabaseTableOptions();
		this.limitSupport = properties.getDatabaseDontSupportLimit() ? LIMIT_SUPPORT_NONE : getLimitSupport();

		switch(limitSupport)
		{
			case LIMIT_SUPPORT_NONE:
			case LIMIT_SUPPORT_CLAUSE_AFTER_SELECT:
			case LIMIT_SUPPORT_CLAUSE_AFTER_WHERE:
			case LIMIT_SUPPORT_ROWNUM:
				break;
			default:
				throw new RuntimeException(Integer.toString(limitSupport));
		}
		//System.out.println("using database "+getClass());
	}
	
	final void addTable(final Table table)
	{
		if(!buildStage)
			throw new RuntimeException();
		tables.add(table);
	}
	
	final void addUniqueConstraint(final String constraintID, final UniqueConstraint constraint)
	{
		if(!buildStage)
			throw new RuntimeException();

		final Object collision = uniqueConstraintsByID.put(constraintID, constraint);
		if(collision!=null)
			throw new RuntimeException("ambiguous unique constraint "+constraint+" trimmed to >"+constraintID+"< colliding with "+collision);
	}
	
	final void addIntegrityConstraint(final ItemColumn column)
	{
		if(!buildStage)
			throw new RuntimeException();
		if(itemColumnsByIntegrityConstraintName.put(column.integrityConstraintName, column)!=null)
			throw new RuntimeException("there is more than one integrity constraint with name "+column.integrityConstraintName);
	}
	
	protected final Statement createStatement()
	{
		return createStatement(true);
	}
	
	protected final Statement createStatement(final boolean qualifyTable)
	{
		return new Statement(prepare, qualifyTable, useDefineColumnTypes);
	}
	
	void createDatabase()
	{
		buildStage = false;
		
		makeSchema().create();
	}

	void createDatabaseConstraints()
	{
		buildStage = false;
		
		makeSchema().createConstraints();
	}

	//private static int checkTableTime = 0;

	void checkDatabase(final Connection connection)
	{
		buildStage = false;

		//final long time = System.currentTimeMillis();
		final Statement bf = createStatement(true);
		bf.append("select count(*) from ").defineColumnInteger();
		boolean first = true;

		for(Iterator i = tables.iterator(); i.hasNext(); )
		{
			if(first)
				first = false;
			else
				bf.append(',');

			final Table table = (Table)i.next();
			bf.append(table.protectedID);
		}
		
		final Long testDate = new Long(System.currentTimeMillis());
		final Integer testDay = new Integer(DayColumn.getTransientNumber(new Day(2005, 9, 26)));
		
		bf.append(" where ");
		first = true;
		for(Iterator i = tables.iterator(); i.hasNext(); )
		{
			if(first)
				first = false;
			else
				bf.append(" and ");

			final Table table = (Table)i.next();

			final Column primaryKey = table.getPrimaryKey();
			bf.append(table.protectedID).
				append('.').
				append(primaryKey.protectedID).
				append('=').
				appendValue(Type.NOT_A_PK);
			
			for(Iterator j = table.getColumns().iterator(); j.hasNext(); )
			{
				final Column column = (Column)j.next();
				bf.append(" and ").
					append(table.protectedID).
					append('.').
					append(column.protectedID).
					append('=');

				if(column instanceof IntegerColumn)
					bf.appendValue(column, ((IntegerColumn)column).longInsteadOfInt ? (Number)new Long(1) : new Integer(1));
				else if(column instanceof DoubleColumn)
					bf.appendValue(column, new Double(2.2));
				else if(column instanceof StringColumn)
					bf.appendValue(column, "z");
				else if(column instanceof TimestampColumn)
					bf.appendValue(column, testDate);
				else if(column instanceof DayColumn)
					bf.appendValue(column, testDay);
				else
					throw new RuntimeException(column.toString());
			}
		}
		
		executeSQLQuery(connection, bf,
			new ResultSetHandler()
			{
				public void run(final ResultSet resultSet) throws SQLException
				{
					if(!resultSet.next())
						throw new RuntimeException();
				}
			},
			false
		);
	}	

	void dropDatabase()
	{
		buildStage = false;

		makeSchema().drop();
	}
	
	void dropDatabaseConstraints()
	{
		buildStage = false;

		makeSchema().dropConstraints();
	}
	
	void tearDownDatabase()
	{
		buildStage = false;

		makeSchema().tearDown();
	}

	void tearDownDatabaseConstraints()
	{
		buildStage = false;

		makeSchema().tearDownConstraints();
	}
	
	void checkEmptyDatabase(final Connection connection)
	{
		buildStage = false;

		//final long time = System.currentTimeMillis();
		for(Iterator i = tables.iterator(); i.hasNext(); )
		{
			final Table table = (Table)i.next();
			final int count = countTable(connection, table);
			if(count>0)
				throw new RuntimeException("there are "+count+" items left for table "+table.id);
		}
		//final long amount = (System.currentTimeMillis()-time);
		//checkEmptyTableTime += amount;
		//System.out.println("CHECK EMPTY TABLES "+amount+"ms  accumulated "+checkEmptyTableTime);
	}
	
	private final void appendLimitClauseAroundPrefix(final Statement bf, final int start, final int count)
	{
		if((start==0&&count==Query.UNLIMITED_COUNT)||(count<=0&&count!=Query.UNLIMITED_COUNT)||start<0)
			throw new RuntimeException(start+"-"+count);

		// TODO: check, whether ROW_NUMBER() OVER is faster,
		// see http://www.php-faq.de/q/q-oracle-limit.html
		bf.append("select * from(");
		if(start>0)
			bf.append("select "+Table.ROWNUM_INNER_VIEW_ALIAS+".*,ROWNUM "+Table.ROWNUM_INNER_ALIAS+" from(");
	}
	
	private final void appendLimitClauseAroundSuffix(final Statement bf, final int start, final int count)
	{
		if((start==0&&count==Query.UNLIMITED_COUNT)||(count<=0&&count!=Query.UNLIMITED_COUNT)||start<0)
			throw new RuntimeException(start+"-"+count);

		bf.append(')');
		if(start>0)
			bf.append(Table.ROWNUM_INNER_VIEW_ALIAS+' ');
		if(count!=Query.UNLIMITED_COUNT)
			bf.append("where ROWNUM<=").appendValue(start+count);
		if(start>0)
			bf.append(")where "+Table.ROWNUM_INNER_ALIAS+'>').appendValue(start);
	}
	
	final ArrayList search(final Connection connection, final Query query, final boolean doCountOnly)
	{
		if ( expectedCalls!=null )
		{
			nextExpectedCall().checkSearch( connection, query );
		}
		
		buildStage = false;

		final int limitStart = query.limitStart;
		final int limitCount = query.limitCount;
		final boolean limitActive = limitStart>0 || limitCount!=Query.UNLIMITED_COUNT;

		final ArrayList queryJoins = query.joins;
		final Statement bf = createStatement(queryJoins!=null);
		bf.setJoinsToAliases(query);
		
		if(!doCountOnly && limitActive && limitSupport==LIMIT_SUPPORT_ROWNUM)
			appendLimitClauseAroundPrefix(bf, limitStart, limitCount);
		
		bf.append("select");
		
		if(!doCountOnly && limitActive && limitSupport==LIMIT_SUPPORT_CLAUSE_AFTER_SELECT)
			appendLimitClause(bf, limitStart, limitCount);
		
		bf.append(' ');

		final Selectable[] selectables = query.selectables;
		final Column[] selectColumns = new Column[selectables.length];
		final Type[] selectTypes = new Type[selectables.length];

		if(doCountOnly)
		{
			bf.append("count(*)");
		}
		else
		{
			for(int selectableIndex = 0; selectableIndex<selectables.length; selectableIndex++)
			{
				final Selectable selectable = selectables[selectableIndex];
				final Column selectColumn;
				final Type selectType;
				final Table selectTable;
				final Column selectPrimaryKey;
				if(selectable instanceof Function)
				{
					final Function selectAttribute = (Function)selectable;
					selectType = selectAttribute.getType();
	
					if(selectableIndex>0)
						bf.append(',');
					
					if(selectable instanceof ObjectAttribute)
					{
						selectColumn = ((ObjectAttribute)selectAttribute).getColumn();
						bf.append((ObjectAttribute)selectable, (Join)null).defineColumn(selectColumn);
					}
					else
					{
						selectColumn = null;
						final ComputedFunction computedFunction = (ComputedFunction)selectable;
						bf.append(computedFunction, (Join)null).defineColumn(computedFunction);
					}
				}
				else
				{
					selectType = (Type)selectable;
					selectTable = selectType.getTable();
					selectPrimaryKey = selectTable.getPrimaryKey();
					selectColumn = selectPrimaryKey;
	
					if(selectableIndex>0)
						bf.append(',');
					
					bf.appendPK(selectType, (Join)null).defineColumn(selectColumn);
	
					if(selectColumn.primaryKey)
					{
						final StringColumn selectTypeColumn = selectColumn.getTypeColumn();
						if(selectTypeColumn!=null)
						{
							bf.append(',').
								append(selectTable.protectedID).
								append('.').
								append(selectTypeColumn.protectedID).defineColumn(selectTypeColumn);
						}
						else
							selectTypes[selectableIndex] = selectType;
					}
					else
						selectTypes[selectableIndex] = selectType;
				}
	
				selectColumns[selectableIndex] = selectColumn;
			}
		}

		bf.append(" from ").
			append(query.type.getTable().protectedID);
		final String fromAlias = bf.getAlias(null);
		if(fromAlias!=null)
		{<commit_msg>reduce unnecesary table qualifiers for selectTypeColumn<commit_after>/*
 * Copyright (C) 2004-2005  exedio GmbH (www.exedio.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package com.exedio.cope;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import com.exedio.cope.search.Condition;
import com.exedio.cope.util.Day;
import com.exedio.dsmf.Driver;
import com.exedio.dsmf.SQLRuntimeException;
import com.exedio.dsmf.Schema;

abstract class Database
{
	private final List tables = new ArrayList();
	private final HashMap uniqueConstraintsByID = new HashMap();
	private final HashMap itemColumnsByIntegrityConstraintName = new HashMap();
	private boolean buildStage = true;
	final Driver driver;
	private final boolean prepare;
	private final boolean useDefineColumnTypes;
	private final boolean log;
	private final boolean butterflyPkSource;
	final ConnectionPool connectionPool;
	private final java.util.Properties forcedNames;
	final java.util.Properties tableOptions;
	final int limitSupport;
	private List expectedCalls = null;
	
	protected Database(final Driver driver, final Properties properties)
	{
		this.driver = driver;
		this.prepare = !properties.getDatabaseDontSupportPreparedStatements();
		this.useDefineColumnTypes = this instanceof DatabaseColumnTypesDefinable;
		this.log = properties.getDatabaseLog();
		this.butterflyPkSource = properties.getPkSourceButterfly();
		this.connectionPool = new ConnectionPool(properties);
		this.forcedNames = properties.getDatabaseForcedNames();
		this.tableOptions = properties.getDatabaseTableOptions();
		this.limitSupport = properties.getDatabaseDontSupportLimit() ? LIMIT_SUPPORT_NONE : getLimitSupport();

		switch(limitSupport)
		{
			case LIMIT_SUPPORT_NONE:
			case LIMIT_SUPPORT_CLAUSE_AFTER_SELECT:
			case LIMIT_SUPPORT_CLAUSE_AFTER_WHERE:
			case LIMIT_SUPPORT_ROWNUM:
				break;
			default:
				throw new RuntimeException(Integer.toString(limitSupport));
		}
		//System.out.println("using database "+getClass());
	}
	
	final void addTable(final Table table)
	{
		if(!buildStage)
			throw new RuntimeException();
		tables.add(table);
	}
	
	final void addUniqueConstraint(final String constraintID, final UniqueConstraint constraint)
	{
		if(!buildStage)
			throw new RuntimeException();

		final Object collision = uniqueConstraintsByID.put(constraintID, constraint);
		if(collision!=null)
			throw new RuntimeException("ambiguous unique constraint "+constraint+" trimmed to >"+constraintID+"< colliding with "+collision);
	}
	
	final void addIntegrityConstraint(final ItemColumn column)
	{
		if(!buildStage)
			throw new RuntimeException();
		if(itemColumnsByIntegrityConstraintName.put(column.integrityConstraintName, column)!=null)
			throw new RuntimeException("there is more than one integrity constraint with name "+column.integrityConstraintName);
	}
	
	protected final Statement createStatement()
	{
		return createStatement(true);
	}
	
	protected final Statement createStatement(final boolean qualifyTable)
	{
		return new Statement(prepare, qualifyTable, useDefineColumnTypes);
	}
	
	void createDatabase()
	{
		buildStage = false;
		
		makeSchema().create();
	}

	void createDatabaseConstraints()
	{
		buildStage = false;
		
		makeSchema().createConstraints();
	}

	//private static int checkTableTime = 0;

	void checkDatabase(final Connection connection)
	{
		buildStage = false;

		//final long time = System.currentTimeMillis();
		final Statement bf = createStatement(true);
		bf.append("select count(*) from ").defineColumnInteger();
		boolean first = true;

		for(Iterator i = tables.iterator(); i.hasNext(); )
		{
			if(first)
				first = false;
			else
				bf.append(',');

			final Table table = (Table)i.next();
			bf.append(table.protectedID);
		}
		
		final Long testDate = new Long(System.currentTimeMillis());
		final Integer testDay = new Integer(DayColumn.getTransientNumber(new Day(2005, 9, 26)));
		
		bf.append(" where ");
		first = true;
		for(Iterator i = tables.iterator(); i.hasNext(); )
		{
			if(first)
				first = false;
			else
				bf.append(" and ");

			final Table table = (Table)i.next();

			final Column primaryKey = table.getPrimaryKey();
			bf.append(table.protectedID).
				append('.').
				append(primaryKey.protectedID).
				append('=').
				appendValue(Type.NOT_A_PK);
			
			for(Iterator j = table.getColumns().iterator(); j.hasNext(); )
			{
				final Column column = (Column)j.next();
				bf.append(" and ").
					append(table.protectedID).
					append('.').
					append(column.protectedID).
					append('=');

				if(column instanceof IntegerColumn)
					bf.appendValue(column, ((IntegerColumn)column).longInsteadOfInt ? (Number)new Long(1) : new Integer(1));
				else if(column instanceof DoubleColumn)
					bf.appendValue(column, new Double(2.2));
				else if(column instanceof StringColumn)
					bf.appendValue(column, "z");
				else if(column instanceof TimestampColumn)
					bf.appendValue(column, testDate);
				else if(column instanceof DayColumn)
					bf.appendValue(column, testDay);
				else
					throw new RuntimeException(column.toString());
			}
		}
		
		executeSQLQuery(connection, bf,
			new ResultSetHandler()
			{
				public void run(final ResultSet resultSet) throws SQLException
				{
					if(!resultSet.next())
						throw new RuntimeException();
				}
			},
			false
		);
	}	

	void dropDatabase()
	{
		buildStage = false;

		makeSchema().drop();
	}
	
	void dropDatabaseConstraints()
	{
		buildStage = false;

		makeSchema().dropConstraints();
	}
	
	void tearDownDatabase()
	{
		buildStage = false;

		makeSchema().tearDown();
	}

	void tearDownDatabaseConstraints()
	{
		buildStage = false;

		makeSchema().tearDownConstraints();
	}
	
	void checkEmptyDatabase(final Connection connection)
	{
		buildStage = false;

		//final long time = System.currentTimeMillis();
		for(Iterator i = tables.iterator(); i.hasNext(); )
		{
			final Table table = (Table)i.next();
			final int count = countTable(connection, table);
			if(count>0)
				throw new RuntimeException("there are "+count+" items left for table "+table.id);
		}
		//final long amount = (System.currentTimeMillis()-time);
		//checkEmptyTableTime += amount;
		//System.out.println("CHECK EMPTY TABLES "+amount+"ms  accumulated "+checkEmptyTableTime);
	}
	
	private final void appendLimitClauseAroundPrefix(final Statement bf, final int start, final int count)
	{
		if((start==0&&count==Query.UNLIMITED_COUNT)||(count<=0&&count!=Query.UNLIMITED_COUNT)||start<0)
			throw new RuntimeException(start+"-"+count);

		// TODO: check, whether ROW_NUMBER() OVER is faster,
		// see http://www.php-faq.de/q/q-oracle-limit.html
		bf.append("select * from(");
		if(start>0)
			bf.append("select "+Table.ROWNUM_INNER_VIEW_ALIAS+".*,ROWNUM "+Table.ROWNUM_INNER_ALIAS+" from(");
	}
	
	private final void appendLimitClauseAroundSuffix(final Statement bf, final int start, final int count)
	{
		if((start==0&&count==Query.UNLIMITED_COUNT)||(count<=0&&count!=Query.UNLIMITED_COUNT)||start<0)
			throw new RuntimeException(start+"-"+count);

		bf.append(')');
		if(start>0)
			bf.append(Table.ROWNUM_INNER_VIEW_ALIAS+' ');
		if(count!=Query.UNLIMITED_COUNT)
			bf.append("where ROWNUM<=").appendValue(start+count);
		if(start>0)
			bf.append(")where "+Table.ROWNUM_INNER_ALIAS+'>').appendValue(start);
	}
	
	final ArrayList search(final Connection connection, final Query query, final boolean doCountOnly)
	{
		if ( expectedCalls!=null )
		{
			nextExpectedCall().checkSearch( connection, query );
		}
		
		buildStage = false;

		final int limitStart = query.limitStart;
		final int limitCount = query.limitCount;
		final boolean limitActive = limitStart>0 || limitCount!=Query.UNLIMITED_COUNT;

		final ArrayList queryJoins = query.joins;
		final Statement bf = createStatement(queryJoins!=null);
		bf.setJoinsToAliases(query);
		
		if(!doCountOnly && limitActive && limitSupport==LIMIT_SUPPORT_ROWNUM)
			appendLimitClauseAroundPrefix(bf, limitStart, limitCount);
		
		bf.append("select");
		
		if(!doCountOnly && limitActive && limitSupport==LIMIT_SUPPORT_CLAUSE_AFTER_SELECT)
			appendLimitClause(bf, limitStart, limitCount);
		
		bf.append(' ');

		final Selectable[] selectables = query.selectables;
		final Column[] selectColumns = new Column[selectables.length];
		final Type[] selectTypes = new Type[selectables.length];

		if(doCountOnly)
		{
			bf.append("count(*)");
		}
		else
		{
			for(int selectableIndex = 0; selectableIndex<selectables.length; selectableIndex++)
			{
				final Selectable selectable = selectables[selectableIndex];
				final Column selectColumn;
				final Type selectType;
				final Table selectTable;
				final Column selectPrimaryKey;
				if(selectable instanceof Function)
				{
					final Function selectAttribute = (Function)selectable;
					selectType = selectAttribute.getType();
	
					if(selectableIndex>0)
						bf.append(',');
					
					if(selectable instanceof ObjectAttribute)
					{
						selectColumn = ((ObjectAttribute)selectAttribute).getColumn();
						bf.append((ObjectAttribute)selectable, (Join)null).defineColumn(selectColumn);
					}
					else
					{
						selectColumn = null;
						final ComputedFunction computedFunction = (ComputedFunction)selectable;
						bf.append(computedFunction, (Join)null).defineColumn(computedFunction);
					}
				}
				else
				{
					selectType = (Type)selectable;
					selectTable = selectType.getTable();
					selectPrimaryKey = selectTable.getPrimaryKey();
					selectColumn = selectPrimaryKey;
	
					if(selectableIndex>0)
						bf.append(',');
					
					bf.appendPK(selectType, (Join)null).defineColumn(selectColumn);
	
					if(selectColumn.primaryKey)
					{
						final StringColumn selectTypeColumn = selectColumn.getTypeColumn();
						if(selectTypeColumn!=null)
						{
							bf.append(',').
								append(selectTypeColumn, (Join)null).defineColumn(selectTypeColumn);
						}
						else
							selectTypes[selectableIndex] = selectType;
					}
					else
						selectTypes[selectableIndex] = selectType;
				}
	
				selectColumns[selectableIndex] = selectColumn;
			}
		}

		bf.append(" from ").
			append(query.type.getTable().protectedID);
		final String fromAlias = bf.getAlias(null);
		if(fromAlias!=null)
		{
{'fraction_non_alphanumeric': 0.11233460616286298, 'fraction_numerical': 0.004440103010389841, 'mean_word_length': 4.3020244821092275, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5920900', 'n_tokens_mistral': 7842, 'n_tokens_neox': 6607, 'n_words': 1584}