<commit_before>package core;

import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import utils.Statistics;
import basicDatatypes.QualifiedName;
import basicDatatypes.Schema;
import columnTypes.ColumnPumper;
import connection.DBMSConnection;

public class Generator{
	
	protected Distribution distribution;
	protected DBMSConnection dbmsConn;
	private Random random;
	
	protected Map<String, Integer> mNumDupsRepetition;
	protected int maxNumDupsRepetition;

	/** Size of the window for duplicates picking **/
	public static int duplicatesWindowSize = 30000;
	public static int maxRepeatDuplicateWindowReads = 5;
	public static int freshDuplicatesSize = 1000;
	private static int maxPrimaryDuplicateValuesBufferSize = 5;
	private boolean pureRandom = false;
	
	private static Logger logger = Logger.getLogger(Generator.class.getCanonicalName());

	public Generator(DBMSConnection dbmsConn) {
		this.dbmsConn = dbmsConn;
		this.distribution = new Distribution(dbmsConn);
		this.random = new Random();
				
		mNumDupsRepetition = new HashMap<String, Integer>();
		maxNumDupsRepetition = 0;

		logger.setLevel(Level.INFO);
	}
	
	public List<Schema> pumpTable(int nRows, Schema schema){
				
		PreparedStatement stmt = null;
		List<Schema> tablesToChase = new LinkedList<Schema>(); // Return value
		
		/** mapping (vn -> v_1, ..., v_n-1) where (v1, ..., vn) is a pk **/
		Map<String, List<List<String>>> mFreshDuplicatesToDuplicatePks = new HashMap<String, List<List<String>>>(); 
		                                                                                                  
		List<String> freshDuplicates = new LinkedList<String>(); // Freshly generated strings from which duplicates can be chosen
		Map<String, List<String>> uncommittedFresh = new HashMap<String, List<String>>(); // Keeps track of uncommitted fresh values
		
		initNullRatios(schema);
		initDuplicateValues(schema, 0);
		initDuplicateRatios(schema);		
		initNumDupsRepetitionCounters();
		increaseChaseCycles(schema);
		
		for( ColumnPumper c : schema.getColumns() ){
			if( c.isPrimary() && c.referencedBy().size() > 0 ){
				uncommittedFresh.put(c.getName(), new ArrayList<String>());
			}
		}
		// templateInsert to be called AFTER the ratios initialization
		// because of the reordering of the columns
		String templateInsert = dbmsConn.createInsertTemplate(schema);
		
		
		stmt = dbmsConn.getPreparedStatement(templateInsert);
		logger.debug(templateInsert);

		// Disable auto-commit
		dbmsConn.setAutoCommit(false);
		
		for( int j = 1; j <= nRows; ++j ){

			/** Keeps track of the DUPLICATE values chosen ---for the current row---
			 *  for columns part of a primary key  **/
			List<String> primaryDuplicateValues = new ArrayList<String>();

			for( ColumnPumper column : schema.getColumns() ){
				boolean terminate = pumpColumn(schema, column, stmt, j, nRows, primaryDuplicateValues, uncommittedFresh, 
						mFreshDuplicatesToDuplicatePks, freshDuplicates, tablesToChase);
				if( terminate )	return new ArrayList<Schema>(); // Stop immediately. Not possible to pump rows (foreign key violations)
			}
			try{
				stmt.addBatch();
			}catch(SQLException e){
				e.printStackTrace();
			}
			if( (j % 350000 == 0) ){ // Let's put a limit to the dimension of the stmt 
				try{
					stmt.executeBatch();	
					dbmsConn.commit();
				}catch(SQLException e){
					e.printStackTrace();
				}
				initUncommittedFresh(schema, uncommittedFresh);					
			}
			if( maxNumDupsRepetition > Generator.maxRepeatDuplicateWindowReads ){
				logger.info("Advancing the set of candidate duplicates");

				try{
					stmt.executeBatch();	
					dbmsConn.commit();
				}catch(SQLException e){
					e.printStackTrace();
				}

				initUncommittedFresh(schema, uncommittedFresh);
				initDuplicateValues(schema, j);
				initNumDupsRepetitionCounters();
				mFreshDuplicatesToDuplicatePks.clear();
				freshDuplicates.clear();
				System.gc();
			}
		}
		try{
			stmt.executeBatch();	
			dbmsConn.commit();
			stmt.close();
		} catch (SQLException e) {
			logger.error(e.getMessage());
			e.printStackTrace();
		}
		dbmsConn.setAutoCommit(true);	
		resetState(schema); // Frees memory
		logger.info("Table '"+ schema.getTableName() + "' pumped with " + nRows +" rows.");
		
		return tablesToChase; 
	}
	
	private boolean pumpColumn(Schema schema, ColumnPumper column,
			PreparedStatement stmt, int nRows,
			int j,
			List<String> primaryDuplicateValues,
			Map<String, List<String>> uncommittedFresh,
			Map<String, List<List<String>>> mFreshDuplicatesToDuplicatePks,
			List<String> freshDuplicates, List<Schema> tablesToChase) {
		
		
		String toInsert = null;
		float dupOrNullToss = random.nextFloat();
		
		boolean stopChase = (column.referencesTo().size() > 0) && column.getMaximumChaseCycles() < column.getCurrentChaseCycle();
		if( stopChase && (column.getDuplicateRatio() > 0 || !column.isPrimary()) ){
			column.setDuplicateRatio(1); // DO NOT generate fresh values. Fresh values trigger new chase steps.
		}	
		if( j == nRows && (toInsert = column.getNextChased(dbmsConn, schema)) != null && 
				(!column.isPrimary() || !uncommittedFresh.get(column.getName()).contains(toInsert)) ){
			dbmsConn.setter(stmt, column.getIndex(), column.getType(), toInsert); 
			addToUncommittedFresh(uncommittedFresh, column, toInsert);
			if( column.hasNextChase() )	++nRows; // I haven't finished yet to insert chased values.
		}
		else if( column.getDuplicateRatio() > dupOrNullToss){
			putDuplicate(schema, column, primaryDuplicateValues, 
					mFreshDuplicatesToDuplicatePks, freshDuplicates, stmt, uncommittedFresh, tablesToChase);	
		}
		else if( column.getNullRatio() > (dupOrNullToss - column.getDuplicateRatio()) ){
			putNull(schema, column, stmt);
		}
		else if( ( 0.8 > random.nextFloat() ) && 
				(toInsert = column.getNextChased(dbmsConn, schema) ) != null && 
				(!column.isPrimary() || !uncommittedFresh.get(column.getName()).contains(toInsert)) 
				){
			dbmsConn.setter(stmt, column.getIndex(), column.getType(), toInsert); 
			addToUncommittedFresh(uncommittedFresh, column, toInsert);
		}
		else if( stopChase ){
			// We cannot take a chase value, neither we can pick a duplicate. The only way out is 
			// to take the necessary number of elements (non-duplicate with this column) from the referenced column(s)
			toInsert = column.getFromReferenced(dbmsConn, schema);
			
			if( toInsert == null ){
				try {
					stmt.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
				return true;
			}
			dbmsConn.setter(stmt, column.getIndex(), column.getType(), toInsert);
		}
		else{ // Add a random value						
			Statistics.addInt(schema.getTableName()+"."+column.getName()+" fresh values", 1);
			String generatedRandom = putFreshRandom(column, stmt, uncommittedFresh);<commit_msg>Debugged<commit_after>package core;

import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import utils.Statistics;
import basicDatatypes.QualifiedName;
import basicDatatypes.Schema;
import columnTypes.ColumnPumper;
import connection.DBMSConnection;

public class Generator{
	
	protected Distribution distribution;
	protected DBMSConnection dbmsConn;
	private Random random;
	
	protected Map<String, Integer> mNumDupsRepetition;
	protected int maxNumDupsRepetition;

	/** Size of the window for duplicates picking **/
	public static int duplicatesWindowSize = 30000;
	public static int maxRepeatDuplicateWindowReads = 5;
	public static int freshDuplicatesSize = 1000;
	private static int maxPrimaryDuplicateValuesBufferSize = 5;
	private boolean pureRandom = false;
	
	private static Logger logger = Logger.getLogger(Generator.class.getCanonicalName());

	public Generator(DBMSConnection dbmsConn) {
		this.dbmsConn = dbmsConn;
		this.distribution = new Distribution(dbmsConn);
		this.random = new Random();
				
		mNumDupsRepetition = new HashMap<String, Integer>();
		maxNumDupsRepetition = 0;

		logger.setLevel(Level.INFO);
	}
	
	public List<Schema> pumpTable(int nRows, Schema schema){
				
		PreparedStatement stmt = null;
		List<Schema> tablesToChase = new LinkedList<Schema>(); // Return value
		
		/** mapping (vn -> v_1, ..., v_n-1) where (v1, ..., vn) is a pk **/
		Map<String, List<List<String>>> mFreshDuplicatesToDuplicatePks = new HashMap<String, List<List<String>>>(); 
		                                                                                                  
		List<String> freshDuplicates = new LinkedList<String>(); // Freshly generated strings from which duplicates can be chosen
		Map<String, List<String>> uncommittedFresh = new HashMap<String, List<String>>(); // Keeps track of uncommitted fresh values
		
		initNullRatios(schema);
		initDuplicateValues(schema, 0);
		initDuplicateRatios(schema);		
		initNumDupsRepetitionCounters();
		increaseChaseCycles(schema);
		
		for( ColumnPumper c : schema.getColumns() ){
			if( c.isPrimary() && c.referencedBy().size() > 0 ){
				uncommittedFresh.put(c.getName(), new ArrayList<String>());
			}
		}
		// templateInsert to be called AFTER the ratios initialization
		// because of the reordering of the columns
		String templateInsert = dbmsConn.createInsertTemplate(schema);
		
		
		stmt = dbmsConn.getPreparedStatement(templateInsert);
		logger.debug(templateInsert);

		// Disable auto-commit
		dbmsConn.setAutoCommit(false);
		
		for( int j = 1; j <= nRows; ++j ){

			/** Keeps track of the DUPLICATE values chosen ---for the current row---
			 *  for columns part of a primary key  **/
			List<String> primaryDuplicateValues = new ArrayList<String>();

			for( ColumnPumper column : schema.getColumns() ){
				boolean terminate = pumpColumn(schema, column, stmt, j, nRows, primaryDuplicateValues, uncommittedFresh, 
						mFreshDuplicatesToDuplicatePks, freshDuplicates, tablesToChase);
				if( terminate )	return new ArrayList<Schema>(); // Stop immediately. Not possible to pump rows (foreign key violations)
			}
			try{
				stmt.addBatch();
			}catch(SQLException e){
				e.printStackTrace();
			}
			if( (j % 350000 == 0) ){ // Let's put a limit to the dimension of the stmt 
				try{
					stmt.executeBatch();	
					dbmsConn.commit();
				}catch(SQLException e){
					e.printStackTrace();
				}
				initUncommittedFresh(schema, uncommittedFresh);					
			}
			if( maxNumDupsRepetition > Generator.maxRepeatDuplicateWindowReads ){
				logger.info("Advancing the set of candidate duplicates");

				try{
					stmt.executeBatch();	
					dbmsConn.commit();
				}catch(SQLException e){
					e.printStackTrace();
				}

				initUncommittedFresh(schema, uncommittedFresh);
				initDuplicateValues(schema, j);
				initNumDupsRepetitionCounters();
				mFreshDuplicatesToDuplicatePks.clear();
				freshDuplicates.clear();
				System.gc();
			}
		}
		try{
			stmt.executeBatch();	
			dbmsConn.commit();
			stmt.close();
		} catch (SQLException e) {
			logger.error(e.getMessage());
			e.printStackTrace();
		}
		dbmsConn.setAutoCommit(true);	
		resetState(schema); // Frees memory
		logger.info("Table '"+ schema.getTableName() + "' pumped with " + nRows +" rows.");
		
		return tablesToChase; 
	}
	
	private boolean pumpColumn(Schema schema, ColumnPumper column,
			PreparedStatement stmt, int nRows,
			int j,
			List<String> primaryDuplicateValues,
			Map<String, List<String>> uncommittedFresh,
			Map<String, List<List<String>>> mFreshDuplicatesToDuplicatePks,
			List<String> freshDuplicates, List<Schema> tablesToChase) {
		
		
		String toInsert = null;
		float dupOrNullToss = random.nextFloat();
		
		boolean stopChase = (column.referencesTo().size() > 0) && column.getMaximumChaseCycles() < column.getCurrentChaseCycle();
		if( stopChase && (column.getDuplicateRatio() > 0 || !column.isPrimary()) ){
			column.setDuplicateRatio(1); // DO NOT generate fresh values. Fresh values trigger new chase steps.
		}	
		if( j == nRows && (toInsert = column.getNextChased(dbmsConn, schema)) != null && 
				(!column.isPrimary() || !uncommittedFresh.get(column.getName()).contains(toInsert)) ){
			dbmsConn.setter(stmt, column.getIndex(), column.getType(), toInsert); 
			addToUncommittedFresh(uncommittedFresh, column, toInsert);
			if( column.hasNextChase() )	++nRows; // I haven't finished yet to insert chased values.
		}
		else if( column.getDuplicateRatio() > dupOrNullToss){
			putDuplicate(schema, column, primaryDuplicateValues, 
					mFreshDuplicatesToDuplicatePks, freshDuplicates, stmt, uncommittedFresh, tablesToChase);	
		}
//		else if( column.getNullRatio() > (dupOrNullToss - column.getDuplicateRatio()) ){
//			putNull(schema, column, stmt);
//		}
		else if( ( 0.8 > random.nextFloat() ) && 
				(toInsert = column.getNextChased(dbmsConn, schema) ) != null && 
				(!column.isPrimary() || !uncommittedFresh.get(column.getName()).contains(toInsert)) 
				){
			dbmsConn.setter(stmt, column.getIndex(), column.getType(), toInsert); 
			addToUncommittedFresh(uncommittedFresh, column, toInsert);
		}
		else if( stopChase ){
			// We cannot take a chase value, neither we can pick a duplicate. The only way out is 
			// to take the necessary number of elements (non-duplicate with this column) from the referenced column(s)
			toInsert = column.getFromReferenced(dbmsConn, schema);
			
			if( toInsert == null ){
				try {
					stmt.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
				return true;
			}
			dbmsConn.setter(stmt, column.getIndex(), column.getType(), toInsert);
		}
		else{ // Add a random value						
			Statistics.addInt(schema.getTableName()+"."+column.getName()+" fresh values", 1);
			String generatedRandom = putFreshRandom(column, stmt, uncommittedFresh);
{'fraction_non_alphanumeric': 0.11852696060973003, 'fraction_numerical': 0.004701189543414773, 'mean_word_length': 3.955877162019061, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1192088', 'n_tokens_mistral': 5029, 'n_tokens_neox': 4229, 'n_words': 1035}