<commit_before>package krishagni.catissueplus.upgrade;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;

import javax.sql.DataSource;

import org.apache.log4j.Logger;
import org.springframework.util.DigestUtils;

import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.krishagni.catissueplus.core.de.domain.SavedQuery;

import edu.common.dynamicextensions.ndao.JdbcDao;
import edu.common.dynamicextensions.ndao.JdbcDaoFactory;
import edu.common.dynamicextensions.ndao.ResultExtractor;
import edu.common.dynamicextensions.ndao.TransactionManager;
import edu.common.dynamicextensions.nutility.IoUtil;
import edu.wustl.common.util.global.CommonServiceLocator;

public class ImportQueries {
	private static final Logger logger = Logger.getLogger(ImportQueries.class);
	
	public static void main(String[] args) 
	throws Exception {
		if (args.length !=2) {
			logger.error("Requires admin username and queries folder as input. Exiting importing queries");
			return;
		}
		
		DataSource ds = DbUtil.getDataSource();
		JdbcDaoFactory.setDataSource(ds);
        TransactionManager.getInstance(ds);
		
        Long userId = getUserId(args[0]);
        if (userId == null) {
        	logger.error("Invalid username: " + args[0]);
        	return;
        }				
                
        importQueries(userId, args[1]);        
	}
				
	private static Long getUserId(String username) 
	throws Exception {
		JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
		List<Object> params = new ArrayList<Object>();
		params.add(username);
		
		Long userId = jdbcDao.getResultSet(GET_USER_ID_SQL, params, new ResultExtractor<Long>() {
			@Override
			public Long extract(ResultSet rs) throws SQLException {
				return rs.next() ? rs.getLong(1) : null;
			}			
		});
		
		if (userId == null) {
			logger.error("No active user with login name: " + username);
			return null;
		}
		
		return userId;
	}
	
	private static void importQueries(Long userId, String queriesDir) 
	throws Exception { 
		File dir = new File(queriesDir);
				
		for (String file : dir.list()) {			
			String filename = queriesDir + File.separator + file;			
			logger.info("Importing query from file: " + filename);
			
			Object[] row = getQueryIdAndMd5Digest(filename);			
			byte[] content = getFileContent(filename); 
			String digest = getMd5Digest(content);
			
			if (row == null) {
				insertQuery(userId, filename, content, digest);
			} else {
				Long queryId = (Long)row[0];
				String dbDigest = (String)row[1];
								
				if (dbDigest != null && dbDigest.equals(digest)) {
					logger.info("No change found in file " + file + " since last import");
					continue;
				}
				
				updateQuery(userId, queryId, filename, content, digest);				
			}
		}		
	}
		
	private static Object[] getQueryIdAndMd5Digest(String filename) {
		JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
		List<Object> params = new ArrayList<Object>();
		params.add(filename);
		params.add(filename);
		
		return jdbcDao.getResultSet(GET_QUERY_ID_AND_MD5_SQL, params, new ResultExtractor<Object[]>() {
			@Override
			public Object[] extract(ResultSet rs) throws SQLException {
				return rs.next() ? new Object[] {rs.getLong(1), rs.getString(2)} : null;
			}			
		});				
	}
	
	private static byte[] getFileContent(String filename) 
	throws Exception {
		FileInputStream fin = null;
		ByteArrayOutputStream bout = null;
		try {
			fin = new FileInputStream(filename);
			bout = new ByteArrayOutputStream();
			IoUtil.copy(fin, bout);
			return bout.toByteArray();
		} finally {
			IoUtil.close(fin);
			IoUtil.close(bout);
		}
	}
	
	private static String getMd5Digest(byte[] content) {
		return DigestUtils.md5DigestAsHex(content);
	}
	
	private static SavedQuery getSavedQuery(byte[] content) 
	throws Exception {
		ObjectMapper mapper = new ObjectMapper();
		mapper.setVisibilityChecker(
			mapper.getSerializationConfig().getDefaultVisibilityChecker()
				.withFieldVisibility(Visibility.ANY)
				.withGetterVisibility(Visibility.NONE)
				.withSetterVisibility(Visibility.NONE)
				.withCreatorVisibility(Visibility.NONE));
		return mapper.readValue(content, SavedQuery.class);				
	}
	
	private static void insertQuery(Long userId, String filename, byte[] queryContent, String md5) {
		 TransactionManager.Transaction txn = TransactionManager.getInstance().newTxn();
		 try {
			 SavedQuery query = getSavedQuery(queryContent);
			 JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
			 List<? extends Object> params = Arrays.asList(
					 query.getTitle(),
					 userId,
					 userId,
					 query.getQueryDefJson(),
					 Calendar.getInstance().getTime());
			 
			 String sql = DbUtil.isOracle() ? INSERT_QUERY_ORA_SQL : (DbUtil.isMySQL() ? INSERT_QUERY_MY_SQL : null);
			 Number id = jdbcDao.executeUpdateAndGetKey(sql, params, "IDENTIFIER");
			 if (id == null) {
				 logger.error("Inserting query definition from file: " + filename + " failed");
				 TransactionManager.getInstance().rollback(txn);
				 return;
			 }
			 
			 insertChangeLog(jdbcDao, filename, md5, "INSERTED", id);
			 TransactionManager.getInstance().commit(txn);			 			 			 
		 } catch (Exception e) {
			 logger.error("Inserting query definition from file: " + filename + " failed", e);
			 TransactionManager.getInstance().rollback(txn);
		 }
	}
	
	private static void updateQuery(Long userId, Long queryId, String filename, byte[] queryContent, String md5) {
		 TransactionManager.Transaction txn = TransactionManager.getInstance().newTxn();
		 try {
			 SavedQuery query = getSavedQuery(queryContent);
			 JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
			 List<? extends Object> params = Arrays.asList(
					 query.getTitle(),
					 userId,
					 query.getQueryDefJson(),
					 Calendar.getInstance().getTime(),
					 queryId);
			 
			 jdbcDao.executeUpdate(UPDATE_QUERY_SQL, params);
			 insertChangeLog(jdbcDao, filename, md5, "UPDATED", queryId);
			 TransactionManager.getInstance().commit(txn);			 			 			 
		 } catch (Exception e) {
			 logger.error("Updating query " + queryId + " with query definition from file: " + filename + " failed", e);
			 TransactionManager.getInstance().rollback(txn);
		 }
	}
		
	private static void insertChangeLog(JdbcDao jdbcDao, String filename, String md5, String status, Number id) {		
		jdbcDao.executeUpdate(
				INSERT_CHANGE_LOG_SQL, 
				Arrays.asList(filename, md5, status, Calendar.getInstance().getTime(), id));<commit_msg>Panga with Oracle timestamp column<commit_after>package krishagni.catissueplus.upgrade;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;

import javax.sql.DataSource;

import org.apache.log4j.Logger;
import org.springframework.util.DigestUtils;

import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.krishagni.catissueplus.core.de.domain.SavedQuery;

import edu.common.dynamicextensions.ndao.JdbcDao;
import edu.common.dynamicextensions.ndao.JdbcDaoFactory;
import edu.common.dynamicextensions.ndao.ResultExtractor;
import edu.common.dynamicextensions.ndao.TransactionManager;
import edu.common.dynamicextensions.nutility.IoUtil;
import edu.wustl.common.util.global.CommonServiceLocator;

public class ImportQueries {
	private static final Logger logger = Logger.getLogger(ImportQueries.class);
	
	public static void main(String[] args) 
	throws Exception {
		if (args.length !=2) {
			logger.error("Requires admin username and queries folder as input. Exiting importing queries");
			return;
		}
		
		DataSource ds = DbUtil.getDataSource();
		JdbcDaoFactory.setDataSource(ds);
        TransactionManager.getInstance(ds);
		
        Long userId = getUserId(args[0]);
        if (userId == null) {
        	logger.error("Invalid username: " + args[0]);
        	return;
        }				
                
        importQueries(userId, args[1]);        
	}
				
	private static Long getUserId(String username) 
	throws Exception {
		JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
		List<Object> params = new ArrayList<Object>();
		params.add(username);
		
		Long userId = jdbcDao.getResultSet(GET_USER_ID_SQL, params, new ResultExtractor<Long>() {
			@Override
			public Long extract(ResultSet rs) throws SQLException {
				return rs.next() ? rs.getLong(1) : null;
			}			
		});
		
		if (userId == null) {
			logger.error("No active user with login name: " + username);
			return null;
		}
		
		return userId;
	}
	
	private static void importQueries(Long userId, String queriesDir) 
	throws Exception { 
		File dir = new File(queriesDir);
				
		for (String file : dir.list()) {			
			String filename = queriesDir + File.separator + file;			
			logger.info("Importing query from file: " + filename);
			
			Object[] row = getQueryIdAndMd5Digest(filename);			
			byte[] content = getFileContent(filename); 
			String digest = getMd5Digest(content);
			
			if (row == null) {
				insertQuery(userId, filename, content, digest);
			} else {
				Long queryId = (Long)row[0];
				String dbDigest = (String)row[1];
								
				if (dbDigest != null && dbDigest.equals(digest)) {
					logger.info("No change found in file " + file + " since last import");
					continue;
				}
				
				updateQuery(userId, queryId, filename, content, digest);				
			}
		}		
	}
		
	private static Object[] getQueryIdAndMd5Digest(String filename) {
		JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
		List<Object> params = new ArrayList<Object>();
		params.add(filename);
		params.add(filename);
		
		return jdbcDao.getResultSet(GET_QUERY_ID_AND_MD5_SQL, params, new ResultExtractor<Object[]>() {
			@Override
			public Object[] extract(ResultSet rs) throws SQLException {
				return rs.next() ? new Object[] {rs.getLong(1), rs.getString(2)} : null;
			}			
		});				
	}
	
	private static byte[] getFileContent(String filename) 
	throws Exception {
		FileInputStream fin = null;
		ByteArrayOutputStream bout = null;
		try {
			fin = new FileInputStream(filename);
			bout = new ByteArrayOutputStream();
			IoUtil.copy(fin, bout);
			return bout.toByteArray();
		} finally {
			IoUtil.close(fin);
			IoUtil.close(bout);
		}
	}
	
	private static String getMd5Digest(byte[] content) {
		return DigestUtils.md5DigestAsHex(content);
	}
	
	private static SavedQuery getSavedQuery(byte[] content) 
	throws Exception {
		ObjectMapper mapper = new ObjectMapper();
		mapper.setVisibilityChecker(
			mapper.getSerializationConfig().getDefaultVisibilityChecker()
				.withFieldVisibility(Visibility.ANY)
				.withGetterVisibility(Visibility.NONE)
				.withSetterVisibility(Visibility.NONE)
				.withCreatorVisibility(Visibility.NONE));
		return mapper.readValue(content, SavedQuery.class);				
	}
	
	private static void insertQuery(Long userId, String filename, byte[] queryContent, String md5) {
		 TransactionManager.Transaction txn = TransactionManager.getInstance().newTxn();
		 try {
			 SavedQuery query = getSavedQuery(queryContent);
			 JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
			 List<? extends Object> params = Arrays.asList(
					 query.getTitle(),
					 userId,
					 userId,
					 query.getQueryDefJson(),
					 new Timestamp(Calendar.getInstance().getTimeInMillis()));
			 
			 String sql = DbUtil.isOracle() ? INSERT_QUERY_ORA_SQL : (DbUtil.isMySQL() ? INSERT_QUERY_MY_SQL : null);
			 Number id = jdbcDao.executeUpdateAndGetKey(sql, params, "IDENTIFIER");
			 if (id == null) {
				 logger.error("Inserting query definition from file: " + filename + " failed");
				 TransactionManager.getInstance().rollback(txn);
				 return;
			 }
			 
			 insertChangeLog(jdbcDao, filename, md5, "INSERTED", id);
			 TransactionManager.getInstance().commit(txn);			 			 			 
		 } catch (Exception e) {
			 logger.error("Inserting query definition from file: " + filename + " failed", e);
			 TransactionManager.getInstance().rollback(txn);
		 }
	}
	
	private static void updateQuery(Long userId, Long queryId, String filename, byte[] queryContent, String md5) {
		 TransactionManager.Transaction txn = TransactionManager.getInstance().newTxn();
		 try {
			 SavedQuery query = getSavedQuery(queryContent);
			 JdbcDao jdbcDao = JdbcDaoFactory.getJdbcDao();
			 List<? extends Object> params = Arrays.asList(
					 query.getTitle(),
					 userId,
					 query.getQueryDefJson(),
					 new Timestamp(Calendar.getInstance().getTimeInMillis()),
					 queryId);
			 
			 jdbcDao.executeUpdate(UPDATE_QUERY_SQL, params);
			 insertChangeLog(jdbcDao, filename, md5, "UPDATED", queryId);
			 TransactionManager.getInstance().commit(txn);			 			 			 
		 } catch (Exception e) {
			 logger.error("Updating query " + queryId + " with query definition from file: " + filename + " failed", e);
			 TransactionManager.getInstance().rollback(txn);
		 }
	}
		
	private static void insertChangeLog(JdbcDao jdbcDao, String filename, String md5, String status, Number id) {		
		jdbcDao.executeUpdate(
				INSERT_CHANGE_LOG_SQL, 
				Arrays.asList(filename, md5, status, Calendar.getInstance().getTime(), id));
{'fraction_non_alphanumeric': 0.11237712243074173, 'fraction_numerical': 0.0032767351802204347, 'mean_word_length': 3.892167577413479, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5830074', 'n_tokens_mistral': 4754, 'n_tokens_neox': 4061, 'n_words': 893}