<commit_before>package com.servinglynk.hmis.warehouse;

import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;

public class ActiveListView  extends Logging {

	 private SyncHBaseProcessor syncHBaseImport;
	 private Logger logger;
	 public ActiveListView(Logger logger) throws Exception {
	        this.logger = logger;
	        this.syncHBaseImport = new SyncHBaseProcessor();
	    }
	 
	public void createHbaseTable(String tableName) {
		syncHBaseImport.createHBASETable(tableName, logger);
	}
	
	
	 private void syncTable(String hbaseTable, String projectGroupCode,String postgresTable,Properties props) {
	       // log.info("Start sync for table: " + postgresTable);
	        ResultSet resultSet;
	        HTable htable;
	        PreparedStatement statement;
	        Connection connection;
	        try {   
	     	      htable = new HTable(HbaseUtil.getConfiguration(), hbaseTable);
	     	      StringBuilder builder  = new StringBuilder();
	     	      builder.append(" SELECT distinct on (t.client_dedup_id) t.client_dedup_id,t.client_id,t.deleted,t.ignore_match_process,survey_score,survey_submission_date FROM housing_inventory.eligible_clients t ");
	     	      builder.append(" inner join (  ");
	     	      builder.append("	  select client_dedup_id,max(survey_submission_date) as maxDate from housing_inventory.eligible_clients tm ");
	     	      builder.append("	  where tm.project_group_code= ? and tm.deleted=false ");
	     	      builder.append(" group by client_dedup_id ) tm ");
	     	      builder.append(" on  t.client_dedup_id = tm.client_dedup_id ");
	     	      builder.append(" and t.survey_submission_date = tm.maxDate and t.project_group_code=? ");
	     	      builder.append(" and t.deleted=false and t.ignore_match_process=false ");
	     	      builder.append(" and t.survey_submission_date >  ? ");
	     	      builder.append(" order by t.client_dedup_id,survey_score desc" );
	     	      
	     	      connection = SyncPostgresProcessor.getConnection();
		          statement = connection.prepareStatement(builder.toString());
		          statement.setString(1, projectGroupCode);
		          statement.setString(2, projectGroupCode);
		          statement.setDate(3, getCutOffDate(props));
		          resultSet = statement.executeQuery();
		            
	     	      List<String> existingKeysInHbase = syncHBaseImport.getAllKeyRecords(htable, logger);
	               List<String> existingKeysInPostgres = new ArrayList<>();

	               List<Put> putsToUpdate = new ArrayList<>();
	               List<Put> putsToInsert = new ArrayList<>();
	               List<String> putsToDelete = new ArrayList<>();
	               while (resultSet.next()) {
	                   Boolean markedForDelete = false;
	                   try{
	                   	markedForDelete = resultSet.getBoolean("deleted");<commit_msg>Looking into the dateupdated verses the survey submission date<commit_after>package com.servinglynk.hmis.warehouse;

import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;

public class ActiveListView  extends Logging {

	 private SyncHBaseProcessor syncHBaseImport;
	 private Logger logger;
	 public ActiveListView(Logger logger) throws Exception {
	        this.logger = logger;
	        this.syncHBaseImport = new SyncHBaseProcessor();
	    }
	 
	public void createHbaseTable(String tableName) {
		syncHBaseImport.createHBASETable(tableName, logger);
	}
	
	
	 private void syncTable(String hbaseTable, String projectGroupCode,String postgresTable,Properties props) {
	       // log.info("Start sync for table: " + postgresTable);
	        ResultSet resultSet;
	        HTable htable;
	        PreparedStatement statement;
	        Connection connection;
	        try {   
	     	      htable = new HTable(HbaseUtil.getConfiguration(), hbaseTable);
	     	      StringBuilder builder  = new StringBuilder();
	     	      builder.append(" SELECT distinct on (t.client_dedup_id) t.client_dedup_id,t.client_id,t.deleted,t.ignore_match_process,survey_score,survey_submission_date FROM housing_inventory.eligible_clients t ");
	     	      builder.append(" inner join (  ");
	     	      builder.append("	  select client_dedup_id,max(survey_submission_date) as maxDate from housing_inventory.eligible_clients tm ");
	     	      builder.append("	  where tm.project_group_code= ? and tm.deleted=false ");
	     	      builder.append(" group by client_dedup_id ) tm ");
	     	      builder.append(" on  t.client_dedup_id = tm.client_dedup_id ");
	     	      builder.append(" and t.survey_submission_date = tm.maxDate and t.project_group_code=? ");
	     	      builder.append(" and t.date_updated >  ? ");
	     	      builder.append(" order by t.client_dedup_id,survey_score desc" );
	     	      
	     	      connection = SyncPostgresProcessor.getConnection();
		          statement = connection.prepareStatement(builder.toString());
		          statement.setString(1, projectGroupCode);
		          statement.setString(2, projectGroupCode);
		          statement.setDate(3, getCutOffDate(props));
		          resultSet = statement.executeQuery();
		            
	     	      List<String> existingKeysInHbase = syncHBaseImport.getAllKeyRecords(htable, logger);
	               List<String> existingKeysInPostgres = new ArrayList<>();

	               List<Put> putsToUpdate = new ArrayList<>();
	               List<Put> putsToInsert = new ArrayList<>();
	               List<String> putsToDelete = new ArrayList<>();
	               while (resultSet.next()) {
	                   Boolean markedForDelete = false;
	                   try{
	                   	markedForDelete = resultSet.getBoolean("deleted");
{'fraction_non_alphanumeric': 0.09525196621031168, 'fraction_numerical': 0.00203903291581707, 'mean_word_length': 3.022847100175747, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7062320', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1935, 'n_words': 386}