<commit_before>package org.arrah.framework.dataquality;

/***************************************************
 *     Copyright to <NAME> 2016         *
 *                                                 *
 * Any part of code or file can be changed,        *
 * redistributed, modified with the copyright      *
 * information intact                              *
 *                                                 *
 * Author$ : <NAME>                           *
 *                                                 *
 **************************************************/

/*
 * This class is used finding duplicate rows from
 * RTM and show the duplicate values
 *
 * This file will also be used for completing 
 * some empty columns like address completion
 * if zip code is provided.
 */
import java.math.BigInteger;
import java.util.Vector;

import org.arrah.framework.ndtable.ReportTableModel;
import org.arrah.framework.ndtable.ResultsetToRTM;

public class DeDuplicateRTM {
	private ReportTableModel duplicate = null;
	
	// Constructor
	public DeDuplicateRTM() {
		// Do nothing
	}
	
	/* this function will take report table model and colI
	 * on which duplicate need to be found.
	 * It will take string values of those colI and will do
	 * MD5 and will do match.
	 * 1 in billion there is possibility that it will be a 
	 * false positive. 
	 * 
	 */
	public ReportTableModel removeDuplicate (ReportTableModel rtm, int[] colI) {
		if (rtm == null) return rtm;
		int rowC = rtm.getModel().getRowCount();
		if (rowC <= 0) return rtm;
		Vector<BigInteger> md5array = new Vector<BigInteger>();
		Vector<Integer> markDel = new Vector<Integer>();
		duplicate = new ReportTableModel(rtm.getAllColName(),true,true);
				
		// Sanity check and Init over
		
		for (int i=0; i < rowC; i++) {
			Object[] rowv = null;
			if (colI != null)
				rowv = rtm.getSelectedColRow(i,colI);
			else
				rowv = rtm.getRow(i);
			
			String rowStr="";
			
			// Create a string for row Values
			for (int j=0;j< rowv.length; j++) {
				if (rowv[j] == null)
					rowStr += "Null";
				else
					rowStr += rowv[j].toString();
			}
			BigInteger md5v = ResultsetToRTM.getMD5(rowStr);
			int indexf = md5array.indexOf(md5v);
			if (indexf == -1){ // not found so uniuqe row
				md5array.add(md5v);	
			} else { // not unique so mark for delete it
				markDel.add(i);
				duplicate.addFillRow(rtm.getRow(i));
			}
		}
		rtm.removeMarkedRows(markDel);
		
		return rtm;
	}
	
	public ReportTableModel showDuplicateModel () {
		
		return duplicate;
	}
	
}<commit_msg>clean up<commit_after>package org.arrah.framework.dataquality;

/***************************************************
 *     Copyright to <NAME> 2016         *
 *                                                 *
 * Any part of code or file can be changed,        *
 * redistributed, modified with the copyright      *
 * information intact                              *
 *                                                 *
 * Author$ : <NAME>                           *
 *                                                 *
 **************************************************/

/*
 * This class is used finding duplicate rows from
 * RTM and show the duplicate values
 *
 * This file will also be used for completing 
 * some empty columns like address completion
 * if zip code is provided.
 */
import java.math.BigInteger;
import java.util.Vector;

import org.arrah.framework.ndtable.ReportTableModel;
import org.arrah.framework.ndtable.ResultsetToRTM;

public class DeDuplicateRTM {
	private ReportTableModel duplicates = null;
	
	// Constructor
	public DeDuplicateRTM() {
		// Do nothing
	}

	public ReportTableModel removeDuplicate(ReportTableModel reportTableModel, int[] columnIndices) {
		return removeDuplicate(reportTableModel, columnIndices, false);
	}

	/* this function will take report table model and colI
	 * on which duplicate need to be found.
	 * It will take string values of those colI and will do
	 * MD5 and will do match.
	 * 1 in billion there is possibility that it will be a 
	 * false positive. 
	 * 
	 */
	public ReportTableModel removeDuplicate (ReportTableModel reportTableModel, int[] columnIndices, boolean ignoreCase) {
		if (reportTableModel == null) {
			return null;
		}

		int rowCount = reportTableModel.getModel().getRowCount();

		if (rowCount <= 0) {
			return reportTableModel;
		}

		Vector<Integer> deletionIndices = new Vector<>();
		Vector<BigInteger> md5array = new Vector<>();

		duplicates = new ReportTableModel(reportTableModel.getAllColName(),true,true);
				
		// Sanity check and Init over
		
		for (int i = 0; i < rowCount; i++) {
			Object[] rowObjectArray;

			if (columnIndices != null) {
				rowObjectArray = reportTableModel.getSelectedColRow(i, columnIndices);
			} else {
				rowObjectArray = reportTableModel.getRow(i);
			}

			StringBuilder row = new StringBuilder();
			
			// Create a string for row Values
			for (Object o : rowObjectArray) {
				if (o == null) {
					row.append("Null");
				} else {
					row.append(o.toString());
				}
			}

			BigInteger md5v = ResultsetToRTM.getMD5(row.toString());

			int md5RowIndex = md5array.indexOf(md5v);

			if (md5RowIndex == -1){ // not found so uniuqe row
				md5array.add(md5v);	
			} else { // not unique so mark for delete it
				deletionIndices.add(i);

				duplicates.addFillRow(reportTableModel.getRow(i));
			}
		}

		reportTableModel.removeMarkedRows(deletionIndices);
		
		return reportTableModel;
	}
	
	public ReportTableModel showDuplicateModel () {
		return duplicates;
	}
	
}
{'fraction_non_alphanumeric': 0.12990990990990992, 'fraction_numerical': 0.006306306306306306, 'mean_word_length': 2.542437779195916, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5044132', 'n_tokens_mistral': 1799, 'n_tokens_neox': 1558, 'n_words': 471}