<commit_before>package org.rcsb.project5;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.rcsb.hadoop.io.SimplePolymerChain;
import org.rcsb.hadoop.io.SimplePolymerType;
import org.rcsb.utils.BlastClustReader;

import scala.Tuple2;

public class SequenceClusterIdComparison {
	private static int NUM_THREADS = 4;
	private static int NUM_TASKS_PER_THREAD = 3; // Spark recommends 2-3 tasks per thread

	public static void main(String[] args) {
		
		int seqIdBig = 100;
		int seqIdSmall = 90;

		// initialize Spark		
		JavaSparkContext sc = getSparkContext();
		
		/*long startTest = System.nanoTime();
		List<Tuple2<Integer, List<Integer>>> list = getSeqIdClustering(90, sc);
		System.out.println(list);
		System.out.println("Time: " + (System.nanoTime() - startTest)/1E9 + " sec.");
		System.exit(-1);*/
		
		long start = System.nanoTime();

		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs = getSequenceClusters(sc, seqIdBig)
				.cache();

		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs2 = getSequenceClusters(sc, seqIdSmall)
				.cache();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters = clusterPairs.groupByKey();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters2 = clusterPairs2.groupByKey();

		int big = getNumSeqClusters(seqIdBig);
		int small = getNumSeqClusters(seqIdSmall);
		int score = big - small;
		
		System.out.println(big);
		System.out.println(small);
		System.out.println("The difference in cluster numbers is " + score);
//		System.exit(-1);
		
		List<Tuple2<Integer, Iterable<String>>> clustersList = clusters.map(t -> t).collect();
		
		List<Integer> dif = clusters2.map(t -> getGroupDifference(t._2, clustersList)).collect();
		
		int sum = 0;
		
		for(Integer i: dif) {
			System.out.println(i);
			sum += i;
		}
		
		System.out.println(big);
		System.out.println(small);
		System.out.println("The difference in cluster numbers is " + score);
		System.out.println("The value is " + sum);
		System.out.println("The difference between score and sum is " + (score - sum));
		
		System.out.println("Time: " + (System.nanoTime() - start)/1E9 + " sec.");
		
	}
	
	public static List<Tuple2<Integer, List<Integer>>> getSeqIdClustering(int seqId, JavaSparkContext sc) {
				//read <sequence cluster id, PdbId.chainId> pairs
				JavaPairRDD<Integer,String> clusterPairs = getSequenceClusters(sc, 100)
						.cache();

				//read <sequence cluster id, PdbId.chainId> pairs
				JavaPairRDD<Integer,String> clusterPairs2 = getSequenceClusters(sc, seqId)
						.cache();

				// group by cluster id
				JavaPairRDD<Integer, Iterable<String>> clusters = clusterPairs.groupByKey();

				// group by cluster id
				JavaPairRDD<Integer, Iterable<String>> clusters2 = clusterPairs2.groupByKey();

				
				List<Tuple2<Integer, Iterable<String>>> clustersList = clusters.map(t -> t).collect();
				
				List<Tuple2<Integer, List<Integer>>> dif = clusters2.map(t -> new Tuple2<Integer, List<Integer>>(t._1, getGroupDifferenceArray(t._2, clustersList))).collect();
				
				return dif;
	}
	
	private static int getGroupDifference(Iterable<String> list, List<Tuple2<Integer, Iterable<String>>> clusters) {
		List<Tuple2<Integer, Iterable<String>>> clusterList = clusters;
		List<Integer> clusterIdList = new ArrayList<Integer>();
		for(Tuple2<Integer, Iterable<String>> tuple: clusterList) {
			for(String s: list) {
				if(contains(tuple, s)) {
					clusterIdList.add(tuple._1);
					break;
				}
			}
		}
		return clusterIdList.size() - 1;
	}
	
	private static List<Integer> getGroupDifferenceArray(Iterable<String> list, List<Tuple2<Integer, Iterable<String>>> clusters) {
		List<Tuple2<Integer, Iterable<String>>> clusterList = clusters;
		List<Integer> clusterIdList = new ArrayList<Integer>();
		for(Tuple2<Integer, Iterable<String>> tuple: clusterList) {
			for(String s: list) {
				if(contains(tuple, s)) {
					clusterIdList.add(tuple._1);
					break;
				}
			}
		}
		return clusterIdList;
	}
	
	private static boolean contains(Tuple2<Integer, Iterable<String>> tuple, String s) {
		Iterator<String> iter = tuple._2().iterator();
		while(iter.hasNext()) {
			if(iter.next().equals(s)) {
				return true;
			}
		}
		return false;
	}
<commit_msg>Added SequenceClusterIdComparison class which helps in finding clusters of altered sequence identity<commit_after>package org.rcsb.project5;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.rcsb.hadoop.io.SimplePolymerChain;
import org.rcsb.hadoop.io.SimplePolymerType;
import org.rcsb.utils.BlastClustReader;

import scala.Tuple2;

public class SequenceClusterIdComparison {
	private static int NUM_THREADS = 4;
	private static int NUM_TASKS_PER_THREAD = 3; // Spark recommends 2-3 tasks per thread

	public static void main(String[] args) {

		int seqIdBig = 100;
		int seqIdSmall = 90;

		// initialize Spark		
		JavaSparkContext sc = getSparkContext();

		/*long startTest = System.nanoTime();
		List<Tuple2<Integer, List<Integer>>> list = getSeqIdClustering(90, sc);
		System.out.println(list);
		System.out.println("Time: " + (System.nanoTime() - startTest)/1E9 + " sec.");
		System.exit(-1);*/

		long start = System.nanoTime();

		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs = getSequenceClusters(sc, seqIdBig)
				.cache();

		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs2 = getSequenceClusters(sc, seqIdSmall)
				.cache();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters = clusterPairs.groupByKey();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters2 = clusterPairs2.groupByKey();

		int big = getNumSeqClusters(seqIdBig);
		int small = getNumSeqClusters(seqIdSmall);
		int score = big - small;

		System.out.println(big);
		System.out.println(small);
		System.out.println("The difference in cluster numbers is " + score);
		//		System.exit(-1);

		List<Tuple2<Integer, Iterable<String>>> clustersList = clusters.map(t -> t).collect();

		List<Integer> dif = clusters2.map(t -> getGroupDifference(t._2, clustersList)).collect();

		int sum = 0;

		for(Integer i: dif) {
			System.out.println(i);
			sum += i;
		}

		System.out.println(big);
		System.out.println(small);
		System.out.println("The difference in cluster numbers is " + score);
		System.out.println("The value is " + sum);
		System.out.println("The difference between score and sum is " + (score - sum));

		System.out.println("Time: " + (System.nanoTime() - start)/1E9 + " sec.");

	}

	public static List<Tuple2<Integer, List<Integer>>> getSeqIdClustering(int seqId, JavaSparkContext sc, int startCluster, int endCluster) {
		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs = getSequenceClusters(sc, 100)
				.cache();

		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs2 = getSequenceClusters(sc, seqId)
				.filter(c -> (c._1 >= startCluster && c._1 <= endCluster))
				.cache();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters = clusterPairs.groupByKey();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters2 = clusterPairs2.groupByKey();


		List<Tuple2<Integer, Iterable<String>>> clustersList = clusters.map(t -> t).collect();

		List<Tuple2<Integer, List<Integer>>> dif = clusters2.map(t -> new Tuple2<Integer, List<Integer>>(t._1, getGroupDifferenceArray(t._2, clustersList))).collect();

		return dif;
	}

	public static List<Integer> getSeq100Clusters(int seqId, JavaSparkContext sc, int startCluster, int endCluster) {
		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs = getSequenceClusters(sc, 100)
				.cache();

		//read <sequence cluster id, PdbId.chainId> pairs
		JavaPairRDD<Integer,String> clusterPairs2 = getSequenceClusters(sc, seqId)
				.filter(c -> (c._1 >= startCluster && c._1 <= endCluster))
				.cache();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters = clusterPairs.groupByKey();

		// group by cluster id
		JavaPairRDD<Integer, Iterable<String>> clusters2 = clusterPairs2.groupByKey();

		
		List<Tuple2<Integer, Iterable<String>>> clustersList = clusters.map(t -> t).collect();
		
		List<Tuple2<Integer, List<Integer>>> dif = clusters2.map(t -> new Tuple2<Integer, List<Integer>>(t._1, getGroupDifferenceArray(t._2, clustersList))).collect();
		
		List<Integer> seq100Clusters = new ArrayList<Integer>();
		
		for(int a = 0; a < dif.size(); a ++) {
			for(int b = 0; b < dif.get(a)._2.size(); b ++) {
				seq100Clusters.add(dif.get(a)._2.get(b));
			}
		}
		
		return seq100Clusters;
	}

	private static int getGroupDifference(Iterable<String> list, List<Tuple2<Integer, Iterable<String>>> clusters) {
		List<Tuple2<Integer, Iterable<String>>> clusterList = clusters;
		List<Integer> clusterIdList = new ArrayList<Integer>();
		for(Tuple2<Integer, Iterable<String>> tuple: clusterList) {
			for(String s: list) {
				if(contains(tuple, s)) {
					clusterIdList.add(tuple._1);
					break;
				}
			}
		}
		return clusterIdList.size() - 1;
	}

	private static List<Integer> getGroupDifferenceArray(Iterable<String> list, List<Tuple2<Integer, Iterable<String>>> clusters) {
		List<Tuple2<Integer, Iterable<String>>> clusterList = clusters;
		List<Integer> clusterIdList = new ArrayList<Integer>();
		for(Tuple2<Integer, Iterable<String>> tuple: clusterList) {
			for(String s: list) {
				if(contains(tuple, s)) {
					clusterIdList.add(tuple._1);
					break;
				}
			}
		}
		return clusterIdList;
	}

	private static boolean contains(Tuple2<Integer, Iterable<String>> tuple, String s) {
		Iterator<String> iter = tuple._2().iterator();
		while(iter.hasNext()) {
			if(iter.next().equals(s)) {
				return true;
			}
		}
		return false;
	}

{'fraction_non_alphanumeric': 0.137744767049291, 'fraction_numerical': 0.012925629400983891, 'mean_word_length': 4.408450704225352, 'pattern_counts': {'":': 0, '<': 146, '<?xml version=': 0, '>': 156, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7659798', 'n_tokens_mistral': 3833, 'n_tokens_neox': 3355, 'n_words': 745}