<commit_before>import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.lang3.builder.ReflectionToStringBuilder;

public class DC {
	// Need to be ordered to get the same ordering from all AWS datacenters when
	// calculating the cloests DC from a Req.
	private static Set<String> allDCsAvailable = new TreeSet<String>();

	private static String localAz = null;
	private static String localDc = null;
	static List<String> remoteDCs = new ArrayList<String>();

	static public void Init() throws Exception {
		try (Cons.MT _ = new Cons.MT("DC.Init() ...")) {
			// Get local and remote DC names from .run/dc-ip-map
			File fn_jar = new File(AcornYoutube.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
			String fn = String.format("%s/.run/dc-ip-map", fn_jar.getParentFile().getParentFile());
			try (BufferedReader br = new BufferedReader(new FileReader(fn))) {
				String line;
				while ((line = br.readLine()) != null) {
					// us-east-1 192.168.3.11
					String[] t = line.split("\\s+");
					if (t.length !=2)
						throw new RuntimeException(String.format("Unexpcted format [%s]", line));
					allDCsAvailable.add(t[0]);
				}
			}
			Cons.P("allDCsAvailable: %s", String.join(", ", allDCsAvailable));

			// Get local DC and AZ names.
			Runtime r = Runtime.getRuntime();
			Process p = r.exec("curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone");
			p.waitFor();
			BufferedReader b = new BufferedReader(new InputStreamReader(p.getInputStream()));
			String line = "";
			while ((line = b.readLine()) != null) {
				localAz = line;
			}
			b.close();
			// Trim the last [a-z].
			localDc = localAz.substring(0, localAz.length() - 1);
			Cons.P("Local DC=%s AZ=%s", localDc, localAz);

			// Calc remote DC names.
			remoteDCs = allDCsAvailable.stream().filter(dc -> (!dc.equals(localDc))).collect(Collectors.toList());
			Cons.P("remoteDCs: %s", String.join(", ", remoteDCs));

			// Cassandra Ec2Snitch sometimes seems to have shorter names, which you
			// could get around with prefix matches.
		}
	}

	static public boolean IsLocalDcTheClosestToReq(YoutubeData.Req r) {
		return GetClosestDcToReq(r).equals(localDc);
	}

	// This won't break the tie.
	//
	//private static Coord localDcCoord = null;
	//
	//static public boolean IsLocalDcTheClosestToReq(YoutubeData.Req r) {
	//	if (localDcCoord == null)
	//		localDcCoord = Conf.acornYoutubeOptions.mapDcCoord.get(localDc);
	//
	//	double localDcToReq = ArcInRadians(localDcCoord, r);
	//
	//	for (String rDc: remoteDcs) {
	//		Coord c = Conf.acornYoutubeOptions.mapDcCoord.get(rDc);
	//		if (c == null)
	//			throw new RuntimeException(String.format("Unexpected: rDc=%s", rDc));
	//
	//		double dcToReq = ArcInRadians(c, r);
	//		// In the case of tie, we won't see the request from any DC. Need to have
	//		// an order to break ties.
	//		if (localDcToReq > dcToReq)
	//			return true;
	//	}
	//	return false;
	//}

	private static Set<String> allDCs = new TreeSet<String>(Arrays.asList(
				"us-east-1"
				, "eu-west-1"
				, "us-west-1"
				, "us-west-2"
				, "eu-central-1"
				, "sa-east-1"
				, "ap-south-1"
				, "ap-southeast-1"
				, "ap-southeast-2"
				, "ap-northeast-1"
				, "ap-northeast-2"
				));

	static public String GetClosestDcToReq(YoutubeData.Req r) {
		String closestDc = null;
		// Initial value doesn't matter. Just to make the compiler quiet.
		double shortestDist = 0;

		Set<String> dcs = null;
		if (Conf.acornYoutubeOptions.use_all_dcs_for_finding_the_local_dc_of_a_req) {
			dcs = allDCs;
		} else {
			dcs = allDCsAvailable;
		}

		for (String dc: dcs) {
			Coord c = Conf.acornYoutubeOptions.mapDcCoord.get(dc);
			if (c == null)
				throw new RuntimeException(String.format("Unexpected: dc=%s", dc));

			double dcToReq = ArcInRadians(c, r);
			if (closestDc == null) {
				closestDc = dc;
				shortestDist = dcToReq;
				continue;
			}
			if (dcToReq < shortestDist) {
				closestDc = dc;
				shortestDist = dcToReq;
			}
		}
		return closestDc;
	}

	static public String GetClosestDcFromCandidates(Set<String> dcs) {
		Coord local_coord = Conf.acornYoutubeOptions.mapDcCoord.get(localDc);
		String closestDc = null;
		double shortestDist = 0;

		for (String dc: dcs) {
			Coord c = Conf.acornYoutubeOptions.mapDcCoord.get(dc);
			if (c == null)
				throw new RuntimeException(String.format("Unexpected: dc=%s", dc));

			// Caching the distance result would be an optimization, but I doubt it
			// would optimize much.
			double dcToLocalDc = ArcInRadians(c, local_coord);
			if (closestDc == null) {
				closestDc = dc;
				shortestDist = dcToLocalDc;
				continue;
			}
			if (dcToLocalDc < shortestDist) {
				closestDc = dc;
				shortestDist = dcToLocalDc;
			}
		}
		return closestDc;
	}<commit_msg>Cass Ec2Snitch returns us-west instead of us-west-1. The sames goes with all the other DCs that end with -1.<commit_after>import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.lang3.builder.ReflectionToStringBuilder;

public class DC {
	// Need to be ordered to get the same ordering from all AWS datacenters when
	// calculating the cloests DC from a Req.
	private static Set<String> allDCsAvailable = new TreeSet<String>();

	private static String localAz = null;
	private static String localDc = null;
	static List<String> remoteDCs = new ArrayList<String>();

	static public void Init() throws Exception {
		try (Cons.MT _ = new Cons.MT("DC.Init() ...")) {
			// Get local and remote DC names from .run/dc-ip-map
			File fn_jar = new File(AcornYoutube.class.getProtectionDomain().getCodeSource().getLocation().toURI().getPath());
			String fn = String.format("%s/.run/dc-ip-map", fn_jar.getParentFile().getParentFile());
			try (BufferedReader br = new BufferedReader(new FileReader(fn))) {
				String line;
				while ((line = br.readLine()) != null) {
					// us-east-1 192.168.3.11
					String[] t = line.split("\\s+");
					if (t.length !=2)
						throw new RuntimeException(String.format("Unexpcted format [%s]", line));
					allDCsAvailable.add(t[0]);
				}
			}
			Cons.P("allDCsAvailable: %s", String.join(", ", allDCsAvailable));

			// Get local DC and AZ names.
			Runtime r = Runtime.getRuntime();
			Process p = r.exec("curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone");
			p.waitFor();
			BufferedReader b = new BufferedReader(new InputStreamReader(p.getInputStream()));
			String line = "";
			while ((line = b.readLine()) != null) {
				localAz = line;
			}
			b.close();
			// Trim the last [a-z].
			localDc = localAz.substring(0, localAz.length() - 1);
			Cons.P("Local DC=%s AZ=%s", localDc, localAz);

			// Calc remote DC names.
			remoteDCs = allDCsAvailable.stream().filter(dc -> (!dc.equals(localDc))).collect(Collectors.toList());
			Cons.P("remoteDCs: %s", String.join(", ", remoteDCs));

			// Cassandra Ec2Snitch sometimes seems to have shorter names, which you
			// could get around with prefix matches.
		}
	}

	static public boolean IsLocalDcTheClosestToReq(YoutubeData.Req r) {
		return GetClosestDcToReq(r).equals(localDc);
	}

	// This won't break the tie.
	//
	//private static Coord localDcCoord = null;
	//
	//static public boolean IsLocalDcTheClosestToReq(YoutubeData.Req r) {
	//	if (localDcCoord == null)
	//		localDcCoord = Conf.acornYoutubeOptions.mapDcCoord.get(localDc);
	//
	//	double localDcToReq = ArcInRadians(localDcCoord, r);
	//
	//	for (String rDc: remoteDcs) {
	//		Coord c = Conf.acornYoutubeOptions.mapDcCoord.get(rDc);
	//		if (c == null)
	//			throw new RuntimeException(String.format("Unexpected: rDc=%s", rDc));
	//
	//		double dcToReq = ArcInRadians(c, r);
	//		// In the case of tie, we won't see the request from any DC. Need to have
	//		// an order to break ties.
	//		if (localDcToReq > dcToReq)
	//			return true;
	//	}
	//	return false;
	//}

	private static Set<String> allDCs = new TreeSet<String>(Arrays.asList(
				"us-east-1"
				, "eu-west-1"
				, "us-west-1"
				, "us-west-2"
				, "eu-central-1"
				, "sa-east-1"
				, "ap-south-1"
				, "ap-southeast-1"
				, "ap-southeast-2"
				, "ap-northeast-1"
				, "ap-northeast-2"
				));

	static public String GetClosestDcToReq(YoutubeData.Req r) {
		String closestDc = null;
		// Initial value doesn't matter. Just to make the compiler quiet.
		double shortestDist = 0;

		Set<String> dcs = null;
		if (Conf.acornYoutubeOptions.use_all_dcs_for_finding_the_local_dc_of_a_req) {
			dcs = allDCs;
		} else {
			dcs = allDCsAvailable;
		}

		for (String dc: dcs) {
			Coord c = Conf.acornYoutubeOptions.mapDcCoord.get(dc);
			if (c == null)
				throw new RuntimeException(String.format("Unexpected: dc=%s", dc));

			double dcToReq = ArcInRadians(c, r);
			if (closestDc == null) {
				closestDc = dc;
				shortestDist = dcToReq;
				continue;
			}
			if (dcToReq < shortestDist) {
				closestDc = dc;
				shortestDist = dcToReq;
			}
		}
		return closestDc;
	}

	private static Pattern _pattern0 = Pattern.compile(".+-.+-\\d$");

	public static String GetClosestDcFromCandidates(Set<String> dcs) {
		Coord local_coord = Conf.acornYoutubeOptions.mapDcCoord.get(localDc);
		String closestDc = null;
		double shortestDist = 0;

		for (String dc: dcs) {
			Matcher matcher = _pattern0.matcher(dc);
			String dc1 = null;
			if (matcher.find()) {
				dc1 = dc;
			} else {
				dc1 = dc + "-1";
			}

			Coord c = Conf.acornYoutubeOptions.mapDcCoord.get(dc1);
			if (c == null)
				throw new RuntimeException(String.format("Unexpected: dc1=%s", dc1));

			// Caching the distance result would be an optimization, but I doubt it
			// would optimize much.
			double dcToLocalDc = ArcInRadians(c, local_coord);
			if (closestDc == null) {
				closestDc = dc;
				shortestDist = dcToLocalDc;
				continue;
			}
			if (dcToLocalDc < shortestDist) {
				closestDc = dc;
				shortestDist = dcToLocalDc;
			}
		}
		return closestDc;
	}
{'fraction_non_alphanumeric': 0.14080459770114942, 'fraction_numerical': 0.00900383141762452, 'mean_word_length': 3.318031430934657, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5990220', 'n_tokens_mistral': 4276, 'n_tokens_neox': 3699, 'n_words': 885}