<commit_before>package com.atlassian.rstocker.cm;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Common {

	private static final String ENTITY = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});";

	// foo: not sure about [ in [
	static final String ESCAPABLE = "[!\"#$%&\'()*+,./:;<=>?@\\[\\\\\\]^_`{|}~-]";

	// foo: not sure about the backslashes here
	private static final Pattern reBackslashOrAmp = Pattern.compile("[\\\\&]");

	// foo: had flags 'gi' before
	private static final Pattern reEntityOrEscapedChar =
			Pattern.compile("\\\\" + ESCAPABLE + '|' + ENTITY, Pattern.CASE_INSENSITIVE);

	private static final String XMLSPECIAL = "[&<>\"]";

	// foo: had flags 'g' before
	private static final Pattern reXmlSpecial = Pattern.compile(XMLSPECIAL);

	// foo: had flags 'gi' before
	private static final Pattern reXmlSpecialOrEntity = Pattern.compile(ENTITY + '|' + XMLSPECIAL,
			Pattern.CASE_INSENSITIVE);

	static String unescapeChar(String s) {
		if (s.charAt(0) == '\\') {
			return s.substring(1);
		} else {
			return Html5Entities.entityToString(s);
		}
	}

	// Replace entities and backslash escapes with literal characters.
	public static String unescapeString(String s) {
		if (reBackslashOrAmp.matcher(s).find()) {
			return replaceAll(reEntityOrEscapedChar, s, match -> unescapeChar(match));
		} else {
			return s;
		}
	}

	public static String normalizeURI(String uri) {
		try {
			// foo: equivalent to encodeURI(decodeURI(uri))?
			uri = uri.replaceAll(" ", "%20");
			uri = uri.replaceAll("\\\\", "%5C");
			uri = uri.replaceAll("`", "%60");
			return new URI(uri).toASCIIString();
		} catch (URISyntaxException e) {
			return uri;
		}
	}

	private static Pattern whitespace = Pattern.compile("[ \t\r\n]+");

	public static String normalizeReference(String input) {
		// foo: is this the same as JS?
		return whitespace.matcher(input.toLowerCase(Locale.ROOT)).replaceAll(" ");
	}

	public static Escaper XML_ESCAPER = (s, preserveEntities) -> {
		Pattern p = preserveEntities ? reXmlSpecialOrEntity : reXmlSpecial;
		return replaceAll(p, s, match -> replaceUnsafeChar(match));
	};

	private static String replaceUnsafeChar(String s) {
		switch (s) {
		case "&":
			return "&amp;";
		case "<":
			return "&lt;";
		case ">":
			return "&gt;";
		case "\"":<commit_msg>Add some more escapes<commit_after>package com.atlassian.rstocker.cm;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.Locale;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Common {

	private static final String ENTITY = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});";

	// foo: not sure about [ in [
	static final String ESCAPABLE = "[!\"#$%&\'()*+,./:;<=>?@\\[\\\\\\]^_`{|}~-]";

	// foo: not sure about the backslashes here
	private static final Pattern reBackslashOrAmp = Pattern.compile("[\\\\&]");

	// foo: had flags 'gi' before
	private static final Pattern reEntityOrEscapedChar =
			Pattern.compile("\\\\" + ESCAPABLE + '|' + ENTITY, Pattern.CASE_INSENSITIVE);

	private static final String XMLSPECIAL = "[&<>\"]";

	// foo: had flags 'g' before
	private static final Pattern reXmlSpecial = Pattern.compile(XMLSPECIAL);

	// foo: had flags 'gi' before
	private static final Pattern reXmlSpecialOrEntity = Pattern.compile(ENTITY + '|' + XMLSPECIAL,
			Pattern.CASE_INSENSITIVE);

	static String unescapeChar(String s) {
		if (s.charAt(0) == '\\') {
			return s.substring(1);
		} else {
			return Html5Entities.entityToString(s);
		}
	}

	// Replace entities and backslash escapes with literal characters.
	public static String unescapeString(String s) {
		if (reBackslashOrAmp.matcher(s).find()) {
			return replaceAll(reEntityOrEscapedChar, s, match -> unescapeChar(match));
		} else {
			return s;
		}
	}

	public static String normalizeURI(String uri) {
		try {
			// foo: equivalent to encodeURI(decodeURI(uri))?
			uri = uri.replaceAll(" ", "%20");
			uri = uri.replaceAll("\\\\", "%5C");
			uri = uri.replaceAll("`", "%60");
			uri = uri.replaceAll("\"", "%22");
			uri = uri.replaceAll("\\]", "%5D");
			uri = uri.replaceAll("\\[", "%5B");
			return new URI(uri).toASCIIString();
		} catch (URISyntaxException e) {
			return uri;
		}
	}

	private static Pattern whitespace = Pattern.compile("[ \t\r\n]+");

	public static String normalizeReference(String input) {
		// foo: is this the same as JS?
		return whitespace.matcher(input.toLowerCase(Locale.ROOT)).replaceAll(" ");
	}

	public static Escaper XML_ESCAPER = (s, preserveEntities) -> {
		Pattern p = preserveEntities ? reXmlSpecialOrEntity : reXmlSpecial;
		return replaceAll(p, s, match -> replaceUnsafeChar(match));
	};

	private static String replaceUnsafeChar(String s) {
		switch (s) {
		case "&":
			return "&amp;";
		case "<":
			return "&lt;";
		case ">":
			return "&gt;";
		case "\"":
{'fraction_non_alphanumeric': 0.17125199362041468, 'fraction_numerical': 0.009170653907496013, 'mean_word_length': 3.9624134520276955, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2007997', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1703, 'n_words': 416}