<commit_before>package com.atlassian.rstocker.cm;

import static com.atlassian.rstocker.cm.Common.ESCAPABLE;
import static com.atlassian.rstocker.cm.Common.normalizeReference;
import static com.atlassian.rstocker.cm.Common.normalizeURI;
import static com.atlassian.rstocker.cm.Common.unescapeString;
import static com.atlassian.rstocker.cm.Html5Entities.entityToString;

import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.atlassian.rstocker.cm.Node.Type;

public class InlineParser {
	// Constants for character codes:

	private static final char C_NEWLINE = 10;
	private static final char C_ASTERISK = 42;
	private static final char C_UNDERSCORE = 95;
	private static final char C_BACKTICK = 96;
	private static final char C_OPEN_BRACKET = 91;
	private static final char C_CLOSE_BRACKET = 93;
	private static final char C_LESSTHAN = 60;
	private static final char C_BANG = 33;
	private static final char C_BACKSLASH = 92;
	private static final char C_AMPERSAND = 38;
	private static final char C_OPEN_PAREN = 40;
	private static final char C_COLON = 58;

	private static final String ESCAPED_CHAR = "\\\\" + ESCAPABLE;
	private static final String REG_CHAR = "[^\\\\()\\x00-\\x20]";
	private static final String IN_PARENS_NOSP = "\\((" + REG_CHAR + '|' + ESCAPED_CHAR + ")*\\)";
	private static final String TAGNAME = "[A-Za-z][A-Za-z0-9]*";
	private static final String ATTRIBUTENAME = "[a-zA-Z_:][a-zA-Z0-9:._-]*";
	private static final String UNQUOTEDVALUE = "[^\"'=<>`\\x00-\\x20]+";
	private static final String SINGLEQUOTEDVALUE = "'[^']*'";
	private static final String DOUBLEQUOTEDVALUE = "\"[^\"]*\"";
	private static final String ATTRIBUTEVALUE = "(?:" + UNQUOTEDVALUE + "|" + SINGLEQUOTEDVALUE
			+ "|" + DOUBLEQUOTEDVALUE + ")";
	private static final String ATTRIBUTEVALUESPEC = "(?:" + "\\s*=" + "\\s*" + ATTRIBUTEVALUE
			+ ")";
	private static final String ATTRIBUTE = "(?:" + "\\s+" + ATTRIBUTENAME + ATTRIBUTEVALUESPEC
			+ "?)";
	private static final String OPENTAG = "<" + TAGNAME + ATTRIBUTE + "*" + "\\s*/?>";
	private static final String CLOSETAG = "</" + TAGNAME + "\\s*[>]";
	private static final String HTMLCOMMENT = "<!---->|<!--(?:-?[^>-])(?:-?[^-])*-->";
	private static final String PROCESSINGINSTRUCTION = "[<][?].*?[?][>]";
	private static final String DECLARATION = "<![A-Z]+" + "\\s+[^>]*>";
	private static final String CDATA = "<!\\[CDATA\\[[\\s\\S]*?\\]\\]>";
	private static final String HTMLTAG = "(?:" + OPENTAG + "|" + CLOSETAG + "|" + HTMLCOMMENT
			+ "|" +
			PROCESSINGINSTRUCTION + "|" + DECLARATION + "|" + CDATA + ")";
	private static final String ENTITY = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});";

	private static final Pattern rePunctuation = Pattern
			.compile("^[\u2000-\u206F\u2E00-\u2E7F\\'!\"#\\$%&\\(\\)\\*\\+,\\-\\./:;<=>\\?@\\[\\]\\^_`\\{\\|\\}~]");

	private static final Pattern reHtmlTag = Pattern.compile('^' + HTMLTAG, Pattern.CASE_INSENSITIVE);

	private static final Pattern reLinkTitle = Pattern.compile(
			"^(?:\"(" + ESCAPED_CHAR + "|[^\"\\x00])*\"" +
					'|' +
					"'(" + ESCAPED_CHAR + "|[^'\\x00])*'" +
					'|' +
					"\\((" + ESCAPED_CHAR + "|[^)\\x00])*\\))");

	private static final Pattern reLinkDestinationBraces = Pattern.compile(
			"^(?:[<](?:[^<>\\n\\\\\\x00]" + '|' + ESCAPED_CHAR + '|' + "\\\\)*[>])");

	private static final Pattern reLinkDestination = Pattern.compile(
			"^(?:" + REG_CHAR + "+|" + ESCAPED_CHAR + '|' + IN_PARENS_NOSP + ")*");

	private static final Pattern reEscapable = Pattern.compile(ESCAPABLE);

	private static final Pattern reEntityHere = Pattern.compile('^' + ENTITY, Pattern.CASE_INSENSITIVE);

	private static final Pattern reTicks = Pattern.compile("`+");

	private static final Pattern reTicksHere = Pattern.compile("^`+");

	private static final Pattern reEmailAutolink = Pattern
			.compile("^<([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>");

	// foo: /i
	private static final Pattern reAutolink = Pattern
			.compile("^<(?:coap|doi|javascript|aaa|aaas|about|acap|cap|cid|crid|data|dav|dict|dns|file|ftp|geo|go|gopher|h323|http|https|iax|icap|im|imap|info|ipp|iris|iris.beep|iris.xpc|iris.xpcs|iris.lwz|ldap|mailto|mid|msrp|msrps|mtqp|mupdate|news|nfs|ni|nih|nntp|opaquelocktoken|pop|pres|rtsp|service|session|shttp|sieve|sip|sips|sms|snmp|soap.beep|soap.beeps|tag|tel|telnet|tftp|thismessage|tn3270|tip|tv|urn|vemmi|ws|wss|xcon|xcon-userid|xmlrpc.beep|xmlrpc.beeps|xmpp|z39.50r|z39.50s|adiumxtra|afp|afs|aim|apt|attachment|aw|beshare|bitcoin|bolo|callto|chrome|chrome-extension|com-eventbrite-attendee|content|cvs|dlna-playsingle|dlna-playcontainer|dtn|dvb|ed2k|facetime|feed|finger|fish|gg|git|gizmoproject|gtalk|hcp|icon|ipn|irc|irc6|ircs|itms|jar|jms|keyparc|lastfm|ldaps|magnet|maps|market|message|mms|ms-help|msnim|mumble|mvn|notes|oid|palm|paparazzi|platform|proxy|psyc|query|res|resource|rmi|rsync|rtmp|secondlife|sftp|sgn|skype|smb|soldat|spotify|ssh|steam|svn|teamspeak|things|udp|unreal|ut2004|ventrilo|view-source|webcal|wtai|wyciwyg|xfire|xri|ymsgr):[^<>\u0000-\u0020]*>");

	private static final Pattern reSpnl = Pattern.compile("^ *(?:\n *)?");

	private static final Pattern reWhitespaceChar = Pattern.compile("^\\s");

	// foo: /g
	private static final Pattern reWhitespace = Pattern.compile("\\s+");
	private static final Pattern reSingleWhitespace = Pattern.compile("\\s");

	private static final Pattern reFinalSpace = Pattern.compile(" *$");

	private static final Pattern reLineEnd = Pattern.compile("^ *(?:\n|$)");

	private static final Pattern reInitialSpace = Pattern.compile("^ *");<commit_msg>Use Unicode whitespace in RE as per spec.txt<commit_after>package com.atlassian.rstocker.cm;

import static com.atlassian.rstocker.cm.Common.ESCAPABLE;
import static com.atlassian.rstocker.cm.Common.normalizeReference;
import static com.atlassian.rstocker.cm.Common.normalizeURI;
import static com.atlassian.rstocker.cm.Common.unescapeString;
import static com.atlassian.rstocker.cm.Html5Entities.entityToString;

import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.atlassian.rstocker.cm.Node.Type;

public class InlineParser {
	// Constants for character codes:

	private static final char C_NEWLINE = 10;
	private static final char C_ASTERISK = 42;
	private static final char C_UNDERSCORE = 95;
	private static final char C_BACKTICK = 96;
	private static final char C_OPEN_BRACKET = 91;
	private static final char C_CLOSE_BRACKET = 93;
	private static final char C_LESSTHAN = 60;
	private static final char C_BANG = 33;
	private static final char C_BACKSLASH = 92;
	private static final char C_AMPERSAND = 38;
	private static final char C_OPEN_PAREN = 40;
	private static final char C_COLON = 58;

	private static final String ESCAPED_CHAR = "\\\\" + ESCAPABLE;
	private static final String REG_CHAR = "[^\\\\()\\x00-\\x20]";
	private static final String IN_PARENS_NOSP = "\\((" + REG_CHAR + '|' + ESCAPED_CHAR + ")*\\)";
	private static final String TAGNAME = "[A-Za-z][A-Za-z0-9]*";
	private static final String ATTRIBUTENAME = "[a-zA-Z_:][a-zA-Z0-9:._-]*";
	private static final String UNQUOTEDVALUE = "[^\"'=<>`\\x00-\\x20]+";
	private static final String SINGLEQUOTEDVALUE = "'[^']*'";
	private static final String DOUBLEQUOTEDVALUE = "\"[^\"]*\"";
	private static final String ATTRIBUTEVALUE = "(?:" + UNQUOTEDVALUE + "|" + SINGLEQUOTEDVALUE
			+ "|" + DOUBLEQUOTEDVALUE + ")";
	private static final String ATTRIBUTEVALUESPEC = "(?:" + "\\s*=" + "\\s*" + ATTRIBUTEVALUE
			+ ")";
	private static final String ATTRIBUTE = "(?:" + "\\s+" + ATTRIBUTENAME + ATTRIBUTEVALUESPEC
			+ "?)";
	private static final String OPENTAG = "<" + TAGNAME + ATTRIBUTE + "*" + "\\s*/?>";
	private static final String CLOSETAG = "</" + TAGNAME + "\\s*[>]";
	private static final String HTMLCOMMENT = "<!---->|<!--(?:-?[^>-])(?:-?[^-])*-->";
	private static final String PROCESSINGINSTRUCTION = "[<][?].*?[?][>]";
	private static final String DECLARATION = "<![A-Z]+" + "\\s+[^>]*>";
	private static final String CDATA = "<!\\[CDATA\\[[\\s\\S]*?\\]\\]>";
	private static final String HTMLTAG = "(?:" + OPENTAG + "|" + CLOSETAG + "|" + HTMLCOMMENT
			+ "|" +
			PROCESSINGINSTRUCTION + "|" + DECLARATION + "|" + CDATA + ")";
	private static final String ENTITY = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});";

	private static final Pattern rePunctuation = Pattern
			.compile("^[\u2000-\u206F\u2E00-\u2E7F\\'!\"#\\$%&\\(\\)\\*\\+,\\-\\./:;<=>\\?@\\[\\]\\^_`\\{\\|\\}~]");

	private static final Pattern reHtmlTag = Pattern.compile('^' + HTMLTAG, Pattern.CASE_INSENSITIVE);

	private static final Pattern reLinkTitle = Pattern.compile(
			"^(?:\"(" + ESCAPED_CHAR + "|[^\"\\x00])*\"" +
					'|' +
					"'(" + ESCAPED_CHAR + "|[^'\\x00])*'" +
					'|' +
					"\\((" + ESCAPED_CHAR + "|[^)\\x00])*\\))");

	private static final Pattern reLinkDestinationBraces = Pattern.compile(
			"^(?:[<](?:[^<>\\n\\\\\\x00]" + '|' + ESCAPED_CHAR + '|' + "\\\\)*[>])");

	private static final Pattern reLinkDestination = Pattern.compile(
			"^(?:" + REG_CHAR + "+|" + ESCAPED_CHAR + '|' + IN_PARENS_NOSP + ")*");

	private static final Pattern reEscapable = Pattern.compile(ESCAPABLE);

	private static final Pattern reEntityHere = Pattern.compile('^' + ENTITY, Pattern.CASE_INSENSITIVE);

	private static final Pattern reTicks = Pattern.compile("`+");

	private static final Pattern reTicksHere = Pattern.compile("^`+");

	private static final Pattern reEmailAutolink = Pattern
			.compile("^<([a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>");

	// foo: /i
	private static final Pattern reAutolink = Pattern
			.compile("^<(?:coap|doi|javascript|aaa|aaas|about|acap|cap|cid|crid|data|dav|dict|dns|file|ftp|geo|go|gopher|h323|http|https|iax|icap|im|imap|info|ipp|iris|iris.beep|iris.xpc|iris.xpcs|iris.lwz|ldap|mailto|mid|msrp|msrps|mtqp|mupdate|news|nfs|ni|nih|nntp|opaquelocktoken|pop|pres|rtsp|service|session|shttp|sieve|sip|sips|sms|snmp|soap.beep|soap.beeps|tag|tel|telnet|tftp|thismessage|tn3270|tip|tv|urn|vemmi|ws|wss|xcon|xcon-userid|xmlrpc.beep|xmlrpc.beeps|xmpp|z39.50r|z39.50s|adiumxtra|afp|afs|aim|apt|attachment|aw|beshare|bitcoin|bolo|callto|chrome|chrome-extension|com-eventbrite-attendee|content|cvs|dlna-playsingle|dlna-playcontainer|dtn|dvb|ed2k|facetime|feed|finger|fish|gg|git|gizmoproject|gtalk|hcp|icon|ipn|irc|irc6|ircs|itms|jar|jms|keyparc|lastfm|ldaps|magnet|maps|market|message|mms|ms-help|msnim|mumble|mvn|notes|oid|palm|paparazzi|platform|proxy|psyc|query|res|resource|rmi|rsync|rtmp|secondlife|sftp|sgn|skype|smb|soldat|spotify|ssh|steam|svn|teamspeak|things|udp|unreal|ut2004|ventrilo|view-source|webcal|wtai|wyciwyg|xfire|xri|ymsgr):[^<>\u0000-\u0020]*>");

	private static final Pattern reSpnl = Pattern.compile("^ *(?:\n *)?");

	private static final Pattern reWhitespaceChar = Pattern.compile("^\\p{IsWhite_Space}");

	// foo: /g
	private static final Pattern reWhitespace = Pattern.compile("\\s+");
	private static final Pattern reSingleWhitespace = Pattern.compile("\\s");

	private static final Pattern reFinalSpace = Pattern.compile(" *$");

	private static final Pattern reLineEnd = Pattern.compile("^ *(?:\n|$)");

	private static final Pattern reInitialSpace = Pattern.compile("^ *");
{'fraction_non_alphanumeric': 0.20390132960111967, 'fraction_numerical': 0.020818754373687895, 'mean_word_length': 5.190037899296156, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5104653', 'n_tokens_mistral': 4782, 'n_tokens_neox': 4398, 'n_words': 731}