<commit_before>#!/usr/bin/env python
# 2014 - <NAME> & <NAME>
# Port of @jgm's JavaScript stmd.js implementation of the CommonMark spec

# Basic usage:
#
# import CommonMark
# parser = CommonMark.DocParser()
# renderer = CommonMark.HtmlRenderer()
# print(renderer.render(parser.parse('Hello *world*')))


import re

#debug#
def dump(obj):
  for attr in dir(obj):
    print("obj.%s = %s" % (attr, getattr(obj, attr)))
 #debug#

# Some of the regexps used in inline parser :<

ESCAPABLE = '[!"#$%&\'()*+,./:;<=>?@[\\\\\\]^_`{|}~-]'
ESCAPED_CHAR = '\\\\' + ESCAPABLE
IN_DOUBLE_QUOTES = '"(' + ESCAPED_CHAR + '|[^"\\x00])*"'
IN_SINGLE_QUOTES = '\'(' + ESCAPED_CHAR + '|[^\'\\x00])*\''
IN_PARENS = '\\((' + ESCAPED_CHAR + '|[^)\\x00])*\\)'
REG_CHAR = '[^\\\\()\\x00-\\x20]'
IN_PARENS_NOSP = '\\((' + REG_CHAR + '|' + ESCAPED_CHAR + ')*\\)'
TAGNAME = '[A-Za-z][A-Za-z0-9]*'
BLOCKTAGNAME = '(?:article|header|aside|hgroup|iframe|blockquote|hr|body|li|map|button|object|canvas|ol|caption|output|col|p|colgroup|pre|dd|progress|div|section|dl|table|td|dt|tbody|embed|textarea|fieldset|tfoot|figcaption|th|figure|thead|footer|footer|tr|form|ul|h1|h2|h3|h4|h5|h6|video|script|style)'
ATTRIBUTENAME = '[a-zA-Z_:][a-zA-Z0-9:._-]*'
UNQUOTEDVALUE = "[^\"'=<>`\\x00-\\x20]+"
SINGLEQUOTEDVALUE = "'[^']*'"
DOUBLEQUOTEDVALUE = '"[^"]*"'
ATTRIBUTEVALUE = "(?:" + UNQUOTEDVALUE + "|" + SINGLEQUOTEDVALUE + "|" + DOUBLEQUOTEDVALUE + ")"
ATTRIBUTEVALUESPEC = "(?:" + "\\s*=" + "\\s*" + ATTRIBUTEVALUE + ")"
ATTRIBUTE = "(?:" + "\\s+" + ATTRIBUTENAME + ATTRIBUTEVALUESPEC + "?)"
OPENTAG = "<" + TAGNAME + ATTRIBUTE + "*" + "\\s*/?>"
CLOSETAG = "</" + TAGNAME + "\\s*[>]"
OPENBLOCKTAG = "<" + BLOCKTAGNAME + ATTRIBUTE + "*" + "\\s*/?>"
CLOSEBLOCKTAG = "</" + BLOCKTAGNAME + "\\s*[>]"
HTMLCOMMENT = "<!--([^-]+|[-][^-]+)*-->"
PROCESSINGINSTRUCTION = "[<][?].*?[?][>]"
DECLARATION = "<![A-Z]+" + "\\s+[^>]*>"
CDATA = "<!\\[CDATA\\[([^\\]]+|\\][^\\]]|\\]\\][^>])*\\]\\]>"
HTMLTAG = "(?:" + OPENTAG + "|" + CLOSETAG + "|" + HTMLCOMMENT + "|" + PROCESSINGINSTRUCTION + "|" + DECLARATION + "|" + CDATA + ")"
HTMLBLOCKOPEN = "<(?:" + BLOCKTAGNAME + "[\\s/>]" + "|" + "/" + BLOCKTAGNAME + "[\\s>]" + "|" + "[?!])"

reHtmlTag = re.compile('^' + HTMLTAG, re.IGNORECASE)
reHtmlBlockOpen = re.compile('^' + HTMLBLOCKOPEN, re.IGNORECASE)
reLinkTitle = re.compile('^(?:"(' + ESCAPED_CHAR + '|[^"\\x00])*"' + '|' + '\'(' + ESCAPED_CHAR + '|[^\'\\x00])*\'' + '|' + '\\((' + ESCAPED_CHAR + '|[^)\\x00])*\\))')
reLinkDestinationBraces = re.compile('^(?:[<](?:[^<>\\n\\\\\\x00]' + '|' + ESCAPED_CHAR + '|' + '\\\\)*[>])')
reLinkDestination = re.compile('^(?:' + REG_CHAR + '+|' + ESCAPED_CHAR + '|' + IN_PARENS_NOSP + ')*')
reEscapable = re.compile(ESCAPABLE)
reAllEscapedChar = '\\\\(' + ESCAPABLE + ')'
reEscapedChar = re.compile('^\\\\(' + ESCAPABLE + ')')
reAllTab = re.compile("\t")
reHrule = re.compile(r"^(?:(?:\* *){3,}|(?:_ *){3,}|(?:- *){3,}) *$")

# Matches a character with a special meaning in markdown,
# or a string of non-special characters.
reMain = r"^(?:[\n`\[\]\\!<&*_]|[^\n`\[\]\\!<&*_]+)"

# Utility functions

def unescape(s):
	""" Replace backslash escapes with literal characters."""
	return re.sub(reAllEscapedChar, r"\g<1>", s)

def isBlank(s):
  """ Returns True if string contains only space characters."""
  return bool(re.compile("^\s*$").match(s))

def normalizeReference(s):
	""" Normalize reference label: collapse internal whitespace to
	 single space, remove leading/trailing whitespace, case fold."""
	return re.sub(r'\s+', ' ', s.strip()).upper()

def matchAt(pattern, s, offset):
	""" Attempt to match a regex in string s at offset offset.
	Return index of match or None."""
	matched = re.search(pattern, s[offset:])
	if matched:
		return offset+s[offset:].index(matched.group(0))
	else:
		return None

def detabLine(text):
	""" Convert tabs to spaces on each line using a 4-space tab stop."""
	if re.match('\t', text) and text.index('\t') == -1:
		return text
	else:
		def tabber(m):
			result = "    "[(m.end()-1-tabber.lastStop)%4:]
			tabber.lastStop = m.end()
			return result
		tabber.lastStop = 0
		text = re.sub("\t", tabber, text)
		return text


class Block(object):

	@staticmethod
	def makeBlock(tag,start_line, start_column):
		# self.t =  tag
		# self.isOpen =  true
		# self.last_line_blank =  false
		# self.start_line =  start_line
		# self.start_column =  start_column
		# self.end_line =  start_line
		# self.children =  []
		# self.parent =  None
		# self.string_content =  ""
		# self.strings =  []
		# self.inline_content =  []
		return Block(t=tag, start_line=start_line, start_column=start_column)

	def __init__(self, t="", c="", destination="", label=[], start_line="", start_column="", title=""):
		self.t = t
		self.c = c
		self.destination = destination
		self.label = label
		self.isOpen =  True
		self.last_line_blank =  False
		self.start_line =  start_line
		self.start_column =  start_column
		self.end_line =  start_line
		self.children =  []
		self.parent =  None
		self.string_content =  ""
		self.strings =  []
		self.inline_content =  []
		self.list_data = {}
		self.title = title
		self.info = ""


class InlineParser(object):
	"""  INLINE PARSER

	 These are methods of an InlineParser class, defined below.
	 An InlineParser keeps track of a subject (a string to be
	 parsed) and a position in that subject.

	 If re matches at current position in the subject, advance
	 position in subject and return the match; otherwise return null."""

	def __init__(self):
		self.subject = ""
		self.label_nest_level = 0
		self.pos = 0
		self.refmap = {}

	def match(self, regexString, reCompileFlags=0):
		#regex = re.compile(regexString, flags=reCompileFlags)
		match = re.search(regexString, self.subject[self.pos:], flags=reCompileFlags)
		if match:
			self.pos += match.end()
			return match.group(0)
		else:
			return None

	def peek(self):
		try:
			return self.subject[self.pos]
		except IndexError:
			return None

	def spnl(self):
		self.match(r"^ *(?:\n *)?")
		return 1

	def parseBackticks(self, inlines):
		startpos = self.pos
		ticks = self.match(r"^`+")
		if not ticks:
			return 0
		afterOpenTicks = self.pos
		foundCode = False
		match = self.match(r"`+", re.MULTILINE)
		while (not foundCode) and (match != None):
			if (match == ticks):
				c = self.subject[afterOpenTicks:(self.pos-len(ticks))]
				c = re.sub(r"[ \n]+", ' ', c)
				c = c.strip()
				inlines.append(Block(t="Code", c=c))
				return (self.pos - startpos)
			match = self.match(r"`+", re.MULTILINE)
		inlines.append(Block(t="Str", c=ticks))
		self.pos = afterOpenTicks
		return (self.pos - startpos)
	

	def parseEscaped(self, inlines):
		subj = self.subject
		pos = self.pos
		if (subj[pos] == "\\"):
			if len(subj) > pos+1 and (subj[pos+1] == "\n"):
				inlines.append(Block(t="Hardbreak"))
				self.pos += 2
				return 2
			elif (reEscapable.search(subj[pos:pos+1])):
				inlines.append(Block(t="Str", c=subj[pos+1:pos+2]))
				self.pos += 2
				return 2
			else:
				self.pos +=1
				inlines.append(Block(t="Str", c="\\"))
				return 1
		else:
			return 0

	def parseAutoLink(self, inlines):
		m = self.match("^<([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>")
		m2 = self.match("^<(?:coap|doi|javascript|aaa|aaas|about|acap|cap|cid|crid|data|dav|dict|dns|file|ftp|geo|go|gopher|h323|http|https|iax|icap|im|imap|info|ipp|iris|iris.beep|iris.xpc|iris.xpcs|iris.lwz|ldap|mailto|mid|msrp|msrps|mtqp|mupdate|news|nfs|ni|nih|nntp|opaquelocktoken|pop|pres|rtsp|service|session|shttp|sieve|sip|sips|sms|snmp|soap.beep|soap.beeps|tag|tel|telnet|tftp|thismessage|tn3270|tip|tv|urn|vemmi|ws|wss|xcon|xcon-userid|xmlrpc.beep|xmlrpc.beeps|xmpp|z39.50r|z39.50s|adiumxtra|afp|afs|aim|apt|attachment|aw|beshare|bitcoin|bolo|callto|chrome|chrome-extension|com-eventbrite-attendee|content|cvs|dlna-playsingle|dlna-playcontainer|dtn|dvb|ed2k|facetime|feed|finger|fish|gg|git|gizmoproject|gtalk|hcp|icon|ipn|irc|irc6|ircs|itms|jar|jms|keyparc|lastfm|ldaps|magnet|maps|market|message|mms|ms-help|msnim|mumble|mvn|notes|oid|palm|paparazzi|platform|proxy|psyc|query|res|resource|rmi|rsync|rtmp|secondlife|sftp|sgn|skype|smb|soldat|spotify|ssh|steam|svn|teamspeak|things|udp|unreal|ut2004|ventrilo|view-source|webcal|wtai|wyciwyg|xfire|xri|ymsgr):[^<>\x00-\x20]*>", re.IGNORECASE) 
		if m:
			# email
			dest = m[1:-1]
			inlines.append(Block(t="Link", label=[Block(t="Str", c=dest)], destination="mailto:"+dest))
			return len(m)
		elif m2:
			# link
			dest2 = m2[1:-1]
			inlines.append(Block(t="Link", label=[Block(t="Str", c=dest2)], destination=dest2))
			return len(m2)
		else:
			return 0

	def parseHtmlTag(self, inlines):
		m = self.match(reHtmlTag)
		if (m):
			inlines.append(Block(t="Html", c=m))
			return len(m)
		else:
			return 0

	def scanDelims(self, c):
		numdelims = 0
		first_close_delims = 0
		char_before = char_after = None
		startpos = self.pos

		char_before = '\n' if self.pos == 0 else self.subject[self.pos - 1]

		while (self.peek() == c):
			numdelims += 1
			self.pos += 1 

		a = self.peek()
		char_after = a if a else "\\n"

		can_open = (numdelims > 0) and (numdelims <=3) and (not re.match("\s", char_after))
		can_close = (numdelims > 0) and (numdelims <=3) and (not re.match("\s", char_before))

		if (c == "_"):
			can_open = can_open and (not re.match("[a-z0-9]", char_before, re.IGNORECASE))
			can_close = can_close and (not re.match("[a-z0-9]", char_after, re.IGNORECASE))
		self.pos = startpos
		return {
			"numdelims": numdelims,
			"can_open": can_open,
			"can_close": can_close
		}

	def parseEmphasis(self, inlines):
		startpos = self.pos
		first_close = 0
		nxt = self.peek()
		if ((nxt == "*") or (nxt == "_")):
			c = nxt
		else:
			return 0

		res = self.scanDelims(c)
		numdelims = res["numdelims"]
		self.pos += numdelims
		if startpos > 0:
			inlines.append(Block(t="Str", c=self.subject[self.pos - numdelims:numdelims+startpos]))
		else:
			inlines.append(Block(t="Str", c=self.subject[self.pos - numdelims:numdelims]))
		delimpos = len(inlines) - 1

		if ((not res["can_open"]) or (numdelims == 0)):
			return 0

		first_close_delims = 0

		if (numdelims == 1):
			while (True):
				res = self.scanDelims(c)
				if (res["numdelims"] >= 1 and res["can_close"]):
					self.pos += 1
					inlines[delimpos].t = "Emph"
					inlines[delimpos].c = inlines[delimpos+1:]
					if len(inlines) > 1:
						for x in range(delimpos+1, len(inlines)):
							inlines.pop(len(inlines)-1)
					break 
				else:
					if (self.parseInline(inlines) == 0):
						break
			return (self.pos - startpos)
		elif (numdelims == 2):
			while (True):
				res = self.scanDelims(c)
				if (res["numdelims"] >= 2 and res["can_close"]):
					self.pos += 2
					inlines[delimpos].t = "Strong"
					inlines[delimpos].c = inlines[delimpos+1:]
					if len(inlines) > 1:
						for x in range(delimpos+1, len(inlines)):
							inlines.pop(len(inlines)-1)
					break
				else:
					if (self.parseInline(inlines) == 0):
						break
			return (self.pos - startpos)
		elif (numdelims == 3):
			while (True):
				res = self.scanDelims(c)
				if (res["numdelims"] >= 1 and res["numdelims"] <= 3 and res["can_close"] and not res["numdelims"] == first_close_delims):
					if first_close_delims == 1 and numdelims > 2:
						res["numdelims"] = 2
					elif first_close_delims == 2:
						res['numdelims'] = 1
					elif res['numdelims'] == 3:
						res['numdelims'] = 1
					self.pos += res['numdelims']

					if first_close > 0:
						inlines[delimpos].t = "Strong" if first_close_delims == 1 else "Emph"
						temp = "Emph" if first_close_delims == 1 else "Strong"
						inlines[delimpos].c = [Block(t=temp, c=inlines[delimpos+1:first_close])]+inlines[first_close+1:] # error on 362?
						if len(inlines) > 1:
							for x in range(delimpos+1, len(inlines)):
								inlines.pop(len(inlines)-1)
						break
					else:
						inlines.append(Block(t="Str", c=self.subject[self.pos-res["numdelims"]:self.pos]))
						first_close = len(inlines)-1
						first_close_delims = res["numdelims"]
				else:
					if self.parseInline(inlines) == 0:
						break
			return (self.pos-startpos)
		else:
			return res

		return 0

	def parseLinkTitle(self):
		title = self.match(reLinkTitle)
		if title:
			return unescape(title[1:-1])
		else:
			return None

	def parseLinkDestination(self):
		res = self.match(reLinkDestinationBraces)
		if res:
			return unescape(res[1:-1])
		else:
			res2 = self.match(reLinkDestination)
			if res2:
				return unescape(res2)
			else:
				return None

	def parseLinkLabel(self):
		if not self.peek() == "[":
			return 0
		startpos = self.pos
		nest_level = 0
		if self.label_nest_level > 0:
			self.label_nest_level -= 1
			return 0
		self.pos += 1
		c = self.peek()
		while ((not c == "]") or (nest_level > 0)) and not c == None: # and (c = self.peek()):
			if c == "`":
				self.parseBackticks([])
				break
			elif c == "<":
				self.parseAutoLink([]) or self.parseHtmlTag([]) or self.parseString([])
				break
			elif c == "[":
				nest_level += 1
				self.pos += 1
				break
			elif c == "]":
				nest_level -= 1
				self.pos += 1
				break
			elif c == "\\":
				self.parseEscaped([])
				break
			else:
				self.parseString([])
			c = self.peek()
		if c == "]":
			self.label_nest_level = 0
			self.pos += 1
			return self.pos-startpos
		else:
			if c == None:
				self.label_nest_level = nest_level
			self.pos = startpos
			return 0

	def parseRawLabel(self, s):
		return InlineParser().parse(s[1:-1])

	def parseLink(self, inlines):
		startpos = self.pos
		n = self.parseLinkLabel()
		if startpos > 0:
			n += startpos

		if n == 0:
			return 0

		afterlabel = self.pos
		rawlabel = self.subject[startpos:n]

		if self.peek() == "(":
			self.pos += 1
			if self.spnl():
				dest = self.parseLinkDestination()
				if dest and self.spnl():
					title = self.parseLinkTitle() or ''
					if re.match(r"^\s", self.subject[self.pos-1]) and (not title == None) or True:
						if (title or True) and self.spnl() and self.match(r"^\)"):
							inlines.append(Block(t="Link", destination=dest, title=title, label=self.parseRawLabel(rawlabel)))
							return self.pos-startpos
						else:
							self.pos = startpos
							return 0
					else:
						self.pos = startpos
						return 0
				else:
					self.pos = startpos
					return 0
			else:
				self.pos = startpos
				return 0

		savepos = self.pos
		self.spnl()
		beforelabel = self.pos
		n = self.parseLinkLabel()
		if n == 2:
			reflabel = rawlabel
		elif n > 0:
			reflabel = self.subject[beforelabel:beforelabel+n]
		else:
			self.pos = savepos
			reflabel = rawlabel
		if normalizeReference(reflabel) in self.refmap:
			link = self.refmap[normalizeReference(reflabel)]
		else: 
			link = None
		if link:
			if link.get("title", None):
			 	title = link['title']
			else:
			 	title = ""
			if link.get("destination", None):
				destination = link['destination']
			else:
				destination = ""
			inlines.append(Block(t="Link", destination=destination, title=title, label=self.parseRawLabel(rawlabel)))
			return self.pos-startpos
		else:
			self.pos = startpos
			return 0
		self.pos = startpos
		return 0


	def parseEntity(self, inlines):
		m = self.match(r"^&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});", re.IGNORECASE)
		if m:
			inlines.append(Block(t="Entity", c=m))
			return len(m)
		else:
			return 0

	def parseString(self, inlines):
		m = self.match(reMain, re.MULTILINE)
		if m:
			inlines.append(Block(t="Str", c=m))
			return len(m)
		else:
			return 0

	def parseNewline(self, inlines):
		if (self.peek() == '\n'):
			self.pos += 1
			last = inlines[len(inlines)-1]
			if last and last.t == "Str" and last.c[-2:] == "  ":
				last.c = re.sub(r' *$', '', last.c)
				inlines.append(Block(t="Hardbreak"))
			else:
				if last and last.t == "Str" and len(last.c) >= 2 and last.c[-2] == " ":
					last.c = last.c[0:-1]
				inlines.append(Block(t="Softbreak"))
			return 1
		else:
			return 0

	def parseImage(self, inlines):
		if (self.match("^!")):
			n = self.parseLink(inlines)
			if (n == 0):
				inlines.append(Block(t="Str", c="!"))
				return 1
			elif (inlines[len(inlines) - 1] and
				(inlines[len(inlines)-1].t == "Link")):
				inlines[len(inlines)-1].t = "Image"
				return n+1<commit_msg>some link stuff<commit_after>#!/usr/bin/env python
# 2014 - <NAME> & <NAME>
# Port of @jgm's JavaScript stmd.js implementation of the CommonMark spec

# Basic usage:
#
# import CommonMark
# parser = CommonMark.DocParser()
# renderer = CommonMark.HtmlRenderer()
# print(renderer.render(parser.parse('Hello *world*')))


import re

#debug#
def dump(obj):
  for attr in dir(obj):
    print("obj.%s = %s" % (attr, getattr(obj, attr)))
 #debug#

# Some of the regexps used in inline parser :<

ESCAPABLE = '[!"#$%&\'()*+,./:;<=>?@[\\\\\\]^_`{|}~-]'
ESCAPED_CHAR = '\\\\' + ESCAPABLE
IN_DOUBLE_QUOTES = '"(' + ESCAPED_CHAR + '|[^"\\x00])*"'
IN_SINGLE_QUOTES = '\'(' + ESCAPED_CHAR + '|[^\'\\x00])*\''
IN_PARENS = '\\((' + ESCAPED_CHAR + '|[^)\\x00])*\\)'
REG_CHAR = '[^\\\\()\\x00-\\x20]'
IN_PARENS_NOSP = '\\((' + REG_CHAR + '|' + ESCAPED_CHAR + ')*\\)'
TAGNAME = '[A-Za-z][A-Za-z0-9]*'
BLOCKTAGNAME = '(?:article|header|aside|hgroup|iframe|blockquote|hr|body|li|map|button|object|canvas|ol|caption|output|col|p|colgroup|pre|dd|progress|div|section|dl|table|td|dt|tbody|embed|textarea|fieldset|tfoot|figcaption|th|figure|thead|footer|footer|tr|form|ul|h1|h2|h3|h4|h5|h6|video|script|style)'
ATTRIBUTENAME = '[a-zA-Z_:][a-zA-Z0-9:._-]*'
UNQUOTEDVALUE = "[^\"'=<>`\\x00-\\x20]+"
SINGLEQUOTEDVALUE = "'[^']*'"
DOUBLEQUOTEDVALUE = '"[^"]*"'
ATTRIBUTEVALUE = "(?:" + UNQUOTEDVALUE + "|" + SINGLEQUOTEDVALUE + "|" + DOUBLEQUOTEDVALUE + ")"
ATTRIBUTEVALUESPEC = "(?:" + "\\s*=" + "\\s*" + ATTRIBUTEVALUE + ")"
ATTRIBUTE = "(?:" + "\\s+" + ATTRIBUTENAME + ATTRIBUTEVALUESPEC + "?)"
OPENTAG = "<" + TAGNAME + ATTRIBUTE + "*" + "\\s*/?>"
CLOSETAG = "</" + TAGNAME + "\\s*[>]"
OPENBLOCKTAG = "<" + BLOCKTAGNAME + ATTRIBUTE + "*" + "\\s*/?>"
CLOSEBLOCKTAG = "</" + BLOCKTAGNAME + "\\s*[>]"
HTMLCOMMENT = "<!--([^-]+|[-][^-]+)*-->"
PROCESSINGINSTRUCTION = "[<][?].*?[?][>]"
DECLARATION = "<![A-Z]+" + "\\s+[^>]*>"
CDATA = "<!\\[CDATA\\[([^\\]]+|\\][^\\]]|\\]\\][^>])*\\]\\]>"
HTMLTAG = "(?:" + OPENTAG + "|" + CLOSETAG + "|" + HTMLCOMMENT + "|" + PROCESSINGINSTRUCTION + "|" + DECLARATION + "|" + CDATA + ")"
HTMLBLOCKOPEN = "<(?:" + BLOCKTAGNAME + "[\\s/>]" + "|" + "/" + BLOCKTAGNAME + "[\\s>]" + "|" + "[?!])"

reHtmlTag = re.compile('^' + HTMLTAG, re.IGNORECASE)
reHtmlBlockOpen = re.compile('^' + HTMLBLOCKOPEN, re.IGNORECASE)
reLinkTitle = re.compile('^(?:"(' + ESCAPED_CHAR + '|[^"\\x00])*"' + '|' + '\'(' + ESCAPED_CHAR + '|[^\'\\x00])*\'' + '|' + '\\((' + ESCAPED_CHAR + '|[^)\\x00])*\\))')
reLinkDestinationBraces = re.compile('^(?:[<](?:[^<>\\n\\\\\\x00]' + '|' + ESCAPED_CHAR + '|' + '\\\\)*[>])')
reLinkDestination = re.compile('^(?:' + REG_CHAR + '+|' + ESCAPED_CHAR + '|' + IN_PARENS_NOSP + ')*')
reEscapable = re.compile(ESCAPABLE)
reAllEscapedChar = '\\\\(' + ESCAPABLE + ')'
reEscapedChar = re.compile('^\\\\(' + ESCAPABLE + ')')
reAllTab = re.compile("\t")
reHrule = re.compile(r"^(?:(?:\* *){3,}|(?:_ *){3,}|(?:- *){3,}) *$")

# Matches a character with a special meaning in markdown,
# or a string of non-special characters.
reMain = r"^(?:[\n`\[\]\\!<&*_]|[^\n`\[\]\\!<&*_]+)"

# Utility functions

def unescape(s):
	""" Replace backslash escapes with literal characters."""
	return re.sub(reAllEscapedChar, r"\g<1>", s)

def isBlank(s):
  """ Returns True if string contains only space characters."""
  return bool(re.compile("^\s*$").match(s))

def normalizeReference(s):
	""" Normalize reference label: collapse internal whitespace to
	 single space, remove leading/trailing whitespace, case fold."""
	return re.sub(r'\s+', ' ', s.strip()).upper()

def matchAt(pattern, s, offset):
	""" Attempt to match a regex in string s at offset offset.
	Return index of match or None."""
	matched = re.search(pattern, s[offset:])
	if matched:
		return offset+s[offset:].index(matched.group(0))
	else:
		return None

def detabLine(text):
	""" Convert tabs to spaces on each line using a 4-space tab stop."""
	if re.match('\t', text) and text.index('\t') == -1:
		return text
	else:
		def tabber(m):
			result = "    "[(m.end()-1-tabber.lastStop)%4:]
			tabber.lastStop = m.end()
			return result
		tabber.lastStop = 0
		text = re.sub("\t", tabber, text)
		return text


class Block(object):

	@staticmethod
	def makeBlock(tag,start_line, start_column):
		# self.t =  tag
		# self.isOpen =  true
		# self.last_line_blank =  false
		# self.start_line =  start_line
		# self.start_column =  start_column
		# self.end_line =  start_line
		# self.children =  []
		# self.parent =  None
		# self.string_content =  ""
		# self.strings =  []
		# self.inline_content =  []
		return Block(t=tag, start_line=start_line, start_column=start_column)

	def __init__(self, t="", c="", destination="", label=[], start_line="", start_column="", title=""):
		self.t = t
		self.c = c
		self.destination = destination
		self.label = label
		self.isOpen =  True
		self.last_line_blank =  False
		self.start_line =  start_line
		self.start_column =  start_column
		self.end_line =  start_line
		self.children =  []
		self.parent =  None
		self.string_content =  ""
		self.strings =  []
		self.inline_content =  []
		self.list_data = {}
		self.title = title
		self.info = ""


class InlineParser(object):
	"""  INLINE PARSER

	 These are methods of an InlineParser class, defined below.
	 An InlineParser keeps track of a subject (a string to be
	 parsed) and a position in that subject.

	 If re matches at current position in the subject, advance
	 position in subject and return the match; otherwise return null."""

	def __init__(self):
		self.subject = ""
		self.label_nest_level = 0
		self.pos = 0
		self.refmap = {}

	def match(self, regexString, reCompileFlags=0):
		match = re.search(regexString, self.subject[self.pos:], flags=reCompileFlags)
		if match:
			self.pos += match.end(0)
			return match.group()
		else:
			return None

	def peek(self):
		try:
			return self.subject[self.pos]
		except IndexError:
			return None

	def spnl(self):
		self.match(r"^ *(?:\n *)?")
		return 1

	def parseBackticks(self, inlines):
		startpos = self.pos
		ticks = self.match(r"^`+")
		if not ticks:
			return 0
		afterOpenTicks = self.pos
		foundCode = False
		match = self.match(r"`+", re.MULTILINE)
		while (not foundCode) and (match != None):
			if (match == ticks):
				c = self.subject[afterOpenTicks:(self.pos-len(ticks))]
				c = re.sub(r"[ \n]+", ' ', c)
				c = c.strip()
				inlines.append(Block(t="Code", c=c))
				return (self.pos - startpos)
			match = self.match(r"`+", re.MULTILINE)
		inlines.append(Block(t="Str", c=ticks))
		self.pos = afterOpenTicks
		return (self.pos - startpos)
	

	def parseEscaped(self, inlines):
		subj = self.subject
		pos = self.pos
		if (subj[pos] == "\\"):
			if len(subj) > pos+1 and (subj[pos+1] == "\n"):
				inlines.append(Block(t="Hardbreak"))
				self.pos += 2
				return 2
			elif (reEscapable.search(subj[pos:pos+1])):
				inlines.append(Block(t="Str", c=subj[pos+1:pos+2]))
				self.pos += 2
				return 2
			else:
				self.pos +=1
				inlines.append(Block(t="Str", c="\\"))
				return 1
		else:
			return 0

	def parseAutoLink(self, inlines):
		m = self.match("^<([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)>")
		m2 = self.match("^<(?:coap|doi|javascript|aaa|aaas|about|acap|cap|cid|crid|data|dav|dict|dns|file|ftp|geo|go|gopher|h323|http|https|iax|icap|im|imap|info|ipp|iris|iris.beep|iris.xpc|iris.xpcs|iris.lwz|ldap|mailto|mid|msrp|msrps|mtqp|mupdate|news|nfs|ni|nih|nntp|opaquelocktoken|pop|pres|rtsp|service|session|shttp|sieve|sip|sips|sms|snmp|soap.beep|soap.beeps|tag|tel|telnet|tftp|thismessage|tn3270|tip|tv|urn|vemmi|ws|wss|xcon|xcon-userid|xmlrpc.beep|xmlrpc.beeps|xmpp|z39.50r|z39.50s|adiumxtra|afp|afs|aim|apt|attachment|aw|beshare|bitcoin|bolo|callto|chrome|chrome-extension|com-eventbrite-attendee|content|cvs|dlna-playsingle|dlna-playcontainer|dtn|dvb|ed2k|facetime|feed|finger|fish|gg|git|gizmoproject|gtalk|hcp|icon|ipn|irc|irc6|ircs|itms|jar|jms|keyparc|lastfm|ldaps|magnet|maps|market|message|mms|ms-help|msnim|mumble|mvn|notes|oid|palm|paparazzi|platform|proxy|psyc|query|res|resource|rmi|rsync|rtmp|secondlife|sftp|sgn|skype|smb|soldat|spotify|ssh|steam|svn|teamspeak|things|udp|unreal|ut2004|ventrilo|view-source|webcal|wtai|wyciwyg|xfire|xri|ymsgr):[^<>\x00-\x20]*>", re.IGNORECASE) 
		if m:
			# email
			dest = m[1:-1]
			inlines.append(Block(t="Link", label=[Block(t="Str", c=dest)], destination="mailto:"+dest))
			return len(m)
		elif m2:
			# link
			dest2 = m2[1:-1]
			inlines.append(Block(t="Link", label=[Block(t="Str", c=dest2)], destination=dest2))
			return len(m2)
		else:
			return 0

	def parseHtmlTag(self, inlines):
		m = self.match(reHtmlTag)
		if (m):
			inlines.append(Block(t="Html", c=m))
			return len(m)
		else:
			return 0

	def scanDelims(self, c):
		numdelims = 0
		first_close_delims = 0
		char_before = char_after = None
		startpos = self.pos

		char_before = '\n' if self.pos == 0 else self.subject[self.pos - 1]

		while (self.peek() == c):
			numdelims += 1
			self.pos += 1 

		a = self.peek()
		char_after = a if a else "\\n"

		can_open = (numdelims > 0) and (numdelims <=3) and (not re.match("\s", char_after))
		can_close = (numdelims > 0) and (numdelims <=3) and (not re.match("\s", char_before))

		if (c == "_"):
			can_open = can_open and (not re.match("[a-z0-9]", char_before, re.IGNORECASE))
			can_close = can_close and (not re.match("[a-z0-9]", char_after, re.IGNORECASE))
		self.pos = startpos
		return {
			"numdelims": numdelims,
			"can_open": can_open,
			"can_close": can_close
		}

	def parseEmphasis(self, inlines):
		startpos = self.pos
		first_close = 0
		nxt = self.peek()
		if ((nxt == "*") or (nxt == "_")):
			c = nxt
		else:
			return 0

		res = self.scanDelims(c)
		numdelims = res["numdelims"]
		self.pos += numdelims
		if startpos > 0:
			inlines.append(Block(t="Str", c=self.subject[self.pos - numdelims:numdelims+startpos]))
		else:
			inlines.append(Block(t="Str", c=self.subject[self.pos - numdelims:numdelims]))
		delimpos = len(inlines) - 1

		if ((not res["can_open"]) or (numdelims == 0)):
			return 0

		first_close_delims = 0

		if (numdelims == 1):
			while (True):
				res = self.scanDelims(c)
				if (res["numdelims"] >= 1 and res["can_close"]):
					self.pos += 1
					inlines[delimpos].t = "Emph"
					inlines[delimpos].c = inlines[delimpos+1:]
					if len(inlines) > 1:
						for x in range(delimpos+1, len(inlines)):
							inlines.pop(len(inlines)-1)
					break 
				else:
					if (self.parseInline(inlines) == 0):
						break
			return (self.pos - startpos)
		elif (numdelims == 2):
			while (True):
				res = self.scanDelims(c)
				if (res["numdelims"] >= 2 and res["can_close"]):
					self.pos += 2
					inlines[delimpos].t = "Strong"
					inlines[delimpos].c = inlines[delimpos+1:]
					if len(inlines) > 1:
						for x in range(delimpos+1, len(inlines)):
							inlines.pop(len(inlines)-1)
					break
				else:
					if (self.parseInline(inlines) == 0):
						break
			return (self.pos - startpos)
		elif (numdelims == 3):
			while (True):
				res = self.scanDelims(c)
				if (res["numdelims"] >= 1 and res["numdelims"] <= 3 and res["can_close"] and not res["numdelims"] == first_close_delims):
					if first_close_delims == 1 and numdelims > 2:
						res["numdelims"] = 2
					elif first_close_delims == 2:
						res['numdelims'] = 1
					elif res['numdelims'] == 3:
						res['numdelims'] = 1
					self.pos += res['numdelims']

					if first_close > 0:
						inlines[delimpos].t = "Strong" if first_close_delims == 1 else "Emph"
						temp = "Emph" if first_close_delims == 1 else "Strong"
						inlines[delimpos].c = [Block(t=temp, c=inlines[delimpos+1:first_close])]+inlines[first_close+1:] # error on 362?
						if len(inlines) > 1:
							for x in range(delimpos+1, len(inlines)):
								inlines.pop(len(inlines)-1)
						break
					else:
						inlines.append(Block(t="Str", c=self.subject[self.pos-res["numdelims"]:self.pos]))
						first_close = len(inlines)-1
						first_close_delims = res["numdelims"]
				else:
					if self.parseInline(inlines) == 0:
						break
			return (self.pos-startpos)
		else:
			return res

		return 0

	def parseLinkTitle(self):
		title = self.match(reLinkTitle)
		if title:
			return unescape(title[1:-1])
		else:
			return None

	def parseLinkDestination(self):
		res = self.match(reLinkDestinationBraces)
		if not res == None:
			return unescape(res[1:len(res)-1])
		else:
			res2 = self.match(reLinkDestination)
			print(":wewe")
			if not res2 == None:
				return unescape(res2)
			else:
				return None

	def parseLinkLabel(self):
		if not self.peek() == "[":
			return 0
		startpos = self.pos
		nest_level = 0
		if self.label_nest_level > 0:
			self.label_nest_level -= 1
			return 0
		self.pos += 1
		c = self.peek()
		while ((not c == "]") or (nest_level > 0)) and not c == None: # and (c = self.peek()):
			if c == "`":
				self.parseBackticks([])
				break
			elif c == "<":
				self.parseAutoLink([]) or self.parseHtmlTag([]) or self.parseString([])
				break
			elif c == "[":
				nest_level += 1
				self.pos += 1
				break
			elif c == "]":
				nest_level -= 1
				self.pos += 1
				break
			elif c == "\\":
				self.parseEscaped([])
				break
			else:
				self.parseString([])
			c = self.peek()
		if c == "]":
			self.label_nest_level = 0
			self.pos += 1
			return self.pos-startpos
		else:
			if c == None:
				self.label_nest_level = nest_level
			self.pos = startpos
			return 0

	def parseRawLabel(self, s):
		return InlineParser().parse(s[1:-1])

	def parseLink(self, inlines):
		startpos = self.pos
		n = self.parseLinkLabel()
		if startpos > 0:
			n += startpos

		if n == 0:
			return 0

		afterlabel = self.pos
		rawlabel = self.subject[startpos:n]

		if self.peek() == "(":
			self.pos += 1
			if self.spnl():
				dest = self.parseLinkDestination()
				if not dest == None and self.spnl():
					title = self.parseLinkTitle() or ''
					if re.match(r"^\s", self.subject[self.pos-1]) and (not title == None) or True:
						if (title or True) and self.spnl() and self.match(r"^\)"):
							inlines.append(Block(t="Link", destination=dest, title=title, label=self.parseRawLabel(rawlabel)))
							return self.pos-startpos
						else:
							self.pos = startpos
							return 0
					else:
						self.pos = startpos
						return 0
				else:
					self.pos = startpos
					return 0
			else:
				self.pos = startpos
				return 0

		savepos = self.pos
		self.spnl()
		beforelabel = self.pos
		n = self.parseLinkLabel()
		if n == 2:
			reflabel = rawlabel
		elif n > 0:
			reflabel = self.subject[beforelabel:beforelabel+n]
		else:
			self.pos = savepos
			reflabel = rawlabel
		if normalizeReference(reflabel) in self.refmap:
			link = self.refmap[normalizeReference(reflabel)]
		else: 
			link = None
		if link:
			if link.get("title", None):
			 	title = link['title']
			else:
			 	title = ""
			if link.get("destination", None):
				destination = link['destination']
			else:
				destination = ""
			inlines.append(Block(t="Link", destination=destination, title=title, label=self.parseRawLabel(rawlabel)))
			return self.pos-startpos
		else:
			self.pos = startpos
			return 0
		self.pos = startpos
		return 0


	def parseEntity(self, inlines):
		m = self.match(r"^&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});", re.IGNORECASE)
		if m:
			inlines.append(Block(t="Entity", c=m))
			return len(m)
		else:
			return 0

	def parseString(self, inlines):
		m = self.match(reMain, re.MULTILINE)
		if m:
			inlines.append(Block(t="Str", c=m))
			return len(m)
		else:
			return 0

	def parseNewline(self, inlines):
		if (self.peek() == '\n'):
			self.pos += 1
			last = inlines[len(inlines)-1]
			if last and last.t == "Str" and last.c[-2:] == "  ":
				last.c = re.sub(r' *$', '', last.c)
				inlines.append(Block(t="Hardbreak"))
			else:
				if last and last.t == "Str" and last.c[-1] == " ":
					print(last.c[0:-1])
					last.c = last.c[0:-1]
				inlines.append(Block(t="Softbreak"))
			return 1
		else:
			return 0

	def parseImage(self, inlines):
		if (self.match("^!")):
			n = self.parseLink(inlines)
			if (n == 0):
				inlines.append(Block(t="Str", c="!"))
				return 1
			elif (inlines[len(inlines) - 1] and
				(inlines[len(inlines)-1].t == "Link")):
				inlines[len(inlines)-1].t = "Image"
				return n+1
{'fraction_non_alphanumeric': 0.19552609776304888, 'fraction_numerical': 0.015311915063364939, 'mean_word_length': 3.613533408833522, 'pattern_counts': {'":': 27, '<': 55, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3337940', 'n_tokens_mistral': 15000, 'n_tokens_neox': 12812, 'n_words': 2465}