<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
This file is part of DragonCreole
Copyright (C) 2013 - 2016 <NAME>

This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
'''

'''
==ABOUT==
DragonCreole is designed as a lightweight markup language, derived from
the Creole markup language (http://wikicreole.org/).  It includes an
expanded syntax and is optimized for high performance.
'''

try:
	from html import escape as escape
except:
	from cgi import escape as escape


import shlex, inspect, re, html2text, time, datetime

from types import *

regex_list = re.compile(r'\*+\s.+|\#+\s.+|\@+\s.+|\!+\s.+')
regex_list2 = re.compile(r'(\*+|\#+|\@+|\!+)')
regex_indent = re.compile(r'(\:+)')
rematch = re.match

class MacroObject():
	'''
		This class is used by macro functions to provide data about the page and a reference to the renderer
	'''
	__slots__ = ("body", "renderer")
	def __init__(self, body="", renderer=None):
		if(body != ""):
			if(body[0] == "\n"): body = body[1:]
		self.body = body
		self.renderer = renderer


class DragonCreole():
	"""
		The main renderer class.
	"""
	__slots__ = ("noMacros", "auto_paragraphs", "link_path", "image_path", "bodied_macros", "non_bodied_macros", "link_path_func", "link_class_func", "out_link_mark")
	def __init__(self, link_path="/", image_path="", bodied_macros={}, non_bodied_macros={}, link_path_func=None, link_class_func=None, out_link_mark = "➚", auto_paragraphs=True, noMacros=False):
		"""
			Returns a DragonCreole parser object.  The following optional arguments may be passed when creating a DragonCreole parser, in order to extend it.
			* link_path: The base path for all wiki links, which will be prepended to them.
			* image_path: The base path for all wiki images, which will be preprended to them.
			* bodied_macros: A dictionary of function references, which are called when the parser encounters a bodied macro.
			* non_bodied_macros: A dictionary of function references, which are called when the parser encounters a non-bodied macro.
			* link_path_func: A function reference which will be called to inspect any wiki link, and may modify its path.
			* link_class_func: A function reference which will be called to inspect any wiki link, and may modify its CSS class.
			* out_link_mark: A character which is appended to external links.  Set to a blank string for no mark.  Can be set to DragonCreole code, if desired, such as when one wishes to use an image or special formatting for the mark.
			* auto_paragraphs: If True, then paragraphs will automatically be created.
			* noMacros: If true, then macros will not be run.
		"""
		
		'''
		assert type(link_path) is str
		assert type(image_path) is str
		
		assert type(bodied_macros) is dict
		for func in bodied_macros:
			assert type(func) is FunctionType
		
		assert type(non_bodied_macros) is dict
		for func in non_bodied_macros:
			assert type(func) is FunctionType
		
		if(link_path_func != None):
			assert type(link_path_func) is FunctionType
		if(link_class_func != None):
			assert type(link_class_func) is FunctionType
		'''
		
		
		self.bodied_macros = {
			"block": self.macro_block
		}
		self.bodied_macros.update(bodied_macros)
		
		self.non_bodied_macros = {
			"datetime": self.macro_datetime
		}
		self.non_bodied_macros.update(non_bodied_macros)
		
		self.link_path = link_path
		self.image_path = image_path
		self.link_path_func = link_path_func
		self.link_class_func = link_class_func
		self.auto_paragraphs = auto_paragraphs
		self.noMacros = noMacros
		if(out_link_mark != ""):
			self.out_link_mark = self.process(out_link_mark)
		else:
			self.out_link_mark = out_link_mark
		return
	__cinit__ = __init__
	
	'''
	Renders a page to HTML
	'''
	def render(self, text, noMacros=None):
		frags = self.fragmentize(text)
		output = []
		i = -1
		skip = -1
		for i, frag in enumerate(frags):
			if(skip > i):
				continue
			nextFrag = ""
			if(i+1 < len(frags)):
				nextFrag = frags[i+1]
			
			if(frag == ""):
				frag = "\n"
			elif(frag[:1] == "="):
				frag = self.handleHeading(frag)
			elif(frag[:2] == "\\\\"):
				frag = "<br>"
			elif(frag[:4] == "----"):
				frag = "<hr>"
			elif(frag[:1] in ">:"):
				frag = self.handleParagraph(frag)
			elif(frag[:1] == "|"):
				if(i+1 < len(frags)):
					i2 = i+1
					while(i2 < len(frags)):
						if(frags[i2][:1] == "|"):
							frag += "\n" + frags[i2]
							i2 += 1
							skip = i2
						else:
							break
				frag = self.handleTables(frag)
			elif(frag.startswith("{{{") and frag.endswith("}}}")):
				frag = self.handlePreformat(frag)[0]
			elif(frag.startswith("<<") and frag.endswith(">>")):
				frag = self.handleMacro(frag)[0]
			elif(rematch(regex_list,frag) != None):
				if(i+1 < len(frags)):
					i2 = i+1
					while(i2 < len(frags)):
						if(rematch(regex_list,frags[i2]) != None):
							frag += "\n" + frags[i2]
							i2 += 1
							skip = i2
						else:
							break
				frag = self.handleLists(frag)
			elif(frag[:1] == ";" and nextFrag[:1] == ":"):
				if(i+1 < len(frags)):
					frag += "\n" + nextFrag
					i += 2
					i2 = i
					while(i2 < len(frags)):
						if(frags[i2][:1] == ";" and frags[i2+1][:1] == ":"):
							frag += "\n" + frags[i2] + "\n" + frags[i2+1]
							i2 += 2
							skip = i2
						else:
							break
				frag = self.handleDefinitionLists(frag)
			else:
				if(frag != ""):
					if(self.auto_paragraphs):
						frag = "<p>" + self.process(frag) + "</p>"
					else:
						frag = self.process(frag)
			
				
			output += [frag]
		return "\n".join(output)
	
	def fragmentize(self, text):
		frags = []
		fragstart = -1
		length = len(text)
		skip = -1
		for i, c in enumerate(text):
			if(skip > i):
				continue
			if(fragstart == -1):
				fragstart = i
			if(c == "{" and text[i:i+3] == "{{{"):
				preEnd = text.find("}}}", i)
				if(preEnd != -1):
					skip = preEnd
					continue
			
			if(c == "\n"):
				frags += [text[fragstart:i].strip()]
				fragstart = -1
			
			if(i+1 == length and fragstart != -1):
				frags += [text[fragstart:].strip()]
		
		return frags
	
	def process(self, text, noMacros=None):
		if(noMacros!=None and type(noMacros) is bool):
			self.noMacros = noMacros
		skip = 0
		output=""
		text = "\n".join([line.rstrip() for line in text.split("\n")])
		length = len(text)-1
		blockStart=-1
		prevChar=""
		nextChar=""
		for i, c in enumerate(text):
			if(skip > 0):
				skip -= 1<commit_msg>Made a small optimization to .fragmentize()<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
This file is part of DragonCreole
Copyright (C) 2013 - 2016 <NAME>

This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
'''

'''
==ABOUT==
DragonCreole is designed as a lightweight markup language, derived from
the Creole markup language (http://wikicreole.org/).  It includes an
expanded syntax and is optimized for high performance.
'''

try:
	from html import escape as escape
except:
	from cgi import escape as escape


import shlex, inspect, re, html2text, time, datetime

from types import *

regex_list = re.compile(r'\*+\s.+|\#+\s.+|\@+\s.+|\!+\s.+')
regex_list2 = re.compile(r'(\*+|\#+|\@+|\!+)')
regex_indent = re.compile(r'(\:+)')
rematch = re.match

class MacroObject():
	'''
		This class is used by macro functions to provide data about the page and a reference to the renderer
	'''
	__slots__ = ("body", "renderer")
	def __init__(self, body="", renderer=None):
		if(body != ""):
			if(body[0] == "\n"): body = body[1:]
		self.body = body
		self.renderer = renderer


class DragonCreole():
	"""
		The main renderer class.
	"""
	__slots__ = ("noMacros", "auto_paragraphs", "link_path", "image_path", "bodied_macros", "non_bodied_macros", "link_path_func", "link_class_func", "out_link_mark")
	def __init__(self, link_path="/", image_path="", bodied_macros={}, non_bodied_macros={}, link_path_func=None, link_class_func=None, out_link_mark = "➚", auto_paragraphs=True, noMacros=False):
		"""
			Returns a DragonCreole parser object.  The following optional arguments may be passed when creating a DragonCreole parser, in order to extend it.
			* link_path: The base path for all wiki links, which will be prepended to them.
			* image_path: The base path for all wiki images, which will be preprended to them.
			* bodied_macros: A dictionary of function references, which are called when the parser encounters a bodied macro.
			* non_bodied_macros: A dictionary of function references, which are called when the parser encounters a non-bodied macro.
			* link_path_func: A function reference which will be called to inspect any wiki link, and may modify its path.
			* link_class_func: A function reference which will be called to inspect any wiki link, and may modify its CSS class.
			* out_link_mark: A character which is appended to external links.  Set to a blank string for no mark.  Can be set to DragonCreole code, if desired, such as when one wishes to use an image or special formatting for the mark.
			* auto_paragraphs: If True, then paragraphs will automatically be created.
			* noMacros: If true, then macros will not be run.
		"""
		
		'''
		assert type(link_path) is str
		assert type(image_path) is str
		
		assert type(bodied_macros) is dict
		for func in bodied_macros:
			assert type(func) is FunctionType
		
		assert type(non_bodied_macros) is dict
		for func in non_bodied_macros:
			assert type(func) is FunctionType
		
		if(link_path_func != None):
			assert type(link_path_func) is FunctionType
		if(link_class_func != None):
			assert type(link_class_func) is FunctionType
		'''
		
		
		self.bodied_macros = {
			"block": self.macro_block
		}
		self.bodied_macros.update(bodied_macros)
		
		self.non_bodied_macros = {
			"datetime": self.macro_datetime
		}
		self.non_bodied_macros.update(non_bodied_macros)
		
		self.link_path = link_path
		self.image_path = image_path
		self.link_path_func = link_path_func
		self.link_class_func = link_class_func
		self.auto_paragraphs = auto_paragraphs
		self.noMacros = noMacros
		if(out_link_mark != ""):
			self.out_link_mark = self.process(out_link_mark)
		else:
			self.out_link_mark = out_link_mark
		return
	__cinit__ = __init__
	
	'''
	Renders a page to HTML
	'''
	def render(self, text, noMacros=None):
		frags = self.fragmentize(text)
		output = []
		i = -1
		skip = -1
		for i, frag in enumerate(frags):
			if(skip > i):
				continue
			nextFrag = ""
			if(i+1 < len(frags)):
				nextFrag = frags[i+1]
			
			if(frag == ""):
				frag = "\n"
			elif(frag[:1] == "="):
				frag = self.handleHeading(frag)
			elif(frag[:2] == "\\\\"):
				frag = "<br>"
			elif(frag[:4] == "----"):
				frag = "<hr>"
			elif(frag[:1] in ">:"):
				frag = self.handleParagraph(frag)
			elif(frag[:1] == "|"):
				if(i+1 < len(frags)):
					i2 = i+1
					while(i2 < len(frags)):
						if(frags[i2][:1] == "|"):
							frag += "\n" + frags[i2]
							i2 += 1
							skip = i2
						else:
							break
				frag = self.handleTables(frag)
			elif(frag.startswith("{{{") and frag.endswith("}}}")):
				frag = self.handlePreformat(frag)[0]
			elif(frag.startswith("<<") and frag.endswith(">>")):
				frag = self.handleMacro(frag)[0]
			elif(rematch(regex_list,frag) != None):
				if(i+1 < len(frags)):
					i2 = i+1
					while(i2 < len(frags)):
						if(rematch(regex_list,frags[i2]) != None):
							frag += "\n" + frags[i2]
							i2 += 1
							skip = i2
						else:
							break
				frag = self.handleLists(frag)
			elif(frag[:1] == ";" and nextFrag[:1] == ":"):
				if(i+1 < len(frags)):
					frag += "\n" + nextFrag
					i += 2
					i2 = i
					while(i2 < len(frags)):
						if(frags[i2][:1] == ";" and frags[i2+1][:1] == ":"):
							frag += "\n" + frags[i2] + "\n" + frags[i2+1]
							i2 += 2
							skip = i2
						else:
							break
				frag = self.handleDefinitionLists(frag)
			else:
				if(frag != ""):
					if(self.auto_paragraphs):
						frag = "<p>" + self.process(frag) + "</p>"
					else:
						frag = self.process(frag)
			
				
			output += [frag]
		return "\n".join(output)
	
	def fragmentize(self, text):
		frags = []
		fragstart = -1
		length = len(text)
		skip = False
		for i, c in enumerate(text):
			if(skip):
				if(c == "}" and text[i:i+3] == "}}}"):
					skip = False
				if(i+1 == length and fragstart != -1):
					frags += [text[fragstart:].strip()]
				continue
			
			if(fragstart == -1):
				fragstart = i
			
			if(c == "{" and text[i:i+3] == "{{{"):
				skip = True
				continue
			
			if(c == "\n"):
				frags += [text[fragstart:i].strip()]
				fragstart = -1
			
			if(i+1 == length and fragstart != -1):
				frags += [text[fragstart:].strip()]
		
		return frags
	
	def process(self, text, noMacros=None):
		if(noMacros!=None and type(noMacros) is bool):
			self.noMacros = noMacros
		skip = 0
		output=""
		text = "\n".join([line.rstrip() for line in text.split("\n")])
		length = len(text)-1
		blockStart=-1
		prevChar=""
		nextChar=""
		for i, c in enumerate(text):
			if(skip > 0):
				skip -= 1
{'fraction_non_alphanumeric': 0.1485080859463974, 'fraction_numerical': 0.011768278794320856, 'mean_word_length': 3.220442165972445, 'pattern_counts': {'":': 8, '<': 31, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '94789', 'n_tokens_mistral': 5611, 'n_tokens_neox': 4639, 'n_words': 1296}