<commit_before>#!/usr/bin/env python
# encoding: utf-8
# <NAME>, 2013 (zougloub)

"""
reStructuredText support (experimental)

Example::

	def configure(conf):
		conf.load('rst')
		if not conf.env.RST2HTML:
			conf.fatal('The program rst2html is required')

	def build(bld):
		bld(
		 features = 'rst',
		 type     = 'rst2html', # rst2html, rst2pdf, ...
		 source   = 'index.rst', # mandatory, the source
		 deps     = 'image.png', # to give additional non-trivial dependencies
		)

By default the tool looks for a set of programs in PATH.
The tools are defined in `rst_progs`.
To configure with a special program use::

	$ RST2HTML=/path/to/rst2html waf configure

This tool is experimental; don't hesitate to contribute to it.

"""

import os, re
from waflib import Node, Utils, Task, Errors, Logs
from waflib.TaskGen import feature, before_method

rst_progs = "rst2html rst2xetex rst2latex rst2xml rst2pdf rst2s5 rst2man rst2odt rst2rtf".split()

class docutils(Task.Task):
	"""
	Compile a rst file.
	"""

	def scan(self):
		"""
		A recursive regex-based scanner that finds rst dependencies.
		"""
		# TODO add extensibility, to handle custom rst include tags...

		nodes = []
		names = []
		seen = []
		
		node = self.inputs[0]

		if not node:
			return (nodes, names)

		def parse_node(node):
			if node in seen:
				return
			seen.append(node)
			code = node.read()
			re_rst = re.compile(r'^\s*.. (?P<type>include|image):: (?P<file>.*)$', re.M)
			for match in re_rst.finditer(code):
				ipath = match.group('file')
				itype = match.group('type')
				Logs.debug("rst: visiting %s: %s" % (itype, ipath))
				found = node.parent.find_resource(ipath)
				nodes.append(found)
				if itype == 'include':
					parse_node(found)
				names.append(ipath)
		parse_node(node)

		for x in nodes:
			x.parent.get_bld().mkdir()

		Logs.debug("rst: found the following : %s and names %s" % (nodes, names))
		return (nodes, names)

	def check_status(self, msg, retcode):
		"""
		Check an exit status and raise an error with a particular message

		:param msg: message to display if the code is non-zero
		:type msg: string
		:param retcode: condition
		:type retcode: boolean
		"""
		if retcode != 0:
			raise Errors.WafError("%r command exit status %r" % (msg, retcode))

	def run(self):
		"""
		Runs the rst compilation using docutils
		"""
		raise NotImplementedError()


class rst2html(docutils):
	color = 'BLUE'

	def scan(self):
		nodes, names = docutils.scan(self)

		stylesheet = getattr(self.generator, 'stylesheet', None)
		if stylesheet is not None:
			names.append(stylesheet)
			ssnode = self.generator.to_nodes(stylesheet)[0]
			nodes.append(ssnode)
			Logs.debug("rst: adding dep to stylesheet %s" % stylesheet)
	
		return nodes, names

	def run(self):
		rst2x = self.generator.env.RST2HTML
		src = self.inputs[0].bldpath()
		dst = self.outputs[0].bldpath()

		cmd = [rst2x, src, dst]
		cmd += getattr(self.generator, 'options', [])
		stylesheet = getattr(self.generator, 'stylesheet', None)
		if stylesheet is not None:
			stylesheet = self.generator.to_nodes(stylesheet)[0]
			cmd += ['--stylesheet', stylesheet.bldpath()]

		return self.exec_command(cmd)

class rst2pdf(docutils):
	color = 'BLUE'
	def run(self):
		rst2x = self.generator.env.RST2PDF
		src = self.inputs[0].bldpath()
		dst = self.outputs[0].bldpath()

		cmd = [rst2x, src, dst]
		cmd += getattr(self.generator, 'options', [])
		
		return self.exec_command(cmd)

@feature('rst')
@before_method('process_source')
def apply_rst(self):
	"""
	Create :py:class:`rst` or other rst-related task objects
	"""

	tsk_target = self.target if self.target != '' else None
	tsk_type = getattr(self, 'type', None)
	
	if tsk_type is not None and tsk_target is None:
		src = self.to_nodes(self.source)
		assert len(src) == 1
		# TODO multiple source nodes ?
		src = src[0]
		if tsk_type.startswith('rst2'):
			ext = tsk_type[4:]
		else:
			raise ValueError("Feature rst could not detect the output file extension")
		tgt = src.change_ext('.%s' % ext)
	elif tsk_type is None and tsk_target is not None:
		src = self.to_nodes(self.source)[0]
		if tsk_target.__class__ == Node.Node:
			tgt = tsk_target
		elif tsk_target.__class__ == str:
			tgt = src.parent.get_bld().make_node(tsk_target)
		else:
			raise NotImplementedError()
		out = tgt.name
		ext = out[out.rfind('.')+1:]
		self.type = 'rst2' + ext
	elif tsk_type is not None and tsk_target is not None:
		pass # the user knows what he wants
	else:
		raise ValueError("Need to indicate task type or target name for %s" % str(self))

	deps_lst = []

	if getattr(self, 'deps', None):
		deps = self.to_list(self.deps)<commit_msg>extras: rst: pimp the scanner<commit_after>#!/usr/bin/env python
# encoding: utf-8
# <NAME>, 2013 (zougloub)

"""
reStructuredText support (experimental)

Example::

	def configure(conf):
		conf.load('rst')
		if not conf.env.RST2HTML:
			conf.fatal('The program rst2html is required')

	def build(bld):
		bld(
		 features = 'rst',
		 type     = 'rst2html', # rst2html, rst2pdf, ...
		 source   = 'index.rst', # mandatory, the source
		 deps     = 'image.png', # to give additional non-trivial dependencies
		)

By default the tool looks for a set of programs in PATH.
The tools are defined in `rst_progs`.
To configure with a special program use::

	$ RST2HTML=/path/to/rst2html waf configure

This tool is experimental; don't hesitate to contribute to it.

"""

import os, re
from waflib import Node, Utils, Task, Errors, Logs
from waflib.TaskGen import feature, before_method

rst_progs = "rst2html rst2xetex rst2latex rst2xml rst2pdf rst2s5 rst2man rst2odt rst2rtf".split()

class docutils(Task.Task):
	"""
	Compile a rst file.
	"""

	def scan(self):
		"""
		A recursive regex-based scanner that finds rst dependencies.
		"""
		# TODO add extensibility, to handle custom rst include tags...

		nodes = []
		names = []
		seen = []

		node = self.inputs[0]

		if not node:
			return (nodes, names)

		def parse_node(node):
			if node in seen:
				return
			seen.append(node)
			code = node.read()
			re_rst = re.compile(r'^\s*.. (?P<subst>\|\S+\|) (?P<type>include|image|figure):: (?P<file>.*)$', re.M)
			for match in re_rst.finditer(code):
				ipath = match.group('file')
				itype = match.group('type')
				Logs.debug("rst: visiting %s: %s" % (itype, ipath))
				found = node.parent.find_resource(ipath)
				if found:
					nodes.append(found)
					if itype == 'include':
						parse_node(found)
				else:
					names.append(ipath)
		parse_node(node)

		Logs.debug("rst: %s: found the following file deps: %s" % (repr(self), nodes))
		if names:
			Logs.warn("rst: %s: could not find the following file deps: %s" % (repr(self), names))

		return (nodes, names)

	def check_status(self, msg, retcode):
		"""
		Check an exit status and raise an error with a particular message

		:param msg: message to display if the code is non-zero
		:type msg: string
		:param retcode: condition
		:type retcode: boolean
		"""
		if retcode != 0:
			raise Errors.WafError("%r command exit status %r" % (msg, retcode))

	def run(self):
		"""
		Runs the rst compilation using docutils
		"""
		raise NotImplementedError()


class rst2html(docutils):
	color = 'BLUE'

	def scan(self):
		nodes, names = docutils.scan(self)

		stylesheet = getattr(self.generator, 'stylesheet', None)
		if stylesheet is not None:
			names.append(stylesheet)
			ssnode = self.generator.to_nodes(stylesheet)[0]
			nodes.append(ssnode)
			Logs.debug("rst: adding dep to stylesheet %s" % stylesheet)

		return nodes, names

	def run(self):
		rst2x = self.generator.env.RST2HTML
		src = self.inputs[0].bldpath()
		dst = self.outputs[0].bldpath()

		cmd = [rst2x]
		cmd += getattr(self.generator, 'options', [])
		stylesheet = getattr(self.generator, 'stylesheet', None)
		if stylesheet is not None:
			stylesheet = self.generator.to_nodes(stylesheet)[0]
			cmd += ['--stylesheet', stylesheet.bldpath()]
		cmd += [src, dst]

		return self.exec_command(cmd)

class rst2pdf(docutils):
	color = 'BLUE'
	def run(self):
		rst2x = self.generator.env.RST2PDF
		src = self.inputs[0].bldpath()
		dst = self.outputs[0].bldpath()

		cmd = [rst2x, src, dst]
		cmd += getattr(self.generator, 'options', [])

		return self.exec_command(cmd)

@feature('rst')
@before_method('process_source')
def apply_rst(self):
	"""
	Create :py:class:`rst` or other rst-related task objects
	"""

	tsk_target = self.target if self.target != '' else None
	tsk_type = getattr(self, 'type', None)

	if tsk_type is not None and tsk_target is None:
		src = self.to_nodes(self.source)
		assert len(src) == 1
		# TODO multiple source nodes ?
		src = src[0]
		if tsk_type.startswith('rst2'):
			ext = tsk_type[4:]
		else:
			raise ValueError("Feature rst could not detect the output file extension")
		tgt = src.change_ext('.%s' % ext)
	elif tsk_type is None and tsk_target is not None:
		src = self.to_nodes(self.source)[0]
		if tsk_target.__class__ == Node.Node:
			tgt = tsk_target
		elif tsk_target.__class__ == str:
			tgt = src.parent.get_bld().make_node(tsk_target)
		else:
			raise NotImplementedError()
		out = tgt.name
		ext = out[out.rfind('.')+1:]
		self.type = 'rst2' + ext
	elif tsk_type is not None and tsk_target is not None:
		pass # the user knows what he wants
	else:
		raise ValueError("Need to indicate task type or target name for %s" % str(self))

	deps_lst = []

	if getattr(self, 'deps', None):
		deps = self.to_list(self.deps)
{'fraction_non_alphanumeric': 0.13404911236313383, 'fraction_numerical': 0.009567343467630487, 'mean_word_length': 3.739546599496222, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1509277', 'n_tokens_mistral': 3773, 'n_tokens_neox': 3350, 'n_words': 949}