<commit_before>##############################################################################
#
# Copyright (c) 2004-2006 TINY SPRL. (http://tiny.be) All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################


import xml.dom.minidom
import os, time
import ir, netsvc
import osv
from interface import  report_rml
import re
import tools
import pooler
import netsvc
import warnings

import copy

parents = {
	'tr':1,
	'li':1,
	'story': 0,
	'section': 0
}

#
# Context: {'node': node.dom}
#
class browse_record_list(list):
	def __init__(self, lst, context):
		super(browse_record_list, self).__init__(lst)
		self.parents = copy.copy(parents)
		self.context = context

	def __getattr__(self, name):
		res = browse_record_list([getattr(x,name) for x in self], self.context)
		return res

	def __str__(self):
		return "browse_record_list("+str(len(self))+")"

	def repeatIn(self, name):
		warnings.warn('Use repeatIn(object_list, \'variable\')', DeprecationWarning)
		node = self.context['_node']
		node.data = ''
		while True:
			if not node.parentNode:
				break
			node = node.parentNode
			if node.nodeType == node.ELEMENT_NODE and node.localName in parents:
				break
		parent_node = node
		if not len(self):
			return None
		nodes = [(0,node)]
		for i in range(1,len(self)):
			newnode = parent_node.cloneNode(1)
			n = parent_node.parentNode
			n.insertBefore(newnode, parent_node)
			nodes.append((i,newnode))
		for i,node in nodes:
			self.context[name] = self[i]
			self.context['_self']._parse_node(node)
		return None

class rml_parse(object):
	def __init__(self, cr, uid, name, context=None):
		if not context:
			context={}
		self.cr = cr
		self.uid = uid
		self.pool = pooler.get_pool(cr.dbname)
		user = self.pool.get('res.users').browse(cr, uid, uid)
		self.localcontext = {
			'user': user,
			'company': user.company_id,
			'repeatIn': self.repeatIn,
			'setLang': self.setLang,
			'setTag': self.setTag,
			'removeParentNode': self.removeParentNode,
			'format': self.format,
		}
		self.localcontext.update(context)
		self.name = name
		self._regex = re.compile('\[\[(.+?)\]\]')
		self._transl_regex = re.compile('(\[\[.+?\]\])')
		self._node = None
#		self.already = {}

	def setTag(self, oldtag, newtag, attrs=None):
		if not attrs:
			attrs={}
		node = self._find_parent(self._node, [oldtag])
		if node:
			node.tagName = newtag
			for key, val in attrs.items():
				node.setAttribute(key, val)
		return None

	def format(self, text, oldtag='para'):
		self._node.data = ''
		node = self._find_parent(self._node, [oldtag])

		pp = node.parentNode
		ns = node.nextSibling
		pp.removeChild(node)
		self._node = pp
		lst = text.split('\n')
		if not len(lst):
			return None
		nodes = []
		for i in range(len(lst)):
			newnode = node.cloneNode(1)
			newnode.tagName='para'
			newnode.__dict__['childNodes'][0].__dict__['data'] = lst[i].decode('utf8')
			if ns:
				pp.insertBefore(newnode, ns)
			else:
				pp.appendChild(newnode)
			nodes.append((i, newnode))

	def removeParentNode(self, tag):
		node = self._find_parent(self._node, [tag])
		if node:
			parentNode = node.parentNode
			parentNode.removeChild(node)
			self._node = parentNode

	def setLang(self, lang):
		self.localcontext['lang'] = lang

	def repeatIn(self, lst, name, nodes_parent=False):
		self._node.data = ''
		node = self._find_parent(self._node, nodes_parent or parents)

		pp = node.parentNode
		ns = node.nextSibling
		pp.removeChild(node)
		self._node = pp

		if not len(lst):
			return None
		nodes = []
		for i in range(len(lst)):
			newnode = node.cloneNode(1)
			if ns:
				pp.insertBefore(newnode, ns)
			else:
				pp.appendChild(newnode)
			nodes.append((i, newnode))
		for i, node in nodes:
			self.node_context[node] = {name: lst[i]}
		return None

	def _eval(self, expr):
		try:
			res = eval(expr, self.localcontext)
			if not res:
				res = ''
		except Exception,e:
			import traceback, sys
			tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
			netsvc.Logger().notifyChannel('report', netsvc.LOG_ERROR, 'report %s:\n%s\n%s\nexpr: %s' % (self.name, tb_s, str(e), expr))
			res = ''
		return res

	def _find_parent(self, node, parents):
		while True:
			if not node.parentNode:
				return False
			node = node.parentNode
			if node.nodeType == node.ELEMENT_NODE and node.localName in parents:
				break
		return node

	def _parse_text(self, text, level=None):
		if not level:
			level=[]
		res = self._regex.findall(text)
		todo = []
		# translate the text<commit_msg>Fix bad encoding in log message<commit_after>##############################################################################
#
# Copyright (c) 2004-2006 TINY SPRL. (http://tiny.be) All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################


import xml.dom.minidom
import os, time
import ir, netsvc
import osv
from interface import  report_rml
import re
import tools
import pooler
import netsvc
import warnings

import copy

parents = {
	'tr':1,
	'li':1,
	'story': 0,
	'section': 0
}

#
# Context: {'node': node.dom}
#
class browse_record_list(list):
	def __init__(self, lst, context):
		super(browse_record_list, self).__init__(lst)
		self.parents = copy.copy(parents)
		self.context = context

	def __getattr__(self, name):
		res = browse_record_list([getattr(x,name) for x in self], self.context)
		return res

	def __str__(self):
		return "browse_record_list("+str(len(self))+")"

	def repeatIn(self, name):
		warnings.warn('Use repeatIn(object_list, \'variable\')', DeprecationWarning)
		node = self.context['_node']
		node.data = ''
		while True:
			if not node.parentNode:
				break
			node = node.parentNode
			if node.nodeType == node.ELEMENT_NODE and node.localName in parents:
				break
		parent_node = node
		if not len(self):
			return None
		nodes = [(0,node)]
		for i in range(1,len(self)):
			newnode = parent_node.cloneNode(1)
			n = parent_node.parentNode
			n.insertBefore(newnode, parent_node)
			nodes.append((i,newnode))
		for i,node in nodes:
			self.context[name] = self[i]
			self.context['_self']._parse_node(node)
		return None

class rml_parse(object):
	def __init__(self, cr, uid, name, context=None):
		if not context:
			context={}
		self.cr = cr
		self.uid = uid
		self.pool = pooler.get_pool(cr.dbname)
		user = self.pool.get('res.users').browse(cr, uid, uid)
		self.localcontext = {
			'user': user,
			'company': user.company_id,
			'repeatIn': self.repeatIn,
			'setLang': self.setLang,
			'setTag': self.setTag,
			'removeParentNode': self.removeParentNode,
			'format': self.format,
		}
		self.localcontext.update(context)
		self.name = name
		self._regex = re.compile('\[\[(.+?)\]\]')
		self._transl_regex = re.compile('(\[\[.+?\]\])')
		self._node = None
#		self.already = {}

	def setTag(self, oldtag, newtag, attrs=None):
		if not attrs:
			attrs={}
		node = self._find_parent(self._node, [oldtag])
		if node:
			node.tagName = newtag
			for key, val in attrs.items():
				node.setAttribute(key, val)
		return None

	def format(self, text, oldtag='para'):
		self._node.data = ''
		node = self._find_parent(self._node, [oldtag])

		pp = node.parentNode
		ns = node.nextSibling
		pp.removeChild(node)
		self._node = pp
		lst = text.split('\n')
		if not len(lst):
			return None
		nodes = []
		for i in range(len(lst)):
			newnode = node.cloneNode(1)
			newnode.tagName='para'
			newnode.__dict__['childNodes'][0].__dict__['data'] = lst[i].decode('utf8')
			if ns:
				pp.insertBefore(newnode, ns)
			else:
				pp.appendChild(newnode)
			nodes.append((i, newnode))

	def removeParentNode(self, tag):
		node = self._find_parent(self._node, [tag])
		if node:
			parentNode = node.parentNode
			parentNode.removeChild(node)
			self._node = parentNode

	def setLang(self, lang):
		self.localcontext['lang'] = lang

	def repeatIn(self, lst, name, nodes_parent=False):
		self._node.data = ''
		node = self._find_parent(self._node, nodes_parent or parents)

		pp = node.parentNode
		ns = node.nextSibling
		pp.removeChild(node)
		self._node = pp

		if not len(lst):
			return None
		nodes = []
		for i in range(len(lst)):
			newnode = node.cloneNode(1)
			if ns:
				pp.insertBefore(newnode, ns)
			else:
				pp.appendChild(newnode)
			nodes.append((i, newnode))
		for i, node in nodes:
			self.node_context[node] = {name: lst[i]}
		return None

	def _eval(self, expr):
		try:
			res = eval(expr, self.localcontext)
			if not res:
				res = ''
		except Exception,e:
			import traceback, sys
			tb_s = reduce(lambda x, y: x+y, traceback.format_exception(sys.exc_type, sys.exc_value, sys.exc_traceback))
			netsvc.Logger().notifyChannel('report', netsvc.LOG_ERROR,
					'report %s:\n%s\n%s\nexpr: %s' % (self.name, tb_s, str(e),
						expr.encode('utf-8')))
			res = ''
		return res

	def _find_parent(self, node, parents):
		while True:
			if not node.parentNode:
				return False
			node = node.parentNode
			if node.nodeType == node.ELEMENT_NODE and node.localName in parents:
				break
		return node

	def _parse_text(self, text, level=None):
		if not level:
			level=[]
		res = self._regex.findall(text)
		todo = []
		# translate the text
{'fraction_non_alphanumeric': 0.1480828558836492, 'fraction_numerical': 0.006081974438078449, 'mean_word_length': 3.961084390030608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6477120', 'n_tokens_mistral': 4248, 'n_tokens_neox': 3770, 'n_words': 1098}