<commit_before>from common import *
from collections import MutableMapping, MutableSequence

is_string = lambda x:isinstance(x,basestring)

__all__ = ['Node']
class Node(MutableMapping, MutableSequence):
	'''An object with a name, children and attributes, used to represent components
	of a document.
	
	>>> rep = lambda x:sorted(vars(x).items())
	>>> Node('x')
	Node('x')
	>>> Node('y', 1, 2, 3, a=5)
	Node('y', 1, 2, 3, _a=5)
	>>> n = Node('n', r='s')
	>>> n.append('o')
	>>> n.extend(['p', 'z'])
	>>> n[-1] = 'q'
	>>> n['r'] = 'r'
	>>> n
	Node('n', 'o', 'p', 'q', _r='r')
	>>> n.update(_s='t', r=0)
	>>> del n['r']
	>>> n.pop(1)
	'p'
	>>> n
	Node('n', 'o', 'q', _s='t')

	An underscore '_' is prepended to all attribute names to allow for attribute
	names that are also python keywords such as 'class', which is a valid concern e.g. in
	xml. To set set a literal attribute '_name', use two underscores: '__name'.
	repr() is unaffected so that eval(repr(Node(...))) evaluates to the original
	Node value.

	>>> a = Node('a', _class='123')
	>>> a
	Node('a', _class='123')
	>>> a['class'] = '234'
	>>> a
	Node('a', _class='234')
	>>> print a
	Node('a', class='234')
	
	If the first argument is another node, that node's values are copied. The 
	other arguments are ignored.
	
	>>> b = Node.copy(a)
	>>> b
	Node('a', _class='234')
	>>> b.children is a.children
	False
	>>> b.attributes is a.attributes
	False
	>>> is_sequence(Node())
	False
	
	Use Node.walk to iterate over its entire tree.
	
	>>> doc = Node('a', Node('b', 'c', Node('d', e='f')))
	>>> for depth, element in doc.walk():
	...   if isinstance(element, Node):
	...     print '  '*depth, element.name, element.attributes
	...   else:
	...     print '  '*depth, `element`
	 a {}
	   b {}
	     'c'
	     d {'_e': 'f'}
	'''
	__sequence__ = False
	def __init__(self, name=None, *children, **attributes):
		self.name = name
		self.children = list(children)
		self.attributes = dict((self._attr_key(k),v) for k,v in attributes.items())
	
	@classmethod
	def copy(cls, node):
		return cls(node.name, *node.children, **node.attributes)
	
	@staticmethod
	def _attr_key(key):
		return key if key[0]=='_' else '_'+key
	
	def __getitem__(self, key):
		if is_string(key):
			return self.__dict__['attributes'][self._attr_key(key)]
		else:
			return self.__dict__['children'][key]
	
	def __setitem__(self, key, value):
		if is_string(key):
			self.__dict__['attributes'][self._attr_key(key)] = value
		else:
			self.__dict__['children'][key] = value
	
	def __delitem__(self, key):
		if is_string(key):
			del self.attributes[self._attr_key(key)]
		else:
			del self.children[key]
	
	def __len__(self):
		return len(self.children)
		
	def insert(self, place, value):
		self.children.insert(place, value)
		
	def _real_attrs(self):
		return dict((k[1:],v) if k[0]=='_' else (k,v) for k,v in self.attributes.iteritems())
		
	def __repr__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self.attributes.items()]))
		
	def __str__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self._real_attrs().items()]))
		  
	def __nonzero__(self):
		return True
		
	def walk(self, filter=None, depth=0):
		yield depth, self
		for element in self.children:
			if isinstance(element, Node):
				for d,sub in element.walk(depth=depth+1):
					yield d,sub<commit_msg>Added __getattr__... behavior to nodes, duck typing in 'walk'<commit_after>from common import *
from collections import MutableMapping, MutableSequence

is_string = lambda x:isinstance(x,basestring)

__all__ = ['Node']
class Node(MutableMapping, MutableSequence):
	'''An object with a name, children and attributes, used to represent components
	of a document.
	
	>>> rep = lambda x:sorted(vars(x).items())
	>>> Node('x')
	Node('x')
	>>> Node('y', 1, 2, 3, a=5)
	Node('y', 1, 2, 3, _a=5)
	>>> n = Node('n', r='s')
	>>> n.append('o')
	>>> n.extend(['p', 'z'])
	>>> n[-1] = 'q'
	>>> n['r'] = 'r'
	>>> n
	Node('n', 'o', 'p', 'q', _r='r')
	>>> n.update(_s='t', r=0)
	>>> del n['r']
	>>> n.pop(1)
	'p'
	>>> n
	Node('n', 'o', 'q', _s='t')

	An underscore '_' is prepended to all attribute names to allow for attribute
	names that are also python keywords such as 'class', which is a valid concern e.g. in
	xml. To set set a literal attribute '_name', use two underscores: '__name'.
	repr() is unaffected so that eval(repr(Node(...))) evaluates to the original
	Node value.

	>>> a = Node('a', _class='123')
	>>> a
	Node('a', _class='123')
	>>> a['class'] = '234'
	>>> a
	Node('a', _class='234')
	>>> print a
	Node('a', class='234')
	
	Set and retrieve attributes on the object that start with an underscore. If
	that attribute doesn't exist, None is returned (similar to dict.get()).
	
	>>> a._class = '345'
	>>> print a
	Node('a', class='345')
	>>> print a._class
	345
	>>> print a._milk
	None
	
	Normal attribute access isn't so lucky

	>>> a.milk
	Traceback (most recent call last):
	 ...
	AttributeError: 'Node' object has no attribute 'milk'
	
	If the first argument is another node, that node's values are copied. The 
	other arguments are ignored.
	
	>>> b = Node.copy(a)
	>>> b
	Node('a', _class='345')
	>>> b.children is a.children
	False
	>>> b.attributes is a.attributes
	False
	>>> is_sequence(Node())
	False
	
	Use Node.walk to iterate over its entire tree.
	
	>>> doc = Node('a', Node('b', 'c', Node('d', e='f')))
	>>> for depth, element in doc.walk():
	...   if isinstance(element, Node):
	...     print '  '*depth, element.name, element.attributes
	...   else:
	...     print '  '*depth, `element`
	 a {}
	   b {}
	     'c'
	     d {'_e': 'f'}
	'''
	__sequence__ = False
	def __init__(self, name=None, *children, **attributes):
		self.name = name
		self.children = list(children)
		self.attributes = dict((self._attr_key(k),v) for k,v in attributes.items())
	
	@classmethod
	def copy(cls, node):
		return cls(node.name, *node.children, **node.attributes)
	
	@staticmethod
	def _attr_key(key):
		return key if key[0]=='_' else '_'+key
	
	def __getitem__(self, key):
		if is_string(key):
			return self.__dict__['attributes'][self._attr_key(key)]
		else:
			return self.__dict__['children'][key]

	def __getattr__(self, key):
		if key[0] == '_':
			return self.get(key)
		else:
			return super(Node, self).__getattribute__(key)
	
	def __setitem__(self, key, value):
		if is_string(key):
			self.__dict__['attributes'][self._attr_key(key)] = value
		else:
			self.__dict__['children'][key] = value

	def __setattr__(self, key, value):
		if key[0] == '_':
			self.__setitem__(key, value)
		else:
			super(Node, self).__setattr__(key, value)
	
	def __delitem__(self, key):
		if is_string(key):
			del self.attributes[self._attr_key(key)]
		else:
			del self.children[key]
	
	def __len__(self):
		return len(self.children)
		
	def insert(self, place, value):
		self.children.insert(place, value)
		
	def _real_attrs(self):
		return dict((k[1:],v) if k[0]=='_' else (k,v) for k,v in self.attributes.iteritems())
		
	def __repr__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self.attributes.items()]))
		
	def __str__(self):
		return '%s(%s)'%(self.__class__.__name__,', '.join([repr(self.name)] + \
		  map(repr,self.children) + \
		  ['%s=%r'%i for i in self._real_attrs().items()]))
		  
	def __nonzero__(self):
		return True
		
	def walk(self, filter=None, depth=0):
		yield depth, self
		for element in self.children:
			if hasattr(element, 'walk'):
				for d,sub in element.walk(depth=depth+1):
					yield d,sub
{'fraction_non_alphanumeric': 0.18912591050988553, 'fraction_numerical': 0.010275754422476586, 'mean_word_length': 3.8541666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 168, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3166023', 'n_tokens_mistral': 3045, 'n_tokens_neox': 2785, 'n_words': 766}