<commit_before># -*- coding: utf-8 -*-
from datetime import datetime

class DictInterface(object):

	_dict_attrs = ()

	@property
	def as_dict(self):
		return self.to_dict()

	def to_dict(self, only=[], date_format='%Y-%m-%d %H:%M:%S'):

		def _obj_to_dict(o, limit=[]):
			names = o._dict_attrs or filter(lambda x: not x.startswith('_'), vars(o))
			if limit:
				names = list(set(limit).intersection(set(names)))
			return {k: getattr(o, k) for k in names}

		def _walk(o, parents=[], limit=[]):
			c = {}
			d = _obj_to_dict(o, limit)
			for attr, value in d.iteritems():
				if isinstance(value, DictInterface):
					if value is o:
						continue
					if value in parents:
						continue
					parents.append(o)
					c[attr] = _walk(value, parents)
				elif hasattr(value, '__iter__'):
					c[attr] = [_walk(v, parents) for v in value]
				elif isinstance(value, datetime):
					c[attr] = value.strftime(date_format)
				else:
					c[attr] = value
			return c

		
		return _walk(self, limit=only)<commit_msg>fixed a very subtle bug in the recursive behavior of to_dict. was not playing nice with certain path traversal scenarios. all set now :)<commit_after># -*- coding: utf-8 -*-

from pprint import pprint

from datetime import datetime

def _step():
	raw_input('Step >')

class DictInterface(object):

	_dict_attrs = ()

	@property
	def as_dict(self):
		return self.to_dict()

	def to_dict(self, only=[], date_format='%Y-%m-%d %H:%M:%S'):

		def _obj_to_dict(o, limit=[], terminate=False):
			names = o._dict_attrs or filter(lambda x: not x.startswith('_'), vars(o))
			if limit:
				names = list(set(limit).intersection(set(names)))

			if terminate:
				return {k: getattr(o, k) for k in names if not isinstance(getattr(o,k), DictInterface)}
			return {k: getattr(o, k) for k in names}

		def _walk(o, path=[], prev=None, limit=[]):
			c = {}
			d = _obj_to_dict(o, limit)
			
			if prev:
				path.append(prev)
			
			for attr, value in d.iteritems():
				if isinstance(value, DictInterface):
					if path and (value is path[0]):
						continue
					if value is prev:
						continue
					c[attr] = _walk(value, path, o)
				elif hasattr(value, '__iter__'):
					c[attr] = [_walk(v, path, o) for v in value]
				elif isinstance(value, datetime):
					c[attr] = value.strftime(date_format)
				else:
					c[attr] = value
			return c

		
		return _walk(self, limit=only)
{'fraction_non_alphanumeric': 0.14393939393939395, 'fraction_numerical': 0.0012626262626262627, 'mean_word_length': 3.385608856088561, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2049322', 'n_tokens_mistral': 1010, 'n_tokens_neox': 829, 'n_words': 243}