<commit_before># -*- coding: utf-8 -*-

from cStringIO import StringIO
from os.path import getsize
from struct import pack, unpack, error as StructError
from .log import log
from .structures import fields


class DBHeader(object):
	"""
	Base DBFile header class
	"""
	def __repr__(self):
		return "%s(%s)" % (self.__class__.__name__, ", ".join("%s=%r" % (k, self.__dict__[k]) for k in self.__dict__))


class DBFile(object):
	"""
	Base class for WDB and DBC files
	"""

	def __init__(self, file, build, structure, environment):
		if isinstance(file, basestring):
			file = open(file, "r")
		self.file = file
		self._addresses = {}
		self._values = {}
		self.environment = environment

		self.__row_dynfields = 0 # Dynamic fields index, used when parsing a row

	def __repr__(self):
		return "%s(file=%r, build=%r)" % (self.__class__.__name__, self.file, self.build)

	def __contains__(self, id):
		return id in self._addresses

	def __getitem__(self, item):
		if isinstance(item, slice):
			keys = sorted(self._addresses.keys())[item]
			return [self[k] for k in keys]

		if item not in self._values:
			self._parse_row(item)

		return self._values[item]

	def __setitem__(self, item, value):
		if not isinstance(item, int):
			raise TypeError("DBFile indices must be integers, not %s" % (type(item)))

		if isinstance(value, DBRow):
			self._values[item] = value
			self._addresses[item] = -1
		else:
			# FIXME technically we should allow DBRow, but this is untested and will need resetting parent
			raise TypeError("Unsupported type for DBFile.__setitem__: %s" % (type(value)))

	def __delitem__(self, item):
		if item in self._values:
			del self._values[item]
		del self._addresses[item]

	def __iter__(self):
		return self._addresses.__iter__()

	def __len__(self):
		return len(self._addresses)

	def _add_row(self, id, address, reclen):
		if id in self._addresses: # Something's wrong here
			log.warning("Multiple instances of row %r found" % (id))
		self._addresses[id] = (address, reclen)

	def _parse_field(self, data, field, row=None):
		"""
		Parse a single field in stream.
		"""
		if field.dyn > self.__row_dynfields:
			return None # The column doesn't exist in this row, we set it to None

		ret = None
		try:
			if isinstance(field, fields.StringField):
				ret = self._parse_string(data)

			elif isinstance(field, fields.DataField): # wowcache.wdb
				length = getattr(row, field.master)
				ret = data.read(length)

			elif isinstance(field, fields.DynamicMaster):
				ret, = unpack("<I", data.read(4))
				self.__row_dynfields = ret

			else:
				ret, = unpack("<%s" % (field.char), data.read(field.size))
		except StructError:
			log.warning("Field %s could not be parsed properly" % (field))
			ret = None

		return ret

	def append(self, row):
		"""
		Append a row at the end of the file.
		If the row does not have an id, one is automatically assigned.
		"""
		i = len(self) + 1 # FIXME this wont work properly in incomplete files
		if "_id" not in row:
			row["_id"] = i
		self[i] = row

	def clear(self):
		"""
		Delete every row in the file
		"""
		for k in self.keys(): # Use key, otherwise we get RuntimeError: dictionary changed size during iteration
			del self[k]
<commit_msg>wdbc: Implement DBFile.supportsSeeking(), rename some internal fields<commit_after># -*- coding: utf-8 -*-

from cStringIO import StringIO
from os.path import getsize
from struct import pack, unpack, error as StructError
from .log import log
from .structures import fields


class DBHeader(object):
	"""
	Base DBFile header class
	"""
	def __repr__(self):
		return "%s(%s)" % (self.__class__.__name__, ", ".join("%s=%r" % (k, self.__dict__[k]) for k in self.__dict__))


class DBFile(object):
	"""
	Base class for WDB and DBC files
	"""

	def __init__(self, file, build, structure, environment):
		if isinstance(file, basestring):
			file = open(file, "r")
		self.file = file
		self.build = build
		self._addresses = {}
		self._values = {}
		self.environment = environment

		self._rowDynamicFields = 0 # Dynamic fields index, used when parsing a row

	def __repr__(self):
		return "%s(file=%r, build=%r)" % (self.__class__.__name__, self.file, self.build)

	def __contains__(self, id):
		return id in self._addresses

	def __getitem__(self, item):
		if isinstance(item, slice):
			keys = sorted(self._addresses.keys())[item]
			return [self[k] for k in keys]

		if item not in self._values:
			self._parse_row(item)

		return self._values[item]

	def __setitem__(self, item, value):
		if not isinstance(item, int):
			raise TypeError("DBFile indices must be integers, not %s" % (type(item)))

		if isinstance(value, DBRow):
			self._values[item] = value
			self._addresses[item] = -1
		else:
			# FIXME technically we should allow DBRow, but this is untested and will need resetting parent
			raise TypeError("Unsupported type for DBFile.__setitem__: %s" % (type(value)))

	def __delitem__(self, item):
		if item in self._values:
			del self._values[item]
		del self._addresses[item]

	def __iter__(self):
		return self._addresses.__iter__()

	def __len__(self):
		return len(self._addresses)

	def _add_row(self, id, address, reclen):
		if id in self._addresses: # Something's wrong here
			log.warning("Multiple instances of row %r found" % (id))
		self._addresses[id] = (address, reclen)

	def _parse_field(self, data, field, row=None):
		"""
		Parse a single field in stream.
		"""
		if field.dyn > self._rowDynamicFields:
			return None # The column doesn't exist in this row, we set it to None

		ret = None
		try:
			if isinstance(field, fields.StringField):
				ret = self._parse_string(data)

			elif isinstance(field, fields.DataField): # wowcache.wdb
				length = getattr(row, field.master)
				ret = data.read(length)

			elif isinstance(field, fields.DynamicMaster):
				ret, = unpack("<I", data.read(4))
				self._rowDynamicFields = ret

			else:
				ret, = unpack("<%s" % (field.char), data.read(field.size))
		except StructError:
			log.warning("Field %s could not be parsed properly" % (field))
			ret = None

		return ret

	def supportsSeeking(self):
		return hasattr(self.file, "seek")

	def append(self, row):
		"""
		Append a row at the end of the file.
		If the row does not have an id, one is automatically assigned.
		"""
		i = len(self) + 1 # FIXME this wont work properly in incomplete files
		if "_id" not in row:
			row["_id"] = i
		self[i] = row

	def clear(self):
		"""
		Delete every row in the file
		"""
		for k in self.keys(): # Use key, otherwise we get RuntimeError: dictionary changed size during iteration
			del self[k]

{'fraction_non_alphanumeric': 0.12385879488740109, 'fraction_numerical': 0.0015216068167985392, 'mean_word_length': 3.81891495601173, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '290884', 'n_tokens_mistral': 2386, 'n_tokens_neox': 2118, 'n_words': 709}