<commit_before># -*- coding: utf-8 -*-

from struct import unpack
from .dbc import DBCFile
from .log import log
from .main import DBHeader
from .structures import getstructure, LocalizedStringField, LocalizedField
from .utils import getfilename

SEEK_CUR = 1 # os.SEEK_CUR

class DB2Header(DBHeader):
	def __len__(self):
		if self.build < 12834:
			return 32
		return 48
	
	def load(self, file):
		file.seek(0)
		self.signature, self.row_count, self.field_count, self.reclen, self.stringblocksize, self.dbhash, self.build, self.timestamp = unpack("<4s7i", file.read(32))
		if self.build >= 12834:
			if self.signature == "WCH2" and self.build < 12942:
				# Work around a bug in cataclysm beta which doesn't take in account the first \0
				log.warning("Old adb file, working around stringblock bug")
				self.stringblocksize += 1
			self.lookup_start, self.lookup_end, self.locale, self.unk3 = unpack("<4i", file.read(16))


class DB2File(DBCFile):
	"""
	New DB format introduced in build 12803
	"""
	
	def __init__(self, file, build, structure, environment):
		super(DBCFile, self).__init__(file, build, structure, environment)
		self.header = DB2Header()
		self.header.load(file)
		self.build = build or self.header.build
		self.__load_structure(structure)
	
	def __load_structure(self, structure):
		name = getfilename(self.file.name)
		try:
			self.structure = getstructure(name, self.build, parent=self)
		except StructureNotFound:
			self.structure = generate_structure(self)
		
		# Generate the Localized Fields
		fieldidx = []
		for i, field in enumerate(self.structure):
			if isinstance(field, LocalizedField):
				fieldidx.append((i, field.name))
		
		if fieldidx:
			from copy import copy
			fields = LocalizedStringField(build=self.build)
			for i, name in reversed(fieldidx):
				# Build a copy of the fields
				toinsert = [copy(field).rename("%s_%s" % (name, field.name)) for field in fields]
				self.structure[i:i+1] = toinsert
		
		log.info("Using %s build %i" % (self.structure, self.build))
		self.check_integrity()
	
	def preload(self):
		f = self.file
		f.seek(len(self.header))
		
		if self.build >= 12834:
			if self.header.lookup_start != self.header.lookup_end:
				size = (self.header.lookup_end - self.header.lookup_start + 1) * 6
				if size < 0:
					log.error("lookup size < 0: %i. This file is corrupt. Expect breakage!" % (size))
				else:
					lookups = f.read(size)
		
		rows = 0
		field = self.structure[0]
		row_header_size = field.size
		reclen = self.header.reclen
		while rows < self.header.row_count:
			address = f.tell() # Get the address of the full row
			id = self._parse_field(f, field)
			
			self._add_row(id, address, reclen)
			
			f.seek(reclen - row_header_size, SEEK_CUR) # minus length of id
			rows += 1<commit_msg>wdbc/db2: Fix incorrect build check in the header for 12880<commit_after># -*- coding: utf-8 -*-

from struct import unpack
from .dbc import DBCFile
from .log import log
from .main import DBHeader
from .structures import getstructure, LocalizedStringField, LocalizedField
from .utils import getfilename

SEEK_CUR = 1 # os.SEEK_CUR

class DB2Header(DBHeader):
	def __len__(self):
		if self.build <= 12880:
			return 32
		return 48
	
	def load(self, file):
		file.seek(0)
		self.signature, self.row_count, self.field_count, self.reclen, self.stringblocksize, self.dbhash, self.build, self.timestamp = unpack("<4s7i", file.read(32))
		if self.build > 12880:
			if self.signature == "WCH2" and self.build < 12942:
				# Work around a bug in cataclysm beta which doesn't take in account the first \0
				log.warning("Old adb file, working around stringblock bug")
				self.stringblocksize += 1
			self.lookup_start, self.lookup_end, self.locale, self.unk3 = unpack("<4i", file.read(16))


class DB2File(DBCFile):
	"""
	New DB format introduced in build 12803
	"""
	
	def __init__(self, file, build, structure, environment):
		super(DBCFile, self).__init__(file, build, structure, environment)
		self.header = DB2Header()
		self.header.load(file)
		self.build = build or self.header.build
		self.__load_structure(structure)
	
	def __load_structure(self, structure):
		name = getfilename(self.file.name)
		try:
			self.structure = getstructure(name, self.build, parent=self)
		except StructureNotFound:
			self.structure = generate_structure(self)
		
		# Generate the Localized Fields
		fieldidx = []
		for i, field in enumerate(self.structure):
			if isinstance(field, LocalizedField):
				fieldidx.append((i, field.name))
		
		if fieldidx:
			from copy import copy
			fields = LocalizedStringField(build=self.build)
			for i, name in reversed(fieldidx):
				# Build a copy of the fields
				toinsert = [copy(field).rename("%s_%s" % (name, field.name)) for field in fields]
				self.structure[i:i+1] = toinsert
		
		log.info("Using %s build %i" % (self.structure, self.build))
		self.check_integrity()
	
	def preload(self):
		f = self.file
		f.seek(len(self.header))
		
		if self.build > 12880:
			if self.header.lookup_start != self.header.lookup_end:
				size = (self.header.lookup_end - self.header.lookup_start + 1) * 6
				if size < 0:
					log.error("lookup size < 0: %i. This file is corrupt. Expect breakage!" % (size))
				else:
					lookups = f.read(size)
		
		rows = 0
		field = self.structure[0]
		row_header_size = field.size
		reclen = self.header.reclen
		while rows < self.header.row_count:
			address = f.tell() # Get the address of the full row
			id = self._parse_field(f, field)
			
			self._add_row(id, address, reclen)
			
			f.seek(reclen - row_header_size, SEEK_CUR) # minus length of id
			rows += 1
{'fraction_non_alphanumeric': 0.11715256662493292, 'fraction_numerical': 0.02038991235914863, 'mean_word_length': 3.7430025445292623, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1716339', 'n_tokens_mistral': 2188, 'n_tokens_neox': 1841, 'n_words': 522}