<commit_before># -*- coding: iso-8859-1 -*-

__all__ = ("FileMerge", "FileWindow", "MakeTIFF", "TIFF", "ExifWrapper",
           "identify_raw", "make_pdirs", "raw_exts", "RawWrapper",
           "CommentWrapper", "DotDict")

class TIFF:
	"""Pretty minimal TIFF container parser"""
	
	types = {1: (1, "B"),  # BYTE
		 2: (1, None), # ASCII
		 3: (2, "H"),  # SHORT
		 4: (4, "I"),  # LONG
		 5: (8, "II"), # RATIONAL
		 # No TIFF6 fields, sorry
		}
	
	def __init__(self, fh, short_header=False):
		from struct import unpack
		self._fh = fh
		d = fh.read(4)
		if short_header:
			if d[:2] not in (b"II", b"MM"): raise Exception("Not TIFF")
		else:
			if d not in (b"II*\0", b"MM\0*"): raise Exception("Not TIFF")
		endian = {b"M": ">", b"I": "<"}[d[0]]
		self._up = lambda fmt, *a: unpack(endian + fmt, *a)
		self._up1 = lambda *a: self._up(*a)[0]
		if short_header:
			next_ifd = short_header
		else:
			next_ifd = self._up1("I", fh.read(4))
		self.reinit_from(next_ifd, short_header)
	
	def reinit_from(self, next_ifd, short_header=False):
		self.ifd = []
		self.subifd = []
		while next_ifd:
			self.ifd.append(self._ifdread(next_ifd))
			if short_header: return
			next_ifd = self._up1("I", self._fh.read(4))
		subifd = self.ifdget(self.ifd[0], 0x14a) or []
		for next_ifd in subifd:
			self.subifd.append(self._ifdread(next_ifd))
	
	def ifdget(self, ifd, tag):
		if tag in ifd:
			type, vc, off = ifd[tag]
			if type not in self.types: return None
			if isinstance(off, int): # offset
				self._fh.seek(off)
				tl, fmt = self.types[type]
				off = self._fh.read(tl * vc)
				if fmt: off = self._up(fmt * vc, off)
			if isinstance(off, basestring):
				off = off.rstrip("\0")
			return off
	
	def _ifdread(self, next_ifd):
		ifd = {}
		self._fh.seek(next_ifd)
		count = self._up1("H", self._fh.read(2))<commit_msg>More robust TIFF parsing, survive loops.<commit_after># -*- coding: iso-8859-1 -*-

from __future__ import print_function

__all__ = ("FileMerge", "FileWindow", "MakeTIFF", "TIFF", "ExifWrapper",
           "identify_raw", "make_pdirs", "raw_exts", "RawWrapper",
           "CommentWrapper", "DotDict")

class TIFF:
	"""Pretty minimal TIFF container parser"""
	
	types = {1: (1, "B"),  # BYTE
		 2: (1, None), # ASCII
		 3: (2, "H"),  # SHORT
		 4: (4, "I"),  # LONG
		 5: (8, "II"), # RATIONAL
		 # No TIFF6 fields, sorry
		}
	
	def __init__(self, fh, short_header=False):
		from struct import unpack
		self._fh = fh
		d = fh.read(4)
		if short_header:
			if d[:2] not in (b"II", b"MM"): raise Exception("Not TIFF")
		else:
			if d not in (b"II*\0", b"MM\0*"): raise Exception("Not TIFF")
		endian = {b"M": ">", b"I": "<"}[d[0]]
		self._up = lambda fmt, *a: unpack(endian + fmt, *a)
		self._up1 = lambda *a: self._up(*a)[0]
		if short_header:
			next_ifd = short_header
		else:
			next_ifd = self._up1("I", fh.read(4))
		self.reinit_from(next_ifd, short_header)
	
	def reinit_from(self, next_ifd, short_header=False):
		self.ifd = []
		self.subifd = []
		seen_ifd = set()
		while next_ifd:
			self.ifd.append(self._ifdread(next_ifd))
			if short_header: return
			next_ifd = self._up1("I", self._fh.read(4))
			if next_ifd in seen_ifd:
				from sys import stderr
				print("WARNING: Looping IFDs", file=stderr)
				break
			seen_ifd.add(next_ifd)
			assert len(self.ifd) < 32 # way too many
		subifd = self.ifdget(self.ifd[0], 0x14a) or []
		assert len(subifd) < 32 # way too many
		for next_ifd in subifd:
			self.subifd.append(self._ifdread(next_ifd))
	
	def ifdget(self, ifd, tag):
		if tag in ifd:
			type, vc, off = ifd[tag]
			if type not in self.types: return None
			if isinstance(off, int): # offset
				self._fh.seek(off)
				tl, fmt = self.types[type]
				off = self._fh.read(tl * vc)
				if fmt: off = self._up(fmt * vc, off)
			if isinstance(off, basestring):
				off = off.rstrip("\0")
			return off
	
	def _ifdread(self, next_ifd):
		ifd = {}
		self._fh.seek(next_ifd)
		count = self._up1("H", self._fh.read(2))
{'fraction_non_alphanumeric': 0.17110266159695817, 'fraction_numerical': 0.018250950570342206, 'mean_word_length': 3.3362637362637364, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1408403', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1611, 'n_words': 396}