<commit_before>"""ttLib/sfnt.py -- low-level module to deal with the sfnt file format.

Defines two public classes:
	SFNTReader
	SFNTWriter

(Normally you don't have to use these classes explicitly; they are
used automatically by ttLib.TTFont.)

The reading and writing of sfnt files is separated in two distinct
classes, since whenever to number of tables changes or whenever
a table's length chages you need to rewrite the whole file anyway.
"""

from io import BytesIO
from fontTools.misc.py23 import Tag
from fontTools.misc import sstruct
from fontTools.ttLib import TTLibError
import struct
from collections import OrderedDict
import logging


log = logging.getLogger(__name__)


class SFNTReader(object):

	def __new__(cls, *args, **kwargs):
		""" Return an instance of the SFNTReader sub-class which is compatible
		with the input file type.
		"""
		if args and cls is SFNTReader:
			infile = args[0]
			infile.seek(0)
			sfntVersion = Tag(infile.read(4))
			infile.seek(0)
			if sfntVersion == "wOF2":
				# return new WOFF2Reader object
				from fontTools.ttLib.woff2 import WOFF2Reader
				return object.__new__(WOFF2Reader)
		# return default object
		return object.__new__(cls)

	def __init__(self, file, checkChecksums=1, fontNumber=-1):
		self.file = file
		self.checkChecksums = checkChecksums

		self.flavor = None
		self.flavorData = None
		self.DirectoryEntry = SFNTDirectoryEntry
		self.file.seek(0)
		self.sfntVersion = self.file.read(4)
		self.file.seek(0)
		if self.sfntVersion == b"ttcf":
			header = readTTCHeader(self.file)
			numFonts = header.numFonts
			if not 0 <= fontNumber < numFonts:
				raise TTLibError("specify a font number between 0 and %d (inclusive)" % (numFonts - 1))
			self.numFonts = numFonts
			self.file.seek(header.offsetTable[fontNumber])
			data = self.file.read(sfntDirectorySize)
			if len(data) != sfntDirectorySize:
				raise TTLibError("Not a Font Collection (not enough data)")
			sstruct.unpack(sfntDirectoryFormat, data, self)
		elif self.sfntVersion == b"wOFF":
			self.flavor = "woff"
			self.DirectoryEntry = WOFFDirectoryEntry
			data = self.file.read(woffDirectorySize)
			if len(data) != woffDirectorySize:
				raise TTLibError("Not a WOFF font (not enough data)")
			sstruct.unpack(woffDirectoryFormat, data, self)
		else:
			data = self.file.read(sfntDirectorySize)
			if len(data) != sfntDirectorySize:
				raise TTLibError("Not a TrueType or OpenType font (not enough data)")
			sstruct.unpack(sfntDirectoryFormat, data, self)
		self.sfntVersion = Tag(self.sfntVersion)

		if self.sfntVersion not in ("\x00\x01\x00\x00", "OTTO", "true"):
			raise TTLibError("Not a TrueType or OpenType font (bad sfntVersion)")
		tables = {}
		for i in range(self.numTables):
			entry = self.DirectoryEntry()
			entry.fromFile(self.file)
			tag = Tag(entry.tag)
			tables[tag] = entry
		self.tables = OrderedDict(sorted(tables.items(), key=lambda i: i[1].offset))

		# Load flavor data if any
		if self.flavor == "woff":
			self.flavorData = WOFFFlavorData(self)

	def has_key(self, tag):
		return tag in self.tables

	__contains__ = has_key

	def keys(self):
		return self.tables.keys()

	def __getitem__(self, tag):
		"""Fetch the raw table data."""
		entry = self.tables[Tag(tag)]
		data = entry.loadData (self.file)
		if self.checkChecksums:
			if tag == 'head':
				# Beh: we have to special-case the 'head' table.
				checksum = calcChecksum(data[:8] + b'\0\0\0\0' + data[12:])
			else:
				checksum = calcChecksum(data)
			if self.checkChecksums > 1:
				# Be obnoxious, and barf when it's wrong
				assert checksum == entry.checkSum, "bad checksum for '%s' table" % tag
			elif checksum != entry.checkSum:
				# Be friendly, and just log a warning.
				log.warning("bad checksum for '%s' table", tag)
		return data

	def __delitem__(self, tag):
		del self.tables[Tag(tag)]

	def close(self):
		self.file.close()

	def __getstate__(self):
		"""Makes SFNTReader pickle/deepcopy-able even with TTFont loaded as lazy=True.

		If SFNTReader holds a reference to an external file object which is not
		pickleable, we copy the file data to an in-memory bytes stream, which is
		pickelable and behaves just like a file.
		"""
		if isinstance(self.file, BytesIO):
			return self.__dict__
		state = self.__dict__.copy()
		state["file"] = _copy_file_to_bytes_stream(state["file"])
		return state


def _copy_file_to_bytes_stream(f) -> BytesIO:
	# Copy bytes from file object to BytesIO stream.
	# The returned stream also records the original file `name` and current position.
	pos = f.tell()
	f.seek(0)
	buf = BytesIO(f.read())
	f.seek(pos)
	buf.seek(pos)
	if hasattr(f, "name"):
		buf.name = f.name
	return buf


# default compression level for WOFF 1.0 tables and metadata
ZLIB_COMPRESSION_LEVEL = 6

# if set to True, use zopfli instead of zlib for compressing WOFF 1.0.
# The Python bindings are available at https://pypi.python.org/pypi/zopfli
USE_ZOPFLI = False

# mapping between zlib's compression levels and zopfli's 'numiterations'.
# Use lower values for files over several MB in size or it will be too slow
ZOPFLI_LEVELS = {
	# 0: 0,  # can't do 0 iterations...
	1: 1,
	2: 3,
	3: 5,
	4: 8,
	5: 10,
	6: 15,
	7: 25,
	8: 50,<commit_msg>SFNTReader: define __getstate__/__setstate__ to reopen external file<commit_after>"""ttLib/sfnt.py -- low-level module to deal with the sfnt file format.

Defines two public classes:
	SFNTReader
	SFNTWriter

(Normally you don't have to use these classes explicitly; they are
used automatically by ttLib.TTFont.)

The reading and writing of sfnt files is separated in two distinct
classes, since whenever to number of tables changes or whenever
a table's length chages you need to rewrite the whole file anyway.
"""

from io import BytesIO
from fontTools.misc.py23 import Tag
from fontTools.misc import sstruct
from fontTools.ttLib import TTLibError
import struct
from collections import OrderedDict
import logging


log = logging.getLogger(__name__)


class SFNTReader(object):

	def __new__(cls, *args, **kwargs):
		""" Return an instance of the SFNTReader sub-class which is compatible
		with the input file type.
		"""
		if args and cls is SFNTReader:
			infile = args[0]
			infile.seek(0)
			sfntVersion = Tag(infile.read(4))
			infile.seek(0)
			if sfntVersion == "wOF2":
				# return new WOFF2Reader object
				from fontTools.ttLib.woff2 import WOFF2Reader
				return object.__new__(WOFF2Reader)
		# return default object
		return object.__new__(cls)

	def __init__(self, file, checkChecksums=1, fontNumber=-1):
		self.file = file
		self.checkChecksums = checkChecksums

		self.flavor = None
		self.flavorData = None
		self.DirectoryEntry = SFNTDirectoryEntry
		self.file.seek(0)
		self.sfntVersion = self.file.read(4)
		self.file.seek(0)
		if self.sfntVersion == b"ttcf":
			header = readTTCHeader(self.file)
			numFonts = header.numFonts
			if not 0 <= fontNumber < numFonts:
				raise TTLibError("specify a font number between 0 and %d (inclusive)" % (numFonts - 1))
			self.numFonts = numFonts
			self.file.seek(header.offsetTable[fontNumber])
			data = self.file.read(sfntDirectorySize)
			if len(data) != sfntDirectorySize:
				raise TTLibError("Not a Font Collection (not enough data)")
			sstruct.unpack(sfntDirectoryFormat, data, self)
		elif self.sfntVersion == b"wOFF":
			self.flavor = "woff"
			self.DirectoryEntry = WOFFDirectoryEntry
			data = self.file.read(woffDirectorySize)
			if len(data) != woffDirectorySize:
				raise TTLibError("Not a WOFF font (not enough data)")
			sstruct.unpack(woffDirectoryFormat, data, self)
		else:
			data = self.file.read(sfntDirectorySize)
			if len(data) != sfntDirectorySize:
				raise TTLibError("Not a TrueType or OpenType font (not enough data)")
			sstruct.unpack(sfntDirectoryFormat, data, self)
		self.sfntVersion = Tag(self.sfntVersion)

		if self.sfntVersion not in ("\x00\x01\x00\x00", "OTTO", "true"):
			raise TTLibError("Not a TrueType or OpenType font (bad sfntVersion)")
		tables = {}
		for i in range(self.numTables):
			entry = self.DirectoryEntry()
			entry.fromFile(self.file)
			tag = Tag(entry.tag)
			tables[tag] = entry
		self.tables = OrderedDict(sorted(tables.items(), key=lambda i: i[1].offset))

		# Load flavor data if any
		if self.flavor == "woff":
			self.flavorData = WOFFFlavorData(self)

	def has_key(self, tag):
		return tag in self.tables

	__contains__ = has_key

	def keys(self):
		return self.tables.keys()

	def __getitem__(self, tag):
		"""Fetch the raw table data."""
		entry = self.tables[Tag(tag)]
		data = entry.loadData (self.file)
		if self.checkChecksums:
			if tag == 'head':
				# Beh: we have to special-case the 'head' table.
				checksum = calcChecksum(data[:8] + b'\0\0\0\0' + data[12:])
			else:
				checksum = calcChecksum(data)
			if self.checkChecksums > 1:
				# Be obnoxious, and barf when it's wrong
				assert checksum == entry.checkSum, "bad checksum for '%s' table" % tag
			elif checksum != entry.checkSum:
				# Be friendly, and just log a warning.
				log.warning("bad checksum for '%s' table", tag)
		return data

	def __delitem__(self, tag):
		del self.tables[Tag(tag)]

	def close(self):
		self.file.close()

	# We define custom __getstate__ and __setstate__ to make SFNTReader pickle-able
	# and deepcopy-able. When a TTFont is loaded as lazy=True, SFNTReader holds a
	# reference to an external file object which is not pickleable. So in __getstate__
	# we store the file name and current position, and in __setstate__ we reopen the
	# same named file after unpickling.

	def __getstate__(self):
		if isinstance(self.file, BytesIO):
			# BytesIO is already pickleable, return the state unmodified
			return self.__dict__

		# remove unpickleable file attribute, and only store its name and pos
		state = self.__dict__.copy()
		del state["file"]
		state["_filename"] = self.file.name
		state["_filepos"] = self.file.tell()
		return state

	def __setstate__(self, state):
		if "file" not in state:
			self.file = open(state.pop("_filename"), "rb")
			self.file.seek(state.pop("_filepos"))
		self.__dict__.update(state)


# default compression level for WOFF 1.0 tables and metadata
ZLIB_COMPRESSION_LEVEL = 6

# if set to True, use zopfli instead of zlib for compressing WOFF 1.0.
# The Python bindings are available at https://pypi.python.org/pypi/zopfli
USE_ZOPFLI = False

# mapping between zlib's compression levels and zopfli's 'numiterations'.
# Use lower values for files over several MB in size or it will be too slow
ZOPFLI_LEVELS = {
	# 0: 0,  # can't do 0 iterations...
	1: 1,
	2: 3,
	3: 5,
	4: 8,
	5: 10,
	6: 15,
	7: 25,
	8: 50,
{'fraction_non_alphanumeric': 0.1042416209051316, 'fraction_numerical': 0.012213595909865555, 'mean_word_length': 3.954502814258912, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1432827', 'n_tokens_mistral': 3998, 'n_tokens_neox': 3568, 'n_words': 1124}