<commit_before>"""Optimise the cache."""

import os
from logging import warn

def _already_linked(a, b):
	ai = os.stat(a)
	bi = os.stat(b)
	return (ai.st_dev, ai.st_ino) == (bi.st_dev, bi.st_ino)

def _byte_identical(a, b):
	af = file(a, 'rb')
	bf = file(b, 'rb')
	while True:
		adata = af.read(100)
		bdata = bf.read(100)
		if adata != bdata:
			return False
		if not adata:
			return True

def _link(a, b, tmpfile):
	"""Keep 'a', delete 'b' and hard-link to 'a'"""
	if not _byte_identical(a, b):
		warn("Files should be identical, but they're not!\n%s\n%s", a, b)

	os.link(a, tmpfile)
	try:
		os.rename(tmpfile, b)
	except:
		os.unlink(tmpfile)
		raise

def optimise(impl_dir):
	"""Scan an implementation cache directory for duplicate files, and
	hard-link any duplicates together to save space.
	@param impl_dir: a $cache/0install.net/implementations directory
	@type impl_dir: str
	@param verbose: display progress messages
	@type verbose: bool
	@return: (unique bytes, duplicated bytes, already linked, manifest size)
	@rtype: (int, int, int, int)"""

	first_copy = {}		# TypeDigest -> Path
	dup_size = uniq_size = already_linked = man_size = 0

	import random

	for x in range(10):
		tmpfile = os.path.join(impl_dir, 'optimise-%d' % random.randint(0, 1000000))
		if not os.path.exists(tmpfile):
			break
	else:
		raise Exception("Can't generate unused tempfile name!")

	for impl in os.listdir(impl_dir):
		if impl.startswith('.') or '=' not in impl:
			warn("Skipping non-implementation '%s'", impl)
			continue
		manifest_path = os.path.join(impl_dir, impl, '.manifest')
		try:
			ms = file(manifest_path)
		except OSError, ex:
			warn("Failed to read manifest file '%s': %s", manifest_path, str(ex))
			continue

		man_size += os.path.getsize(manifest_path)

		alg = impl.split('=', 1)[0]
		if alg == 'sha1': continue

		dir = ""
		for line in ms:
			if line[0] == 'D':
				type, path = line.split(' ', 1)
				assert path.startswith('/')
				dir = path[1:-1]	# Strip slash and newline
				continue

			if line[0] == "S":<commit_msg>Don't count the size of manifests for skipped implementations.<commit_after>"""Optimise the cache."""

import os
from logging import warn

def _already_linked(a, b):
	ai = os.stat(a)
	bi = os.stat(b)
	return (ai.st_dev, ai.st_ino) == (bi.st_dev, bi.st_ino)

def _byte_identical(a, b):
	af = file(a, 'rb')
	bf = file(b, 'rb')
	while True:
		adata = af.read(100)
		bdata = bf.read(100)
		if adata != bdata:
			return False
		if not adata:
			return True

def _link(a, b, tmpfile):
	"""Keep 'a', delete 'b' and hard-link to 'a'"""
	if not _byte_identical(a, b):
		warn("Files should be identical, but they're not!\n%s\n%s", a, b)

	os.link(a, tmpfile)
	try:
		os.rename(tmpfile, b)
	except:
		os.unlink(tmpfile)
		raise

def optimise(impl_dir):
	"""Scan an implementation cache directory for duplicate files, and
	hard-link any duplicates together to save space.
	@param impl_dir: a $cache/0install.net/implementations directory
	@type impl_dir: str
	@param verbose: display progress messages
	@type verbose: bool
	@return: (unique bytes, duplicated bytes, already linked, manifest size)
	@rtype: (int, int, int, int)"""

	first_copy = {}		# TypeDigest -> Path
	dup_size = uniq_size = already_linked = man_size = 0

	import random

	for x in range(10):
		tmpfile = os.path.join(impl_dir, 'optimise-%d' % random.randint(0, 1000000))
		if not os.path.exists(tmpfile):
			break
	else:
		raise Exception("Can't generate unused tempfile name!")

	for impl in os.listdir(impl_dir):
		if impl.startswith('.') or '=' not in impl:
			warn("Skipping non-implementation '%s'", impl)
			continue
		manifest_path = os.path.join(impl_dir, impl, '.manifest')
		try:
			ms = file(manifest_path)
		except OSError, ex:
			warn("Failed to read manifest file '%s': %s", manifest_path, str(ex))
			continue

		alg = impl.split('=', 1)[0]
		if alg == 'sha1': continue

		man_size += os.path.getsize(manifest_path)

		dir = ""
		for line in ms:
			if line[0] == 'D':
				type, path = line.split(' ', 1)
				assert path.startswith('/')
				dir = path[1:-1]	# Strip slash and newline
				continue

			if line[0] == "S":
{'fraction_non_alphanumeric': 0.14614081780788774, 'fraction_numerical': 0.012581659811275103, 'mean_word_length': 3.8294392523364484, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4991421', 'n_tokens_mistral': 1700, 'n_tokens_neox': 1521, 'n_words': 457}