<commit_before>'use strict'

const path = require('path')
const zlib = require('zlib')

const sha1 = require('sha1')

const nfs = require('./nice-fs')


const basePath = '.ndr/objects'

function store (value) {
	const hash = sha1(value)
	const prefix = hash.slice(0, 2)
	const suffix = hash.slice(2)

	const deflated = zlib.deflateSync(value)

	const directory = path.join(basePath, prefix)
	if (!nfs.exists(directory)) {
		nfs.mkdir(directory)
	}

	const file = path.join(directory, suffix)
	if (!nfs.exists(file)) {
		nfs.write(file, deflated)
	}

	return hash
}

function retrieve (hash) {
	const prefix = hash.slice(0, 2)
	const suffix = hash.slice(2)

	const file = path.join(basePath, prefix, suffix)

	const deflated = nfs.readBinary(file)

	return zlib.inflateSync(deflated).toString()
}

function storeBlob (value) {
	const size = Buffer.byteLength(value, 'utf8')

	return store(`blob ${size}\0${value}`)
}

function retrieveBlob (hash) {
	const content = retrieve(hash)

	return content.slice(content.indexOf('\0') + 1)
}

module.exports = {
	storeBlob,
	retrieveBlob,
}<commit_msg>Use git file format for trees<commit_after>'use strict'

const path = require('path')
const zlib = require('zlib')

const sha1 = require('sha1')

const nfs = require('./nice-fs')


const basePath = '.ndr/objects'

function store (value) {
	const hash = sha1(value)
	const prefix = hash.slice(0, 2)
	const suffix = hash.slice(2)

	const deflated = zlib.deflateSync(value)

	const directory = path.join(basePath, prefix)
	if (!nfs.exists(directory)) {
		nfs.mkdir(directory)
	}

	const file = path.join(directory, suffix)
	if (!nfs.exists(file)) {
		nfs.write(file, deflated)
	}

	return hash
}

function retrieve (hash) {
	const prefix = hash.slice(0, 2)
	const suffix = hash.slice(2)

	const file = path.join(basePath, prefix, suffix)

	const deflated = nfs.readBinary(file)

	return zlib.inflateSync(deflated).toString()
}

function storeBlob (value) {
	const size = Buffer.byteLength(value, 'utf8')

	return store(`blob ${size}\0${value}`)
}

function retrieveBlob (hash) {
	const content = retrieve(hash)

	return content.slice(content.indexOf('\0') + 1)
}

function packHash (string) {
	return string.replace(/../g, (match) =>
		String.fromCharCode(parseInt(match, 16))
	)
}

function unpackHash (binary) {
	return binary.replace(/[^]/g, (match) => {
		const code = match.charCodeAt(0)

		return code < 16
			? `0${code.toString(16)}`
			: code.toString(16)
	})
}

function storeTree (entries) {
	const content = entries.map(
		({ hash, file, type }) => {
			const mode = type === 'blob' ? '100644' : '40000'

			return `${mode} ${file}\0${packHash(hash)}`
		}
	).join('')

	return store(`tree ${content.length}\0${content}`)
}

function retrieveTree (hash) {
	const content = retrieve(hash)

	const regex = /(100644|40000) ([^\0]+)\0([^]{20})/g

	const entries = []

	let match
	while (match = regex.exec(content)) {
		entries.push({
			type: match[1] === '100644' ? 'blob' : 'tree',
			file: match[2],
			hash: unpackHash(match[3]),
		})
	}

	return entries
}

module.exports = {
	storeBlob,
	retrieveBlob,
	storeTree,
	retrieveTree,
}
{'fraction_non_alphanumeric': 0.16310245770826684, 'fraction_numerical': 0.023300351101180977, 'mean_word_length': 4.276094276094276, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793713', 'n_tokens_mistral': 1211, 'n_tokens_neox': 1118, 'n_words': 264}