<commit_before>#pragma once

////////////////////////////////////////////////////////////////////////////////
// The MIT License (MIT)
//
// Copyright (c) 2020 <NAME> & Animation Compression Library contributors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
////////////////////////////////////////////////////////////////////////////////

#include "acl/compression/impl/clip_context.h"
#include "acl/compression/impl/segment_context.h"
#include "acl/compression/impl/write_stream_data.h"
#include "acl/core/iallocator.h"
#include "acl/core/impl/compiler_utils.h"

#include <cstdint>

ACL_IMPL_FILE_PRAGMA_PUSH

namespace acl
{
	// TODO: Add 32 bit integer per segment header with which sample is dropped
	// TODO: Add 32 bit segment index into database
	// TODO: Database needs an array of segment metadata to track which segment samples are loaded
	// TODO: Database chunks need an array of segment indices contained and the samples contained
	// TODO: Samples loaded can be AND/ORed together easily for quick scanning
	// TODO: Each database asset contains data from only one tier and is split into chunks
	// TODO: Each chunk contains multiple clips/segments, chunks are contiguous

	// TODO: To decompress, grab clip UID, segment index, find loaded chunk that contains the samples we need
	//       if not loaded, check nearest samples
	//       From the chunk and local segment/sample indices, we can find where the data begins

	//       When we load a chunk, iterate over every clip/segment and update the loaded samples mapping

	//       Mapping needs a 32 bit offset per chunk, a 32 bit set sample list per chunk, these are per clip/segment

	//       With the clip ID, we find the list of segment metadata. Each segment can be split into at most 2-4 tiers
	//       and thus contains at most 2-4 chunk offsets (keep in mind tier 0 is the clip itself).
	//       Using our sample index, find the nearest loaded sample for each chunk (clz, ctz, popcount). Pick best.
	//       Calculate the frame offset with pop-count and the pose bit size stored in the clip.
	//       Try and fit 2-3 segments per cache line, or 32 bytes. This helps minimize cache misses even if we straddle
	//       segments. When we look for which chunks are loaded, we'll hit a single cache line for the segment header.
	//       We'll also need a cache miss for the database itself for the pointer we read from but it might be hot from
	//       some other decompression call to another clip from the same database.

	//       Clip ID should be unique for database and is the offset into the segment headers for the first header.

	//       Database should be a single blob with optional padding for region that can be memmapped. Everything is inline
	//       for simplicity. Loading can write directly in the database or memmap the region directly.
	//       Chunks will be 1 MB max (user controlled size?).

	//       Database chunks are sorted by tier, lower tiers stream first (with 0 always in memory in the clip itself).
	//       Database is split into two parts, the metadata + headers, and the bulk tier data. The metadata must always be loaded.
	//       Optionally, the bulk tier data could be loaded into a separate buffer.

	//       A database context is used at runtime for decompression and streaming management. The compressed database is read only
	//       and contains metadata and bulk data. UE4 will separate them. The metadata into the UObject package and the bulk data aside
	//       for async streaming. We'll allocate everything up front.

	//       We need a function to split the database into the metadata and bulk data so they can be serialized separately.
	//       The database context is thus bound with a pointer to the metadata and another (optional) to the bulk data
	//       and we can assert/fail init if the two mismatch or the bulk data is expected or not.

	//       When we look up the clip metadata in the database, can we check the clip hash matches the database entry for safety?

	// Compressed database contains the following:
	//    - Buffer headers/metadata for safety (tag, hash, etc)
	//    - Chunk metadata list: offset, size in bulk data per chunk

	// Bulk data contains the following:
	//    - Multiple chunks, max size 1 MB
	//        - Each chunk has a header: num segments, chunk index
	//        - A list of segment metadata contained in the chunk: clip ID, sample indices, chunk offset (sample data), clip header offset (runtime clip header), segment header offset (runtime segment header)
	//        - The frame data for each segment

	// Runtime data contains the following:
	//    - A bit set for loaded chunks
	//    - A bit set for in progress chunk loads
	//    - Clip and segment headers: clip ID, sample indices, chunk offsets for tier 1 and 2

	// Runtime context has a istreamer interface pointer. When we request to load a chunk on the context, it will validate invariants
	// and call the appropriate method on the streamer: load_chunk(chunk id, offset, size, write_ptr, continuation)
	// When the load is complete, the streamer calls the continuation with the same arguments passed to load_chunk along with a status result
	// with success/failure (error_result?). Runtime context will atomically update bit sets and after the load we parse the chunk metadata
	// to update atomically the loaded chunks: write chunk offset first (no need for atomic), then sample indices (no need for atomic, but ordered after offset)

	// We can have a sync_file_streamer, async_file_streamer, mmap_streamer, ue4_streamer, etc

	// If a database hasn't streamed in a given tier, we shouldn't pay the memory cost (unmap memory, free tier data)
	// To do this, add metadata to track how many chunks each tier contains, where the data starts, etc

	namespace acl_impl
	{
		inline void partition_into_database(iallocator& allocator, clip_context& clip)
		{
			for (SegmentContext& segment : clip.segment_iterator())
			{
				if (segment.sample_tiers == nullptr)
					segment.sample_tiers = allocate_type_array<database_tier8>(allocator, segment.num_samples);

				for (uint32_t sample_index = 0; sample_index < segment.num_samples; ++sample_index)
					segment.sample_tiers[sample_index] = database_tier8::high_importance;

				if (segment.num_samples < 3)
					continue;	// Not enough samples, nothing to drop

				// If we have an odd number of samples, we retain the first and last samples and drop every other sample (the odd samples)
				//    e.g: [0, 1, 2, 3, 4] = high [0, 2, 4] + low [1, 3]
				// If we have an even number of samples, we retain the first two and last samples and drop every other sample (the even samples)
				//    e.g: [0, 1, 2, 3, 4, 5] = high [0, 1, 3, 5] + low [2, 4]
				const bool drop_odd_samples = (segment.num_samples % 2) != 0;

				const uint32_t first_sample = drop_odd_samples ? 0 : 1;
				for (uint32_t sample_index = first_sample; sample_index < segment.num_samples - 1; ++sample_index)
				{<commit_msg>style(compression): remove old comment to track progress<commit_after>#pragma once

////////////////////////////////////////////////////////////////////////////////
// The MIT License (MIT)
//
// Copyright (c) 2020 <NAME> & Animation Compression Library contributors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
////////////////////////////////////////////////////////////////////////////////

#include "acl/compression/impl/clip_context.h"
#include "acl/compression/impl/segment_context.h"
#include "acl/compression/impl/write_stream_data.h"
#include "acl/core/iallocator.h"
#include "acl/core/impl/compiler_utils.h"

#include <cstdint>

ACL_IMPL_FILE_PRAGMA_PUSH

namespace acl
{
	namespace acl_impl
	{
		inline void partition_into_database(iallocator& allocator, clip_context& clip)
		{
			for (SegmentContext& segment : clip.segment_iterator())
			{
				if (segment.sample_tiers == nullptr)
					segment.sample_tiers = allocate_type_array<database_tier8>(allocator, segment.num_samples);

				for (uint32_t sample_index = 0; sample_index < segment.num_samples; ++sample_index)
					segment.sample_tiers[sample_index] = database_tier8::high_importance;

				if (segment.num_samples < 3)
					continue;	// Not enough samples, nothing to drop

				// If we have an odd number of samples, we retain the first and last samples and drop every other sample (the odd samples)
				//    e.g: [0, 1, 2, 3, 4] = high [0, 2, 4] + low [1, 3]
				// If we have an even number of samples, we retain the first two and last samples and drop every other sample (the even samples)
				//    e.g: [0, 1, 2, 3, 4, 5] = high [0, 1, 3, 5] + low [2, 4]
				const bool drop_odd_samples = (segment.num_samples % 2) != 0;

				const uint32_t first_sample = drop_odd_samples ? 0 : 1;
				for (uint32_t sample_index = first_sample; sample_index < segment.num_samples - 1; ++sample_index)
				{
{'fraction_non_alphanumeric': 0.09903802107191938, 'fraction_numerical': 0.009711406321575813, 'mean_word_length': 3.035489833641405, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2975540', 'n_tokens_mistral': 3047, 'n_tokens_neox': 2775, 'n_words': 1446}