<commit_before>#include "audio_importer.h"
#include "halley/tools/assets/import_assets_database.h"
#include "halley/file/byte_serializer.h"
#include "halley/resources/metadata.h"
#include "halley/audio/vorbis_dec.h"
#include "halley/audio/resampler.h"

#include <ogg/ogg.h>
#include <vorbis/codec.h>
#include <vorbis/vorbisenc.h>

using namespace Halley;

std::vector<Path> AudioImporter::import(const ImportingAsset& asset, Path dstDir, ProgressReporter reporter, AssetCollector collector)
{
	Path mainFile = asset.inputFiles.at(0).name;
	auto& rawData = asset.inputFiles[0].data;
	auto resData = std::make_shared<ResourceDataStatic>(rawData.data(), rawData.size(), mainFile.string(), false);
	Bytes encodedData;
	gsl::span<const gsl::byte> fileData(gsl::as_bytes(gsl::span<const Halley::Byte>(rawData)));

	std::vector<std::vector<float>> samples;
	int numChannels = 0;
	int sampleRate = 0;
	bool needsEncoding = false;
	bool needsResampling = false;

	if (mainFile.getExtension() == ".ogg") { // assuming Ogg Vorbis
		// Load vorbis data
		VorbisData vorbis(resData);
		numChannels = vorbis.getNumChannels();
		size_t numSamples = vorbis.getNumSamples();
		sampleRate = vorbis.getSampleRate();
		samples.resize(numChannels);
		for (size_t i = 0; i < numChannels; ++i) {
			samples[i].resize(numSamples);
		}

		// Decode
		if (sampleRate != 48000) {
			vorbis.read(samples);
			needsResampling = true;
		}
	} else {
		throw Exception("Unsupported audio format: " + mainFile.getExtension());
	}

	// Resample
	if (needsResampling) {
		// Resample
		for (size_t i = 0; i < numChannels; ++i) {
			samples[i] = resampleChannel(sampleRate, 48000, samples[i]);
		}
		sampleRate = 48000;
		needsEncoding = true;
	}

	// Encode to vorbis
	if (needsEncoding) {
		encodedData = encodeVorbis(numChannels, sampleRate, samples);
		fileData = gsl::as_bytes(gsl::span<const Halley::Byte>(encodedData));
		samples.clear();
	}

	// Simply save file
	FileSystem::writeFile(dstDir / mainFile, fileData);

	// Write metafile
	Metadata meta;
	if (asset.metadata) {
		meta = *asset.metadata;
	}
	meta.set("channels", numChannels);
	meta.set("sampleRate", sampleRate);
	Path metaPath = mainFile.replaceExtension(mainFile.getExtension() + ".meta");
	FileSystem::writeFile(dstDir / metaPath, Serializer::toBytes(meta));<commit_msg>Audio importer will resample audio in parallel.<commit_after>#include "audio_importer.h"
#include "halley/tools/assets/import_assets_database.h"
#include "halley/file/byte_serializer.h"
#include "halley/resources/metadata.h"
#include "halley/audio/vorbis_dec.h"
#include "halley/audio/resampler.h"

#include <ogg/ogg.h>
#include <vorbis/codec.h>
#include <vorbis/vorbisenc.h>
#include "halley/concurrency/concurrent.h"

using namespace Halley;

std::vector<Path> AudioImporter::import(const ImportingAsset& asset, Path dstDir, ProgressReporter reporter, AssetCollector collector)
{
	Path mainFile = asset.inputFiles.at(0).name;
	auto& rawData = asset.inputFiles[0].data;
	auto resData = std::make_shared<ResourceDataStatic>(rawData.data(), rawData.size(), mainFile.string(), false);
	Bytes encodedData;
	gsl::span<const gsl::byte> fileData(gsl::as_bytes(gsl::span<const Halley::Byte>(rawData)));

	std::vector<std::vector<float>> samples;
	int numChannels = 0;
	int sampleRate = 0;
	bool needsEncoding = false;
	bool needsResampling = false;

	if (mainFile.getExtension() == ".ogg") { // assuming Ogg Vorbis
		// Load vorbis data
		VorbisData vorbis(resData);
		numChannels = vorbis.getNumChannels();
		size_t numSamples = vorbis.getNumSamples();
		sampleRate = vorbis.getSampleRate();
		samples.resize(numChannels);
		for (size_t i = 0; i < numChannels; ++i) {
			samples[i].resize(numSamples);
		}

		// Decode
		if (sampleRate != 48000) {
			vorbis.read(samples);
			needsResampling = true;
		}
	} else {
		throw Exception("Unsupported audio format: " + mainFile.getExtension());
	}

	// Resample
	if (needsResampling) {
		// Resample
		Concurrent::foreach(std::begin(samples), std::end(samples), [&] (std::vector<float>& s) {
			s = resampleChannel(sampleRate, 48000, s);
		});
		sampleRate = 48000;
		needsEncoding = true;
	}

	// Encode to vorbis
	if (needsEncoding) {
		encodedData = encodeVorbis(numChannels, sampleRate, samples);
		fileData = gsl::as_bytes(gsl::span<const Halley::Byte>(encodedData));
		samples.clear();
	}

	// Simply save file
	FileSystem::writeFile(dstDir / mainFile, fileData);

	// Write metafile
	Metadata meta;
	if (asset.metadata) {
		meta = *asset.metadata;
	}
	meta.set("channels", numChannels);
	meta.set("sampleRate", sampleRate);
	Path metaPath = mainFile.replaceExtension(mainFile.getExtension() + ".meta");
	FileSystem::writeFile(dstDir / metaPath, Serializer::toBytes(meta));
{'fraction_non_alphanumeric': 0.14811680067710536, 'fraction_numerical': 0.008675412611087601, 'mean_word_length': 5.277556440903054, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3102674', 'n_tokens_mistral': 1748, 'n_tokens_neox': 1654, 'n_words': 324}