<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*- vim:set ts=4 sw=4 sts=4 noet: */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <string>
#include <sstream>
#include <vector>
#include <boost/program_options.hpp>
#include <boost/scoped_array.hpp>
#include <boost/scoped_ptr.hpp>
#include "isdf/isdf.h"
#include "isdf/reader.h"
#include "sys/temporary_path.h"

namespace po = boost::program_options;

using std::cerr;
using std::cout;
using std::endl;
using std::ifstream;
using std::ios;
using std::ofstream;
using std::ostream;
using std::string;
using std::vector;
using boost::scoped_array;
using boost::scoped_ptr;

namespace {

bool ExtractConstantColumns(const char *input,
							boost::uint32_t *num_columns,
							vector<boost::uint32_t> *cv)
{
	ifstream ifs(input, ios::in|ios::binary);
	if (!ifs.is_open()) {
		cerr << "could not open input file: " << input << endl;
		return false;
	}
	isdf::Reader reader;
	if ( !reader.ReadHeader(&ifs) ||
		 !reader.SkipComment(&ifs) ||
		 !reader.SkipDescriptions(&ifs) ||
		 !reader.SkipUnits(&ifs) ) {
		ifs.close();
		return false;
	}

	boost::uint32_t num_objs = reader.num_objs();
	*num_columns = num_objs;

	size_t buf_size = num_objs * sizeof(double);
	scoped_array<char> buf0(new char[buf_size]);
	ifs.read(buf0.get(), buf_size);
	if (ifs.fail()) {
		if (ifs.gcount() == 0) {
			ifs.close();
			// there seems no data, so let's keep all columns
			for (boost::uint32_t i=0;i<num_objs;i++) {
				cv->push_back(i);
			}
			return true;
		} else {
			cerr << "failed to read data: " << input << endl;
		}
		ifs.close();
		return false;
	}

	for (boost::uint32_t i=0;i<num_objs;i++) {
		cv->push_back(i);
	}
	scoped_array<char> buf1(new char[buf_size]);
	for (;;) {
		ifs.read(buf1.get(), buf_size);
		if (ifs.fail()) {
			if (ifs.gcount() == 0) {
				// done
				break;
			} else {
				cerr << "failed to read data: " << input << endl;
				ifs.close();
				return false;
			}
		}
		vector<boost::uint32_t>::iterator it = cv->begin();
		while (it != cv->end()) {
			size_t n = (*it) * sizeof(double);
			// we can ignore endianness by using memcmp()
			if (memcmp(buf0.get()+n, buf1.get()+n, sizeof(double)) == 0) {
				++it;
			} else {
				it = cv->erase(it);
			}
		}
		if (cv->empty()) break;
	}
	ifs.close();

	return true;
}

class StepFilter {
public:
	StepFilter(boost::uint32_t num_objs, const vector<boost::uint32_t> &cv, ostream *os)
		: num_objs_(num_objs),
		  cv_(cv),
		  os_(os),
		  buf_size_((num_objs-cv.size())*sizeof(double)),
		  buf_(new char[buf_size_])
	{}
	~StepFilter() {
		delete buf_;
	}

	int GetStep(size_t, const char *buf) {
		const char *b0 = buf;
		char *b1 = buf_;
		vector<boost::uint32_t>::const_iterator it = cv_.begin();
		for (boost::uint32_t i=0;i<num_objs_;i++) {
			if (it != cv_.end() && i == *it) { // skip this entry
				++it;
			} else {
				memcpy(b1, b0, sizeof(double));
				b1 += sizeof(double);
			}
			b0 += sizeof(double);
		}
		os_->write(buf_, buf_size_);
		if (!os_->good()) {
			cerr << "failed to write data" << endl;
			return -1;
		}
		return 1;
	}

private:
	boost::uint32_t num_objs_;
	const vector<boost::uint32_t> &cv_;
	ostream *os_;
	size_t buf_size_;
	char *buf_;
};

int Filter(const char *input,
		   const vector<boost::uint32_t> &cv,
		   ostream *os)
{
	ifstream ifs(input, ios::in|ios::binary);
	if (!ifs.is_open()) {
		cerr << "could not open file: " << input << endl;
		return EXIT_FAILURE;
	}
	isdf::Reader reader;
	if ( !reader.ReadHeader(&ifs) ||
		 !reader.SkipComment(&ifs) ||
		 !reader.ReadDescriptions(&ifs) ||
		 !reader.ReadUnits(&ifs) ) {
		ifs.close();
		return EXIT_FAILURE;
	}

	const char *rd = reader.descriptions();
	scoped_array<char> descs(new char[reader.num_bytes_descs()]);
	char *d = descs.get();
	vector<boost::uint32_t>::const_iterator it = cv.begin();
	for (boost::uint32_t i=0;i<reader.num_objs();i++) {
		boost::uint32_t bd = *reinterpret_cast<const boost::uint32_t *>(rd);
		if (it != cv.end() && i == *it) { // skip this entry
			++it;
		} else {
			memcpy(d, rd, sizeof(bd) + bd);
			d += sizeof(bd) + bd;<commit_msg>fix alloc-dealloc-mismatch<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*- vim:set ts=4 sw=4 sts=4 noet: */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <iomanip>
#include <fstream>
#include <string>
#include <sstream>
#include <vector>
#include <boost/program_options.hpp>
#include <boost/scoped_array.hpp>
#include <boost/scoped_ptr.hpp>
#include "isdf/isdf.h"
#include "isdf/reader.h"
#include "sys/temporary_path.h"

namespace po = boost::program_options;

using std::cerr;
using std::cout;
using std::endl;
using std::ifstream;
using std::ios;
using std::ofstream;
using std::ostream;
using std::string;
using std::vector;
using boost::scoped_array;
using boost::scoped_ptr;

namespace {

bool ExtractConstantColumns(const char *input,
							boost::uint32_t *num_columns,
							vector<boost::uint32_t> *cv)
{
	ifstream ifs(input, ios::in|ios::binary);
	if (!ifs.is_open()) {
		cerr << "could not open input file: " << input << endl;
		return false;
	}
	isdf::Reader reader;
	if ( !reader.ReadHeader(&ifs) ||
		 !reader.SkipComment(&ifs) ||
		 !reader.SkipDescriptions(&ifs) ||
		 !reader.SkipUnits(&ifs) ) {
		ifs.close();
		return false;
	}

	boost::uint32_t num_objs = reader.num_objs();
	*num_columns = num_objs;

	size_t buf_size = num_objs * sizeof(double);
	scoped_array<char> buf0(new char[buf_size]);
	ifs.read(buf0.get(), buf_size);
	if (ifs.fail()) {
		if (ifs.gcount() == 0) {
			ifs.close();
			// there seems no data, so let's keep all columns
			for (boost::uint32_t i=0;i<num_objs;i++) {
				cv->push_back(i);
			}
			return true;
		} else {
			cerr << "failed to read data: " << input << endl;
		}
		ifs.close();
		return false;
	}

	for (boost::uint32_t i=0;i<num_objs;i++) {
		cv->push_back(i);
	}
	scoped_array<char> buf1(new char[buf_size]);
	for (;;) {
		ifs.read(buf1.get(), buf_size);
		if (ifs.fail()) {
			if (ifs.gcount() == 0) {
				// done
				break;
			} else {
				cerr << "failed to read data: " << input << endl;
				ifs.close();
				return false;
			}
		}
		vector<boost::uint32_t>::iterator it = cv->begin();
		while (it != cv->end()) {
			size_t n = (*it) * sizeof(double);
			// we can ignore endianness by using memcmp()
			if (memcmp(buf0.get()+n, buf1.get()+n, sizeof(double)) == 0) {
				++it;
			} else {
				it = cv->erase(it);
			}
		}
		if (cv->empty()) break;
	}
	ifs.close();

	return true;
}

class StepFilter {
public:
	StepFilter(boost::uint32_t num_objs, const vector<boost::uint32_t> &cv, ostream *os)
		: num_objs_(num_objs),
		  cv_(cv),
		  os_(os),
		  buf_size_((num_objs-cv.size())*sizeof(double)),
		  buf_(new char[buf_size_])
	{}

	int GetStep(size_t, const char *buf) {
		const char *b0 = buf;
		char *b1 = buf_.get();
		vector<boost::uint32_t>::const_iterator it = cv_.begin();
		for (boost::uint32_t i=0;i<num_objs_;i++) {
			if (it != cv_.end() && i == *it) { // skip this entry
				++it;
			} else {
				memcpy(b1, b0, sizeof(double));
				b1 += sizeof(double);
			}
			b0 += sizeof(double);
		}
		os_->write(buf_.get(), buf_size_);
		if (!os_->good()) {
			cerr << "failed to write data" << endl;
			return -1;
		}
		return 1;
	}

private:
	boost::uint32_t num_objs_;
	const vector<boost::uint32_t> &cv_;
	ostream *os_;
	size_t buf_size_;
	boost::scoped_array<char> buf_;
};

int Filter(const char *input,
		   const vector<boost::uint32_t> &cv,
		   ostream *os)
{
	ifstream ifs(input, ios::in|ios::binary);
	if (!ifs.is_open()) {
		cerr << "could not open file: " << input << endl;
		return EXIT_FAILURE;
	}
	isdf::Reader reader;
	if ( !reader.ReadHeader(&ifs) ||
		 !reader.SkipComment(&ifs) ||
		 !reader.ReadDescriptions(&ifs) ||
		 !reader.ReadUnits(&ifs) ) {
		ifs.close();
		return EXIT_FAILURE;
	}

	const char *rd = reader.descriptions();
	scoped_array<char> descs(new char[reader.num_bytes_descs()]);
	char *d = descs.get();
	vector<boost::uint32_t>::const_iterator it = cv.begin();
	for (boost::uint32_t i=0;i<reader.num_objs();i++) {
		boost::uint32_t bd = *reinterpret_cast<const boost::uint32_t *>(rd);
		if (it != cv.end() && i == *it) { // skip this entry
			++it;
		} else {
			memcpy(d, rd, sizeof(bd) + bd);
			d += sizeof(bd) + bd;
{'fraction_non_alphanumeric': 0.18591251175917214, 'fraction_numerical': 0.014111006585136407, 'mean_word_length': 3.9247249565720903, 'pattern_counts': {'":': 0, '<': 114, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2931257', 'n_tokens_mistral': 3718, 'n_tokens_neox': 3359, 'n_words': 727}