<commit_before>/*
* Copyright 2016 Nu-book Inc.
* Copyright 2017 <NAME>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "BlackboxTestRunner.h"
#include "ImageLoader.h"
#include "TestReader.h"
#include "GenericLuminanceSource.h"
#include "HybridBinarizer.h"
#include "TextUtfEncoding.h"
#include "ZXContainerAlgorithms.h"
#include "lodepng.h"

#include <iostream>
#include <set>

#if __has_include(<filesystem>)
#  include <filesystem>
#  ifdef __cpp_lib_filesystem
     namespace fs = std::filesystem;
#    define ZXING_HAS_FILESYSTEM
#  endif
#endif
#if !defined(ZXING_HAS_FILESYSTEM) && __has_include(<experimental/filesystem>)
#  include <experimental/filesystem>
#  ifdef __cpp_lib_experimental_filesystem
     namespace fs = std::experimental::filesystem;
#    define ZXING_HAS_FILESYSTEM
#  endif
#endif
#if defined(__APPLE__) && !defined(__clang__) &&                                                                       \
	__has_include(<dirent.h>) && defined(__SANITIZE_ADDRESS__) && defined(ZXING_HAS_FILESYSTEM)
#  warning detected gcc + macos + <filesystem> + address_sanitizer == bad -> disable <filesystem>
#  undef ZXING_HAS_FILESYSTEM
#endif
#if !defined(ZXING_HAS_FILESYSTEM)
#  if __has_include(<dirent.h>)
#    include <dirent.h>
#  else
#    error need <filesystem> or <dirent.h>
#  endif
#endif

// compiling this with clang (e.g. version 6) might require linking against libc++experimental.a or libc++fs.a.
// E.g.: CMAKE_EXE_LINKER_FLAGS = -L/usr/local/Cellar/llvm/6.0.1/lib -lc++experimental

using namespace ZXing;
using namespace ZXing::Test;

static bool HasSuffix(const std::string& s, const std::string& suffix)
{
	return s.size() >= suffix.size() && std::equal(suffix.rbegin(), suffix.rend(), s.rbegin());
}

class GenericImageLoader : public ImageLoader
{
public:
	virtual std::shared_ptr<LuminanceSource> load(const std::wstring& filename) const override
	{
		std::vector<unsigned char> buffer;
		unsigned width, height;
		unsigned error = lodepng::decode(buffer, width, height, TextUtfEncoding::ToUtf8(filename));
		if (error) {
			throw std::runtime_error("Failed to read image");
		}
		return std::make_shared<GenericLuminanceSource>((int)width, (int)height, buffer.data(), width*4, 4, 0, 1, 2);
	}
};

class GenericBlackboxTestRunner : public BlackboxTestRunner
{
public:
	GenericBlackboxTestRunner(const std::wstring& pathPrefix)
		: BlackboxTestRunner(pathPrefix, std::make_shared<GenericImageLoader>())
	{
	}

	virtual std::vector<std::wstring> getImagesInDirectory(const std::wstring& dirPath) override
	{
		std::vector<std::wstring> result;
#ifndef ZXING_HAS_FILESYSTEM
		if (auto dir = opendir(TextUtfEncoding::ToUtf8(pathPrefix() + L"/" + dirPath).c_str())) {
			while (auto entry = readdir(dir)) {
				if (HasSuffix(entry->d_name, ".png")) {
					result.push_back(dirPath + L"/" + TextUtfEncoding::FromUtf8(entry->d_name));
				}
			}
			closedir(dir);<commit_msg>minor clang-tidy fix in TestReaderMain<commit_after>/*
* Copyright 2016 Nu-book Inc.
* Copyright 2017 <NAME>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*      http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "BlackboxTestRunner.h"
#include "ImageLoader.h"
#include "TestReader.h"
#include "GenericLuminanceSource.h"
#include "HybridBinarizer.h"
#include "TextUtfEncoding.h"
#include "ZXContainerAlgorithms.h"
#include "lodepng.h"

#include <iostream>
#include <set>

#if __has_include(<filesystem>)
#  include <filesystem>
#  ifdef __cpp_lib_filesystem
     namespace fs = std::filesystem;
#    define ZXING_HAS_FILESYSTEM
#  endif
#endif
#if !defined(ZXING_HAS_FILESYSTEM) && __has_include(<experimental/filesystem>)
#  include <experimental/filesystem>
#  ifdef __cpp_lib_experimental_filesystem
     namespace fs = std::experimental::filesystem;
#    define ZXING_HAS_FILESYSTEM
#  endif
#endif
#if defined(__APPLE__) && !defined(__clang__) &&                                                                       \
	__has_include(<dirent.h>) && defined(__SANITIZE_ADDRESS__) && defined(ZXING_HAS_FILESYSTEM)
#  warning detected gcc + macos + <filesystem> + address_sanitizer == bad -> disable <filesystem>
#  undef ZXING_HAS_FILESYSTEM
#endif
#if !defined(ZXING_HAS_FILESYSTEM)
#  if __has_include(<dirent.h>)
#    include <dirent.h>
#  else
#    error need <filesystem> or <dirent.h>
#  endif
#endif

// compiling this with clang (e.g. version 6) might require linking against libc++experimental.a or libc++fs.a.
// E.g.: CMAKE_EXE_LINKER_FLAGS = -L/usr/local/Cellar/llvm/6.0.1/lib -lc++experimental

using namespace ZXing;
using namespace ZXing::Test;

static bool HasSuffix(const std::string& s, const std::string& suffix)
{
	return s.size() >= suffix.size() && std::equal(suffix.rbegin(), suffix.rend(), s.rbegin());
}

class GenericImageLoader : public ImageLoader
{
public:
	std::shared_ptr<LuminanceSource> load(const std::wstring& filename) const final
	{
		std::vector<unsigned char> buffer;
		unsigned width, height;
		unsigned error = lodepng::decode(buffer, width, height, TextUtfEncoding::ToUtf8(filename));
		if (error) {
			throw std::runtime_error("Failed to read image");
		}
		return std::make_shared<GenericLuminanceSource>((int)width, (int)height, buffer.data(), width*4, 4, 0, 1, 2);
	}
};

class GenericBlackboxTestRunner : public BlackboxTestRunner
{
public:
	GenericBlackboxTestRunner(const std::wstring& pathPrefix)
		: BlackboxTestRunner(pathPrefix, std::make_shared<GenericImageLoader>())
	{
	}

	std::vector<std::wstring> getImagesInDirectory(const std::wstring& dirPath) final
	{
		std::vector<std::wstring> result;
#ifndef ZXING_HAS_FILESYSTEM
		if (auto dir = opendir(TextUtfEncoding::ToUtf8(pathPrefix() + L"/" + dirPath).c_str())) {
			while (auto entry = readdir(dir)) {
				if (HasSuffix(entry->d_name, ".png")) {
					result.push_back(dirPath + L"/" + TextUtfEncoding::FromUtf8(entry->d_name));
				}
			}
			closedir(dir);
{'fraction_non_alphanumeric': 0.12490869247626005, 'fraction_numerical': 0.007012417823228634, 'mean_word_length': 4.782094594594595, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6632776', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2206, 'n_words': 561}