<commit_before>#include <iostream>
#include <cstdlib>
#include <cstring>
#include <libgen.h>
#include <unistd.h>
#include "ExecUtil.h"
#include "PdfUtil.h"
#include "util.h"


void Usage() {
    std::cerr << "usage: " << progname << " pdf_image_file_name [language_code_or_codes]\n";
    std::cerr << "       When no language code has been specified, \"deu\" is used as a default.\n";
    std::exit(EXIT_FAILURE);
}


const std::string BASH_HELPER("pdf_images_to_text.sh");


int main(int argc, char *argv[]) {
    progname = argv[0];

    try {
	if (argc != 2 and argc != 3)
	    Usage();
	const std::string language_codes(argc == 2 ? "deu" : argv[2]);
	const std::string input_filename(argv[1]);

	if (::access(input_filename.c_str(), R_OK) != 0)
	    Error("can't read \"" + input_filename + "\"!");

	if (not PdfDocContainsNoText(input_filename))
	    Error("input file \"" + input_filename + "\" contains text!");

	char output_filename[] = "ORC_OUT_XXXXXX";
	const int output_fd(::mkstemp(output_filename));
	if (output_fd == -1)
	    Error("failed to create a temporary file!");

	char path[std::strlen(argv[0]) + 1];
	std::strcpy(argv[0], path);
	const std::string dir_path(::dirname(path));

	if (Exec(dir_path + "/" + BASH_HELPER, { input_filename, output_filename, language_codes },
		 "", 20 /* seconds */) != 0)
	    {
		::unlink(output_filename);
		Error("failed to execute conversion script!");
	    }

	std::string extracted_text;
	if (not ReadFile(output_filename, &extracted_text)) {
	    ::unlink(output_filename);
	    Error("failed to read contents of \"" + std::string(output_filename) + "\"!");
	}
	::unlink(output_filename);

	if (extracted_text.empty())
	    Error("No text was extracted from \"" + input_filename + "\"!");
<commit_msg>Fixed a harmless typo.<commit_after>#include <iostream>
#include <cstdlib>
#include <cstring>
#include <libgen.h>
#include <unistd.h>
#include "ExecUtil.h"
#include "PdfUtil.h"
#include "util.h"


void Usage() {
    std::cerr << "usage: " << progname << " pdf_image_file_name [language_code_or_codes]\n";
    std::cerr << "       When no language code has been specified, \"deu\" is used as a default.\n";
    std::exit(EXIT_FAILURE);
}


const std::string BASH_HELPER("pdf_images_to_text.sh");


int main(int argc, char *argv[]) {
    progname = argv[0];

    try {
	if (argc != 2 and argc != 3)
	    Usage();
	const std::string language_codes(argc == 2 ? "deu" : argv[2]);
	const std::string input_filename(argv[1]);

	if (::access(input_filename.c_str(), R_OK) != 0)
	    Error("can't read \"" + input_filename + "\"!");

	if (not PdfDocContainsNoText(input_filename))
	    Error("input file \"" + input_filename + "\" contains text!");

	char output_filename[] = "OCR_OUT_XXXXXX";
	const int output_fd(::mkstemp(output_filename));
	if (output_fd == -1)
	    Error("failed to create a temporary file!");

	char path[std::strlen(argv[0]) + 1];
	std::strcpy(argv[0], path);
	const std::string dir_path(::dirname(path));

	if (Exec(dir_path + "/" + BASH_HELPER, { input_filename, output_filename, language_codes },
		 "", 20 /* seconds */) != 0)
	    {
		::unlink(output_filename);
		Error("failed to execute conversion script!");
	    }

	std::string extracted_text;
	if (not ReadFile(output_filename, &extracted_text)) {
	    ::unlink(output_filename);
	    Error("failed to read contents of \"" + std::string(output_filename) + "\"!");
	}
	::unlink(output_filename);

	if (extracted_text.empty())
	    Error("No text was extracted from \"" + input_filename + "\"!");

{'fraction_non_alphanumeric': 0.17700368167657887, 'fraction_numerical': 0.007929764939110734, 'mean_word_length': 4.408882082695253, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6066770', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1220, 'n_words': 283}