<commit_before>package org.icij.extract.core;

import java.util.Map;
import java.util.Set;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;

import java.util.logging.Level;
import java.util.logging.Logger;

import java.nio.file.Path;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;

import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParsingReader;
import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.parser.ocr.TesseractOCRConfig;
import org.apache.tika.parser.ocr.TesseractOCRParser;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.mime.MediaType;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.exception.TikaException;

import org.xml.sax.ContentHandler;

/**
 * Extract
 *
 * @author <NAME> <<EMAIL>>
 * @version 1.0.0-beta
 * @since 1.0.0-beta
 */
public class Extractor {
	private final Logger logger;

	private Charset outputEncoding = StandardCharsets.UTF_8;
	private String ocrLanguage = "eng";
	private boolean ocrDisabled = false;

	private final Path file;

	private final TikaConfig config = TikaConfig.getDefaultConfig();
	private final Collection<Class<TesseractOCRParser>> excludedParsers = new ArrayList<Class<TesseractOCRParser>>();

	public Extractor (Logger logger, Path file) {
		this.logger = logger;
		this.file = file;
	}

	public void setOutputEncoding(Charset outputEncoding) {
		this.outputEncoding = outputEncoding;
	}

	public Charset getOutputEncoding() {
		return outputEncoding;
	}

	public void setOcrLanguage(String ocrLanguage) {
		this.ocrLanguage = ocrLanguage;
	}

	public void disableOcr() {
		if (!ocrDisabled) {
			excludedParsers.add(TesseractOCRParser.class);
			ocrDisabled = true;
		}
	}

	public ParsingReader extract(final Path file) throws IOException, FileNotFoundException, TikaException {
		final Metadata metadata = new Metadata();

		final ParseContext context = new ParseContext();
		final PDFParserConfig pdfConfig = new PDFParserConfig();

		excludeParsers();

		final AutoDetectParser parser = new AutoDetectParser(config);

		if (!ocrDisabled) {
			final TesseractOCRConfig ocrConfig = new TesseractOCRConfig();

			ocrConfig.setLanguage(ocrLanguage);
			context.set(TesseractOCRConfig.class, ocrConfig);

			// Run OCR on images contained within PDFs.
			pdfConfig.setExtractInlineImages(true);

			// By default, only the object IDs are used for determining uniqueness.
			// In scanned documents under test from the Panama registry, different embedded images had the same ID, leading to incomplete OCRing when uniqueness detection was turned on.
			pdfConfig.setExtractUniqueInlineImagesOnly(false);
		}

		pdfConfig.setUseNonSequentialParser(true);
		context.set(PDFParserConfig.class, pdfConfig);

		parser.setFallback(new Parser() {
			public Set<MediaType> getSupportedTypes(ParseContext context) {
				return parser.getSupportedTypes(context);
			}

			public void parse(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, ParseContext context) throws TikaException {
				throw new TikaException("Unsupported media type: " + metadata.get(HttpHeaders.CONTENT_TYPE) + ".");
			}
		});

		// Set up recursive parsing of archives and documents with embedded images.
		context.set(Parser.class, parser);
		context.set(EmbeddedDocumentExtractor.class, new EmbedExtractor(logger, file, context));

		metadata.set(TikaMetadataKeys.RESOURCE_NAME_KEY, file.getFileName().toString());

		final TikaInputStream stream = TikaInputStream.get(new FileInputStream(file.toString()));
		final ParsingReader reader = new ParsingReader(parser, stream, metadata, context);

		return reader;
	}

	private void excludeParsers() {
		final CompositeParser compositeParser = (CompositeParser) config.getParser();
		final Map<MediaType, Parser> parsers = compositeParser.getParsers();

		final Iterator iterator = parsers.entrySet().iterator();

		if (excludedParsers.size() == 0) {
			return;
		}

		while (iterator.hasNext()) {
			Map.Entry pair = (Map.Entry) iterator.next();

			if (excludedParsers.contains(pair.getValue().getClass())) {
				iterator.remove();
			}
		}

		compositeParser.setParsers(parsers);
	}
}<commit_msg>Disable in setter<commit_after>package org.icij.extract.core;

import java.util.Map;
import java.util.Set;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;

import java.util.logging.Level;
import java.util.logging.Logger;

import java.nio.file.Path;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;

import org.apache.tika.config.TikaConfig;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParsingReader;
import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.parser.ocr.TesseractOCRConfig;
import org.apache.tika.parser.ocr.TesseractOCRParser;
import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.extractor.EmbeddedDocumentExtractor;
import org.apache.tika.mime.MediaType;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.exception.TikaException;

import org.xml.sax.ContentHandler;

/**
 * Extract
 *
 * @author <NAME> <<EMAIL>>
 * @version 1.0.0-beta
 * @since 1.0.0-beta
 */
public class Extractor {
	private final Logger logger;

	private Charset outputEncoding = StandardCharsets.UTF_8;
	private String ocrLanguage = "eng";
	private boolean ocrDisabled = false;

	private final Path file;

	private final TikaConfig config = TikaConfig.getDefaultConfig();

	public Extractor (Logger logger, Path file) {
		this.logger = logger;
		this.file = file;
	}

	public void setOutputEncoding(Charset outputEncoding) {
		this.outputEncoding = outputEncoding;
	}

	public Charset getOutputEncoding() {
		return outputEncoding;
	}

	public void setOcrLanguage(String ocrLanguage) {
		this.ocrLanguage = ocrLanguage;
	}

	public void disableOcr() {
		if (!ocrDisabled) {
			excludeParsers(TesseractOCRParser.class);
			ocrDisabled = true;
		}
	}

	public ParsingReader extract(final Path file) throws IOException, FileNotFoundException, TikaException {
		final Metadata metadata = new Metadata();

		final ParseContext context = new ParseContext();
		final PDFParserConfig pdfConfig = new PDFParserConfig();
		final AutoDetectParser parser = new AutoDetectParser(config);

		if (!ocrDisabled) {
			final TesseractOCRConfig ocrConfig = new TesseractOCRConfig();

			ocrConfig.setLanguage(ocrLanguage);
			context.set(TesseractOCRConfig.class, ocrConfig);

			// Run OCR on images contained within PDFs.
			pdfConfig.setExtractInlineImages(true);

			// By default, only the object IDs are used for determining uniqueness.
			// In scanned documents under test from the Panama registry, different embedded images had the same ID, leading to incomplete OCRing when uniqueness detection was turned on.
			pdfConfig.setExtractUniqueInlineImagesOnly(false);
		}

		pdfConfig.setUseNonSequentialParser(true);
		context.set(PDFParserConfig.class, pdfConfig);

		parser.setFallback(new Parser() {
			public Set<MediaType> getSupportedTypes(ParseContext context) {
				return parser.getSupportedTypes(context);
			}

			public void parse(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, ParseContext context) throws TikaException {
				throw new TikaException("Unsupported media type: " + metadata.get(HttpHeaders.CONTENT_TYPE) + ".");
			}
		});

		// Set up recursive parsing of archives and documents with embedded images.
		context.set(Parser.class, parser);
		context.set(EmbeddedDocumentExtractor.class, new EmbedExtractor(logger, file, context));

		metadata.set(TikaMetadataKeys.RESOURCE_NAME_KEY, file.getFileName().toString());

		final TikaInputStream stream = TikaInputStream.get(new FileInputStream(file.toString()));
		final ParsingReader reader = new ParsingReader(parser, stream, metadata, context);

		return reader;
	}

	private void excludeParsers(Class... excluded) {
		final CompositeParser composite = (CompositeParser) config.getParser();
		final Map<MediaType, Parser> parsers = composite.getParsers();

		final Iterator iterator = parsers.entrySet().iterator();

		while (iterator.hasNext()) {
			Map.Entry pair = (Map.Entry) iterator.next();

			if (Arrays.asList(excluded).contains(pair.getValue().getClass())) {
				iterator.remove();
			}
		}

		composite.setParsers(parsers);
	}
}
{'fraction_non_alphanumeric': 0.10247123988069877, 'fraction_numerical': 0.0015977844056242012, 'mean_word_length': 5.176973684210527, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3132805', 'n_tokens_mistral': 3012, 'n_tokens_neox': 2792, 'n_words': 648}