<commit_before>package org.icij.extract.solr;

import org.icij.extract.solr.SolrDefaults;

import java.util.Map;
import java.util.HashMap;

import java.util.logging.Level;
import java.util.logging.Logger;

import java.io.IOException;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import java.util.regex.Pattern;

import javax.xml.bind.DatatypeConverter;

import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.util.ClientUtils;

public class SolrRehashConsumer extends SolrMachineConsumer {

	private final SolrClient client;
	private final String idAlgorithm;
	private Pattern pattern = null;
	private String replacement = "";
	private Charset outputEncoding = StandardCharsets.UTF_8;
	private String pathField = SolrDefaults.DEFAULT_PATH_FIELD;

	public SolrRehashConsumer(final Logger logger, final SolrClient client, final String idAlgorithm) {
		super(logger);
		this.client = client;
		this.idAlgorithm = idAlgorithm;
	}

	public void setOutputEncoding(final Charset outputEncoding) {
		this.outputEncoding = outputEncoding;
	}

	public void setOutputEncoding(final String outputEncoding) {
		setOutputEncoding(Charset.forName(outputEncoding));
	}

	public void setPathField(final String pathField) {
		this.pathField = pathField;
	}

	public void setPattern(final String pattern) {
		this.pattern = Pattern.compile(pattern);
	}

	public void setReplacement(final String replacement) {
		this.replacement = replacement;
	}

	@Override
	protected void consume(final SolrDocument input) throws SolrServerException, IOException, NoSuchAlgorithmException {
		final SolrInputDocument output = ClientUtils.toSolrInputDocument(input);
		final String inputPath = (String) input.getFieldValue(pathField);
		final String outputPath;

		if (null != pattern && null != replacement) {
			outputPath = pattern.matcher(inputPath).replaceAll(replacement);
		} else {
			outputPath = inputPath;
		}

		final String inputId = (String) input.getFieldValue(idField);
		final String outputId = DatatypeConverter.printHexBinary(MessageDigest.getInstance(idAlgorithm)
			.digest(outputPath.getBytes(outputEncoding)));

		// Only replace if the IDs are different.
		if (!inputId.equals(outputId)) {
			output.setField("_version_", "-1"); // The new document must not exist.
			output.setField(idField, outputId);

			logger.info(String.format("Replacing path \"%s\" with \"%s\" and rehashing ID from %s to %s.",
				inputPath, outputPath, inputId, outputId));
			client.add(output);
			client.deleteById((String) input.getFieldValue(idField));
		}
	}
}
<commit_msg>The new document may or may not exist.<commit_after>package org.icij.extract.solr;

import org.icij.extract.solr.SolrDefaults;

import java.util.Map;
import java.util.HashMap;

import java.util.logging.Level;
import java.util.logging.Logger;

import java.io.IOException;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import java.util.regex.Pattern;

import javax.xml.bind.DatatypeConverter;

import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.util.ClientUtils;

public class SolrRehashConsumer extends SolrMachineConsumer {

	private final SolrClient client;
	private final String idAlgorithm;
	private Pattern pattern = null;
	private String replacement = "";
	private Charset outputEncoding = StandardCharsets.UTF_8;
	private String pathField = SolrDefaults.DEFAULT_PATH_FIELD;

	public SolrRehashConsumer(final Logger logger, final SolrClient client, final String idAlgorithm) {
		super(logger);
		this.client = client;
		this.idAlgorithm = idAlgorithm;
	}

	public void setOutputEncoding(final Charset outputEncoding) {
		this.outputEncoding = outputEncoding;
	}

	public void setOutputEncoding(final String outputEncoding) {
		setOutputEncoding(Charset.forName(outputEncoding));
	}

	public void setPathField(final String pathField) {
		this.pathField = pathField;
	}

	public void setPattern(final String pattern) {
		this.pattern = Pattern.compile(pattern);
	}

	public void setReplacement(final String replacement) {
		this.replacement = replacement;
	}

	@Override
	protected void consume(final SolrDocument input) throws SolrServerException, IOException, NoSuchAlgorithmException {
		final SolrInputDocument output = ClientUtils.toSolrInputDocument(input);
		final String inputPath = (String) input.getFieldValue(pathField);
		final String outputPath;

		if (null != pattern && null != replacement) {
			outputPath = pattern.matcher(inputPath).replaceAll(replacement);
		} else {
			outputPath = inputPath;
		}

		final String inputId = (String) input.getFieldValue(idField);
		final String outputId = DatatypeConverter.printHexBinary(MessageDigest.getInstance(idAlgorithm)
			.digest(outputPath.getBytes(outputEncoding)));

		// Only replace if the IDs are different.
		if (!inputId.equals(outputId)) {
			output.setField("_version_", "0"); // The document may or may not exist.
			output.setField(idField, outputId);

			logger.info(String.format("Replacing path \"%s\" with \"%s\" and rehashing ID from %s to %s.",
				inputPath, outputPath, inputId, outputId));
			client.add(output);
			client.deleteById((String) input.getFieldValue(idField));
		}
	}
}

{'fraction_non_alphanumeric': 0.09915907498248074, 'fraction_numerical': 0.000700770847932726, 'mean_word_length': 5.364548494983278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3313297', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1688, 'n_words': 394}