<commit_before>package org.sagebionetworks.tool.searchupdater.job;

import java.util.Set;
import java.util.concurrent.Callable;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.joda.time.DateTime;
import org.json.JSONArray;
import org.sagebionetworks.repo.model.search.Document;
import org.sagebionetworks.repo.model.search.DocumentTypeNames;
import org.sagebionetworks.schema.adapter.org.json.EntityFactory;
import org.sagebionetworks.tool.migration.ClientFactory;
import org.sagebionetworks.tool.migration.Configuration;
import org.sagebionetworks.tool.migration.Progress.BasicProgress;
import org.sagebionetworks.tool.migration.job.WorkerResult;
import org.sagebionetworks.tool.searchupdater.CloudSearchClient;
import org.sagebionetworks.tool.searchupdater.SearchUpdaterConfigurationImpl;

/**
 * A worker that will execute a single CloudSearch delete entity job.
 * 
 * @author deflaux
 * 
 */
public class SearchDocumentDeleteWorker implements Callable<WorkerResult> {

	static private Log log = LogFactory
			.getLog(SearchDocumentDeleteWorker.class);

	SearchUpdaterConfigurationImpl configuration = null;
	Set<String> entities = null;
	BasicProgress progress = null;

	/**
	 * Create a new delete worker
	 * 
	 * @param configuration
	 * @param clientFactory
	 * @param entities
	 * @param progress
	 */
	public SearchDocumentDeleteWorker(Configuration configuration,
			ClientFactory clientFactory, Set<String> entities,
			BasicProgress progress) {
		super();
		this.configuration = (SearchUpdaterConfigurationImpl) configuration;
		this.entities = entities;
		this.progress = progress;
	}

	@Override
	public WorkerResult call() throws Exception {
		DateTime now = DateTime.now();
		try {
			// Note that we cannot use a JSONEntity here because the format is
			// just a JSON array
			JSONArray documentBatch = new JSONArray();
			for (String entityId : this.entities) {
				Document document = new Document();
				document.setType(DocumentTypeNames.delete);
				document.setId(entityId);
				document.setVersion(now.getMillis() / 1000);
				document.setLang("en"); // TODO this should have been set via "default" in the schema for this
				documentBatch.put(EntityFactory.createJSONObjectForEntity(document));
			}

			log.info("Deleting from search index "
					+ documentBatch.toString(4));
<commit_msg>Remove the lang field from the delete SDF<commit_after>package org.sagebionetworks.tool.searchupdater.job;

import java.util.Set;
import java.util.concurrent.Callable;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.joda.time.DateTime;
import org.json.JSONArray;
import org.sagebionetworks.repo.model.search.Document;
import org.sagebionetworks.repo.model.search.DocumentTypeNames;
import org.sagebionetworks.schema.adapter.org.json.EntityFactory;
import org.sagebionetworks.tool.migration.ClientFactory;
import org.sagebionetworks.tool.migration.Configuration;
import org.sagebionetworks.tool.migration.Progress.BasicProgress;
import org.sagebionetworks.tool.migration.job.WorkerResult;
import org.sagebionetworks.tool.searchupdater.CloudSearchClient;
import org.sagebionetworks.tool.searchupdater.SearchUpdaterConfigurationImpl;

/**
 * A worker that will execute a single CloudSearch delete entity job.
 * 
 * @author deflaux
 * 
 */
public class SearchDocumentDeleteWorker implements Callable<WorkerResult> {

	static private Log log = LogFactory
			.getLog(SearchDocumentDeleteWorker.class);

	SearchUpdaterConfigurationImpl configuration = null;
	Set<String> entities = null;
	BasicProgress progress = null;

	/**
	 * Create a new delete worker
	 * 
	 * @param configuration
	 * @param clientFactory
	 * @param entities
	 * @param progress
	 */
	public SearchDocumentDeleteWorker(Configuration configuration,
			ClientFactory clientFactory, Set<String> entities,
			BasicProgress progress) {
		super();
		this.configuration = (SearchUpdaterConfigurationImpl) configuration;
		this.entities = entities;
		this.progress = progress;
	}

	@Override
	public WorkerResult call() throws Exception {
		DateTime now = DateTime.now();
		try {
			// Note that we cannot use a JSONEntity here because the format is
			// just a JSON array
			JSONArray documentBatch = new JSONArray();
			for (String entityId : this.entities) {
				Document document = new Document();
				document.setType(DocumentTypeNames.delete);
				document.setId(entityId);
				document.setVersion(now.getMillis() / 1000);
				documentBatch.put(EntityFactory.createJSONObjectForEntity(document));
			}

			log.info("Deleting from search index "
					+ documentBatch.toString(4));

{'fraction_non_alphanumeric': 0.0926839733962669, 'fraction_numerical': 0.002145462347135808, 'mean_word_length': 5.007731958762887, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6653772', 'n_tokens_mistral': 1449, 'n_tokens_neox': 1339, 'n_words': 309}