<commit_before>/**
 * Copyright (C) 2009-2013 Simonsoft Nordic AB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package se.simonsoft.cms.indexing.xml;

import java.util.HashSet;
import java.util.Set;

import javax.inject.Inject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import se.repos.indexing.IndexingDoc;
import se.repos.indexing.item.IndexingItemHandler;
import se.repos.indexing.item.IndexingItemProgress;
import se.repos.indexing.item.ItemPathinfo;
import se.repos.indexing.item.ItemProperties;
import se.simonsoft.cms.item.RepoRevision;
import se.simonsoft.cms.item.events.change.CmsChangesetItem;
import se.simonsoft.xmltracking.source.XmlSourceElement;
import se.simonsoft.xmltracking.source.XmlSourceHandler;
import se.simonsoft.xmltracking.source.XmlSourceReader;

public class IndexingItemHandlerXml implements IndexingItemHandler {

	public final Logger logger = LoggerFactory.getLogger(this.getClass());
	
	private XmlFileFilter xmlFileFilter = new XmlFileFilterExtensionAndSvnMimeType(); // TODO inject, mportant customization point
	
	private SupportLegacySchema supportLegacySchema = new SupportLegacySchema(); // TODO do away with gradually
	
	private XmlSourceReader sourceReader = null;
	
	private Set<XmlIndexFieldExtraction> fieldExtraction = null;

	private XmlIndexWriter indexWriter;
	
	/**
	 * @param fieldExtraction a sequence of pluggable extractors that add fields
	 * @param xmlSourceReader that processes the XML into {@link XmlSourceElement}s for the extractors
	 */
	@Inject
	public void setDependenciesXml(
			Set<XmlIndexFieldExtraction> fieldExtraction,
			XmlSourceReader xmlSourceReader
			) {
		this.fieldExtraction = fieldExtraction;
		this.sourceReader = xmlSourceReader;
	}

	@Inject
	public void setDependenciesIndexing(
			XmlIndexWriter indexAddProvider) {
		this.indexWriter = indexAddProvider;
	}
	
	@Override
	public void handle(IndexingItemProgress progress) {
		CmsChangesetItem c = progress.getItem();
		if (c.isOverwritten()) {
			logger.debug("XML index only contains HEAD so skipping later overwritten {}", c.getPath());
			return;
		}
		if (c.isContent()) {
			// TODO here we should probably read mime type too, or probably after conversion to CmsItem
			if (xmlFileFilter.isXml(c, progress.getFields())) {
				logger.debug("Changeset content update item {} found", c);
				if (c.isDelete()) {
					indexWriter.deletePath(progress.getRepository(), c);
				} else {
					indexWriter.deletePath(progress.getRepository(), c);
					index(progress);
				}
			} else {
				logger.debug("Ignoring content update item {}, not an XML candidate file type", c);
			}
		} else {
			logger.debug("Ignoring changeset item {}, not a file", c);
		}
		// TODO until we have notification on revision end we commit always
		onRevisionEnd(progress.getRevision());
	}

	protected void index(IndexingItemProgress progress) {
		IndexingDoc itemDoc = cloneItemFields(progress.getFields());
		supportLegacySchema.handle(itemDoc);
		XmlIndexAddSession docHandler = indexWriter.get();
		XmlSourceHandler sourceHandler = new XmlSourceHandlerFieldExtractors(itemDoc, fieldExtraction, docHandler);
		sourceReader.read(progress.getContents(), sourceHandler);
	}<commit_msg>Must index for both contents and property changes.<commit_after>/**
 * Copyright (C) 2009-2013 Simonsoft Nordic AB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package se.simonsoft.cms.indexing.xml;

import java.util.HashSet;
import java.util.Set;

import javax.inject.Inject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import se.repos.indexing.IndexingDoc;
import se.repos.indexing.item.IndexingItemHandler;
import se.repos.indexing.item.IndexingItemProgress;
import se.repos.indexing.item.ItemPathinfo;
import se.repos.indexing.item.ItemProperties;
import se.simonsoft.cms.item.RepoRevision;
import se.simonsoft.cms.item.events.change.CmsChangesetItem;
import se.simonsoft.xmltracking.source.XmlSourceElement;
import se.simonsoft.xmltracking.source.XmlSourceHandler;
import se.simonsoft.xmltracking.source.XmlSourceReader;

public class IndexingItemHandlerXml implements IndexingItemHandler {

	public final Logger logger = LoggerFactory.getLogger(this.getClass());
	
	private XmlFileFilter xmlFileFilter = new XmlFileFilterExtensionAndSvnMimeType(); // TODO inject, mportant customization point
	
	private SupportLegacySchema supportLegacySchema = new SupportLegacySchema(); // TODO do away with gradually
	
	private XmlSourceReader sourceReader = null;
	
	private Set<XmlIndexFieldExtraction> fieldExtraction = null;

	private XmlIndexWriter indexWriter;
	
	/**
	 * @param fieldExtraction a sequence of pluggable extractors that add fields
	 * @param xmlSourceReader that processes the XML into {@link XmlSourceElement}s for the extractors
	 */
	@Inject
	public void setDependenciesXml(
			Set<XmlIndexFieldExtraction> fieldExtraction,
			XmlSourceReader xmlSourceReader
			) {
		this.fieldExtraction = fieldExtraction;
		this.sourceReader = xmlSourceReader;
	}

	@Inject
	public void setDependenciesIndexing(
			XmlIndexWriter indexAddProvider) {
		this.indexWriter = indexAddProvider;
	}
	
	@Override
	public void handle(IndexingItemProgress progress) {
		CmsChangesetItem c = progress.getItem();
		if (c.isOverwritten()) {
			logger.debug("XML index only contains HEAD so skipping later overwritten {}", c.getPath());
			return;
		}
		if (c.isFile()) {
			// TODO here we should probably read mime type too, or probably after conversion to CmsItem
			if (xmlFileFilter.isXml(c, progress.getFields())) {
				logger.debug("Changeset content update item {} found", c);
				if (c.isDelete()) {
					indexWriter.deletePath(progress.getRepository(), c);
				} else {
					indexWriter.deletePath(progress.getRepository(), c);
					index(progress);
				}
			} else {
				logger.debug("Ignoring content update item {}, not an XML candidate file type", c);
			}
		} else {
			logger.debug("Ignoring changeset item {}, not a file", c);
		}
		// TODO until we have notification on revision end we commit always
		onRevisionEnd(progress.getRevision());
	}

	protected void index(IndexingItemProgress progress) {
		IndexingDoc itemDoc = cloneItemFields(progress.getFields());
		supportLegacySchema.handle(itemDoc);
		XmlIndexAddSession docHandler = indexWriter.get();
		XmlSourceHandler sourceHandler = new XmlSourceHandlerFieldExtractors(itemDoc, fieldExtraction, docHandler);
		sourceReader.read(progress.getContents(), sourceHandler);
	}
{'fraction_non_alphanumeric': 0.08966160405009325, 'fraction_numerical': 0.0037303490540900614, 'mean_word_length': 4.82389449185415, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6962508', 'n_tokens_mistral': 2318, 'n_tokens_neox': 2084, 'n_words': 619}