<commit_before>/*
 * Copyright 2011-2015 B2i Healthcare Pte Ltd, http://b2i.sg
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.b2international.snowowl.datastore.server.index;

import static com.google.common.base.Preconditions.checkNotNull;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Set;

import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;

import com.b2international.snowowl.core.api.BranchPath;
import com.b2international.snowowl.datastore.index.IndexUtils;
import com.b2international.snowowl.datastore.server.internal.lucene.store.CompositeDirectory;
import com.b2international.snowowl.datastore.server.internal.lucene.store.ReadOnlyDirectory;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

public abstract class AbstractDirectoryManager implements IDirectoryManager {

	protected final String repositoryUuid;
	protected final File indexPath;

	/**
	 * @param repositoryUuid the repository identifier
	 * @param indexPath the <b>absolute</b> path to the terminology index directories
	 */
	protected AbstractDirectoryManager(final String repositoryUuid, final File indexPath) {
		this.repositoryUuid = checkNotNull(repositoryUuid, "repositoryUuid");
		this.indexPath = checkNotNull(indexPath, "indexPath");
	}

	protected File getIndexSubDirectory(final String subDirectoryPath) throws IOException {
		return new File(indexPath, subDirectoryPath);
	}

	@Override
	public final IndexDirectory openDirectory(final BranchPath branchPath, final boolean readOnly) throws IOException {
		return new IndexDirectory(openLuceneDirectory(branchPath, readOnly));
	}

	private final Directory openLuceneDirectory(final BranchPath branchPath, final boolean readOnly) throws IOException {
		final File indexSubDirectory = getIndexSubDirectory(branchPath.path());

		if (branchPath.isMain()) {
			final Directory mainDirectory = openWritableLuceneDirectory(indexSubDirectory);
			return readOnly ? new ReadOnlyDirectory(mainDirectory) : mainDirectory;
		}

		// Don't bother wrapping the parents in a read-only instance
		final Directory parentDirectory = openLuceneDirectory(branchPath.parent(), false);
		final IndexCommit parentCommit = getParentCommit(parentDirectory, branchPath);
		final Set<String> visibleFiles = Sets.newHashSet(parentCommit.getFileNames());
		
		// Make index commit files for ancestors visible as well (these should only refer to files already in parentCommit)
		for (BranchPath ancestorPath = branchPath.parent(); !ancestorPath.isMain(); ancestorPath = ancestorPath.parent()) {
			final IndexCommit ancestorCommit = getParentCommit(parentDirectory, ancestorPath);
			visibleFiles.add(ancestorCommit.getSegmentsFileName());
			if (!visibleFiles.containsAll(ancestorCommit.getFileNames())) {
				throw new IllegalStateException("Files referenced in commit " + ancestorPath + " introduce new files.");
			}
		}
		
		final Directory parentCommitDirectory = new ReadOnlyDirectory(parentDirectory, visibleFiles);

		if (readOnly) {
			return parentCommitDirectory;
		} else {
			final Directory writeableDirectory = openWritableLuceneDirectory(indexSubDirectory);
			return new CompositeDirectory(parentCommitDirectory, writeableDirectory);
		}
	}

	protected abstract Directory openWritableLuceneDirectory(final File folderForBranchPath) throws IOException;

	private IndexCommit getParentCommit(final Directory parentDirectory, final BranchPath branchPath) throws IOException {
		final List<IndexCommit> indexCommits = Lists.<IndexCommit>reverse(DirectoryReader.listCommits(parentDirectory));
		final String path = branchPath.path();

		for (final IndexCommit indexCommit : indexCommits) {
			if (path.equals(indexCommit.getUserData().get(IndexUtils.INDEX_CDO_BRANCH_PATH_KEY))) {
				return indexCommit;
			}
		}<commit_msg>SO-1677: Get all files in IndexCommits when computing the base part...<commit_after>/*
 * Copyright 2011-2015 B2i Healthcare Pte Ltd, http://b2i.sg
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.b2international.snowowl.datastore.server.index;

import static com.google.common.base.Preconditions.checkNotNull;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Set;

import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;

import com.b2international.snowowl.core.api.BranchPath;
import com.b2international.snowowl.datastore.index.IndexUtils;
import com.b2international.snowowl.datastore.server.internal.lucene.store.CompositeDirectory;
import com.b2international.snowowl.datastore.server.internal.lucene.store.ReadOnlyDirectory;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

public abstract class AbstractDirectoryManager implements IDirectoryManager {

	protected final String repositoryUuid;
	protected final File indexPath;

	/**
	 * @param repositoryUuid the repository identifier
	 * @param indexPath the <b>absolute</b> path to the terminology index directories
	 */
	protected AbstractDirectoryManager(final String repositoryUuid, final File indexPath) {
		this.repositoryUuid = checkNotNull(repositoryUuid, "repositoryUuid");
		this.indexPath = checkNotNull(indexPath, "indexPath");
	}

	protected File getIndexSubDirectory(final String subDirectoryPath) throws IOException {
		return new File(indexPath, subDirectoryPath);
	}

	@Override
	public final IndexDirectory openDirectory(final BranchPath branchPath, final boolean readOnly) throws IOException {
		return new IndexDirectory(openLuceneDirectory(branchPath, readOnly));
	}

	private final Directory openLuceneDirectory(final BranchPath branchPath, final boolean readOnly) throws IOException {
		final File indexSubDirectory = getIndexSubDirectory(branchPath.path());

		if (branchPath.isMain()) {
			final Directory mainDirectory = openWritableLuceneDirectory(indexSubDirectory);
			return readOnly ? new ReadOnlyDirectory(mainDirectory) : mainDirectory;
		}

		// Don't bother wrapping the parents in a read-only instance
		final Directory parentDirectory = openLuceneDirectory(branchPath.parent(), false);
		final Set<String> visibleFiles = Sets.newHashSet();
		
		// Make files in index commits for ancestors visible as well
		for (BranchPath ancestorPath = branchPath; !ancestorPath.isMain(); ancestorPath = ancestorPath.parent()) {
			final IndexCommit ancestorCommit = getParentCommit(parentDirectory, ancestorPath);
			/* 
			 * XXX: .del files are not incremental. If they were, we could start by adding files 
			 * referenced in the immediate parent commit, and check that ancestors are only contributing 
			 * segment_n files to the existing set. 
			 */
			visibleFiles.addAll(ancestorCommit.getFileNames());
		}
		
		final Directory parentCommitDirectory = new ReadOnlyDirectory(parentDirectory, visibleFiles);

		if (readOnly) {
			return parentCommitDirectory;
		} else {
			final Directory writeableDirectory = openWritableLuceneDirectory(indexSubDirectory);
			return new CompositeDirectory(parentCommitDirectory, writeableDirectory);
		}
	}

	protected abstract Directory openWritableLuceneDirectory(final File folderForBranchPath) throws IOException;

	private IndexCommit getParentCommit(final Directory parentDirectory, final BranchPath branchPath) throws IOException {
		final List<IndexCommit> indexCommits = Lists.<IndexCommit>reverse(DirectoryReader.listCommits(parentDirectory));
		final String path = branchPath.path();

		for (final IndexCommit indexCommit : indexCommits) {
			if (path.equals(indexCommit.getUserData().get(IndexUtils.INDEX_CDO_BRANCH_PATH_KEY))) {
				return indexCommit;
			}
		}
{'fraction_non_alphanumeric': 0.08020904339922745, 'fraction_numerical': 0.004771642808452625, 'mean_word_length': 5.554728220402085, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '989717', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2310, 'n_words': 683}