<commit_before>package ikube.action.rule;

import ikube.action.index.IndexManager;
import ikube.model.IndexContext;
import ikube.toolkit.FileUtilities;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.CompositeReaderContext;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.MMapDirectory;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * This rule checks whether there are indexes that are created but are not yet opened. This typically
 * needs to be checked if an index is still in the process of being generated. In this case when the index
 * is finished being created the searcher should be opened on all the index directories.
 *
 * @author <NAME>
 * @version 01.00
 * @since 12-02-2011
 */
public class AreUnopenedIndexes extends ARule<IndexContext<?>> {

    /**
     * {@inheritDoc}
     */
    @Override
    @SuppressWarnings("ConstantConditions")
    public boolean evaluate(final IndexContext<?> indexContext) {
        IndexSearcher indexSearcher = indexContext.getMultiSearcher();
        if (indexSearcher == null) {
            return new AreIndexesCreated().evaluate(indexContext);
        }
        String indexDirectoryPath = IndexManager.getIndexDirectoryPath(indexContext);
        File latestIndexDirectory = IndexManager.getLatestIndexDirectory(indexDirectoryPath);
        if (latestIndexDirectory == null ||
                latestIndexDirectory.listFiles() == null ||
                latestIndexDirectory.listFiles().length == 0) {
            return Boolean.FALSE;
        }
        // This block checks that all the directories that have
        // indexes are in fact opened and are accessible in the index readers
        // of the index searcher
        logger.debug("Checking latest index directory for new indexes : {} ", latestIndexDirectory);
        List<File> indexDirectories = new ArrayList<>(Arrays.asList(latestIndexDirectory.listFiles()));
        MultiReader multiReader = (MultiReader) indexSearcher.getIndexReader();
        CompositeReaderContext compositeReaderContext = multiReader.getContext();
        List<AtomicReaderContext> atomicReaderContexts = compositeReaderContext.leaves();
        for (final AtomicReaderContext atomicReaderContext : atomicReaderContexts) {
            SegmentReader atomicReader = (SegmentReader) atomicReaderContext.reader();
            MMapDirectory directory = (MMapDirectory) atomicReader.directory();
            logger.info("Directory : " + directory.getDirectory());
            for (int i = 0; i < indexDirectories.size(); i++) {
                File indexDirectory = indexDirectories.get(i);
                File openedIndexDirectory = directory.getDirectory();
                String l = FileUtilities.cleanFilePath(indexDirectory.getAbsolutePath());
                String p = FileUtilities.cleanFilePath(openedIndexDirectory.getAbsolutePath());
                boolean isOpened = l.equals(p);
                logger.info("        : " + isOpened + ", " + indexDirectory);
                if (isOpened) {
                    indexDirectories.remove(i);
                }
            }<commit_msg>Bug fixing and system testing.<commit_after>package ikube.action.rule;

import ikube.action.index.IndexManager;
import ikube.model.IndexContext;
import ikube.toolkit.FileUtilities;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.CompositeReaderContext;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.MMapDirectory;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * This rule checks whether there are indexes that are created but are not yet opened. This typically
 * needs to be checked if an index is still in the process of being generated. In this case when the index
 * is finished being created the searcher should be opened on all the index directories.
 *
 * @author <NAME>
 * @version 01.00
 * @since 12-02-2011
 */
public class AreUnopenedIndexes extends ARule<IndexContext<?>> {

    /**
     * {@inheritDoc}
     */
    @Override
    @SuppressWarnings("ConstantConditions")
    public boolean evaluate(final IndexContext<?> indexContext) {
        IndexSearcher indexSearcher = indexContext.getMultiSearcher();
        if (indexSearcher == null) {
            return new AreIndexesCreated().evaluate(indexContext);
        }
        String indexDirectoryPath = IndexManager.getIndexDirectoryPath(indexContext);
        File latestIndexDirectory = IndexManager.getLatestIndexDirectory(indexDirectoryPath);
        if (latestIndexDirectory == null ||
                latestIndexDirectory.listFiles() == null ||
                latestIndexDirectory.listFiles().length == 0) {
            return Boolean.FALSE;
        }
        // This block checks that all the directories that have
        // indexes are in fact opened and are accessible in the index readers
        // of the index searcher
        logger.debug("Checking latest index directory for new indexes : {} ", latestIndexDirectory);
        List<File> indexDirectories = new ArrayList<>(Arrays.asList(latestIndexDirectory.listFiles()));
        MultiReader multiReader = (MultiReader) indexSearcher.getIndexReader();
        CompositeReaderContext compositeReaderContext = multiReader.getContext();
        List<AtomicReaderContext> atomicReaderContexts = compositeReaderContext.leaves();
        for (final AtomicReaderContext atomicReaderContext : atomicReaderContexts) {
            SegmentReader atomicReader = (SegmentReader) atomicReaderContext.reader();
            MMapDirectory directory = (MMapDirectory) atomicReader.directory();
            File openedIndexDirectory = directory.getDirectory();
            for (int i = 0; i < indexDirectories.size(); i++) {
                File indexDirectory = indexDirectories.get(i);
                String l = FileUtilities.cleanFilePath(indexDirectory.getAbsolutePath());
                String p = FileUtilities.cleanFilePath(openedIndexDirectory.getAbsolutePath());
                boolean isOpened = l.equals(p);
                logger.info(" l = p : " + isOpened + ", " + l + ", " + p + ", " + indexDirectory);
                if (isOpened) {
                    indexDirectories.remove(i);
                }
            }
{'fraction_non_alphanumeric': 0.09197681513117754, 'fraction_numerical': 0.004270896888346553, 'mean_word_length': 3.1473750790638837, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6647216', 'n_tokens_mistral': 1672, 'n_tokens_neox': 1628, 'n_words': 432}