<commit_before>/*
 * Copyright (c) 2002-2009 "Neo Technology,"
 *     Network Engine for Objects in Lund AB [http://neotechnology.com]
 *
 * This file is part of Neo4j.
 * 
 * Neo4j is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
package org.neo4j.index.lucene;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.neo4j.kernel.impl.cache.LruCache;
import org.neo4j.kernel.impl.transaction.xaframework.XaCommand;
import org.neo4j.kernel.impl.transaction.xaframework.XaCommandFactory;
import org.neo4j.kernel.impl.transaction.xaframework.XaConnection;
import org.neo4j.kernel.impl.transaction.xaframework.XaContainer;
import org.neo4j.kernel.impl.transaction.xaframework.XaDataSource;
import org.neo4j.kernel.impl.transaction.xaframework.XaLogicalLog;
import org.neo4j.kernel.impl.transaction.xaframework.XaTransaction;
import org.neo4j.kernel.impl.transaction.xaframework.XaTransactionFactory;
import org.neo4j.kernel.impl.util.ArrayMap;

/**
 * An {@link XaDataSource} optimized for the {@link LuceneIndexService}.
 * This class is public because the XA framework requires it.
 */
public class LuceneDataSource extends XaDataSource
{
    /**
     * Default {@link Analyzer} for fulltext parsing.
     */
    public static final Analyzer LOWER_CASE_WHITESPACE_ANALYZER =
        new Analyzer()
    {
        @Override
        public TokenStream tokenStream( String fieldName, Reader reader )
        {
            return new LowerCaseFilter( new WhitespaceTokenizer( reader ) );
        }
    };

    
    private final ArrayMap<String,IndexSearcherRef> indexSearchers = 
        new ArrayMap<String,IndexSearcherRef>( 6, true, true );

    private final XaContainer xaContainer;
    private final String storeDir;
    private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); 
    private final Analyzer fieldAnalyzer;
    private final LuceneIndexStore store;
    private LuceneIndexService indexService;
    
    private Map<String,LruCache<String,Collection<Long>>> caching = 
        Collections.synchronizedMap( 
            new HashMap<String,LruCache<String,Collection<Long>>>() );

    /**
     * Constructs this data source.
     * 
     * @param params XA parameters.
     * @throws InstantiationException if the data source couldn't be
     * instantiated
     */
    public LuceneDataSource( Map<Object,Object> params ) 
        throws InstantiationException
    {
        super( params );
        this.storeDir = (String) params.get( "dir" );
        this.fieldAnalyzer = instantiateAnalyzer();
        String dir = storeDir;
        File file = new File( dir );
        if ( !file.exists() )
        {
            try
            {
                autoCreatePath( dir );
            }
            catch ( IOException e )
            {
                throw new RuntimeException(
                    "Unable to create directory " + dir, e );
            }
        }
        this.store = new LuceneIndexStore( storeDir + "/lucene-store.db" );
        XaCommandFactory cf = new LuceneCommandFactory();
        XaTransactionFactory tf = new LuceneTransactionFactory( store );
        xaContainer = XaContainer.create( dir + "/lucene.log", cf, tf, params );
        try
        {
            xaContainer.openLogicalLog();
        }
        catch ( IOException e )
        {
            throw new RuntimeException( "Unable to open lucene log in " + dir,
                e );
        }
    }
    
    /**
     * This is here so that {@link LuceneIndexService#formQuery(String, Object)}
     * can be used when getting stuff from inside a transaction.
     * @param indexService the {@link LuceneIndexService} instance which
     * created it.
     */
    protected void setIndexService( LuceneIndexService indexService )
    {
        this.indexService = indexService;
    }
    
    protected LuceneIndexService getIndexService()
    {
        return this.indexService;
    }

    private Analyzer instantiateAnalyzer()
    {
        return LOWER_CASE_WHITESPACE_ANALYZER;
    }

    private void autoCreatePath( String dirs ) throws IOException
    {
        File directories = new File( dirs );
        if ( !directories.exists() )
        {
            if ( !directories.mkdirs() )<commit_msg>Fixed a problem with LuceneFulltextIndexService in the shell (index command)<commit_after>/*
 * Copyright (c) 2002-2009 "Neo Technology,"
 *     Network Engine for Objects in Lund AB [http://neotechnology.com]
 *
 * This file is part of Neo4j.
 * 
 * Neo4j is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
package org.neo4j.index.lucene;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.neo4j.kernel.impl.cache.LruCache;
import org.neo4j.kernel.impl.transaction.xaframework.XaCommand;
import org.neo4j.kernel.impl.transaction.xaframework.XaCommandFactory;
import org.neo4j.kernel.impl.transaction.xaframework.XaConnection;
import org.neo4j.kernel.impl.transaction.xaframework.XaContainer;
import org.neo4j.kernel.impl.transaction.xaframework.XaDataSource;
import org.neo4j.kernel.impl.transaction.xaframework.XaLogicalLog;
import org.neo4j.kernel.impl.transaction.xaframework.XaTransaction;
import org.neo4j.kernel.impl.transaction.xaframework.XaTransactionFactory;
import org.neo4j.kernel.impl.util.ArrayMap;

/**
 * An {@link XaDataSource} optimized for the {@link LuceneIndexService}.
 * This class is public because the XA framework requires it.
 */
public class LuceneDataSource extends XaDataSource
{
    /**
     * Default {@link Analyzer} for fulltext parsing.
     */
    public static final Analyzer LOWER_CASE_WHITESPACE_ANALYZER =
        new Analyzer()
    {
        @Override
        public TokenStream tokenStream( String fieldName, Reader reader )
        {
            return new LowerCaseFilter( new WhitespaceTokenizer( reader ) );
        }
    };

    
    private final ArrayMap<String,IndexSearcherRef> indexSearchers = 
        new ArrayMap<String,IndexSearcherRef>( 6, true, true );

    private final XaContainer xaContainer;
    private final String storeDir;
    private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); 
    private final Analyzer fieldAnalyzer;
    private final LuceneIndexStore store;
    private LuceneIndexService indexService;
    
    private Map<String,LruCache<String,Collection<Long>>> caching = 
        Collections.synchronizedMap( 
            new HashMap<String,LruCache<String,Collection<Long>>>() );

    /**
     * Constructs this data source.
     * 
     * @param params XA parameters.
     * @throws InstantiationException if the data source couldn't be
     * instantiated
     */
    public LuceneDataSource( Map<Object,Object> params ) 
        throws InstantiationException
    {
        super( params );
        this.storeDir = (String) params.get( "dir" );
        this.fieldAnalyzer = instantiateAnalyzer();
        String dir = storeDir;
        File file = new File( dir );
        if ( !file.exists() )
        {
            try
            {
                autoCreatePath( dir );
            }
            catch ( IOException e )
            {
                throw new RuntimeException(
                    "Unable to create directory " + dir, e );
            }
        }
        this.store = new LuceneIndexStore( storeDir + "/lucene-store.db" );
        XaCommandFactory cf = new LuceneCommandFactory();
        XaTransactionFactory tf = new LuceneTransactionFactory( store );
        xaContainer = XaContainer.create( dir + "/lucene.log", cf, tf, params );
        try
        {
            xaContainer.openLogicalLog();
        }
        catch ( IOException e )
        {
            throw new RuntimeException( "Unable to open lucene log in " + dir,
                e );
        }
    }
    
    /**
     * This is here so that {@link LuceneIndexService#formQuery(String, Object)}
     * can be used when getting stuff from inside a transaction.
     * @param indexService the {@link LuceneIndexService} instance which
     * created it.
     */
    protected void setIndexService( LuceneIndexService indexService )
    {
        this.indexService = indexService;
    }
    
    public LuceneIndexService getIndexService()
    {
        return this.indexService;
    }

    private Analyzer instantiateAnalyzer()
    {
        return LOWER_CASE_WHITESPACE_ANALYZER;
    }

    private void autoCreatePath( String dirs ) throws IOException
    {
        File directories = new File( dirs );
        if ( !directories.exists() )
        {
            if ( !directories.mkdirs() )
{'fraction_non_alphanumeric': 0.08795669824086604, 'fraction_numerical': 0.0038903924221921514, 'mean_word_length': 3.060782967032967, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5214884', 'n_tokens_mistral': 3421, 'n_tokens_neox': 3273, 'n_words': 887}