<commit_before>package org.neo4j.rdf.sail;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;

import org.neo4j.api.core.EmbeddedNeo;
import org.neo4j.api.core.NeoService;
import org.neo4j.rdf.store.RdfStore;
import org.neo4j.rdf.store.VerboseQuadStore;
import org.neo4j.util.index.IndexService;
import org.neo4j.util.index.LuceneIndexService;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.rio.RDFFormat;
import org.openrdf.sail.Sail;

public class BatchInserter
{
	private NeoService neo;
	private RdfStore store;
	
	public BatchInserter( NeoService neo, RdfStore store )
	{
		this.neo = neo;
		this.store = store;
	}
	
	public void insert( File... files ) throws Exception
	{
		SimpleTimer timer = new SimpleTimer();
		Sail sail = new NeoSail( neo, store );
		try
		{
			sail.initialize();
			Repository repo = new SailRepository( sail );
			RepositoryConnection rc = repo.getConnection();
			for ( File file : files )
			{
				rc.add( file, "", RDFFormat.TRIG );
			}
			rc.commit();
			rc.close();
		}
		finally
		{
			sail.shutDown();
			timer.end();
		}
	}
	
	public static void main( final String[] args ) throws Exception
	{
		final NeoService neo = new EmbeddedNeo( "var/neo" );
		final IndexService indexService = new LuceneIndexService( neo );
		VerboseQuadStore store = new VerboseQuadStore( neo, indexService );
		try
		{
			new BatchInserter( neo, store ).insert( listFiles( args ) );<commit_msg>Uses the caching lucene index<commit_after>package org.neo4j.rdf.sail;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;

import org.neo4j.api.core.EmbeddedNeo;
import org.neo4j.api.core.NeoService;
import org.neo4j.rdf.store.CachingLuceneIndexService;
import org.neo4j.rdf.store.RdfStore;
import org.neo4j.rdf.store.VerboseQuadStore;
import org.neo4j.util.index.IndexService;
import org.openrdf.repository.Repository;
import org.openrdf.repository.RepositoryConnection;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.rio.RDFFormat;
import org.openrdf.sail.Sail;

public class BatchInserter
{
	private NeoService neo;
	private RdfStore store;
	
	public BatchInserter( NeoService neo, RdfStore store )
	{
		this.neo = neo;
		this.store = store;
	}
	
	public void insert( File... files ) throws Exception
	{
		SimpleTimer timer = new SimpleTimer();
		Sail sail = new NeoSail( neo, store );
		try
		{
			sail.initialize();
			Repository repo = new SailRepository( sail );
			RepositoryConnection rc = repo.getConnection();
			for ( File file : files )
			{
				rc.add( file, "", RDFFormat.TRIG );
			}
			rc.commit();
			rc.close();
		}
		finally
		{
			sail.shutDown();
			timer.end();
		}
	}
	
	public static void main( final String[] args ) throws Exception
	{
		final NeoService neo = new EmbeddedNeo( "var/neo" );
		final IndexService indexService = new CachingLuceneIndexService( neo );
		VerboseQuadStore store = new VerboseQuadStore( neo, indexService );
		try
		{
			new BatchInserter( neo, store ).insert( listFiles( args ) );
{'fraction_non_alphanumeric': 0.11142948105698822, 'fraction_numerical': 0.004457179242279528, 'mean_word_length': 4.043338683788122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6075254', 'n_tokens_mistral': 1213, 'n_tokens_neox': 1089, 'n_words': 250}