<commit_before>package hep.lcio.util;

import hep.lcio.event.LCCollection;
import hep.lcio.event.LCEvent;
import hep.lcio.event.LCIO;
import hep.lcio.event.SimCalorimeterHit;
import hep.lcio.implementation.event.ILCCollection;
import hep.lcio.implementation.event.ILCEvent;
import hep.lcio.implementation.event.IMCParticle;
import hep.lcio.implementation.event.ISimCalorimeterHit;
import hep.lcio.implementation.event.ISimTrackerHit;
import hep.lcio.implementation.io.LCFactory;
import hep.lcio.io.LCReader;
import hep.lcio.io.LCWriter;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;

/**
 * Utility methods for merging together LCIO 
 * files, events, and collections, with optional 
 * application of a delta time.
 * 
 * @author jeremym
 * @version $Id: MergeUtil.java,v 1.7 2007-02-01 19:38:16 jeremy Exp $
 */
abstract public class MergeUtil
{
	/** 
	 * Merge events from mergeFiles into a single event in outfile,
	 * until all the files in mergeFiles are exhausted, or maxevents 
	 * are created. 
	 * @return The number of merged events that were created.
	 * @param outfile Target file that will contain the merged events.
	 * @param A list of @see MergeFileOptions objects.
	 * @param maxEvents Set the maximum number of merged events.
	 */
	public static int mergeFiles(File outfile, List mergeFiles, int maxEvents) throws IOException
	{
		// Create the writer for the merged events.
		LCWriter writer = LCFactory.getInstance().createLCWriter();

		// Open the writer.
		writer.open(outfile.getCanonicalPath(), LCIO.WRITE_NEW);

		// Count of total merged events that were created.
		int nevents = 0;

		// File read loop.
		for (;;)
		{			
			// Check if max output events is reached.
			if (nevents >= maxEvents)
				break;
			
			System.out.println("merge - processing event <"+(nevents+1)+">");

			// Create the new output event.
			ILCEvent targetEvent = new ILCEvent();

			// On the first time through, the event header needs to
			// be set from the first LCEvent that is read.
			boolean setEventHeader = true;

			// Total events merged in from all files.
			int totmerged = 0;

			// Loop over the files to be merged in.
			for (Iterator iter = mergeFiles.iterator(); iter.hasNext();)
			{
				// Get the next set of merge file parameters.
				MergeFileParameters mfile = (MergeFileParameters) iter.next();

				// Get the number of events to read.
				int ntoread = mfile.nreads();

				// Get the reader.
				LCReader reader = mfile.reader();

				// Get the starting time.
				float startt = mfile.startt();

				// Get the delta time.
				float dt = mfile.dt();

				// Merge ntoread events from this reader into target,
				// using starting time of startt, delta time of dt.
				int nmerged = MergeUtil.mergeEvents(targetEvent, reader, ntoread, setEventHeader, startt, dt);

				// Increment total merged.
				totmerged += nmerged;

				// Next time, don't need to set the header.
				setEventHeader = false;
			}

			// Write out the combined event if something got merged in.
			if (totmerged > 0)
			{
				writer.writeEvent(targetEvent);
				nevents++;
			}
			else
			{
				// Done!
				break;
			}
		} // file read loop

		System.out.println("Created " + nevents + " merged events.");

		// Close the writer.
		writer.close();
<commit_msg>JM: End the job and discard the bad event when any input files reaches EOF.<commit_after>package hep.lcio.util;

import hep.lcio.event.LCCollection;
import hep.lcio.event.LCEvent;
import hep.lcio.event.LCIO;
import hep.lcio.event.SimCalorimeterHit;
import hep.lcio.implementation.event.ILCCollection;
import hep.lcio.implementation.event.ILCEvent;
import hep.lcio.implementation.event.IMCParticle;
import hep.lcio.implementation.event.ISimCalorimeterHit;
import hep.lcio.implementation.event.ISimTrackerHit;
import hep.lcio.implementation.io.LCFactory;
import hep.lcio.io.LCReader;
import hep.lcio.io.LCWriter;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;

/**
 * Utility methods for merging together LCIO 
 * files, events, and collections, with optional 
 * application of a delta time.
 * 
 * @author jeremym
 * @version $Id: MergeUtil.java,v 1.8 2007-02-01 20:50:11 jeremy Exp $
 */
abstract public class MergeUtil
{
	/** 
	 * Merge events from mergeFiles into a single event in outfile,
	 * until all the files in mergeFiles are exhausted, or maxevents 
	 * are created. 
	 * @return The number of merged events that were created.
	 * @param outfile Target file that will contain the merged events.
	 * @param A list of @see MergeFileOptions objects.
	 * @param maxEvents Set the maximum number of merged events.
	 */
	public static int mergeFiles(File outfile, List mergeFiles, int maxEvents) throws IOException
	{
		// Create the writer for the merged events.
		LCWriter writer = LCFactory.getInstance().createLCWriter();

		// Open the writer.
		writer.open(outfile.getCanonicalPath(), LCIO.WRITE_NEW);

		// Count of total merged events that were created.
		int nevents = 0;
		
		boolean done=false;

		// File read loop.
		for (;;)
		{			
			// Check if max output events is reached.
			if (nevents >= maxEvents)
				break;
			
			System.out.println("merge - processing event <"+(nevents+1)+">");

			// Create the new output event.
			ILCEvent targetEvent = new ILCEvent();

			// On the first time through, the event header needs to
			// be set from the first LCEvent that is read.
			boolean setEventHeader = true;

			// Total events merged in from all files.
			int totmerged = 0;

			// Loop over the files to be merged in.
			for (Iterator iter = mergeFiles.iterator(); iter.hasNext();)
			{
				// Get the next set of merge file parameters.
				MergeFileParameters mfile = (MergeFileParameters) iter.next();

				// Get the number of events to read.
				int ntoread = mfile.nreads();

				// Get the reader.
				LCReader reader = mfile.reader();

				// Get the starting time.
				float startt = mfile.startt();

				// Get the delta time.
				float dt = mfile.dt();

				// Merge ntoread events from this reader into target,
				// using starting time of startt, delta time of dt.
				int nmerged = MergeUtil.mergeEvents(targetEvent, reader, ntoread, setEventHeader, startt, dt);
				
				// Check if the input file ran out of events.
				if (nmerged==0) {
				
					System.out.println("EOF on <"+mfile.file().getCanonicalPath()+">.  This merge event will be discarded.");
				
					// Invalidate this event.
					totmerged=0;
					
					// Stop processing this event.
					break;
				}

				// Increment total merged.
				totmerged += nmerged;

				// Next time, don't need to set the header.
				setEventHeader = false;
			}

			// Write out the combined event if it is valid.
			// The totmerged counter will be reset if any
			// of the input events runs out of events.
			if (totmerged > 0)
			{
				writer.writeEvent(targetEvent);
				nevents++;
			}
			else
			{
				// Done!
				break;
			}
		} // file read loop

		System.out.println("Created " + nevents + " merged events.");

		// Close the writer.
		writer.close();

{'fraction_non_alphanumeric': 0.10239865338757188, 'fraction_numerical': 0.005891429372983589, 'mean_word_length': 3.0350877192982457, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1771330', 'n_tokens_mistral': 2593, 'n_tokens_neox': 2130, 'n_words': 746}