<commit_before>package org.broadinstitute.sting.utils.threading;

import net.sf.picard.reference.IndexedFastaSequenceFile;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.HasGenomeLocation;
import org.broadinstitute.sting.utils.SimpleTimer;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Abstract base class to coordinating data processing by a collecting for processes / threads.
 *
 * Conceptually, the genome is viewed as a collection of non-overlapping genome location:
 *
 * chr1:1-10
 * chr1:11-20
 * chr1:21-30
 * etc.
 *
 * This class, and it's concrete derived classes, provide the ability to claim individual locations
 * as "mine", and exclude other processes / threads from processing them.  At the lowest-level this
 * is implemented by the claimOwnership(loc, name) function, that returns true if loc free (unclaimed)
 * and makes name the owner of loc.  High-level, and more efficient operations provide claiming
 * iterators over streams of objects implementing the HasGenomeLocation interface, so that you can
 * write code that looks like:
 *
 * for ( GenomeLoc ownedLoc : onlyOwned(allLocsToProcess.iterator) ) {
 *   doSomeWork(ownedLoc)
 *
 * Much of the code in this class is actually surrounding debugging and performance metrics code.
 * The actual synchronization code is separated out into the ClosableReentrantLock() system
 * and the two abstract functions:
 *
 *  protected abstract void registerNewLocs(Collection<ProcessingLoc> plocs);
 *  protected abstract Collection<ProcessingLoc> readNewLocs();
 *
 * That maintain the state of the tracker.
 *
 * That is, the ProcessingTracker is made of two components: a thread / process locking system and
 * a subclass that implements the methods to record new claimed state changes and to read out updates
 * that may have occurred by another thread or process.
 *
 * NOTE: this class assumes that all threads / processes are working with the same set of potential
 * GenomeLocs to own.  Claiming chr1:1-10 and then chr1:5-6 is allowed by the system.  Basically,
 * you only can stake claim to GenomeLocs that are .equal().
 */
public abstract class GenomeLocProcessingTracker {
    private final static Logger logger = Logger.getLogger(FileBackedGenomeLocProcessingTracker.class);
    private final static SimpleDateFormat STATUS_FORMAT = new SimpleDateFormat("HH:mm:ss,SSS");
    private final static int DEFAULT_OWNERSHIP_ITERATOR_SIZE = 20;

    /**
     * Useful state strings for printing status
     */
    private final static String GOING_FOR_LOCK = "going_for_lock";
    private final static String RELEASING_LOCK = "releasing_lock";
    private final static String HAVE_LOCK = "have_lock";
    private final static String RUNNING = "running";

    /**
     * A map, for efficiency, that allows quick lookup of the processing loc for a
     * given GenomeLoc.  The map points from loc -> loc / owner as a ProcessingLoc
     */
    private final Map<GenomeLoc, ProcessingLoc> processingLocs;

    /**
     * The locking object used to protect data from simulatanous access by multiple
     * threads or processes.
     */
    private final ClosableReentrantLock lock;

    /** A stream for writing status messages.  Can be null if we aren't writing status */
    private final PrintStream status;

    //
    // Timers for recording performance information
    //<commit_msg>Disable caching of ReadShards by the GenomeLocProcessingTracker (at least  temporarily).  Unfortunately this does not completely fix the IndelRealigner  exception that Ryan is seeing, but it helps things quite a bit.<commit_after>package org.broadinstitute.sting.utils.threading;

import net.sf.picard.reference.IndexedFastaSequenceFile;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.utils.GenomeLoc;
import org.broadinstitute.sting.utils.GenomeLocParser;
import org.broadinstitute.sting.utils.HasGenomeLocation;
import org.broadinstitute.sting.utils.SimpleTimer;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * Abstract base class to coordinating data processing by a collecting for processes / threads.
 *
 * Conceptually, the genome is viewed as a collection of non-overlapping genome location:
 *
 * chr1:1-10
 * chr1:11-20
 * chr1:21-30
 * etc.
 *
 * This class, and it's concrete derived classes, provide the ability to claim individual locations
 * as "mine", and exclude other processes / threads from processing them.  At the lowest-level this
 * is implemented by the claimOwnership(loc, name) function, that returns true if loc free (unclaimed)
 * and makes name the owner of loc.  High-level, and more efficient operations provide claiming
 * iterators over streams of objects implementing the HasGenomeLocation interface, so that you can
 * write code that looks like:
 *
 * for ( GenomeLoc ownedLoc : onlyOwned(allLocsToProcess.iterator) ) {
 *   doSomeWork(ownedLoc)
 *
 * Much of the code in this class is actually surrounding debugging and performance metrics code.
 * The actual synchronization code is separated out into the ClosableReentrantLock() system
 * and the two abstract functions:
 *
 *  protected abstract void registerNewLocs(Collection<ProcessingLoc> plocs);
 *  protected abstract Collection<ProcessingLoc> readNewLocs();
 *
 * That maintain the state of the tracker.
 *
 * That is, the ProcessingTracker is made of two components: a thread / process locking system and
 * a subclass that implements the methods to record new claimed state changes and to read out updates
 * that may have occurred by another thread or process.
 *
 * NOTE: this class assumes that all threads / processes are working with the same set of potential
 * GenomeLocs to own.  Claiming chr1:1-10 and then chr1:5-6 is allowed by the system.  Basically,
 * you only can stake claim to GenomeLocs that are .equal().
 */
public abstract class GenomeLocProcessingTracker {
    private final static Logger logger = Logger.getLogger(FileBackedGenomeLocProcessingTracker.class);
    private final static SimpleDateFormat STATUS_FORMAT = new SimpleDateFormat("HH:mm:ss,SSS");
    private final static int DEFAULT_OWNERSHIP_ITERATOR_SIZE = 1;

    /**
     * Useful state strings for printing status
     */
    private final static String GOING_FOR_LOCK = "going_for_lock";
    private final static String RELEASING_LOCK = "releasing_lock";
    private final static String HAVE_LOCK = "have_lock";
    private final static String RUNNING = "running";

    /**
     * A map, for efficiency, that allows quick lookup of the processing loc for a
     * given GenomeLoc.  The map points from loc -> loc / owner as a ProcessingLoc
     */
    private final Map<GenomeLoc, ProcessingLoc> processingLocs;

    /**
     * The locking object used to protect data from simulatanous access by multiple
     * threads or processes.
     */
    private final ClosableReentrantLock lock;

    /** A stream for writing status messages.  Can be null if we aren't writing status */
    private final PrintStream status;

    //
    // Timers for recording performance information
    //
{'fraction_non_alphanumeric': 0.06806898353801934, 'fraction_numerical': 0.006140580088842436, 'mean_word_length': 4.21813224267212, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1376981', 'n_tokens_mistral': 2068, 'n_tokens_neox': 1992, 'n_words': 809}