<commit_before>package polyglot.frontend;

import polyglot.ast.*;
import polyglot.types.*;
import polyglot.util.*;
import polyglot.visit.*;
import polyglot.main.Options;
import polyglot.main.Report;

import java.io.*;
import java.util.*;

/**
 * This is an abstract <code>ExtensionInfo</code>.
 */
public abstract class AbstractExtensionInfo implements ExtensionInfo {
    protected Compiler compiler;
    private Options options;
    protected TypeSystem ts = null;
    protected NodeFactory nf = null;
    protected SourceLoader source_loader = null;
    protected TargetFactory target_factory = null;
    protected Stats stats;

    /**
     * A list of all active (that is, uncompleted) <code>SourceJob</code>s.
     */
    protected LinkedList worklist;

    /**
     * A map from <code>Source</code>s to <code>SourceJob</code>s or to
     * the <code>COMPLETED_JOB</code> object if the SourceJob previously existed
     * but has now finished. The map contains entries for all
     * <code>Source</code>s that have had <code>Job</code>s added for them.
     */
    protected Map jobs;

    protected static final Object COMPLETED_JOB = "COMPLETED JOB";

    /** The currently running job, or null if no job is running. */
    protected Job currentJob;

    public Options getOptions() {
        if (this.options == null) {
            this.options = createOptions();
        }
        return options;
    }

    protected Options createOptions() {
        return new Options(this);
    }

    /** Return a Stats object to accumulate and report statistics. */
    public Stats getStats() {
        if (this.stats == null) {
            this.stats = new Stats(this);
        }
        return stats;
    }

    public Compiler compiler() {
        return compiler;
    }

    public void initCompiler(Compiler compiler) {
	this.compiler = compiler;
        jobs = new HashMap();
        worklist = new LinkedList();

        // Register the extension with the compiler.
        compiler.addExtension(this);

        currentJob = null;

        // Create the type system and node factory.
        typeSystem();
        nodeFactory();

        initTypeSystem();
    }

    protected abstract void initTypeSystem();

    /**
     * Run all jobs in the work list (and any children they have) to
     * completion. This method returns <code>true</code> if all jobs were
     * successfully completed. If all jobs were successfully completed, then
     * the worklist will be empty.
     *
     * The scheduling of <code>Job</code>s uses two methods to maintain
     * scheduling invariants: <code>selectJobFromWorklist</code> selects
     * a <code>SourceJob</code> from <code>worklist</code> (a list of
     * jobs that still need to be processed); <code>enforceInvariants</code> is
     * called before a pass is performed on a <code>SourceJob</code> and is
     * responsible for ensuring all dependencies are satisfied before the
     * pass proceeds, i.e. enforcing any scheduling invariants.
     */
    public boolean runToCompletion() {
        boolean okay = true;

        while (okay && ! worklist.isEmpty()) {
            SourceJob job = selectJobFromWorklist();

            if (Report.should_report(Report.frontend, 1)) {
		Report.report(1, "Running job " + job);
            }

            okay &= runAllPasses(job);

            if (job.completed()) {
                // the job has finished. Let's remove it from the map so it
                // can be garbage collected, and free up the AST.
                jobs.put(job.source(), COMPLETED_JOB);

                if (Report.should_report(Report.frontend, 1)) {
                    Report.report(1, "Completed job " + job);
                }
            }
            else {
                // the job is not yet completed (although, it really
                // should be...)
                if (Report.should_report(Report.frontend, 1)) {
                    Report.report(1, "Failed to complete job " + job);
                }
                worklist.add(job);
            }
        }

        if (Report.should_report(Report.frontend, 1))
	    Report.report(1, "Finished all passes -- " +
                        (okay ? "okay" : "failed"));

        return okay;
    }

    /**
     * Select and remove a <code>SourceJob</code> from the non-empty
     * <code>worklist</code>. Return the selected <code>SourceJob</code>
     * which will be scheduled to run all of its remaining passes.
     */
    protected SourceJob selectJobFromWorklist() {
        return (SourceJob)worklist.remove(0);
    }

    /**
     * Read a source file and compile it up to the the current job's last
     * barrier.
     */
    public boolean readSource(FileSource source) {
        // Add a new SourceJob for the given source. If a Job for the source
        // already exists, then we will be given the existing job.
        SourceJob job = addJob(source);

        if (job == null) {
            // addJob returns null if the job has already been completed, in
            // which case we can just ignore the request to read in the source.
            return true;
        }

        // Run the new job up to the currentJob's SourceJob's last barrier, to
        // make sure that dependencies are satisfied.
        Pass.ID barrier;

        if (currentJob != null) {
            if (currentJob.sourceJob().lastBarrier() == null) {
                throw new InternalCompilerError("A Source Job which has " +
                            "not reached a barrier cannot read another " +
                            "source file.");
            }

            barrier = currentJob.sourceJob().lastBarrier().id();
        }
        else {
            barrier = Pass.FIRST_BARRIER;
        }

        return runToPass(job, barrier);
    }

    /**
     * Run all pending passes on <code>job</code>.
     */
    public boolean runAllPasses(Job job) {
        List pending = job.pendingPasses();

        // Run until there are no more passes.
        if (!pending.isEmpty()) {
            Pass lastPass = (Pass)pending.get(pending.size() - 1);
            return runToPass(job, lastPass);
        }

        return true;
    }

    /**
     * Run a job until the <code>goal</code> pass completes.
     */
    public boolean runToPass(Job job, Pass.ID goal) {
        if (Report.should_report(Report.frontend, 1))
            Report.report(1, "Running " + job + " to pass named " + goal);

        if (job.completed(goal)) {
            return true;
        }

        Pass pass = job.passByID(goal);<commit_msg>Fixed bug in readSource where the source file was sometimes not compiled far enough to create types before they are needed.<commit_after>package polyglot.frontend;

import polyglot.ast.*;
import polyglot.types.*;
import polyglot.util.*;
import polyglot.visit.*;
import polyglot.main.Options;
import polyglot.main.Report;

import java.io.*;
import java.util.*;

/**
 * This is an abstract <code>ExtensionInfo</code>.
 */
public abstract class AbstractExtensionInfo implements ExtensionInfo {
    protected Compiler compiler;
    private Options options;
    protected TypeSystem ts = null;
    protected NodeFactory nf = null;
    protected SourceLoader source_loader = null;
    protected TargetFactory target_factory = null;
    protected Stats stats;

    /**
     * A list of all active (that is, uncompleted) <code>SourceJob</code>s.
     */
    protected LinkedList worklist;

    /**
     * A map from <code>Source</code>s to <code>SourceJob</code>s or to
     * the <code>COMPLETED_JOB</code> object if the SourceJob previously existed
     * but has now finished. The map contains entries for all
     * <code>Source</code>s that have had <code>Job</code>s added for them.
     */
    protected Map jobs;

    protected static final Object COMPLETED_JOB = "COMPLETED JOB";

    /** The currently running job, or null if no job is running. */
    protected Job currentJob;

    public Options getOptions() {
        if (this.options == null) {
            this.options = createOptions();
        }
        return options;
    }

    protected Options createOptions() {
        return new Options(this);
    }

    /** Return a Stats object to accumulate and report statistics. */
    public Stats getStats() {
        if (this.stats == null) {
            this.stats = new Stats(this);
        }
        return stats;
    }

    public Compiler compiler() {
        return compiler;
    }

    public void initCompiler(Compiler compiler) {
	this.compiler = compiler;
        jobs = new HashMap();
        worklist = new LinkedList();

        // Register the extension with the compiler.
        compiler.addExtension(this);

        currentJob = null;

        // Create the type system and node factory.
        typeSystem();
        nodeFactory();

        initTypeSystem();
    }

    protected abstract void initTypeSystem();

    /**
     * Run all jobs in the work list (and any children they have) to
     * completion. This method returns <code>true</code> if all jobs were
     * successfully completed. If all jobs were successfully completed, then
     * the worklist will be empty.
     *
     * The scheduling of <code>Job</code>s uses two methods to maintain
     * scheduling invariants: <code>selectJobFromWorklist</code> selects
     * a <code>SourceJob</code> from <code>worklist</code> (a list of
     * jobs that still need to be processed); <code>enforceInvariants</code> is
     * called before a pass is performed on a <code>SourceJob</code> and is
     * responsible for ensuring all dependencies are satisfied before the
     * pass proceeds, i.e. enforcing any scheduling invariants.
     */
    public boolean runToCompletion() {
        boolean okay = true;

        while (okay && ! worklist.isEmpty()) {
            SourceJob job = selectJobFromWorklist();

            if (Report.should_report(Report.frontend, 1)) {
		Report.report(1, "Running job " + job);
            }

            okay &= runAllPasses(job);

            if (job.completed()) {
                // the job has finished. Let's remove it from the map so it
                // can be garbage collected, and free up the AST.
                jobs.put(job.source(), COMPLETED_JOB);

                if (Report.should_report(Report.frontend, 1)) {
                    Report.report(1, "Completed job " + job);
                }
            }
            else {
                // the job is not yet completed (although, it really
                // should be...)
                if (Report.should_report(Report.frontend, 1)) {
                    Report.report(1, "Failed to complete job " + job);
                }
                worklist.add(job);
            }
        }

        if (Report.should_report(Report.frontend, 1))
	    Report.report(1, "Finished all passes -- " +
                        (okay ? "okay" : "failed"));

        return okay;
    }

    /**
     * Select and remove a <code>SourceJob</code> from the non-empty
     * <code>worklist</code>. Return the selected <code>SourceJob</code>
     * which will be scheduled to run all of its remaining passes.
     */
    protected SourceJob selectJobFromWorklist() {
        return (SourceJob)worklist.remove(0);
    }

    /**
     * Read a source file and compile it up to the the current job's last
     * barrier.
     */
    public boolean readSource(FileSource source) {
        // Add a new SourceJob for the given source. If a Job for the source
        // already exists, then we will be given the existing job.
        SourceJob job = addJob(source);

        if (job == null) {
            // addJob returns null if the job has already been completed, in
            // which case we can just ignore the request to read in the source.
            return true;
        }

        // Run the new job up to the currentJob's SourceJob's last barrier, to
        // make sure that dependencies are satisfied.
        Pass.ID barrier;

        if (currentJob != null) {
            if (currentJob.sourceJob().lastBarrier() == null) {
                throw new InternalCompilerError("A Source Job which has " +
                            "not reached a barrier cannot read another " +
                            "source file.");
            }

            barrier = currentJob.sourceJob().lastBarrier().id();
        }
        else {
            barrier = Pass.FIRST_BARRIER;
        }

        // Make sure we reach at least the first barrier defined
        // in the base compiler.  This forces types to be constructed.
        // If FIRST_BARRIER is before "barrier",
        // then the second runToPass will just return true.
        return runToPass(job, barrier) && runToPass(job, Pass.FIRST_BARRIER);
    }

    /**
     * Run all pending passes on <code>job</code>.
     */
    public boolean runAllPasses(Job job) {
        List pending = job.pendingPasses();

        // Run until there are no more passes.
        if (!pending.isEmpty()) {
            Pass lastPass = (Pass)pending.get(pending.size() - 1);
            return runToPass(job, lastPass);
        }

        return true;
    }

    /**
     * Run a job until the <code>goal</code> pass completes.
     */
    public boolean runToPass(Job job, Pass.ID goal) {
        if (Report.should_report(Report.frontend, 1))
            Report.report(1, "Running " + job + " to pass named " + goal);

        if (job.completed(goal)) {
            return true;
        }

        Pass pass = job.passByID(goal);
{'fraction_non_alphanumeric': 0.09805372604158959, 'fraction_numerical': 0.0017760674905646415, 'mean_word_length': 2.029365613091235, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4816602', 'n_tokens_mistral': 3690, 'n_tokens_neox': 3524, 'n_words': 1246}