<commit_before>package org.metaborg.spoofax.core.syntax.jsglr;

import java.io.IOException;

import org.apache.commons.vfs2.FileObject;
import org.metaborg.spoofax.core.language.ILanguage;
import org.metaborg.spoofax.core.syntax.ParseResult;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.ITermFactory;
import org.spoofax.jsglr.client.Asfix2TreeBuilder;
import org.spoofax.jsglr.client.Disambiguator;
import org.spoofax.jsglr.client.FilterException;
import org.spoofax.jsglr.client.StartSymbolException;
import org.spoofax.jsglr.client.imploder.NullTokenizer;
import org.spoofax.jsglr.client.imploder.TermTreeFactory;
import org.spoofax.jsglr.client.imploder.TreeBuilder;
import org.spoofax.jsglr.io.SGLR;
import org.spoofax.jsglr.shared.SGLRException;
import org.spoofax.terms.attachments.ParentTermFactory;

public class JSGLRI implements IParser<IStrategoTerm> {
    private final IParserConfig config;
    private final ITermFactory termFactory;
    private final ILanguage language;
    private final ILanguage dialect;
    private final FileObject resource;
    private final String input;

    private final SGLR parser;

    private Disambiguator disambiguator;
    private int cursorLocation = Integer.MAX_VALUE;
    private boolean useRecovery = true;
    private boolean implodeEnabled = true;


    public JSGLRI(IParserConfig config, ITermFactory termFactory, ILanguage language, ILanguage dialect,
        FileObject resource, String input) throws IOException {
        this.config = config;
        this.termFactory = termFactory;
        this.language = language;
        this.dialect = dialect;
        this.resource = resource;
        this.input = input;

        final TermTreeFactory factory = new TermTreeFactory(new ParentTermFactory(termFactory));
        this.parser = new SGLR(new TreeBuilder(factory), config.getParseTableProvider().parseTable());

        resetState();
    }


    public void setCursorLocation(int cursorLocation) {
        this.cursorLocation = cursorLocation;
    }

    public void setUseRecovery(boolean useRecovery) {
        this.useRecovery = useRecovery;
        parser.setUseStructureRecovery(useRecovery);
    }

    public void setImplodeEnabled(boolean implode) {
        this.implodeEnabled = implode;
        resetState();
    }

    private void resetState() {
        parser.setTimeout(config.getTimeout());

        if(disambiguator != null) {
            parser.setDisambiguator(disambiguator);
        } else {
            disambiguator = parser.getDisambiguator();
        }

        setUseRecovery(useRecovery);

        if(!implodeEnabled) {
            parser.setTreeBuilder(new Asfix2TreeBuilder(termFactory));
        }
    }

    @Override public ParseResult<IStrategoTerm> parse() throws IOException {
        final String fileName = resource.getName().getPath();

        IStrategoTerm ast = null;

        final JSGLRParseErrorHandler errorHandler =
            new JSGLRParseErrorHandler(this, termFactory, resource, config.getParseTableProvider().parseTable()
                .hasRecovers());

        try {
            ast = actuallyParse(input, fileName);
        } catch(SGLRException | InterruptedException e) {
            errorHandler.setRecoveryFailed(useRecovery);
            errorHandler.processFatalException(new NullTokenizer(input, fileName), e);
        }

        if(ast != null) {
            errorHandler.setRecoveryFailed(false);
            errorHandler.gatherNonFatalErrors(ast);
            if(resource != null) {
                SourceAttachment.putSource(ast, resource, config);
            }
        }

        // GTODO: measure parse time
        return new ParseResult<IStrategoTerm>(ast, resource, errorHandler.messages(), -1, language, dialect);
    }

    public IStrategoTerm actuallyParse(String text, String filename) throws SGLRException, InterruptedException {
        IStrategoTerm result;
        try {
            result = (IStrategoTerm) parser.parse(text, filename, config.getStartSymbol(), false, cursorLocation);
        } catch(FilterException e) {
            if(e.getCause() == null && parser.getDisambiguator().getFilterPriorities()) {
                disambiguator.setFilterPriorities(false);
                try {
                    result = (IStrategoTerm) parser.parse(text, filename, config.getStartSymbol());
                } finally {
                    disambiguator.setFilterPriorities(true);
                }
            } else {
                throw e;
            }
        } catch(StartSymbolException e) {
            result = (IStrategoTerm) parser.parse(text, filename, null, false, cursorLocation);
        }
        return result;
    }

    public IParserConfig getConfig() {<commit_msg>Parsing with concrete syntax extensions requires heuristic filters in the disambiguator.<commit_after>package org.metaborg.spoofax.core.syntax.jsglr;

import java.io.IOException;

import org.apache.commons.vfs2.FileObject;
import org.metaborg.spoofax.core.language.ILanguage;
import org.metaborg.spoofax.core.syntax.ParseResult;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.ITermFactory;
import org.spoofax.jsglr.client.Asfix2TreeBuilder;
import org.spoofax.jsglr.client.Disambiguator;
import org.spoofax.jsglr.client.FilterException;
import org.spoofax.jsglr.client.StartSymbolException;
import org.spoofax.jsglr.client.imploder.NullTokenizer;
import org.spoofax.jsglr.client.imploder.TermTreeFactory;
import org.spoofax.jsglr.client.imploder.TreeBuilder;
import org.spoofax.jsglr.io.SGLR;
import org.spoofax.jsglr.shared.SGLRException;
import org.spoofax.terms.attachments.ParentTermFactory;

public class JSGLRI implements IParser<IStrategoTerm> {
    private final IParserConfig config;
    private final ITermFactory termFactory;
    private final ILanguage language;
    private final ILanguage dialect;
    private final FileObject resource;
    private final String input;

    private final SGLR parser;

    private Disambiguator disambiguator;
    private int cursorLocation = Integer.MAX_VALUE;
    private boolean useRecovery = true;
    private boolean implodeEnabled = true;


    public JSGLRI(IParserConfig config, ITermFactory termFactory, ILanguage language, ILanguage dialect,
        FileObject resource, String input) throws IOException {
        this.config = config;
        this.termFactory = termFactory;
        this.language = language;
        this.dialect = dialect;
        this.resource = resource;
        this.input = input;

        final TermTreeFactory factory = new TermTreeFactory(new ParentTermFactory(termFactory));
        this.parser = new SGLR(new TreeBuilder(factory), config.getParseTableProvider().parseTable());

        resetState();
    }


    public void setCursorLocation(int cursorLocation) {
        this.cursorLocation = cursorLocation;
    }

    public void setUseRecovery(boolean useRecovery) {
        this.useRecovery = useRecovery;
        parser.setUseStructureRecovery(useRecovery);
    }

    public void setImplodeEnabled(boolean implode) {
        this.implodeEnabled = implode;
        resetState();
    }

    private void resetState() {
        parser.setTimeout(config.getTimeout());

        if(disambiguator != null) {
            parser.setDisambiguator(disambiguator);
        } else {
            disambiguator = parser.getDisambiguator();
        }

        setUseRecovery(useRecovery);

        if(!implodeEnabled) {
            parser.setTreeBuilder(new Asfix2TreeBuilder(termFactory));
        }
    }

    @Override public ParseResult<IStrategoTerm> parse() throws IOException {
        final String fileName = resource.getName().getPath();

        IStrategoTerm ast = null;

        final JSGLRParseErrorHandler errorHandler =
            new JSGLRParseErrorHandler(this, termFactory, resource, config.getParseTableProvider().parseTable()
                .hasRecovers());

        try {
            ast = actuallyParse(input, fileName);
        } catch(SGLRException | InterruptedException e) {
            errorHandler.setRecoveryFailed(useRecovery);
            errorHandler.processFatalException(new NullTokenizer(input, fileName), e);
        }

        if(ast != null) {
            errorHandler.setRecoveryFailed(false);
            errorHandler.gatherNonFatalErrors(ast);
            if(resource != null) {
                SourceAttachment.putSource(ast, resource, config);
            }
        }

        // GTODO: measure parse time
        return new ParseResult<IStrategoTerm>(ast, resource, errorHandler.messages(), -1, language, dialect);
    }

    public IStrategoTerm actuallyParse(String text, String filename) throws SGLRException, InterruptedException {
        if(dialect != null) {
            disambiguator.setHeuristicFilters(true);
        } else {
            disambiguator.setHeuristicFilters(false);
        }
        
        IStrategoTerm result;
        try {
            result = (IStrategoTerm) parser.parse(text, filename, config.getStartSymbol(), false, cursorLocation);
        } catch(FilterException e) {
            if(e.getCause() == null && disambiguator.getFilterPriorities()) {
                disambiguator.setFilterPriorities(false);
                try {
                    result = (IStrategoTerm) parser.parse(text, filename, config.getStartSymbol());
                } finally {
                    disambiguator.setFilterPriorities(true);
                }
            } else {
                throw e;
            }
        } catch(StartSymbolException e) {
            result = (IStrategoTerm) parser.parse(text, filename, null, false, cursorLocation);
        }
        return result;
    }

    public IParserConfig getConfig() {
{'fraction_non_alphanumeric': 0.08726752503576538, 'fraction_numerical': 0.0008174943797261394, 'mean_word_length': 2.8714398734177213, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4234900', 'n_tokens_mistral': 2768, 'n_tokens_neox': 2681, 'n_words': 535}