<commit_before>package org.strategoxt.imp.testing.cmd.strategies;

import static org.spoofax.interpreter.core.Tools.asJavaString;
import static org.spoofax.interpreter.core.Tools.isTermList;
import static org.spoofax.interpreter.core.Tools.termAt;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.getLeftToken;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.getRightToken;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.getTokenizer;
import static org.spoofax.terms.Term.tryGetConstructor;

import java.io.IOException;
import java.util.Iterator;

import org.metaborg.spoofax.core.language.ILanguage;
import org.metaborg.spoofax.core.language.ILanguageService;
import org.metaborg.spoofax.core.syntax.jsglr.JSGLRI;
import org.metaborg.spoofax.core.syntax.jsglr.ParserConfig;
import org.metaborg.sunshine.environment.LaunchConfiguration;
import org.metaborg.sunshine.environment.ServiceRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spoofax.interpreter.terms.IStrategoConstructor;
import org.spoofax.interpreter.terms.IStrategoList;
import org.spoofax.interpreter.terms.IStrategoString;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.ITermFactory;
import org.spoofax.jsglr.client.imploder.ImploderAttachment;
import org.spoofax.jsglr.client.imploder.Tokenizer;
import org.spoofax.jsglr.shared.SGLRException;
import org.spoofax.terms.StrategoListIterator;
import org.spoofax.terms.TermTransformer;
import org.spoofax.terms.TermVisitor;
import org.spoofax.terms.attachments.ParentAttachment;
import org.spoofax.terms.attachments.ParentTermFactory;

public class SpoofaxTestingJSGLRI extends JSGLRI {

    private static final int PARSE_TIMEOUT = 20 * 1000;

    private static ITermFactory factory = ServiceRegistry.INSTANCE().getService(LaunchConfiguration.class).termFactory;
    private static final IStrategoConstructor INPUT_4 = factory.makeConstructor("Input", 4);

    private static final IStrategoConstructor OUTPUT_4 = factory.makeConstructor("Output", 4);

    private static final IStrategoConstructor ERROR_1 = factory.makeConstructor("Error", 1);

    private static final IStrategoConstructor LANGUAGE_1 = factory.makeConstructor("Language", 1);

    private static final IStrategoConstructor TARGET_LANGUAGE_1 = factory.makeConstructor("TargetLanguage", 1);

    private static final IStrategoConstructor SETUP_3 = factory.makeConstructor("Setup", 3);

    private static final IStrategoConstructor TARGET_SETUP_3 = factory.makeConstructor("TargetSetup", 3);

    private static final IStrategoConstructor TOPSORT_1 = factory.makeConstructor("TopSort", 1);

    private static final IStrategoConstructor TARGET_TOPSORT_1 = factory.makeConstructor("TargetTopSort", 1);

    private static final Logger LOG = LoggerFactory.getLogger(SpoofaxTestingJSGLRI.class);

    private final FragmentParser fragmentParser = new FragmentParser(SETUP_3, TOPSORT_1);

    private final FragmentParser outputFragmentParser = new FragmentParser(TARGET_SETUP_3, TARGET_TOPSORT_1);

    private final SelectionFetcher selections = new SelectionFetcher();

    public SpoofaxTestingJSGLRI(JSGLRI template) {
        super(new ParserConfig(template.getConfig().getStartSymbol(), template.getConfig().getParseTableProvider(),
            PARSE_TIMEOUT), factory, template.getLanguage(), template.getResource(), template.getInput());
        setUseRecovery(true);
    }

    @Override public IStrategoTerm actuallyParse(String input, String filename) throws InterruptedException,
        SGLRException {
        IStrategoTerm ast = super.actuallyParse(input, filename);
        try {
            return parseTestedFragments(ast);
        } catch(IOException e) {
            throw new SGLRException(getParser(), "Cannot parse", e);
        }
    }

    private IStrategoTerm parseTestedFragments(final IStrategoTerm root) throws IOException {
        final Tokenizer oldTokenizer = (Tokenizer) getTokenizer(root);
        final Retokenizer retokenizer = new Retokenizer(oldTokenizer);
        final ITermFactory nonParentFactory = factory;
        final ITermFactory factory = new ParentTermFactory(nonParentFactory);
        final FragmentParser testedParser = configureFragmentParser(root, getLanguage(root), fragmentParser);
        final FragmentParser outputParser =
            getTargetLanguage(root) == null ? testedParser : configureFragmentParser(root, getTargetLanguage(root),
                outputFragmentParser);
        assert !(nonParentFactory instanceof ParentTermFactory);

        if(testedParser == null || !testedParser.isInitialized() || outputParser == null
            || !outputParser.isInitialized()) {
            return root;<commit_msg>Adapt to new Spoofax core.<commit_after>package org.strategoxt.imp.testing.cmd.strategies;

import static org.spoofax.interpreter.core.Tools.asJavaString;
import static org.spoofax.interpreter.core.Tools.isTermList;
import static org.spoofax.interpreter.core.Tools.termAt;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.getLeftToken;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.getRightToken;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.getTokenizer;
import static org.spoofax.terms.Term.tryGetConstructor;

import java.io.IOException;
import java.util.Iterator;

import org.metaborg.spoofax.core.language.ILanguage;
import org.metaborg.spoofax.core.language.ILanguageService;
import org.metaborg.spoofax.core.syntax.jsglr.JSGLRI;
import org.metaborg.spoofax.core.syntax.jsglr.ParserConfig;
import org.metaborg.sunshine.environment.LaunchConfiguration;
import org.metaborg.sunshine.environment.ServiceRegistry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.spoofax.interpreter.terms.IStrategoConstructor;
import org.spoofax.interpreter.terms.IStrategoList;
import org.spoofax.interpreter.terms.IStrategoString;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.ITermFactory;
import org.spoofax.jsglr.client.imploder.ImploderAttachment;
import org.spoofax.jsglr.client.imploder.Tokenizer;
import org.spoofax.jsglr.shared.SGLRException;
import org.spoofax.terms.StrategoListIterator;
import org.spoofax.terms.TermTransformer;
import org.spoofax.terms.TermVisitor;
import org.spoofax.terms.attachments.ParentAttachment;
import org.spoofax.terms.attachments.ParentTermFactory;

public class SpoofaxTestingJSGLRI extends JSGLRI {

    private static final int PARSE_TIMEOUT = 20 * 1000;

    private static ITermFactory factory = ServiceRegistry.INSTANCE().getService(LaunchConfiguration.class).termFactory;
    private static final IStrategoConstructor INPUT_4 = factory.makeConstructor("Input", 4);

    private static final IStrategoConstructor OUTPUT_4 = factory.makeConstructor("Output", 4);

    private static final IStrategoConstructor ERROR_1 = factory.makeConstructor("Error", 1);

    private static final IStrategoConstructor LANGUAGE_1 = factory.makeConstructor("Language", 1);

    private static final IStrategoConstructor TARGET_LANGUAGE_1 = factory.makeConstructor("TargetLanguage", 1);

    private static final IStrategoConstructor SETUP_3 = factory.makeConstructor("Setup", 3);

    private static final IStrategoConstructor TARGET_SETUP_3 = factory.makeConstructor("TargetSetup", 3);

    private static final IStrategoConstructor TOPSORT_1 = factory.makeConstructor("TopSort", 1);

    private static final IStrategoConstructor TARGET_TOPSORT_1 = factory.makeConstructor("TargetTopSort", 1);

    private static final Logger LOG = LoggerFactory.getLogger(SpoofaxTestingJSGLRI.class);

    private final FragmentParser fragmentParser = new FragmentParser(SETUP_3, TOPSORT_1);

    private final FragmentParser outputFragmentParser = new FragmentParser(TARGET_SETUP_3, TARGET_TOPSORT_1);

    private final SelectionFetcher selections = new SelectionFetcher();

    public SpoofaxTestingJSGLRI(JSGLRI template) throws IOException {
        super(new ParserConfig(template.getConfig().getStartSymbol(), template.getConfig().getParseTableProvider(),
            PARSE_TIMEOUT), factory, template.getLanguage(), template.getResource(), template.getInput());
        setUseRecovery(true);
    }

    @Override public IStrategoTerm actuallyParse(String input, String filename) throws InterruptedException,
        SGLRException {
        IStrategoTerm ast = super.actuallyParse(input, filename);
        try {
            return parseTestedFragments(ast);
        } catch(IOException e) {
            throw new SGLRException(getParser(), "Cannot parse", e);
        }
    }

    private IStrategoTerm parseTestedFragments(final IStrategoTerm root) throws IOException {
        final Tokenizer oldTokenizer = (Tokenizer) getTokenizer(root);
        final Retokenizer retokenizer = new Retokenizer(oldTokenizer);
        final ITermFactory nonParentFactory = factory;
        final ITermFactory factory = new ParentTermFactory(nonParentFactory);
        final FragmentParser testedParser = configureFragmentParser(root, getLanguage(root), fragmentParser);
        final FragmentParser outputParser =
            getTargetLanguage(root) == null ? testedParser : configureFragmentParser(root, getTargetLanguage(root),
                outputFragmentParser);
        assert !(nonParentFactory instanceof ParentTermFactory);

        if(testedParser == null || !testedParser.isInitialized() || outputParser == null
            || !outputParser.isInitialized()) {
            return root;
{'fraction_non_alphanumeric': 0.0850864403249323, 'fraction_numerical': 0.006248698187877526, 'mean_word_length': 5.1875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5049697', 'n_tokens_mistral': 2857, 'n_tokens_neox': 2720, 'n_words': 537}