<commit_before>package org.spoofax.jsglr2.imploder.treefactory;

import static org.spoofax.interpreter.terms.IStrategoTerm.MUTABLE;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.putImploderAttachment;

import java.util.Collections;
import java.util.List;
import java.util.Objects;

import org.metaborg.parsetable.symbols.IMetaVarSymbol;
import org.metaborg.parsetable.symbols.ISymbol;
import org.spoofax.interpreter.terms.IStrategoConstructor;
import org.spoofax.interpreter.terms.IStrategoList;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.ITermFactory;
import org.spoofax.jsglr.client.imploder.IToken;
import org.spoofax.jsglr.client.imploder.ImploderAttachment;
import org.spoofax.terms.TermFactory;

public class TokenizedTermTreeFactory implements ITokenizedTreeFactory<IStrategoTerm> {

    private final ITermFactory termFactory;

    public TokenizedTermTreeFactory() {
        this.termFactory = new TermFactory().getFactoryWithStorageType(MUTABLE);
    }

    @Override public IStrategoTerm createStringTerminal(ISymbol symbol, String value, IToken token) {
        IStrategoTerm stringTerminalTerm = termFactory.makeString(value);

        configure(stringTerminalTerm, null, token, token);

        return stringTerminalTerm;
    }

    @Override public IStrategoTerm createMetaVar(IMetaVarSymbol symbol, String value, IToken token) {
        IStrategoTerm stringTerm = termFactory.makeString(value);
        IStrategoTerm metaVarTerm = termFactory.makeAppl(symbol.metaVarCardinality().constructor, stringTerm);

        configure(stringTerm, null, token, token);
        configure(metaVarTerm, null, token, token);

        return metaVarTerm;
    }

    @Override public IStrategoTerm createNonTerminal(ISymbol symbol, String constructor, List<IStrategoTerm> childASTs,
        IToken leftToken, IToken rightToken) {
        IStrategoConstructor constructorTerm =<commit_msg>Save the lexical sort on lexical AST nodes<commit_after>package org.spoofax.jsglr2.imploder.treefactory;

import static org.spoofax.interpreter.terms.IStrategoTerm.MUTABLE;
import static org.spoofax.jsglr.client.imploder.ImploderAttachment.putImploderAttachment;

import java.util.Collections;
import java.util.List;
import java.util.Objects;

import org.metaborg.parsetable.symbols.IMetaVarSymbol;
import org.metaborg.parsetable.symbols.ISymbol;
import org.spoofax.interpreter.terms.IStrategoConstructor;
import org.spoofax.interpreter.terms.IStrategoList;
import org.spoofax.interpreter.terms.IStrategoTerm;
import org.spoofax.interpreter.terms.ITermFactory;
import org.spoofax.jsglr.client.imploder.IToken;
import org.spoofax.jsglr.client.imploder.ImploderAttachment;
import org.spoofax.terms.TermFactory;

public class TokenizedTermTreeFactory implements ITokenizedTreeFactory<IStrategoTerm> {

    private final ITermFactory termFactory;

    public TokenizedTermTreeFactory() {
        this.termFactory = new TermFactory().getFactoryWithStorageType(MUTABLE);
    }

    @Override public IStrategoTerm createStringTerminal(ISymbol symbol, String value, IToken token) {
        IStrategoTerm stringTerminalTerm = termFactory.makeString(value);

        configure(stringTerminalTerm, ISymbol.getSort(symbol), token, token);

        return stringTerminalTerm;
    }

    @Override public IStrategoTerm createMetaVar(IMetaVarSymbol symbol, String value, IToken token) {
        IStrategoTerm stringTerm = termFactory.makeString(value);
        IStrategoTerm metaVarTerm = termFactory.makeAppl(symbol.metaVarCardinality().constructor, stringTerm);

        configure(stringTerm, ISymbol.getSort(symbol), token, token);
        configure(metaVarTerm, null, token, token);

        return metaVarTerm;
    }

    @Override public IStrategoTerm createNonTerminal(ISymbol symbol, String constructor, List<IStrategoTerm> childASTs,
        IToken leftToken, IToken rightToken) {
        IStrategoConstructor constructorTerm =
{'fraction_non_alphanumeric': 0.08050314465408805, 'fraction_numerical': 0.0005031446540880503, 'mean_word_length': 5.2125, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5401529', 'n_tokens_mistral': 1237, 'n_tokens_neox': 1195, 'n_words': 217}