<commit_before>package de.huberlin.cuneiform.dax.repl;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.UUID;

import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;

import de.huberlin.cuneiform.dax.semanticmodel.DaxSemanticModelListener;
import de.huberlin.cuneiform.libdax.parser.DaxLexer;
import de.huberlin.cuneiform.libdax.parser.DaxParser;
import de.huberlin.wbi.cuneiform.core.repl.BaseRepl;
import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr;
import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext;
import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor;

public class DaxRepl extends BaseRepl {

	public DaxRepl( TicketSrcActor ticketSrc ) {
		super( ticketSrc );
	}
	
	@Override
	public synchronized int interpret( String input ) {
		
		DaxSemanticModelListener adag;
		TopLevelContext tlc;
		
		adag = process( input );
		tlc = adag.toTopLevelContext();
		
		return interpret( tlc );
		
	}

	@Override
	public void queryFailedPost( UUID queryId, long ticketId, Exception e,
			String script, String stdOut, String stdErr ) {}

	@Override
	public void queryFinishedPost( UUID queryId, CompoundExpr result ) {}

	@Override
	public void queryStartedPost( UUID runId ) {}

	public static DaxSemanticModelListener process( String input ) {
		
		ANTLRInputStream instream;
		DaxLexer lexer;
		CommonTokenStream tokenStream;
		DaxParser parser;
		ParseTree tree;
		ParseTreeWalker walker;
		DaxSemanticModelListener adag;
		
		walker = new ParseTreeWalker();
		
		// parse original content
		instream = new ANTLRInputStream( input );
		
		lexer = new DaxLexer( instream );
		lexer.removeErrorListeners();

		tokenStream = new CommonTokenStream( lexer );
		
		parser = new DaxParser( tokenStream );
		parser.removeErrorListeners();
		
		adag = new DaxSemanticModelListener();
		lexer.addErrorListener( adag );
		parser.addErrorListener( adag );

		
		tree = parser.adag();
		
		walker.walk( adag, tree );
		
		return adag;
	}
	

}
<commit_msg>Warnings removed.<commit_after>package de.huberlin.cuneiform.dax.repl;

import java.util.UUID;

import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;

import de.huberlin.cuneiform.dax.semanticmodel.DaxSemanticModelListener;
import de.huberlin.cuneiform.libdax.parser.DaxLexer;
import de.huberlin.cuneiform.libdax.parser.DaxParser;
import de.huberlin.wbi.cuneiform.core.repl.BaseRepl;
import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr;
import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext;
import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor;

public class DaxRepl extends BaseRepl {

	public DaxRepl( TicketSrcActor ticketSrc ) {
		super( ticketSrc );
	}
	
	@Override
	public synchronized int interpret( String input ) {
		
		DaxSemanticModelListener adag;
		TopLevelContext tlc;
		
		adag = process( input );
		tlc = adag.toTopLevelContext();
		
		return interpret( tlc );
		
	}

	@Override
	public void queryFailedPost( UUID queryId, long ticketId, Exception e,
			String script, String stdOut, String stdErr ) {}

	@Override
	public void queryFinishedPost( UUID queryId, CompoundExpr result ) {}

	@Override
	public void queryStartedPost( UUID runId ) {}

	public static DaxSemanticModelListener process( String input ) {
		
		ANTLRInputStream instream;
		DaxLexer lexer;
		CommonTokenStream tokenStream;
		DaxParser parser;
		ParseTree tree;
		ParseTreeWalker walker;
		DaxSemanticModelListener adag;
		
		walker = new ParseTreeWalker();
		
		// parse original content
		instream = new ANTLRInputStream( input );
		
		lexer = new DaxLexer( instream );
		lexer.removeErrorListeners();

		tokenStream = new CommonTokenStream( lexer );
		
		parser = new DaxParser( tokenStream );
		parser.removeErrorListeners();
		
		adag = new DaxSemanticModelListener();
		lexer.addErrorListener( adag );
		parser.addErrorListener( adag );

		
		tree = parser.adag();
		
		walker.walk( adag, tree );
		
		return adag;
	}
	

}

{'fraction_non_alphanumeric': 0.0928220715454758, 'fraction_numerical': 0.0018704699555763386, 'mean_word_length': 4.52, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2774351', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1419, 'n_words': 297}