<commit_before>/*******************************************************************************
 * Copyright (c) 2008 <NAME>.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors: 
 *    <NAME> - initial API and implementation
 *******************************************************************************/
package clojuredev.utils.editors.antlrbased;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Lexer;
import org.antlr.runtime.Token;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.ITokenScanner;

import clojuredev.ClojuredevPlugin;
import clojuredev.editors.antlrbased.TokenData;

abstract public class AntlrBasedTokenScanner implements ITokenScanner {
	private static int ANTLR_EOF = -1;
	private Lexer lexer;
	private final List<TokenData> tokensData;
	private int currentTokenIndex;
	private final Map<Integer, IToken> antlrTokenTypeToJFaceToken;
	private String text;
	private boolean initialized = false;

	public AntlrBasedTokenScanner(Lexer lexer) {
		this.lexer = lexer;
		
		tokensData = new ArrayList<TokenData>();
		
		antlrTokenTypeToJFaceToken = new HashMap<Integer, IToken>();
		initAntlrTokenTypeToJFaceTokenMap();
		antlrTokenTypeToJFaceToken.put(ANTLR_EOF, org.eclipse.jface.text.rules.Token.EOF);
		initialized = true;
	}
	
	abstract protected void initAntlrTokenTypeToJFaceTokenMap();
	
	public final void addTokenType(int tokenIndex, org.eclipse.jface.text.rules.Token token) {
		if (initialized) throw lifeCycleError();
		antlrTokenTypeToJFaceToken.put(tokenIndex, token);
	}
	
	public final void addTokenType(int tokenIndex, TextAttribute textAttribute) {
		if (initialized) throw lifeCycleError();
		addTokenType(tokenIndex, new org.eclipse.jface.text.rules.Token(textAttribute));
	}

	public final void addToken(int tokenIndex, String tokenData) {
		if (initialized) throw lifeCycleError();
		addTokenType(tokenIndex, new org.eclipse.jface.text.rules.Token(tokenData));
	}
	
	private RuntimeException lifeCycleError() {
		return new RuntimeException("Object Lifecycle error: method called at an inappropriate time");
	}

	public final int getTokenLength() {
		return tokensData.get(currentTokenIndex).length;
	}

	public final int getTokenOffset() {
		return tokensData.get(currentTokenIndex).offset;
	}

	public final IToken nextToken() {
		int nextIndex = currentTokenIndex + 1;
		if ( nextIndex >= tokensData.size() ) {
			return org.eclipse.jface.text.rules.Token.EOF;
		}
		currentTokenIndex = nextIndex;
		TokenData token = tokensData.get(currentTokenIndex);
		if( token != null ){
			return token.iToken;
		} else {
			ClojuredevPlugin.logError("nextToken called but null token retrieved ? ! Returning UNDEFINED");
			return org.eclipse.jface.text.rules.Token.UNDEFINED;
		}
	}

	public final void setRange(IDocument document, int offset, int length) {
		if (!document.get().equals(text)) {
			tokensData.clear();
			currentTokenIndex = -1;
			text = document.get();
			
			lexer.setCharStream(new ANTLRStringStream(text));

			while (true) {
				Token token = lexer.nextToken();
				if( token.getType() == ANTLR_EOF ){
					break;
				}
				addTokenInfo((CommonToken) token);
			}
		}
		repositionCurrentTokenAtOffset(offset);
	}

	private void repositionCurrentTokenAtOffset(int offset) {
		for( int i = tokensData.size() - 1 ; i >= 0 ; i-- ){
			TokenData tokenInfo = tokensData.get(i);
			if( tokenInfo.offset < offset ){<commit_msg>Added paren rainbow to the editor.<commit_after>/*******************************************************************************
 * Copyright (c) 2008 <NAME>.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors: 
 *    <NAME> - initial API and implementation
 *******************************************************************************/
package clojuredev.utils.editors.antlrbased;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.Lexer;
import org.antlr.runtime.Token;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.ITokenScanner;
import org.eclipse.swt.SWT;
import org.eclipse.swt.widgets.Display;

import clojuredev.ClojuredevPlugin;
import clojuredev.editors.antlrbased.TokenData;

abstract public class AntlrBasedTokenScanner implements ITokenScanner {
	private static int ANTLR_EOF = -1;
	private Lexer lexer;
	private final List<TokenData> tokensData;
	private int currentTokenIndex;
	private final Map<Integer, IToken> antlrTokenTypeToJFaceToken;
	private String text;
	private boolean initialized = false;
	
	private IToken[] parenLevelTokens = new IToken[] {
	        new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_RED))),
            new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_BLUE))),
            new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_CYAN))),
            new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_GREEN))),
            new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_BLACK))),
            new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_DARK_MAGENTA))),
            new org.eclipse.jface.text.rules.Token(new TextAttribute(Display.getDefault().getSystemColor(SWT.COLOR_DARK_YELLOW))),
	};
    private int currentParenLevel = 0;

	public AntlrBasedTokenScanner(Lexer lexer) {
		this.lexer = lexer;
		
		tokensData = new ArrayList<TokenData>();
		
		antlrTokenTypeToJFaceToken = new HashMap<Integer, IToken>();
		initAntlrTokenTypeToJFaceTokenMap();
		antlrTokenTypeToJFaceToken.put(ANTLR_EOF, org.eclipse.jface.text.rules.Token.EOF);
		initialized = true;
	}
	
	abstract protected void initAntlrTokenTypeToJFaceTokenMap();
	
	public final void addTokenType(int tokenIndex, org.eclipse.jface.text.rules.Token token) {
		if (initialized) throw lifeCycleError();
		antlrTokenTypeToJFaceToken.put(tokenIndex, token);
	}
	
	public final void addTokenType(int tokenIndex, TextAttribute textAttribute) {
		if (initialized) throw lifeCycleError();
		addTokenType(tokenIndex, new org.eclipse.jface.text.rules.Token(textAttribute));
	}

	public final void addToken(int tokenIndex, String tokenData) {
		if (initialized) throw lifeCycleError();
		addTokenType(tokenIndex, new org.eclipse.jface.text.rules.Token(tokenData));
	}
	
	private RuntimeException lifeCycleError() {
		return new RuntimeException("Object Lifecycle error: method called at an inappropriate time");
	}

	public final int getTokenLength() {
		return tokensData.get(currentTokenIndex).length;
	}

	public final int getTokenOffset() {
		return tokensData.get(currentTokenIndex).offset;
	}

	public final IToken nextToken() {
		int nextIndex = currentTokenIndex + 1;
		if ( nextIndex >= tokensData.size() ) {
			return org.eclipse.jface.text.rules.Token.EOF;
		}
		currentTokenIndex = nextIndex;
		TokenData token = tokensData.get(currentTokenIndex);
		if( token != null ){
		    if (token.text.equals("(")) {
		        return parenLevelTokens[Math.abs((currentParenLevel++)) % parenLevelTokens.length];
		    } else if (token.text.equals(")")) {
		        return parenLevelTokens[Math.abs((--currentParenLevel)) % parenLevelTokens.length];
		    } else {
		        return token.iToken;
		    }
		} else {
			ClojuredevPlugin.logError("nextToken called but null token retrieved ? ! Returning UNDEFINED");
			return org.eclipse.jface.text.rules.Token.UNDEFINED;
		}
	}

	public final void setRange(IDocument document, int offset, int length) {
		if (!document.get().equals(text)) {
			tokensData.clear();
			currentTokenIndex = -1;
			text = document.get();
			currentParenLevel = 0;
			
			lexer.setCharStream(new ANTLRStringStream(text));

			while (true) {
				Token token = lexer.nextToken();
				if( token.getType() == ANTLR_EOF ){
					break;
				}
				addTokenInfo((CommonToken) token);
			}
		}
		repositionCurrentTokenAtOffset(offset);
	}

	private void repositionCurrentTokenAtOffset(int offset) {
		for( int i = tokensData.size() - 1 ; i >= 0 ; i-- ){
			TokenData tokenInfo = tokensData.get(i);
			if( tokenInfo.offset < offset ){
{'fraction_non_alphanumeric': 0.13930293730819815, 'fraction_numerical': 0.003068829460762823, 'mean_word_length': 4.812101910828026, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5452142', 'n_tokens_mistral': 2982, 'n_tokens_neox': 2735, 'n_words': 601}