<commit_before>/**
 *  Copyright (c) 2015-2017 <NAME>.
 *  All rights reserved. This program and the accompanying materials
 *  are made available under the terms of the Eclipse Public License v1.0
 *  which accompanies this distribution, and is available at
 *  http://www.eclipse.org/legal/epl-v10.html
 *
 *  Contributors:
 *  <NAME> <<EMAIL>> - initial API and implementation
 */
package org.eclipse.tm4e.ui.text;

import java.util.List;

import org.eclipse.core.runtime.Assert;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.content.IContentType;
import org.eclipse.core.runtime.preferences.IEclipsePreferences;
import org.eclipse.core.runtime.preferences.IEclipsePreferences.IPreferenceChangeListener;
import org.eclipse.core.runtime.preferences.IEclipsePreferences.PreferenceChangeEvent;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.DocumentEvent;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.IRegion;
import org.eclipse.jface.text.ITextInputListener;
import org.eclipse.jface.text.ITextListener;
import org.eclipse.jface.text.ITextViewer;
import org.eclipse.jface.text.ITextViewerExtension5;
import org.eclipse.jface.text.Region;
import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.TextEvent;
import org.eclipse.jface.text.TextPresentation;
import org.eclipse.jface.text.presentation.IPresentationDamager;
import org.eclipse.jface.text.presentation.IPresentationReconciler;
import org.eclipse.jface.text.presentation.IPresentationRepairer;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.Token;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.StyleRange;
import org.eclipse.swt.widgets.Control;
import org.eclipse.tm4e.core.TMCorePlugin;
import org.eclipse.tm4e.core.grammar.IGrammar;
import org.eclipse.tm4e.core.model.IModelTokensChangedListener;
import org.eclipse.tm4e.core.model.ITMModel;
import org.eclipse.tm4e.core.model.ModelTokensChangedEvent;
import org.eclipse.tm4e.core.model.Range;
import org.eclipse.tm4e.core.model.TMToken;
import org.eclipse.tm4e.ui.TMUIPlugin;
import org.eclipse.tm4e.ui.internal.model.DocumentHelper;
import org.eclipse.tm4e.ui.internal.model.TMModel;
import org.eclipse.tm4e.ui.internal.themes.ThemeManager;
import org.eclipse.tm4e.ui.model.ITMModelManager;
import org.eclipse.tm4e.ui.themes.ITokenProvider;

/**
 * TextMate presentation reconcilier which must be initialized with:
 * 
 * <ul>
 * <li>a TextMate grammar {@link IGrammar} used to initialize the TextMate model
 * {@link ITMModel}.</li>
 * <li>a token provider {@link ITokenProvider} to retrieve the {@link IToken}
 * from a TextMate token type .</li>
 * </ul>
 * 
 */
public class TMPresentationReconciler implements IPresentationReconciler {

	/**
	 * The default text attribute if non is returned as data by the current
	 * token
	 */
	protected Token defaultToken = new Token(null);

	/** The target viewer. */
	private ITextViewer viewer;
	/** The internal listener. */
	private InternalListener internalListener = new InternalListener();
	private IGrammar grammar;
	private ITokenProvider tokenProvider;

	private TextAttribute fDefaultTextAttribute = new TextAttribute(null);

	private IPreferenceChangeListener e4CSSThemeChangeListener;

	private boolean forcedTheme;

	/**
	 * Listener to recolorize editors when E4 Theme from General / Appearance
	 * preferences changed.
	 *
	 */
	private class E4CSSThemeChangeListener implements IPreferenceChangeListener {

		@Override
		public void preferenceChange(PreferenceChangeEvent event) {
			if (ThemeManager.E4_THEME_ID.equals(event.getKey())) {
				IDocument document = viewer.getDocument();
				if (document == null) {
					return;
				}
				if (forcedTheme) {
					// The theme was forced, don't update it.
					return;
				}
				ITokenProvider oldTheme = tokenProvider;
				// Select the well TextMate theme from the given E4 theme id.
				ITokenProvider newTheme = ThemeManager.getInstance()
						.getThemeForE4Theme(event.getNewValue() != null ? event.getNewValue().toString() : null);
				if (!newTheme.equals(oldTheme)) {
					// Theme has changed, recolorize
					tokenProvider = newTheme;
					ITMModel model = getTMModelManager().connect(document);
					colorize(0, document.getNumberOfLines() - 1, null, (TMModel) model);
				}
			}
		}
	};

	/**
	 * Internal listener class.
	 */
	class InternalListener implements ITextInputListener, IModelTokensChangedListener, ITextListener {

		@Override
		public void inputDocumentAboutToBeChanged(IDocument oldDocument, IDocument newDocument) {
			if (oldDocument != null) {
				viewer.removeTextListener(this);
				getTMModelManager().disconnect(oldDocument);
			}
		}

		@Override
		public void inputDocumentChanged(IDocument oldDocument, IDocument newDocument) {
			if (newDocument != null) {
				// Connect a TextModel to the new document.
				ITMModel model = getTMModelManager().connect(newDocument);
				try {
					viewer.addTextListener(this);
					// Update theme + grammar
					IContentType[] contentTypes = DocumentHelper.getContentTypes(newDocument);
					updateTokenProvider(contentTypes);
					model.setGrammar(getGrammar(contentTypes));
					// Add model listener
					model.addModelTokensChangedListener(this);
				} catch (CoreException e) {
					e.printStackTrace();
				}
			}
		}

		@Override
		public void textChanged(TextEvent e) {
			if (!e.getViewerRedrawState()) {
				return;
			}
			// textChanged is called in 2 cases:
			// 1) when the content of the document has changed. In this case,
			// colorization must not be done here. The colorization is done with
			// modelTokensChanged listener.
			// 2) when TextViewer#invalidateTextPresentation is called (because
			// of folding, etc)
			if (e.getDocumentEvent() != null) {
				// case 1), ignore colorization
				return;
			}

			// case 2), do the colorization.
			IDocument document = viewer.getDocument();
			if (document != null) {
				IRegion region = null;
				int fromLineNumber = -1;
				int toLineNumber = -1;
				if (e.getOffset() == 0 && e.getLength() == 0 && e.getText() == null) {
					// redraw state change, damage the whole document
					fromLineNumber = 0;
					toLineNumber = document.getNumberOfLines() - 1;
				} else {
					region = widgetRegion2ModelRegion(e);
					if (region != null) {
						try {
							String text = document.get(region.getOffset(), region.getLength());
							DocumentEvent de = new DocumentEvent(document, region.getOffset(), region.getLength(),
									text);
							fromLineNumber = DocumentHelper.getStartLine(de);
							toLineNumber = DocumentHelper.getEndLine(de, false);
						} catch (BadLocationException x) {
						}
					}
				}
				ITMModel model = getTMModelManager().connect(document);
				colorize(fromLineNumber, toLineNumber, region, (TMModel) model);
			}
		}

		/**
		 * Translates the given text event into the corresponding range of the
		 * viewer's document.
		 * 
		 * @param e
		 *            the text event
		 * @return the widget region corresponding the region of the given event
		 *         or <code>null</code> if none
		 * @since 2.1
		 */
		private IRegion widgetRegion2ModelRegion(TextEvent e) {

			String text = e.getText();
			int length = text == null ? 0 : text.length();

			if (viewer instanceof ITextViewerExtension5) {
				ITextViewerExtension5 extension = (ITextViewerExtension5) viewer;
				return extension.widgetRange2ModelRange(new Region(e.getOffset(), length));
			}

			IRegion visible = viewer.getVisibleRegion();
			IRegion region = new Region(e.getOffset() + visible.getOffset(), length);
			return region;
		}

		private IGrammar getGrammar(IContentType[] contentTypes) throws CoreException {
			if (grammar != null) {
				return grammar;
			}
			// Discover the well grammar from the contentTypes
			return TMCorePlugin.getGrammarRegistryManager().getGrammarFor(contentTypes);
		}

		private void updateTokenProvider(IContentType[] contentTypes) throws CoreException {
			if (tokenProvider == null) {
				tokenProvider = TMUIPlugin.getThemeManager().getThemeFor(contentTypes);
			}
		}

		@Override
		public void modelTokensChanged(ModelTokensChangedEvent e) {
			Control control = viewer.getTextWidget();
			if (control != null) {
				control.getDisplay().asyncExec(new Runnable() {
					@Override
					public void run() {
						if (viewer != null)
							colorize(e);
					}
				});
			}
		}
	}

	public void setGrammar(IGrammar grammar) {
		this.grammar = grammar;
	}

	public IGrammar getGrammar() {
		return grammar;
	}

	public ITokenProvider getTokenProvider() {
		return tokenProvider;
	}

	public void setTokenProvider(ITokenProvider tokenProvider) {
		this.tokenProvider = tokenProvider;
		this.forcedTheme = true;
	}

	/**
	 * Force the TextMate theme id to use for the editor.
	 * 
	 * @param themeId
	 */
	public void setThemeId(String themeId) {
		setTokenProvider(TMUIPlugin.getThemeManager().getThemeById(themeId));
	}

	@Override
	public void install(ITextViewer viewer) {
		Assert.isNotNull(viewer);

		this.viewer = viewer;
		viewer.addTextInputListener(internalListener);

		IDocument document = viewer.getDocument();
		if (document != null) {
			internalListener.inputDocumentChanged(null, document);
		}
		IEclipsePreferences preferences = ThemeManager.getInstance().getPreferenceE4CSSTheme();
		if (preferences != null) {
			e4CSSThemeChangeListener = new E4CSSThemeChangeListener();
			preferences.addPreferenceChangeListener(e4CSSThemeChangeListener);
		}
	}

	@Override
	public void uninstall() {
		viewer.removeTextInputListener(internalListener);
		// Ensure we uninstall all listeners
		internalListener.inputDocumentAboutToBeChanged(viewer.getDocument(), null);
		getTMModelManager().disconnect(viewer.getDocument());
		if (e4CSSThemeChangeListener != null) {
			ThemeManager.getInstance().getPreferenceE4CSSTheme()
					.removePreferenceChangeListener(e4CSSThemeChangeListener);
		}
		e4CSSThemeChangeListener = null;
	}

	@Override
	public IPresentationDamager getDamager(String contentType) {
		return null;
	}

	@Override
	public IPresentationRepairer getRepairer(String contentType) {
		return null;
	}

	private ITMModelManager getTMModelManager() {
		return TMUIPlugin.getTMModelManager();
	}

	private void colorize(ModelTokensChangedEvent e) {
		for (Range range : e.getRanges()) {
			colorize(range.fromLineNumber - 1, range.toLineNumber - 1, null, ((TMModel) e.getModel()));
		}
	}

	private void colorize(int fromLineNumber, int toLineNumber, IRegion damage, TMModel model) {
		// Refresh the UI Presentation
		System.err.println("Render from: " + fromLineNumber + " to: " + toLineNumber);
		try {
			IDocument document = model.getDocument();
			TextPresentation presentation = new TextPresentation(
					damage != null ? damage : DocumentHelper.getRegion(document, fromLineNumber, toLineNumber), 1000);

			int lastStart = presentation.getExtent().getOffset();
			int length = 0;
			boolean firstToken = true;
			IToken lastToken = Token.UNDEFINED;
			TextAttribute lastAttribute = getTokenTextAttribute(lastToken);

			List<TMToken> tokens = null;
			for (int line = fromLineNumber; line <= toLineNumber; line++) {
				tokens = model.getLineTokens(line);
				int startLineOffset = document.getLineOffset(line);
				for (int i = 0; i < tokens.size(); i++) {
					TMToken currentToken = tokens.get(i);
					TMToken nextToken = (i + 1 < tokens.size()) ? tokens.get(i + 1) : null;
					int tokenStartIndex = currentToken.startIndex + startLineOffset;

					if (damage != null) {
						// Damage region is setted (this case comes from when
						// hyperlink, occurences, folding are processed and call
						// TextViewer#invalidateTextPresentation)
						if (isBeforeRegion(currentToken, startLineOffset, damage)) {
							// The token is before the damage region
							if (nextToken != null) {
								if (isBeforeRegion(nextToken, startLineOffset, damage)) {
									// ignore it
									continue;
								} else {
									tokenStartIndex = damage.getOffset();
								}
							} else {
								// , ignore it
								continue;
							}
						} else if (isAfterRegion(currentToken, startLineOffset, damage)) {
							// The token is after the damage region, stop the<commit_msg>Fix bug with TextViewer#invalidateTextPresentation with one token.<commit_after>/**
 *  Copyright (c) 2015-2017 <NAME>.
 *  All rights reserved. This program and the accompanying materials
 *  are made available under the terms of the Eclipse Public License v1.0
 *  which accompanies this distribution, and is available at
 *  http://www.eclipse.org/legal/epl-v10.html
 *
 *  Contributors:
 *  <NAME> <<EMAIL>> - initial API and implementation
 */
package org.eclipse.tm4e.ui.text;

import java.util.List;

import org.eclipse.core.runtime.Assert;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.content.IContentType;
import org.eclipse.core.runtime.preferences.IEclipsePreferences;
import org.eclipse.core.runtime.preferences.IEclipsePreferences.IPreferenceChangeListener;
import org.eclipse.core.runtime.preferences.IEclipsePreferences.PreferenceChangeEvent;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.DocumentEvent;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.IRegion;
import org.eclipse.jface.text.ITextInputListener;
import org.eclipse.jface.text.ITextListener;
import org.eclipse.jface.text.ITextViewer;
import org.eclipse.jface.text.ITextViewerExtension5;
import org.eclipse.jface.text.Region;
import org.eclipse.jface.text.TextAttribute;
import org.eclipse.jface.text.TextEvent;
import org.eclipse.jface.text.TextPresentation;
import org.eclipse.jface.text.presentation.IPresentationDamager;
import org.eclipse.jface.text.presentation.IPresentationReconciler;
import org.eclipse.jface.text.presentation.IPresentationRepairer;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.Token;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.StyleRange;
import org.eclipse.swt.widgets.Control;
import org.eclipse.tm4e.core.TMCorePlugin;
import org.eclipse.tm4e.core.grammar.IGrammar;
import org.eclipse.tm4e.core.model.IModelTokensChangedListener;
import org.eclipse.tm4e.core.model.ITMModel;
import org.eclipse.tm4e.core.model.ModelTokensChangedEvent;
import org.eclipse.tm4e.core.model.Range;
import org.eclipse.tm4e.core.model.TMToken;
import org.eclipse.tm4e.ui.TMUIPlugin;
import org.eclipse.tm4e.ui.internal.model.DocumentHelper;
import org.eclipse.tm4e.ui.internal.model.TMModel;
import org.eclipse.tm4e.ui.internal.themes.ThemeManager;
import org.eclipse.tm4e.ui.model.ITMModelManager;
import org.eclipse.tm4e.ui.themes.ITokenProvider;

/**
 * TextMate presentation reconcilier which must be initialized with:
 * 
 * <ul>
 * <li>a TextMate grammar {@link IGrammar} used to initialize the TextMate model
 * {@link ITMModel}.</li>
 * <li>a token provider {@link ITokenProvider} to retrieve the {@link IToken}
 * from a TextMate token type .</li>
 * </ul>
 * 
 */
public class TMPresentationReconciler implements IPresentationReconciler {

	/**
	 * The default text attribute if non is returned as data by the current
	 * token
	 */
	protected Token defaultToken = new Token(null);

	/** The target viewer. */
	private ITextViewer viewer;
	/** The internal listener. */
	private InternalListener internalListener = new InternalListener();
	private IGrammar grammar;
	private ITokenProvider tokenProvider;

	private TextAttribute fDefaultTextAttribute = new TextAttribute(null);

	private IPreferenceChangeListener e4CSSThemeChangeListener;

	private boolean forcedTheme;

	/**
	 * Listener to recolorize editors when E4 Theme from General / Appearance
	 * preferences changed.
	 *
	 */
	private class E4CSSThemeChangeListener implements IPreferenceChangeListener {

		@Override
		public void preferenceChange(PreferenceChangeEvent event) {
			if (ThemeManager.E4_THEME_ID.equals(event.getKey())) {
				IDocument document = viewer.getDocument();
				if (document == null) {
					return;
				}
				if (forcedTheme) {
					// The theme was forced, don't update it.
					return;
				}
				ITokenProvider oldTheme = tokenProvider;
				// Select the well TextMate theme from the given E4 theme id.
				ITokenProvider newTheme = ThemeManager.getInstance()
						.getThemeForE4Theme(event.getNewValue() != null ? event.getNewValue().toString() : null);
				if (!newTheme.equals(oldTheme)) {
					// Theme has changed, recolorize
					tokenProvider = newTheme;
					ITMModel model = getTMModelManager().connect(document);
					colorize(0, document.getNumberOfLines() - 1, null, (TMModel) model);
				}
			}
		}
	};

	/**
	 * Internal listener class.
	 */
	class InternalListener implements ITextInputListener, IModelTokensChangedListener, ITextListener {

		@Override
		public void inputDocumentAboutToBeChanged(IDocument oldDocument, IDocument newDocument) {
			if (oldDocument != null) {
				viewer.removeTextListener(this);
				getTMModelManager().disconnect(oldDocument);
			}
		}

		@Override
		public void inputDocumentChanged(IDocument oldDocument, IDocument newDocument) {
			if (newDocument != null) {
				// Connect a TextModel to the new document.
				ITMModel model = getTMModelManager().connect(newDocument);
				try {
					viewer.addTextListener(this);
					// Update theme + grammar
					IContentType[] contentTypes = DocumentHelper.getContentTypes(newDocument);
					updateTokenProvider(contentTypes);
					model.setGrammar(getGrammar(contentTypes));
					// Add model listener
					model.addModelTokensChangedListener(this);
				} catch (CoreException e) {
					e.printStackTrace();
				}
			}
		}

		@Override
		public void textChanged(TextEvent e) {
			if (!e.getViewerRedrawState()) {
				return;
			}
			// textChanged is called in 2 cases:
			// 1) when the content of the document has changed. In this case,
			// colorization must not be done here. The colorization is done with
			// modelTokensChanged listener.
			// 2) when TextViewer#invalidateTextPresentation is called (because
			// of folding, etc)
			if (e.getDocumentEvent() != null) {
				// case 1), ignore colorization
				return;
			}

			// case 2), do the colorization.
			IDocument document = viewer.getDocument();
			if (document != null) {
				IRegion region = null;
				int fromLineNumber = -1;
				int toLineNumber = -1;
				if (e.getOffset() == 0 && e.getLength() == 0 && e.getText() == null) {
					// redraw state change, damage the whole document
					fromLineNumber = 0;
					toLineNumber = document.getNumberOfLines() - 1;
				} else {
					region = widgetRegion2ModelRegion(e);
					if (region != null) {
						try {
							String text = document.get(region.getOffset(), region.getLength());
							DocumentEvent de = new DocumentEvent(document, region.getOffset(), region.getLength(),
									text);
							fromLineNumber = DocumentHelper.getStartLine(de);
							toLineNumber = DocumentHelper.getEndLine(de, false);
						} catch (BadLocationException x) {
						}
					}
				}
				ITMModel model = getTMModelManager().connect(document);
				colorize(fromLineNumber, toLineNumber, region, (TMModel) model);
			}
		}

		/**
		 * Translates the given text event into the corresponding range of the
		 * viewer's document.
		 * 
		 * @param e
		 *            the text event
		 * @return the widget region corresponding the region of the given event
		 *         or <code>null</code> if none
		 * @since 2.1
		 */
		private IRegion widgetRegion2ModelRegion(TextEvent e) {

			String text = e.getText();
			int length = text == null ? 0 : text.length();

			if (viewer instanceof ITextViewerExtension5) {
				ITextViewerExtension5 extension = (ITextViewerExtension5) viewer;
				return extension.widgetRange2ModelRange(new Region(e.getOffset(), length));
			}

			IRegion visible = viewer.getVisibleRegion();
			IRegion region = new Region(e.getOffset() + visible.getOffset(), length);
			return region;
		}

		private IGrammar getGrammar(IContentType[] contentTypes) throws CoreException {
			if (grammar != null) {
				return grammar;
			}
			// Discover the well grammar from the contentTypes
			return TMCorePlugin.getGrammarRegistryManager().getGrammarFor(contentTypes);
		}

		private void updateTokenProvider(IContentType[] contentTypes) throws CoreException {
			if (tokenProvider == null) {
				tokenProvider = TMUIPlugin.getThemeManager().getThemeFor(contentTypes);
			}
		}

		@Override
		public void modelTokensChanged(ModelTokensChangedEvent e) {
			Control control = viewer.getTextWidget();
			if (control != null) {
				control.getDisplay().asyncExec(new Runnable() {
					@Override
					public void run() {
						if (viewer != null)
							colorize(e);
					}
				});
			}
		}
	}

	public void setGrammar(IGrammar grammar) {
		this.grammar = grammar;
	}

	public IGrammar getGrammar() {
		return grammar;
	}

	public ITokenProvider getTokenProvider() {
		return tokenProvider;
	}

	public void setTokenProvider(ITokenProvider tokenProvider) {
		this.tokenProvider = tokenProvider;
		this.forcedTheme = true;
	}

	/**
	 * Force the TextMate theme id to use for the editor.
	 * 
	 * @param themeId
	 */
	public void setThemeId(String themeId) {
		setTokenProvider(TMUIPlugin.getThemeManager().getThemeById(themeId));
	}

	@Override
	public void install(ITextViewer viewer) {
		Assert.isNotNull(viewer);

		this.viewer = viewer;
		viewer.addTextInputListener(internalListener);

		IDocument document = viewer.getDocument();
		if (document != null) {
			internalListener.inputDocumentChanged(null, document);
		}
		IEclipsePreferences preferences = ThemeManager.getInstance().getPreferenceE4CSSTheme();
		if (preferences != null) {
			e4CSSThemeChangeListener = new E4CSSThemeChangeListener();
			preferences.addPreferenceChangeListener(e4CSSThemeChangeListener);
		}
	}

	@Override
	public void uninstall() {
		viewer.removeTextInputListener(internalListener);
		// Ensure we uninstall all listeners
		internalListener.inputDocumentAboutToBeChanged(viewer.getDocument(), null);
		getTMModelManager().disconnect(viewer.getDocument());
		if (e4CSSThemeChangeListener != null) {
			ThemeManager.getInstance().getPreferenceE4CSSTheme()
					.removePreferenceChangeListener(e4CSSThemeChangeListener);
		}
		e4CSSThemeChangeListener = null;
	}

	@Override
	public IPresentationDamager getDamager(String contentType) {
		return null;
	}

	@Override
	public IPresentationRepairer getRepairer(String contentType) {
		return null;
	}

	private ITMModelManager getTMModelManager() {
		return TMUIPlugin.getTMModelManager();
	}

	private void colorize(ModelTokensChangedEvent e) {
		for (Range range : e.getRanges()) {
			colorize(range.fromLineNumber - 1, range.toLineNumber - 1, null, ((TMModel) e.getModel()));
		}
	}

	private void colorize(int fromLineNumber, int toLineNumber, IRegion damage, TMModel model) {
		// Refresh the UI Presentation
		System.err.println("Render from: " + fromLineNumber + " to: " + toLineNumber);
		try {
			IDocument document = model.getDocument();
			TextPresentation presentation = new TextPresentation(
					damage != null ? damage : DocumentHelper.getRegion(document, fromLineNumber, toLineNumber), 1000);

			int lastStart = presentation.getExtent().getOffset();
			int length = 0;
			boolean firstToken = true;
			IToken lastToken = Token.UNDEFINED;
			TextAttribute lastAttribute = getTokenTextAttribute(lastToken);

			List<TMToken> tokens = null;
			for (int line = fromLineNumber; line <= toLineNumber; line++) {
				tokens = model.getLineTokens(line);
				int startLineOffset = document.getLineOffset(line);
				for (int i = 0; i < tokens.size(); i++) {
					TMToken currentToken = tokens.get(i);
					TMToken nextToken = (i + 1 < tokens.size()) ? tokens.get(i + 1) : null;
					int tokenStartIndex = currentToken.startIndex + startLineOffset;

					if (damage != null) {
						// Damage region is setted (this case comes from when
						// hyperlink, occurences, folding are processed and call
						// TextViewer#invalidateTextPresentation)
						if (isBeforeRegion(currentToken, startLineOffset, damage)) {
							// The token is before the damage region
							if (nextToken != null) {
								if (isBeforeRegion(nextToken, startLineOffset, damage)) {
									// ignore it
									continue;
								} else {
									tokenStartIndex = damage.getOffset();
								}
							} else {
								IToken token = toToken(currentToken);
								lastAttribute = getTokenTextAttribute(token);
								length += getTokenLengh(currentToken, nextToken, line, document);
								// ignore it
								continue;
							}
						} else if (isAfterRegion(currentToken, startLineOffset, damage)) {
							// The token is after the damage region, stop the
{'fraction_non_alphanumeric': 0.10356782906672042, 'fraction_numerical': 0.005885910098770409, 'mean_word_length': 3.9218253968253967, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1120778', 'n_tokens_mistral': 8684, 'n_tokens_neox': 7289, 'n_words': 1771}