<commit_before>/*******************************************************************************
 * Copyright (c) 2010 itemis AG (http://www.itemis.eu) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.nodemodel.impl;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.emf.ecore.util.EcoreUtil;
import org.eclipse.xtext.nodemodel.BidiIterator;
import org.eclipse.xtext.nodemodel.BidiTreeIterable;
import org.eclipse.xtext.nodemodel.BidiTreeIterator;
import org.eclipse.xtext.nodemodel.ICompositeNode;
import org.eclipse.xtext.nodemodel.ILeafNode;
import org.eclipse.xtext.nodemodel.INode;
import org.eclipse.xtext.nodemodel.SyntaxErrorMessage;
import org.eclipse.xtext.nodemodel.serialization.DeserializationConversionContext;
import org.eclipse.xtext.nodemodel.serialization.SerializationConversionContext;
import org.eclipse.xtext.nodemodel.serialization.SerializationUtil;
import org.eclipse.xtext.nodemodel.util.NodeTreeIterator;
import org.eclipse.xtext.nodemodel.util.ReversedBidiTreeIterable;
import org.eclipse.xtext.util.Strings;

import com.google.common.collect.Iterators;

/**
 * @author <NAME> - Initial contribution and API
 * @author <NAME> - Serialization support
 * @noextend This class is not intended to be subclassed by clients.
 */
public abstract class AbstractNode implements INode, BidiTreeIterable<INode> {
	
	private CompositeNode parent;

	private AbstractNode prev;
	
	private AbstractNode next;
	
	private Object grammarElementOrArray;
	
	public ICompositeNode getParent() {
		if (parent != null)
			return parent.resolveAsParent();
		return null;
	}
	
	protected CompositeNode basicGetParent() {
		return parent;
	}
	
	protected void basicSetParent(CompositeNode parent) {
		this.parent = parent;
	}
	
	public BidiTreeIterable<INode> getAsTreeIterable() {
		return this;
	}
	
	public BidiTreeIterator<INode> iterator() {
		return new NodeTreeIterator(this);
	}
	
	public BidiTreeIterable<INode> reverse() {
		return new ReversedBidiTreeIterable<INode>(this);
	}
	
	public Iterable<ILeafNode> getLeafNodes() {
		return new Iterable<ILeafNode>() {
			public Iterator<ILeafNode> iterator() {
				return Iterators.filter(basicIterator(), ILeafNode.class);
			}
		};
	}
	
	public BidiTreeIterator<AbstractNode> basicIterator() {
		return new BasicNodeTreeIterator(this);
	}

	public String getText() {
		INode rootNode = getRootNode();
		if (rootNode != null) {
			int offset = getTotalOffset();
			int length = getTotalLength();
			return rootNode.getText().substring(offset, offset + length);
		}
		return null;
	}
	
	public int getTotalStartLine() {
		INode rootNode = getRootNode();
		if (rootNode != null) {
			int offset = getTotalOffset();
			return basicGetLineOfOffset(rootNode, offset);
		}
		return 1;
	}

	/**
	 * @since 2.0
	 */
	protected int basicGetLineOfOffset(INode rootNode, int offset) {
		if (rootNode instanceof RootNode) {
			int[] lineBreakOffsets = ((RootNode) rootNode).basicGetLineBreakOffsets();
			int insertionPoint = Arrays.binarySearch(lineBreakOffsets, offset);
			if (insertionPoint >= 0) {
				return insertionPoint + 1;
			} else {
				return -insertionPoint;
			}
		}
		String leadingText = rootNode.getText().substring(0, offset);
		int result = Strings.countLines(leadingText);
		return result + 1;
	}
	
	public int getStartLine() {
		INode rootNode = getRootNode();
		if (rootNode != null) {
			int offset = getOffset();
			return basicGetLineOfOffset(rootNode, offset);
		}
		return 1;
	}
	
	public int getEndLine() {
		int offset = getOffset();
		int length = getLength();
		INode rootNode = getRootNode();
		if (rootNode != null)
			return basicGetLineOfOffset(rootNode, offset + length);
		return 1;
	}
	
	public int getTotalEndLine() {
		int offset = getTotalEndOffset();
		INode rootNode = getRootNode();
		if (rootNode != null)
			return basicGetLineOfOffset(rootNode, offset);
		return 1;
	}
	
	public int getOffset() {
		Iterator<ILeafNode> leafIter = Iterators.filter(basicIterator(), ILeafNode.class);
		int firstLeafOffset = -1;
		while(leafIter.hasNext()) {
			ILeafNode leaf = leafIter.next();
			if (firstLeafOffset == -1) {
				firstLeafOffset = leaf.getTotalOffset();
			}
			if (!leaf.isHidden())
				return leaf.getTotalOffset();
		}
		if (firstLeafOffset != -1)
			return firstLeafOffset;
		return getTotalOffset();
	}
	<commit_msg>[perf] misc minor performance and memory improvements.<commit_after>/*******************************************************************************
 * Copyright (c) 2010 itemis AG (http://www.itemis.eu) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.nodemodel.impl;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.emf.ecore.util.EcoreUtil;
import org.eclipse.xtext.nodemodel.BidiIterator;
import org.eclipse.xtext.nodemodel.BidiTreeIterable;
import org.eclipse.xtext.nodemodel.BidiTreeIterator;
import org.eclipse.xtext.nodemodel.ICompositeNode;
import org.eclipse.xtext.nodemodel.ILeafNode;
import org.eclipse.xtext.nodemodel.INode;
import org.eclipse.xtext.nodemodel.SyntaxErrorMessage;
import org.eclipse.xtext.nodemodel.serialization.DeserializationConversionContext;
import org.eclipse.xtext.nodemodel.serialization.SerializationConversionContext;
import org.eclipse.xtext.nodemodel.serialization.SerializationUtil;
import org.eclipse.xtext.nodemodel.util.NodeTreeIterator;
import org.eclipse.xtext.nodemodel.util.ReversedBidiTreeIterable;
import org.eclipse.xtext.util.Strings;

import com.google.common.collect.Iterators;

/**
 * @author <NAME> - Initial contribution and API
 * @author <NAME> - Serialization support
 * @noextend This class is not intended to be subclassed by clients.
 */
public abstract class AbstractNode implements INode, BidiTreeIterable<INode> {
	
	private CompositeNode parent;

	private AbstractNode prev;
	
	private AbstractNode next;
	
	private Object grammarElementOrArray;
	
	public ICompositeNode getParent() {
		if (parent != null)
			return parent.resolveAsParent();
		return null;
	}
	
	protected CompositeNode basicGetParent() {
		return parent;
	}
	
	protected void basicSetParent(CompositeNode parent) {
		this.parent = parent;
	}
	
	public BidiTreeIterable<INode> getAsTreeIterable() {
		return this;
	}
	
	public BidiTreeIterator<INode> iterator() {
		return new NodeTreeIterator(this);
	}
	
	public BidiTreeIterable<INode> reverse() {
		return new ReversedBidiTreeIterable<INode>(this);
	}
	
	public Iterable<ILeafNode> getLeafNodes() {
		return new Iterable<ILeafNode>() {
			public Iterator<ILeafNode> iterator() {
				return Iterators.filter(basicIterator(), ILeafNode.class);
			}
		};
	}
	
	public BidiTreeIterator<AbstractNode> basicIterator() {
		return new BasicNodeTreeIterator(this);
	}

	public String getText() {
		INode rootNode = getRootNode();
		if (rootNode != null) {
			int offset = getTotalOffset();
			int length = getTotalLength();
			return rootNode.getText().substring(offset, offset + length);
		}
		return null;
	}
	
	public int getTotalStartLine() {
		INode rootNode = getRootNode();
		if (rootNode != null) {
			int offset = getTotalOffset();
			return basicGetLineOfOffset(rootNode, offset);
		}
		return 1;
	}

	/**
	 * @since 2.0
	 */
	protected int basicGetLineOfOffset(INode rootNode, int offset) {
		if (rootNode instanceof RootNode) {
			int[] lineBreakOffsets = ((RootNode) rootNode).basicGetLineBreakOffsets();
			int insertionPoint = Arrays.binarySearch(lineBreakOffsets, offset);
			if (insertionPoint >= 0) {
				return insertionPoint + 1;
			} else {
				return -insertionPoint;
			}
		}
		String leadingText = rootNode.getText().substring(0, offset);
		int result = Strings.countLines(leadingText);
		return result + 1;
	}
	
	public int getStartLine() {
		INode rootNode = getRootNode();
		if (rootNode != null) {
			int offset = getOffset();
			return basicGetLineOfOffset(rootNode, offset);
		}
		return 1;
	}
	
	public int getEndLine() {
		int offset = getOffset();
		int length = getLength();
		INode rootNode = getRootNode();
		if (rootNode != null)
			return basicGetLineOfOffset(rootNode, offset + length);
		return 1;
	}
	
	public int getTotalEndLine() {
		int offset = getTotalEndOffset();
		INode rootNode = getRootNode();
		if (rootNode != null)
			return basicGetLineOfOffset(rootNode, offset);
		return 1;
	}
	
	public int getOffset() {
		Iterator<AbstractNode> leafIter = basicIterator();
		int firstLeafOffset = -1;
		while(leafIter.hasNext()) {
			AbstractNode node = leafIter.next();
			if (node instanceof ILeafNode) {
				ILeafNode leaf = (ILeafNode) node;
				if (firstLeafOffset == -1) {
					firstLeafOffset = leaf.getTotalOffset();
				}
				if (!leaf.isHidden())
					return leaf.getTotalOffset();
			}
		}
		if (firstLeafOffset != -1)
			return firstLeafOffset;
		return getTotalOffset();
	}
	
{'fraction_non_alphanumeric': 0.13683033445156043, 'fraction_numerical': 0.004269594388533089, 'mean_word_length': 4.703188405797102, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6081337', 'n_tokens_mistral': 3371, 'n_tokens_neox': 3003, 'n_words': 707}