<commit_before>/*******************************************************************************
 * Copyright (c) 2016 TypeFox GmbH (http://www.typefox.io) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.serializer.sequencer;

import java.util.Collections;
import java.util.List;

import org.eclipse.emf.common.util.TreeIterator;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.xtext.CrossReference;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.nodemodel.ICompositeNode;
import org.eclipse.xtext.nodemodel.ILeafNode;
import org.eclipse.xtext.nodemodel.INode;
import org.eclipse.xtext.parsetree.reconstr.impl.NodeIterator;

import com.google.common.collect.Lists;

/**
 * @author <NAME> - Initial contribution and API
 */
public class EmitterNodeUtil {

	private static boolean isEmitter(INode node, EObject grammarElement) {
		if (node instanceof ILeafNode) {
			ILeafNode leaf = (ILeafNode) node;
			if (leaf.isHidden())
				return false;
			return true;
		} else if (node instanceof ICompositeNode) {
			return GrammarUtil.isDatatypeRuleCall(grammarElement) || grammarElement instanceof CrossReference;
		}
		return false;
	}

	private static boolean isAbsorber(EObject grammarElement) {
		return grammarElement != null && GrammarUtil.isAssigned(grammarElement);
	}

	public static List<INode> collectEmitterNodes(INode from, INode to) {
		if (from == null) {
			return Collections.emptyList();
		}
		TreeIterator<INode> iterator;
		if (from == to) {
			iterator = from.getAsTreeIterable().iterator();
			iterator.next();
		} else {
			iterator = new NodeIterator(from);
		}
		List<INode> result = null;
		while (iterator.hasNext()) {
			INode next = iterator.next();
			if (next == to)
				break;
			EObject grammarElement = next.getGrammarElement();
			if (isEmitter(next, grammarElement)) {
				if (isAbsorber(grammarElement))
					break;
				iterator.prune();
				if (result == null) {
					result = Lists.newArrayList();
				}
				result.add(next);
			}
		}
		if (result == null) {
			return Collections.emptyList();
		}
		return result;
	}

}<commit_msg>fix eclipse/xtext-extras#63 broken serializer tests<commit_after>/*******************************************************************************
 * Copyright (c) 2016 TypeFox GmbH (http://www.typefox.io) and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *******************************************************************************/
package org.eclipse.xtext.serializer.sequencer;

import java.util.Collections;
import java.util.List;

import org.eclipse.emf.ecore.EObject;
import org.eclipse.xtext.CrossReference;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.nodemodel.ICompositeNode;
import org.eclipse.xtext.nodemodel.ILeafNode;
import org.eclipse.xtext.nodemodel.INode;

import com.google.common.collect.Lists;

/**
 * @author <NAME> - Initial contribution and API
 */
public class EmitterNodeUtil {

	public static List<INode> collectEmitterNodes(INode from, INode to) {
		List<INode> actual = collectEmitterNodesInternal(from, to);
		// helpful code to test this implementation
		List<INode> expected = Lists.newArrayList(new EmitterNodeIterator(from, to, false, false));
		if (!expected.equals(actual))
			throw new IllegalStateException();
		return actual;
	}

	private static List<INode> collectEmitterNodesInternal(INode from, INode to) {
		if (from == null) {
			return Collections.emptyList();
		}
		INode current = findNext(from, to);
		List<INode> result = null;
		while (current != null) {
			if (current == to)
				break;
			EObject grammarElement = current.getGrammarElement();
			if (isEmitter(current, grammarElement)) {
				if (isAbsorber(grammarElement))
					break;
				if (result == null) {
					result = Lists.newArrayList();
				}
				result.add(current);
				current = findNextSibling(current, to);
			} else {
				current = findNext(current, to);
			}
		}
		if (result == null) {
			return Collections.emptyList();
		}
		return result;
	}

	private static INode findNext(INode node, INode to) {
		if (node instanceof ICompositeNode) {
			INode firstChild = ((ICompositeNode) node).getFirstChild();
			if (firstChild != null) {
				if (firstChild == to) {
					return null;
				}
				return firstChild;
			}
		}
		return findNextSibling(node, to);
	}

	private static INode findNextSibling(INode node, INode to) {
		INode successor = node.getNextSibling();
		if (successor != null) {
			if (successor == to) {
				return null;
			}
			return successor;
		}
		ICompositeNode parent = node.getParent();
		if (parent == null || parent == to) {
			return null;
		}
		return findNextSibling(parent, to);
	}

	private static boolean isAbsorber(EObject grammarElement) {
		return grammarElement != null && GrammarUtil.isAssigned(grammarElement);
	}

	private static boolean isEmitter(INode node, EObject grammarElement) {
		if (node instanceof ILeafNode) {
			ILeafNode leaf = (ILeafNode) node;
			if (leaf.isHidden())
				return false;
			return true;
		} else if (node instanceof ICompositeNode) {
			return GrammarUtil.isDatatypeRuleCall(grammarElement) || grammarElement instanceof CrossReference;
		}
		return false;
	}

}
{'fraction_non_alphanumeric': 0.1612108412530799, 'fraction_numerical': 0.0031678986272439284, 'mean_word_length': 4.366383380547687, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '262474', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1707, 'n_words': 434}