<commit_before>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package net.colar.netbeans.fan;

import java.io.File;
import java.lang.reflect.Field;
import java.lang.reflect.Member;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Vector;
import javax.swing.text.Document;
import net.colar.netbeans.fan.indexer.FanIndexerFactory;
import net.colar.netbeans.fan.indexer.model.FanMethodParam;
import net.colar.netbeans.fan.indexer.model.FanSlot;
import net.colar.netbeans.fan.scope.FanAstScopeVar;
import net.colar.netbeans.fan.parboiled.FanLexAstUtils;
import net.colar.netbeans.fan.indexer.model.FanType;
import net.colar.netbeans.fan.parboiled.AstKind;
import net.colar.netbeans.fan.parboiled.AstNode;
import net.colar.netbeans.fan.parboiled.FantomParser;
import net.colar.netbeans.fan.parboiled.FantomLexerTokens.TokenName;
import net.colar.netbeans.fan.parboiled.pred.NodeKindPredicate;
import net.colar.netbeans.fan.scope.FanAstScopeVarBase;
import net.colar.netbeans.fan.scope.FanAstScopeVarBase.VarKind;
import net.colar.netbeans.fan.scope.FanLocalScopeVar;
import net.colar.netbeans.fan.scope.FanMethodScopeVar;
import net.colar.netbeans.fan.scope.FanTypeScopeVar;
import net.colar.netbeans.fan.types.FanResolvedFuncType;
import net.colar.netbeans.fan.types.FanResolvedListType;
import net.colar.netbeans.fan.types.FanResolvedMapType;
import net.colar.netbeans.fan.types.FanResolvedNullType;
import net.colar.netbeans.fan.types.FanResolvedType;
import net.colar.netbeans.fan.types.FanUnknownType;
import org.netbeans.modules.csl.api.Error;
import org.netbeans.modules.csl.api.OffsetRange;
import org.netbeans.modules.csl.api.Severity;
import org.netbeans.modules.csl.spi.DefaultError;
import org.netbeans.modules.csl.spi.ParserResult;
import org.netbeans.modules.parsing.api.Snapshot;
import org.openide.filesystems.FileObject;
import org.openide.filesystems.FileUtil;
import org.parboiled.Node;
import org.parboiled.Parboiled;
import org.parboiled.RecoveringParseRunner;
import org.parboiled.errors.ErrorUtils;
import org.parboiled.errors.ParseError;
import org.parboiled.support.ParseTreeUtils;
import org.parboiled.support.ParsingResult;

/**
 * Parse a fan file and holds the results
 * parse() parses the file
 * parseScope() adds to the tree scope vraiables etc...
 * @author  tcolar
 */
public class FanParserTask extends ParserResult
{

	boolean invalidated = false;
	public boolean dumpTree = false; // debug
	List<Error> errors = new Vector<Error>(); // -> use parsingResult.errors ?
	// full path of the source file
	private final FileObject sourceFile;
	// simple name of the source file
	private final String sourceName;
	// pod name
	private final String pod;
	// once parse() is called, will contain the parboiled parsing result
	private ParsingResult<AstNode> parsingResult;
	private AstNode astRoot;
	// Cache types resolution, for performance
	private HashMap<String, FanType> typeCache = new HashMap<String, FanType>();
	// Cache slots resolution, for performance
	private HashMap<String, List<FanSlot>> typeSlotsCache = new HashMap<String, List<FanSlot>>();
	private FantomParser parser;
	private boolean localScopeDone;

	public FanParserTask(Snapshot snapshot)
	{
		super(snapshot);
		invalidated = false;
		sourceName = snapshot==null?null:snapshot.getSource().getFileObject().getName();
		sourceFile = snapshot==null?null:FileUtil.toFileObject(new File(snapshot.getSource().getFileObject().getPath()));
		pod = snapshot==null?null:FanUtilities.getPodForPath(sourceFile.getPath());
		parser = Parboiled.createParser(FantomParser.class, this);
	}

	@Override
	public List<? extends Error> getDiagnostics()
	{
		return errors;
	}

	@Override
	protected void invalidate()
	{
	}

	/**
	 * Return AST tree generated by this parsing
	 * @return
	 */
	public Node<AstNode> getParseNodeTree()
	{
		if (parsingResult != null)
		{
			return parsingResult.parseTreeRoot;
		}
		return null;
	}

	public AstNode getAstTree()
	{
		Node<AstNode> nd = getParseNodeTree();
		return nd == null ? null : nd.getValue();
	}

	/**
	 * Dump AST tree 
	 */
	public void dumpTree()
	{
		FanUtilities.GENERIC_LOGGER.trace("-------------------Start AST Tree dump-----------------------");
		ParseTreeUtils.printNodeTree(parsingResult);
		FanUtilities.GENERIC_LOGGER.trace("-------------------End AST Tree dump-----------------------");
	}

	/**
	 * Shotcut method for getSnapshot().getSource().getDocument(true);
	 * @return
	 */
	public Document getDocument()
	{
		return getSnapshot().getSource().getDocument(true);
	}

	public void addError(String title, Throwable t)
	{
		// "High level error"
		Error error = DefaultError.createDefaultError(title, title, title, null, 0, 0, true, Severity.ERROR);
		errors.add(error);
	}

	/**
	 * The root scope
	 * @return
	 */
	public AstNode getRootScope()
	{
		return astRoot;
	}

	public FileObject getSourceFile()
	{
		return sourceFile;
	}

	/**
	 * Add an error (not the parser errors, but others like semantic etc..)
	 * @param info
	 * @param node
	 */
	public void addError(String info, AstNode node)
	{
		if (node == null)
		{
			return;
		}
		String key = "FanParserTask";
		OffsetRange range = node.getRelevantTextRange();
		int start = range.getStart();
		int end = range.getEnd();
		//System.out.println("Start: "+start+"End:"+end);
		Error error = DefaultError.createDefaultError(key, info, "Syntax Error", sourceFile, start, end, true, Severity.ERROR);
		errors.add(error);
	}

	/**
	 * Parse the file (using parboiled FantomParser)
	 */
	public void parse()
	{
		long start = new Date().getTime();
		System.out.println("Starting parsing of: " + sourceName);

		try
		{
			//TODO: can we drop out of there if invalidated ?
			try
			{
				parsingResult = RecoveringParseRunner.run(parser.compilationUnit(), getSnapshot().getText().toString());
			} catch (IllegalStateException e)
			{
				// The task was cancelled, bailing out
				System.out.print("Parser task was cancelled.");
				return;
			}
			// Copy parboiled parse error into a CSL errrors
			for (ParseError err : parsingResult.parseErrors)
			{
				// key, displayName, description, file, start, end, lineError?, severity
				String msg = ErrorUtils.printParseError(err, parsingResult.inputBuffer);
				Error error = DefaultError.createDefaultError(msg, msg, msg,
						sourceFile, err.getErrorLocation().getIndex(), err.getErrorLocation().getIndex() + err.getErrorCharCount(),
						false, Severity.ERROR);
				errors.add(error);
			}
			if (parsingResult.parseTreeRoot != null)
			{
				astRoot = parsingResult.parseTreeRoot.getValue();
				// Removed orphaned(backtracked) AST nodes.
				prune(astRoot);
			}
		} catch (Exception e)
		{
			addError("Parser error", e);
			e.printStackTrace();
		}

		FanUtilities.GENERIC_LOGGER.info("Parsing completed in " + (new Date().getTime() - start) + " for : " + sourceName);
	}

	/**
	 * Call after parsing to add scope variables / type resolution to the AST tree
	 * This does the "global" items - as needed by the indexer
	 * such as types, slots & their params
	 */
	@SuppressWarnings("unchecked")
	public void parseGlobalScope()
	{
		long start = new Date().getTime();
		FanUtilities.GENERIC_LOGGER.debug("Starting parsing scope of: " + sourceName);
		if (astRoot == null)
		{
			return;
		}
		// First run : lookup using statements
		for (AstNode node : astRoot.getChildren())
		{
			if (invalidated)
			{
				return; // bailing out if task cancelled
			}
			switch (node.getKind())
			{
				case AST_INC_USING:
					addError("Incomplete import statement", node);
					break;

				case AST_USING:
					addUsing(node);
					break;

			}
		}

		// Second pass, lookup types and slots
		for (AstNode node : astRoot.getChildren())
		{
			if (invalidated)
			{
				return; // bailing out if task cancelled
			}
			switch (node.getKind())
			{
				case AST_TYPE_DEF:
					String name = FanLexAstUtils.getFirstChildText(node, new NodeKindPredicate(AstKind.AST_ID));
					if (name == null)
					{
						break;
					}
					FanTypeScopeVar var = new FanTypeScopeVar(node, name);
					AstNode scopeNode = FanLexAstUtils.getScopeNode(node.getRoot());
					// We parse the type base first and add it to scope right away
					// So that parseSlots() can later resolve this & super.
					System.out.print(name);
					var.parse();
					if (scopeNode.getAllScopeVars().containsKey(name)
							&& // If we have a "suing" with the same name, we take precedence
							scopeNode.getAllScopeVars().get(name).getKind() != VarKind.IMPORT)
					{
						addError("Duplicated type name", node);
					} else
					{
						scopeNode.getLocalScopeVars().put(name, var);
					}
					// Parse the slot definitions
					var.parseSlots(this);
					break;
			}
		}

		if (dumpTree)
		{
			FanLexAstUtils.dumpTree(astRoot, 0);
		}
		FanUtilities.GENERIC_LOGGER.info("Parsing of scope completed in " + (new Date().getTime() - start) + " for : " + sourceName);
	}
	//TODO: don't show the whole stack of errors, but just the base.
	// esp. for expressions, calls etc...

	/**
	 * This parses the local scopes (inside slots)
	 * This is not needed by the indexer, so we don't do it when called
	 * from the indexer (see NBFanParser)
	 * It highlights errors as well
	 */
	@SuppressWarnings("unchecked")
	public void parseLocalScopes()
	{
		// don't allow multiple calls
		synchronized (this)
		{
			if (localScopeDone)
			{
				return;
			}
			localScopeDone = true;
		}
		// Now do all the local scopes / variables
		for (AstNode node : astRoot.getChildren())
		{
			if (node.getKind() == AstKind.AST_TYPE_DEF)
			{
				for (FanAstScopeVarBase var : node.getLocalScopeVars().values())
				{
					if (invalidated)
					{
						return; // bailing out if task cancelled
					}					// should be slots
					AstNode bkNode = var.getNode();
					AstNode blockNode = FanLexAstUtils.getFirstChild(bkNode, new NodeKindPredicate(AstKind.AST_BLOCK));
					if (blockNode != null)
					{
						try
						{
							parseVars(blockNode, null);
						} catch (IllegalStateException e)
						{
							return; // task cancelled
						}
					}
				}
			}
		}
	}

	/**
	 * Recursive - called by parseLocalScope
	 * Parse the blocks / expressions
	 * @param node
	 * @param type
	 */
	@SuppressWarnings("unchecked")
	public void parseVars(AstNode node, FanResolvedType type)
	{
		if (invalidated)
		{
			throw new IllegalStateException("Parser task was invalidated");
		}

		if (node == null)
		{
			return;
		}
		// If base type is unknown ... so are child
		if (type instanceof FanUnknownType)
		{
			node.setType(type);
			// Note: all children(if any) will be "unknown" as well.
			for (AstNode nd : node.getChildren())
			{
				parseVars(nd, type);
			}
			return;
		}

		String text = node.getNodeText(true);
		try
		{
			switch (node.getKind())
			{
				/*case AST_FIELD_ACCESSOR:
				AstNode fNode = FanLexAstUtils.findParentNode(node, AstKind.AST_FIELD_DEF);
				if (fNode != null)
				{
				AstNode fieldType = FanLexAstUtils.getFirstChild(fNode, new NodeKindPredicate(AstKind.AST_TYPE));
				// introduce "it" to scope
				FanAstScopeVarBase itVar = new FanLocalScopeVar(node, VarKind.IMPLIED, "it", fieldType.getType());
				node.addScopeVar(itVar, true);
				}
				parseChildren(node);
				break;*/
				case AST_EXPR_INDEX:
					AstNode indexEpr = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_EXPR));
					type = doIndexExpr(indexEpr, type);
					break;
				case AST_LIST:
					// Index expressions, sometimes get parsed as Lists because the parser doesn't know a Type vs a variable
					// so str[0] gets parsed as a list (like Str[0] would be) rather than an index expr
					// so dolist takes care of that issue.
					type = doList(node, type);
					break;
				case AST_MAP:
					type = doMap(node, type);
					break;
				case AST_EXPR:
				case AST_EXPR_ASSIGN: // with the assignment we need reset type to null (start a new expression)
				case AST_EXPR_MULT:
				case AST_EXPR_ADD:
					type = doExpr(node, type);
					break;
				case AST_CALL_EXPR:
					// there is also an operator node, but we ignore it - will be used in doCall()
					AstNode callNode = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_CALL));
					parseVars(callNode, type);
					type = callNode.getType();
					break;
				case AST_CLOSURE:
					// Only comne here for a closure that is NOT part of a call (function def)
					// closure calls are handled by docall()
					type = doClosure(node, null);
					break;
				case AST_CALL:
					type = doCall(node, type, null);
					break;
				case AST_ARG:
					// arg contains one expression - parse it to check for errors
					AstNode argExprNode = node.getChildren().get(0);
					parseVars(argExprNode, null);
					type = argExprNode.getType();
					break;
				case AST_CHILD: // a wrapper node (takes type from wrapped node)
					parseChildren(node);
					if (node.getChildren().size() != 1)
					{
						throw new RuntimeException("AST_CHILD should have only one child node");
					}
					type = node.getChildren().get(0).getType();
					break;
				case AST_EXR_CAST:
					// take the cast type
					AstNode castTypeNode = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_TYPE));
					parseVars(castTypeNode, null);
					type = castTypeNode.getType();
					// then parse the expression separately
					AstNode castExpr = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_EXPR));
					parseVars(castExpr, null);
					//TODO: check if cast is valid
					break;
				case AST_EXPR_TYPE_CHECK:
					type = doTypeCheckExpr(node, type);
					break;
				case AST_EXPR_RANGE:
					// if only one child, then it's really not a rangeExpr, but an addExpr
					if (node.getChildren().size() == 1)
					{
						parseVars(node.getChildren().get(0), type);
						type = node.getChildren().get(0).getType();
					} else
					{
						type = doRangeExpr(node, type);
					}
					break;
				case AST_IT_BLOCK:
					// Note that closure itBlocks are dealt wiht separately, see doExpr().
					type = doItBlock(node, type);
					break;
				case AST_CATCH_BLOCK:
					type = doCatchBlock(node, type);
					break;
				case AST_EXPR_LIT_BASE:
					Node<AstNode> parseNode = node.getParseNode().getChildren().get(0); // firstOf
					type = resolveLitteral(node, parseNode);
					break;
				case AST_ID:
					type = FanResolvedType.makeFromTypeSig(node, text);
					break;
				case AST_TYPE:
					type = FanResolvedType.makeFromTypeSig(node, text);
					break;
				case AST_LOCAL_DEF: // special case, since it introduces scope vars
					type = doLocalDef(node, type);
					break;
				case AST_TYPE_LITTERAL: // 'Type#' or '#slot' or 'Type#slot'
					type = doTypeLitteral(node, type);
					break;
				default:
					// recurse into children
					parseChildren(node);
			}
		} catch (Exception e)
		{
			// We don't want exception to be propagated to user as an exception (prevents fixing it in IDE)
			// Do mark a global parsing error however
			type = FanResolvedType.makeUnresolved(node);
			addError("Unexpected Parsing error: " + e.toString(), node);
			FanUtilities.GENERIC_LOGGER.exception("Error parsing node: " + text, e);
		}
		node.setType(type);
		if (type != null && !type.isResolved())
		{
			addError("Could not resolve item -> " + text, node);

			FanUtilities.GENERIC_LOGGER.info(">Unresolved node");
			FanLexAstUtils.dumpTree(node, 0);
			FanUtilities.GENERIC_LOGGER.info("<Unresolved node");
		}
	}

	private void parseChildren(AstNode node)
	{
		for (AstNode child : node.getChildren())
		{
			parseVars(child, null);
		}
	}

	public FanResolvedType resolveLitteral(AstNode astNode, Node<AstNode> parseNode)
	{
		FanResolvedType type = FanResolvedType.makeUnresolved(astNode);
		String lbl = parseNode.getLabel();
		String txt = astNode.getNodeText(true);
		if (lbl.equalsIgnoreCase(TokenName.ID.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, txt);
		} else if (lbl.equalsIgnoreCase(TokenName.CHAR_.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Int");
		} else if (lbl.equalsIgnoreCase(TokenName.NUMBER.name()))
		{
			char lastChar = txt.charAt(txt.length() - 1);
			if (lastChar == 'f' || lastChar == 'F')
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Float");
			} else if (lastChar == 'd' || lastChar == 'D')
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Decimal");
			} else if (Character.isLetter(lastChar) && lastChar != '_')
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Duration");
			} else if (txt.indexOf(".") != -1)
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Float");
			} else
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Int"); // Default
			}
		} else if (lbl.equalsIgnoreCase(TokenName.STRS.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Str");
		} else if (lbl.equalsIgnoreCase(TokenName.URI.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Uri");
		} else if (lbl.equals("true") || lbl.equals("false"))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Bool");
		} else if (lbl.equals("null"))
		{
			type = new FanResolvedNullType(astNode);
		} else if (lbl.equals("it"))
		{
			FanAstScopeVarBase var = astNode.getAllScopeVars().get("it");
			if (var != null)
			{
				type = var.getType();
			}
			type = type.asStaticContext(false);
		} else if (lbl.equals("this"))
		{
			type = FanResolvedType.resolveThisType(astNode);
			type = type.asStaticContext(false);
		} else if (lbl.equals("super"))
		{
			type = FanResolvedType.resolveSuper(astNode);
			type = type.asStaticContext(false);
		}
		return type;
	}

	@SuppressWarnings("unchecked")
	private void addUsing(AstNode usingNode)
	{
		String type = FanLexAstUtils.getFirstChildText(usingNode, new NodeKindPredicate(AstKind.AST_ID));
		String as = FanLexAstUtils.getFirstChildText(usingNode, new NodeKindPredicate(AstKind.AST_USING_AS));
		String ffi = FanLexAstUtils.getFirstChildText(usingNode, new NodeKindPredicate(AstKind.AST_USING_FFI));

		String name = as != null ? as : type;
		if (name.indexOf("::") > -1)
		{
			name = name.substring(name.indexOf("::") + 2);
		}

		if (ffi != null && ffi.toLowerCase().equals("java"))
		{
			if (type.indexOf("::") != -1)
			{
				// Individual Item
				String qname = type.replaceAll("::", "\\.");

				if (findCachedQualifiedType(qname) == null)
				{
					addError("Unresolved Java Item: " + qname, usingNode);
				} else
				{
					addUsing(name, qname, usingNode);
				}
			} else
			{
				// whole package
				if (!FanType.hasPod(name))
				{
					addError("Unresolved Java package: " + name, usingNode);
				} else
				{
					Vector<FanType> items = FanType.findPodTypes(name, "");
					for (FanType t : items)
					{
						addUsing(t.getSimpleName(), t.getQualifiedName(), usingNode);
					}
				}
			}
		} else
		{
			if (type.indexOf("::") > 0)
			{
				// Adding a specific type
				String[] data = type.split("::");
				if (!FanType.hasPod(data[0]))
				{
					addError("Unresolved Pod: " + data[0], usingNode);
				} else if (findCachedQualifiedType(type) == null)
				{
					addError("Unresolved Type: " + type, usingNode);
				}

				//Type t = FanPodIndexer.getInstance().getPodType(data[0], data[1]);
				addUsing(name, type, usingNode);
			} else
			{
				// Adding all the types of a Pod
				if (name.equalsIgnoreCase("sys")) // sys is always avail.
				{
					return;
				}
				if (!FanType.hasPod(name))
				{
					addError("Unresolved Pod: " + name, usingNode);
				} else
				{
					Vector<FanType> items = FanType.findPodTypes(name, "");
					for (FanType t : items)
					{
						addUsing(t.getSimpleName(), t.getQualifiedName(), usingNode);
					}
				}
			}
		}
	}

	private void addUsing(String name, String qType, AstNode node)
	{
		AstNode scopeNode = FanLexAstUtils.getScopeNode(node);
		if (scopeNode == null)
		{
			return;
		}
		if (scopeNode.getLocalScopeVars().containsKey(name))
		{
			// This is 'legal' ... maybe show a warning later ?
			//addError("Duplicated using: " + qType + " / " + scopeNode.getLocalScopeVars().get(name), node);
			System.out.println("Already have a using called: " + qType + " (" + scopeNode.getLocalScopeVars().get(name) + ")");
			// Note: only keeping the 'last' definition (ie: override)
		}
		FanResolvedType rType = FanResolvedType.makeFromDbType(node, qType);
		rType = rType.asStaticContext(true);
		scopeNode.addScopeVar(name, FanAstScopeVar.VarKind.IMPORT, rType, true);
	}

	public ParsingResult<AstNode> getParsingResult()
	{
		return parsingResult;
	}

	public String getPod()
	{
		return pod;
	}

	/**
	 * TODO: this whole prunning stuff is a bit ugly
	 * Should try to buod the AST properly using technizues here:
	 * http://parboiled.hostingdelivered.com/viewtopic.php?f=3&t=9
	 *
	 * During ParseNode construction, some astNodes that migth have been constructed from
	 * some parseNode that where then "backtracked" (not the whoel sequence matched)
	 * This looks for and remove those unwanted nodes.
	 * @param node
	 */
	private void prune(AstNode node)
	{
		Node<AstNode> rtNode = astRoot.getParseNode();
		String rootLabel = "n/a";
		while (rtNode != null)
		{
			rootLabel = rtNode.getLabel();
			rtNode = rtNode.getParent();
		}
		List<AstNode> children = node.getChildren();
		List<AstNode> toBepruned = new ArrayList<AstNode>();
		for (AstNode child : children)
		{
			Node<AstNode> parseNode = child.getParseNode();
			// If the node is orphaned (no link back to the root), that means it was backtracked out of.
			String label = "N/A";
			while (parseNode != null)
			{
				label = parseNode.getLabel();
				parseNode = parseNode.getParent();
			}
			if (!rootLabel.equals(label))
			{
				toBepruned.add(child);
			} else
			{
				// recurse into children
				prune(child);
			}
		}
		// Drop the orphaned nodes
		for (AstNode nd : toBepruned)
		{
			children.remove(nd);
		}
	}

	@SuppressWarnings("unchecked")
	private FanResolvedType doExpr(AstNode node, FanResolvedType type)
	{
		// TODO: validate assignment type is compatible.
		boolean first = true;
		type = null;
		List<AstNode> children = node.getChildren();
		for (int i = 0; i != children.size(); i++)
		{
			// Check for special procedure
			// when a itBlock follows a call, it's actually a sort of closure call (with block)
			AstNode child = children.get(i);
			if (child.getKind() == AstKind.AST_CALL_EXPR)
			{
				if (i + 1 < children.size())
				{
					AstNode nextChild = children.get(i + 1);
					if (nextChild.getKind() == AstKind.AST_IT_BLOCK)
					{
						AstNode callNode = FanLexAstUtils.getFirstChild(child, new NodeKindPredicate(AstKind.AST_CALL));
						// parse the whole thing as a closure call
						type = doCall(callNode, type, nextChild);
						// skp next child since we juts did it
						i++;
						continue;
					}
				}
			}

			// Normal procedure
			parseVars(child, type);
			// we take type of right hand side if first in expression
			// and not one of the special type we don't want the rhs for
			if (first || (child.getKind() != AstKind.AST_EXPR_ADD // add/mult operation cannot chnage the type
					&& child.getKind() != AstKind.AST_EXPR_MULT //same
					/*&& child.getKind() != AstKind.AST_EXPR*/)) // new/sub expression
			{
				type = child.getType();
				// If part of the expr chain is unresolved (error), mark it unresolved
				// This avoids getting errors for the whole expression chain
				if (type != null && !type.isResolved())
				{
					type = new FanUnknownType(node, child.getNodeText(true));
				}
			} //else
			//{
			//	System.out.println("Not taking rhs type for "+node.getKind());
			// TODO: check what doesn't since it's probably less common than what does
			//}
			first = false;
		}
		return type;
	}

	/**
	 *
	 * @param node
	 * @param type
	 * @param forcedClosure :  when we have an itBlock, we need to send it as if it was a closure
	 * @return
	 */
	@SuppressWarnings("unchecked")
	private FanResolvedType doCall(AstNode node, FanResolvedType type, AstNode forcedClosure)
	{
		// saving the base type, because we need it for closures
		FanResolvedType baseType = type;
		AstNode callChild = node.getChildren().get(0);
		String name = callChild.getNodeText(true);

		//if a direct call like doThis(), then search scope for type/slot
		if (type == null)
		{
			baseType = FanResolvedType.resolveItType(callChild);
			// constructor call or local slot
			type = FanResolvedType.makeFromTypeSig(callChild, name);
			type = type.parameterize(baseType, callChild);
			if (type.isStaticContext())
			{ // It was a constructor call
				baseType = type;
				name = "make";
			}
		} else
		// otherwise a slot of the base type like var.toStr()
		{
			// checking what call op we are dealing with
			String op = ".";
			AstNode callExpr = FanLexAstUtils.findParentNode(node, AstKind.AST_CALL_EXPR);
			if (callExpr != null)
			{
				AstNode operator = FanLexAstUtils.getFirstChild(callExpr, new NodeKindPredicate(AstKind.LBL_OP));
				if (operator != null)
				{
					op = operator.getNodeText(true);
				}
			}
			// Can't resolve dynamic calls -> unknow type
			if (op.endsWith("->"))
			{
				return new FanUnknownType(node, sourceName);
			}

			// If using null check call on non nullable object, show an error, but continue resolution
			if (!type.isNullable() && op.startsWith("?"))
			{
				addError("Null check call on non-nullable: " + type.toString(), node);
			}

			//if(! (type instanceof FanUnknownType))
			type = type.resolveSlotType(name, this);
			type = type.parameterize(baseType, node);
		}

		// parse args
		List<AstNode> args = FanLexAstUtils.getChildren(node, new NodeKindPredicate(AstKind.AST_ARG));
		int argIndex = 0;
		for (AstNode arg : args)
		{
			if (FanLexAstUtils.isWrappingNode(arg, new NodeKindPredicate(AstKind.AST_CLOSURE)))
			{
				// If the argument is a closure, we need to call doClosure directly
				AstNode closure = FanLexAstUtils.getFirstChildRecursive(arg, new NodeKindPredicate(AstKind.AST_CLOSURE));
				doClosureCall(closure, baseType, name, argIndex);
			} else
			{
				// otherwsie, do normal parsing
				parseVars(arg, null);
			}
			argIndex++;
		}

		// deal with the sepcial trailing closure argument (if any)
		//TODO: Check that param types matches slot declaration
		AstNode closure = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_CLOSURE));
		if (closure != null || forcedClosure != null)
		{
			doClosureCall(forcedClosure != null ? forcedClosure : closure, baseType, name, argIndex);
		}
		// we need to parameterize the result
		// if baseType is null, then it probably couldn't return generics and if it did, then it would be "this", so leave it alone either way
		if (baseType != null)
		{
			type = type.parameterize(baseType, node);
		}
		return type;
	}

	private void doClosureCall(AstNode closureNode, FanResolvedType baseType, String slotName, int argIndex)
	{
		//TODO: Check that param types matches slot declaration
		FanResolvedType slotBase = baseType.resolveSlotBaseType(slotName, this);
		FanSlot slot = FanSlot.findByTypeAndName(slotBase.getQualifiedType(), slotName);
		if (slot == null)
		{
			addError("Can't call closure on " + slotBase.getQualifiedType() + "." + slotName, closureNode);
		} else
		{
			List<FanMethodParam> params = FanMethodParam.findAllForSlot(slot.getId());
			if (argIndex >= params.size())
			{
				addError("Can't call closure (too many parameters) on " + slotBase.getQualifiedType() + "." + slotName, closureNode);
			} else
			{
				FanResolvedType func = FanResolvedType.makeFromTypeSig(closureNode, params.get(argIndex).getQualifiedType());
				if (!((func instanceof FanResolvedFuncType) || func.isTypeCompatible(FanResolvedType.makeFromDbType(closureNode, "sys::Func"))))
				{
					addError("Closure call, but not expecting a function at parameter " + (argIndex + 1) + " in " + slotBase.getQualifiedType() + "." + slotName, closureNode);
				} else
				{
					// build lits of expected params for doClosure to use for inferrence resolution
					List<FanResolvedType> infTypes = new ArrayList<FanResolvedType>();
					if (func instanceof FanResolvedFuncType)
					{ // Otherwise it's a generic sys::Func -> no inference possible
						for (FanResolvedType t : ((FanResolvedFuncType) func).getTypes())
						{
							// might be a generic type
							t = t.parameterize(baseType, closureNode);
							infTypes.add(t);
						}
					}
					// parse the closure (but does not take it's type)
					if (closureNode.getKind() == AstKind.AST_IT_BLOCK)
					{
						if (infTypes.size() == 0)
						{
							addError("No parmeters expected in function, can't do itBlock closure", closureNode);
						} else
						{
							doItBlock(closureNode, infTypes.get(0));
						}
					} else
					{
						doClosure(closureNode, infTypes);
					}
				}
			}
		}
	}

	/**
	 * Simple closure like |Str a, Int b->| { echo("hi there") }
	 * a.k.a closure definition
	 * @param node
	 * @param callAgainst : When called against a function (call) pass the expected parameters (for inference)
	 * @return
	 */
	@SuppressWarnings("unchecked")
	private FanResolvedFuncType doClosure(AstNode node, List<FanResolvedType> paramTypes)
	{
		FanResolvedType retType = FanResolvedType.makeFromDbType(node, "sys::Void");
		AstNode closureBlock = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_BLOCK));
		// Will store resolved types
		List<FanResolvedType> formalTypes = new ArrayList<FanResolvedType>();
		int cpt = 0;
		for (AstNode child : node.getChildren())
		{
			if (child.getKind() == AstKind.AST_FORMAL)
			{
				FanResolvedType t = doFormal(child, closureBlock, paramTypes, cpt);
				if (t != null && !t.isResolved())
				{
					addError("Couldn't resolve formal definition: " + node.getNodeText(true), node);
				}
				formalTypes.add(t);
			} else if (child.getKind() == AstKind.AST_TYPE)
			{
				// save the returned type
				parseVars(child, null);
				retType = child.getType();
			} else if (child.getKind() == AstKind.AST_BLOCK)
			{
				// parse the block content
				parseVars(child, null);
			}
			cpt++;
		}
		return new FanResolvedFuncType(node, formalTypes, retType);
	}

	/**
	 * Deal with a formal (as used in a call)
	 * @param node
	 * @param baseType
	 * @param closureBlock
	 * @param formalTypes
	 * @param index
	 * @return
	 */
	@SuppressWarnings("unchecked")
	private FanResolvedType doFormal(AstNode node, AstNode closureBlock, List<FanResolvedType> inferredTypes, int index)
	{
		AstNode formalName = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_ID));
		AstNode formalTypeAndId = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_TYPE_AND_ID));
		FanResolvedType fType = FanResolvedType.makeFromDbType(node, "sys::Void");
		if (formalTypeAndId != null)
		{	// typed formal
			formalName = FanLexAstUtils.getFirstChild(formalTypeAndId, new NodeKindPredicate(AstKind.AST_ID));
			AstNode formalType = FanLexAstUtils.getFirstChild(formalTypeAndId, new NodeKindPredicate(AstKind.AST_TYPE));
			parseVars(formalType, null);
			fType = formalType.getType().asStaticContext(false);
			// TODO: check that it matches callParams
		} else
		{
			if (inferredTypes == null)
			{
				addError("Can't have inferred formals in closure definition.", node);
				fType = FanResolvedType.makeUnresolved(node);
			} else if (index > inferredTypes.size())
			{
				addError("More inferred formals than expected.", node);
				fType = FanResolvedType.makeUnresolved(node);
			} else
			{<commit_msg>Couple of closure typing fixes: - fixed bug when closure not 1st arg - allowing for inference is closure signature (Obj?)<commit_after>/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package net.colar.netbeans.fan;

import java.io.File;
import java.lang.reflect.Field;
import java.lang.reflect.Member;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Vector;
import javax.swing.text.Document;
import net.colar.netbeans.fan.indexer.FanIndexerFactory;
import net.colar.netbeans.fan.indexer.model.FanMethodParam;
import net.colar.netbeans.fan.indexer.model.FanSlot;
import net.colar.netbeans.fan.scope.FanAstScopeVar;
import net.colar.netbeans.fan.parboiled.FanLexAstUtils;
import net.colar.netbeans.fan.indexer.model.FanType;
import net.colar.netbeans.fan.parboiled.AstKind;
import net.colar.netbeans.fan.parboiled.AstNode;
import net.colar.netbeans.fan.parboiled.FantomParser;
import net.colar.netbeans.fan.parboiled.FantomLexerTokens.TokenName;
import net.colar.netbeans.fan.parboiled.pred.NodeKindPredicate;
import net.colar.netbeans.fan.scope.FanAstScopeVarBase;
import net.colar.netbeans.fan.scope.FanAstScopeVarBase.VarKind;
import net.colar.netbeans.fan.scope.FanLocalScopeVar;
import net.colar.netbeans.fan.scope.FanMethodScopeVar;
import net.colar.netbeans.fan.scope.FanTypeScopeVar;
import net.colar.netbeans.fan.types.FanResolvedFuncType;
import net.colar.netbeans.fan.types.FanResolvedListType;
import net.colar.netbeans.fan.types.FanResolvedMapType;
import net.colar.netbeans.fan.types.FanResolvedNullType;
import net.colar.netbeans.fan.types.FanResolvedType;
import net.colar.netbeans.fan.types.FanUnknownType;
import org.netbeans.modules.csl.api.Error;
import org.netbeans.modules.csl.api.OffsetRange;
import org.netbeans.modules.csl.api.Severity;
import org.netbeans.modules.csl.spi.DefaultError;
import org.netbeans.modules.csl.spi.ParserResult;
import org.netbeans.modules.parsing.api.Snapshot;
import org.openide.filesystems.FileObject;
import org.openide.filesystems.FileUtil;
import org.parboiled.Node;
import org.parboiled.Parboiled;
import org.parboiled.RecoveringParseRunner;
import org.parboiled.errors.ErrorUtils;
import org.parboiled.errors.ParseError;
import org.parboiled.support.ParseTreeUtils;
import org.parboiled.support.ParsingResult;

/**
 * Parse a fan file and holds the results
 * parse() parses the file
 * parseScope() adds to the tree scope vraiables etc...
 * @author  tcolar
 */
public class FanParserTask extends ParserResult
{

	boolean invalidated = false;
	public boolean dumpTree = false; // debug
	List<Error> errors = new Vector<Error>(); // -> use parsingResult.errors ?
	// full path of the source file
	private final FileObject sourceFile;
	// simple name of the source file
	private final String sourceName;
	// pod name
	private final String pod;
	// once parse() is called, will contain the parboiled parsing result
	private ParsingResult<AstNode> parsingResult;
	private AstNode astRoot;
	// Cache types resolution, for performance
	private HashMap<String, FanType> typeCache = new HashMap<String, FanType>();
	// Cache slots resolution, for performance
	private HashMap<String, List<FanSlot>> typeSlotsCache = new HashMap<String, List<FanSlot>>();
	private FantomParser parser;
	private boolean localScopeDone;

	public FanParserTask(Snapshot snapshot)
	{
		super(snapshot);
		invalidated = false;
		sourceName = snapshot==null?null:snapshot.getSource().getFileObject().getName();
		sourceFile = snapshot==null?null:FileUtil.toFileObject(new File(snapshot.getSource().getFileObject().getPath()));
		pod = snapshot==null?null:FanUtilities.getPodForPath(sourceFile.getPath());
		parser = Parboiled.createParser(FantomParser.class, this);
	}

	@Override
	public List<? extends Error> getDiagnostics()
	{
		return errors;
	}

	@Override
	protected void invalidate()
	{
	}

	/**
	 * Return AST tree generated by this parsing
	 * @return
	 */
	public Node<AstNode> getParseNodeTree()
	{
		if (parsingResult != null)
		{
			return parsingResult.parseTreeRoot;
		}
		return null;
	}

	public AstNode getAstTree()
	{
		Node<AstNode> nd = getParseNodeTree();
		return nd == null ? null : nd.getValue();
	}

	/**
	 * Dump AST tree 
	 */
	public void dumpTree()
	{
		FanUtilities.GENERIC_LOGGER.trace("-------------------Start AST Tree dump-----------------------");
		ParseTreeUtils.printNodeTree(parsingResult);
		FanUtilities.GENERIC_LOGGER.trace("-------------------End AST Tree dump-----------------------");
	}

	/**
	 * Shotcut method for getSnapshot().getSource().getDocument(true);
	 * @return
	 */
	public Document getDocument()
	{
		return getSnapshot().getSource().getDocument(true);
	}

	public void addError(String title, Throwable t)
	{
		// "High level error"
		Error error = DefaultError.createDefaultError(title, title, title, null, 0, 0, true, Severity.ERROR);
		errors.add(error);
	}

	/**
	 * The root scope
	 * @return
	 */
	public AstNode getRootScope()
	{
		return astRoot;
	}

	public FileObject getSourceFile()
	{
		return sourceFile;
	}

	/**
	 * Add an error (not the parser errors, but others like semantic etc..)
	 * @param info
	 * @param node
	 */
	public void addError(String info, AstNode node)
	{
		if (node == null)
		{
			return;
		}
		String key = "FanParserTask";
		OffsetRange range = node.getRelevantTextRange();
		int start = range.getStart();
		int end = range.getEnd();
		//System.out.println("Start: "+start+"End:"+end);
		Error error = DefaultError.createDefaultError(key, info, "Syntax Error", sourceFile, start, end, true, Severity.ERROR);
		errors.add(error);
	}

	/**
	 * Parse the file (using parboiled FantomParser)
	 */
	public void parse()
	{
		long start = new Date().getTime();
		System.out.println("Starting parsing of: " + sourceName);

		try
		{
			//TODO: can we drop out of there if invalidated ?
			try
			{
				parsingResult = RecoveringParseRunner.run(parser.compilationUnit(), getSnapshot().getText().toString());
			} catch (IllegalStateException e)
			{
				// The task was cancelled, bailing out
				System.out.print("Parser task was cancelled.");
				return;
			}
			// Copy parboiled parse error into a CSL errrors
			for (ParseError err : parsingResult.parseErrors)
			{
				// key, displayName, description, file, start, end, lineError?, severity
				String msg = ErrorUtils.printParseError(err, parsingResult.inputBuffer);
				Error error = DefaultError.createDefaultError(msg, msg, msg,
						sourceFile, err.getErrorLocation().getIndex(), err.getErrorLocation().getIndex() + err.getErrorCharCount(),
						false, Severity.ERROR);
				errors.add(error);
			}
			if (parsingResult.parseTreeRoot != null)
			{
				astRoot = parsingResult.parseTreeRoot.getValue();
				// Removed orphaned(backtracked) AST nodes.
				prune(astRoot);
			}
		} catch (Exception e)
		{
			addError("Parser error", e);
			e.printStackTrace();
		}

		FanUtilities.GENERIC_LOGGER.info("Parsing completed in " + (new Date().getTime() - start) + " for : " + sourceName);
	}

	/**
	 * Call after parsing to add scope variables / type resolution to the AST tree
	 * This does the "global" items - as needed by the indexer
	 * such as types, slots & their params
	 */
	@SuppressWarnings("unchecked")
	public void parseGlobalScope()
	{
		long start = new Date().getTime();
		FanUtilities.GENERIC_LOGGER.debug("Starting parsing scope of: " + sourceName);
		if (astRoot == null)
		{
			return;
		}
		// First run : lookup using statements
		for (AstNode node : astRoot.getChildren())
		{
			if (invalidated)
			{
				return; // bailing out if task cancelled
			}
			switch (node.getKind())
			{
				case AST_INC_USING:
					addError("Incomplete import statement", node);
					break;

				case AST_USING:
					addUsing(node);
					break;

			}
		}

		// Second pass, lookup types and slots
		for (AstNode node : astRoot.getChildren())
		{
			if (invalidated)
			{
				return; // bailing out if task cancelled
			}
			switch (node.getKind())
			{
				case AST_TYPE_DEF:
					String name = FanLexAstUtils.getFirstChildText(node, new NodeKindPredicate(AstKind.AST_ID));
					if (name == null)
					{
						break;
					}
					FanTypeScopeVar var = new FanTypeScopeVar(node, name);
					AstNode scopeNode = FanLexAstUtils.getScopeNode(node.getRoot());
					// We parse the type base first and add it to scope right away
					// So that parseSlots() can later resolve this & super.
					System.out.print(name);
					var.parse();
					if (scopeNode.getAllScopeVars().containsKey(name)
							&& // If we have a "suing" with the same name, we take precedence
							scopeNode.getAllScopeVars().get(name).getKind() != VarKind.IMPORT)
					{
						addError("Duplicated type name", node);
					} else
					{
						scopeNode.getLocalScopeVars().put(name, var);
					}
					// Parse the slot definitions
					var.parseSlots(this);
					break;
			}
		}

		if (dumpTree)
		{
			FanLexAstUtils.dumpTree(astRoot, 0);
		}
		FanUtilities.GENERIC_LOGGER.info("Parsing of scope completed in " + (new Date().getTime() - start) + " for : " + sourceName);
	}
	//TODO: don't show the whole stack of errors, but just the base.
	// esp. for expressions, calls etc...

	/**
	 * This parses the local scopes (inside slots)
	 * This is not needed by the indexer, so we don't do it when called
	 * from the indexer (see NBFanParser)
	 * It highlights errors as well
	 */
	@SuppressWarnings("unchecked")
	public void parseLocalScopes()
	{
		// don't allow multiple calls
		synchronized (this)
		{
			if (localScopeDone)
			{
				return;
			}
			localScopeDone = true;
		}
		// Now do all the local scopes / variables
		for (AstNode node : astRoot.getChildren())
		{
			if (node.getKind() == AstKind.AST_TYPE_DEF)
			{
				for (FanAstScopeVarBase var : node.getLocalScopeVars().values())
				{
					if (invalidated)
					{
						return; // bailing out if task cancelled
					}					// should be slots
					AstNode bkNode = var.getNode();
					AstNode blockNode = FanLexAstUtils.getFirstChild(bkNode, new NodeKindPredicate(AstKind.AST_BLOCK));
					if (blockNode != null)
					{
						try
						{
							parseVars(blockNode, null);
						} catch (IllegalStateException e)
						{
							return; // task cancelled
						}
					}
				}
			}
		}
	}

	/**
	 * Recursive - called by parseLocalScope
	 * Parse the blocks / expressions
	 * @param node
	 * @param type
	 */
	@SuppressWarnings("unchecked")
	public void parseVars(AstNode node, FanResolvedType type)
	{
		if (invalidated)
		{
			throw new IllegalStateException("Parser task was invalidated");
		}

		if (node == null)
		{
			return;
		}
		// If base type is unknown ... so are child
		if (type instanceof FanUnknownType)
		{
			node.setType(type);
			// Note: all children(if any) will be "unknown" as well.
			for (AstNode nd : node.getChildren())
			{
				parseVars(nd, type);
			}
			return;
		}

		String text = node.getNodeText(true);
		try
		{
			switch (node.getKind())
			{
				/*case AST_FIELD_ACCESSOR:
				AstNode fNode = FanLexAstUtils.findParentNode(node, AstKind.AST_FIELD_DEF);
				if (fNode != null)
				{
				AstNode fieldType = FanLexAstUtils.getFirstChild(fNode, new NodeKindPredicate(AstKind.AST_TYPE));
				// introduce "it" to scope
				FanAstScopeVarBase itVar = new FanLocalScopeVar(node, VarKind.IMPLIED, "it", fieldType.getType());
				node.addScopeVar(itVar, true);
				}
				parseChildren(node);
				break;*/
				case AST_EXPR_INDEX:
					AstNode indexEpr = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_EXPR));
					type = doIndexExpr(indexEpr, type);
					break;
				case AST_LIST:
					// Index expressions, sometimes get parsed as Lists because the parser doesn't know a Type vs a variable
					// so str[0] gets parsed as a list (like Str[0] would be) rather than an index expr
					// so dolist takes care of that issue.
					type = doList(node, type);
					break;
				case AST_MAP:
					type = doMap(node, type);
					break;
				case AST_EXPR:
				case AST_EXPR_ASSIGN: // with the assignment we need reset type to null (start a new expression)
				case AST_EXPR_MULT:
				case AST_EXPR_ADD:
					type = doExpr(node, type);
					break;
				case AST_CALL_EXPR:
					// there is also an operator node, but we ignore it - will be used in doCall()
					AstNode callNode = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_CALL));
					parseVars(callNode, type);
					type = callNode.getType();
					break;
				case AST_CLOSURE:
					// Only comne here for a closure that is NOT part of a call (function def)
					// closure calls are handled by docall()
					type = doClosure(node, null, 0);
					break;
				case AST_CALL:
					type = doCall(node, type, null);
					break;
				case AST_ARG:
					// arg contains one expression - parse it to check for errors
					AstNode argExprNode = node.getChildren().get(0);
					parseVars(argExprNode, null);
					type = argExprNode.getType();
					break;
				case AST_CHILD: // a wrapper node (takes type from wrapped node)
					parseChildren(node);
					if (node.getChildren().size() != 1)
					{
						throw new RuntimeException("AST_CHILD should have only one child node");
					}
					type = node.getChildren().get(0).getType();
					break;
				case AST_EXR_CAST:
					// take the cast type
					AstNode castTypeNode = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_TYPE));
					parseVars(castTypeNode, null);
					type = castTypeNode.getType();
					// then parse the expression separately
					AstNode castExpr = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_EXPR));
					parseVars(castExpr, null);
					//TODO: check if cast is valid
					break;
				case AST_EXPR_TYPE_CHECK:
					type = doTypeCheckExpr(node, type);
					break;
				case AST_EXPR_RANGE:
					// if only one child, then it's really not a rangeExpr, but an addExpr
					if (node.getChildren().size() == 1)
					{
						parseVars(node.getChildren().get(0), type);
						type = node.getChildren().get(0).getType();
					} else
					{
						type = doRangeExpr(node, type);
					}
					break;
				case AST_IT_BLOCK:
					// Note that closure itBlocks are dealt wiht separately, see doExpr().
					type = doItBlock(node, type);
					break;
				case AST_CATCH_BLOCK:
					type = doCatchBlock(node, type);
					break;
				case AST_EXPR_LIT_BASE:
					Node<AstNode> parseNode = node.getParseNode().getChildren().get(0); // firstOf
					type = resolveLitteral(node, parseNode);
					break;
				case AST_ID:
					type = FanResolvedType.makeFromTypeSig(node, text);
					break;
				case AST_TYPE:
					type = FanResolvedType.makeFromTypeSig(node, text);
					break;
				case AST_LOCAL_DEF: // special case, since it introduces scope vars
					type = doLocalDef(node, type);
					break;
				case AST_TYPE_LITTERAL: // 'Type#' or '#slot' or 'Type#slot'
					type = doTypeLitteral(node, type);
					break;
				default:
					// recurse into children
					parseChildren(node);
			}
		} catch (Exception e)
		{
			// We don't want exception to be propagated to user as an exception (prevents fixing it in IDE)
			// Do mark a global parsing error however
			type = FanResolvedType.makeUnresolved(node);
			addError("Unexpected Parsing error: " + e.toString(), node);
			FanUtilities.GENERIC_LOGGER.exception("Error parsing node: " + text, e);
		}
		node.setType(type);
		if (type != null && !type.isResolved())
		{
			addError("Could not resolve item -> " + text, node);

			FanUtilities.GENERIC_LOGGER.info(">Unresolved node");
			FanLexAstUtils.dumpTree(node, 0);
			FanUtilities.GENERIC_LOGGER.info("<Unresolved node");
		}
	}

	private void parseChildren(AstNode node)
	{
		for (AstNode child : node.getChildren())
		{
			parseVars(child, null);
		}
	}

	public FanResolvedType resolveLitteral(AstNode astNode, Node<AstNode> parseNode)
	{
		FanResolvedType type = FanResolvedType.makeUnresolved(astNode);
		String lbl = parseNode.getLabel();
		String txt = astNode.getNodeText(true);
		if (lbl.equalsIgnoreCase(TokenName.ID.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, txt);
		} else if (lbl.equalsIgnoreCase(TokenName.CHAR_.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Int");
		} else if (lbl.equalsIgnoreCase(TokenName.NUMBER.name()))
		{
			char lastChar = txt.charAt(txt.length() - 1);
			if (lastChar == 'f' || lastChar == 'F')
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Float");
			} else if (lastChar == 'd' || lastChar == 'D')
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Decimal");
			} else if (Character.isLetter(lastChar) && lastChar != '_')
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Duration");
			} else if (txt.indexOf(".") != -1)
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Float");
			} else
			{
				type = FanResolvedType.makeFromTypeSig(astNode, "sys::Int"); // Default
			}
		} else if (lbl.equalsIgnoreCase(TokenName.STRS.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Str");
		} else if (lbl.equalsIgnoreCase(TokenName.URI.name()))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Uri");
		} else if (lbl.equals("true") || lbl.equals("false"))
		{
			type = FanResolvedType.makeFromTypeSig(astNode, "sys::Bool");
		} else if (lbl.equals("null"))
		{
			type = new FanResolvedNullType(astNode);
		} else if (lbl.equals("it"))
		{
			FanAstScopeVarBase var = astNode.getAllScopeVars().get("it");
			if (var != null)
			{
				type = var.getType();
			}
			type = type.asStaticContext(false);
		} else if (lbl.equals("this"))
		{
			type = FanResolvedType.resolveThisType(astNode);
			type = type.asStaticContext(false);
		} else if (lbl.equals("super"))
		{
			type = FanResolvedType.resolveSuper(astNode);
			type = type.asStaticContext(false);
		}
		return type;
	}

	@SuppressWarnings("unchecked")
	private void addUsing(AstNode usingNode)
	{
		String type = FanLexAstUtils.getFirstChildText(usingNode, new NodeKindPredicate(AstKind.AST_ID));
		String as = FanLexAstUtils.getFirstChildText(usingNode, new NodeKindPredicate(AstKind.AST_USING_AS));
		String ffi = FanLexAstUtils.getFirstChildText(usingNode, new NodeKindPredicate(AstKind.AST_USING_FFI));

		String name = as != null ? as : type;
		if (name.indexOf("::") > -1)
		{
			name = name.substring(name.indexOf("::") + 2);
		}

		if (ffi != null && ffi.toLowerCase().equals("java"))
		{
			if (type.indexOf("::") != -1)
			{
				// Individual Item
				String qname = type.replaceAll("::", "\\.");

				if (findCachedQualifiedType(qname) == null)
				{
					addError("Unresolved Java Item: " + qname, usingNode);
				} else
				{
					addUsing(name, qname, usingNode);
				}
			} else
			{
				// whole package
				if (!FanType.hasPod(name))
				{
					addError("Unresolved Java package: " + name, usingNode);
				} else
				{
					Vector<FanType> items = FanType.findPodTypes(name, "");
					for (FanType t : items)
					{
						addUsing(t.getSimpleName(), t.getQualifiedName(), usingNode);
					}
				}
			}
		} else
		{
			if (type.indexOf("::") > 0)
			{
				// Adding a specific type
				String[] data = type.split("::");
				if (!FanType.hasPod(data[0]))
				{
					addError("Unresolved Pod: " + data[0], usingNode);
				} else if (findCachedQualifiedType(type) == null)
				{
					addError("Unresolved Type: " + type, usingNode);
				}

				//Type t = FanPodIndexer.getInstance().getPodType(data[0], data[1]);
				addUsing(name, type, usingNode);
			} else
			{
				// Adding all the types of a Pod
				if (name.equalsIgnoreCase("sys")) // sys is always avail.
				{
					return;
				}
				if (!FanType.hasPod(name))
				{
					addError("Unresolved Pod: " + name, usingNode);
				} else
				{
					Vector<FanType> items = FanType.findPodTypes(name, "");
					for (FanType t : items)
					{
						addUsing(t.getSimpleName(), t.getQualifiedName(), usingNode);
					}
				}
			}
		}
	}

	private void addUsing(String name, String qType, AstNode node)
	{
		AstNode scopeNode = FanLexAstUtils.getScopeNode(node);
		if (scopeNode == null)
		{
			return;
		}
		if (scopeNode.getLocalScopeVars().containsKey(name))
		{
			// This is 'legal' ... maybe show a warning later ?
			//addError("Duplicated using: " + qType + " / " + scopeNode.getLocalScopeVars().get(name), node);
			System.out.println("Already have a using called: " + qType + " (" + scopeNode.getLocalScopeVars().get(name) + ")");
			// Note: only keeping the 'last' definition (ie: override)
		}
		FanResolvedType rType = FanResolvedType.makeFromDbType(node, qType);
		rType = rType.asStaticContext(true);
		scopeNode.addScopeVar(name, FanAstScopeVar.VarKind.IMPORT, rType, true);
	}

	public ParsingResult<AstNode> getParsingResult()
	{
		return parsingResult;
	}

	public String getPod()
	{
		return pod;
	}

	/**
	 * TODO: this whole prunning stuff is a bit ugly
	 * Should try to buod the AST properly using technizues here:
	 * http://parboiled.hostingdelivered.com/viewtopic.php?f=3&t=9
	 *
	 * During ParseNode construction, some astNodes that migth have been constructed from
	 * some parseNode that where then "backtracked" (not the whoel sequence matched)
	 * This looks for and remove those unwanted nodes.
	 * @param node
	 */
	private void prune(AstNode node)
	{
		Node<AstNode> rtNode = astRoot.getParseNode();
		String rootLabel = "n/a";
		while (rtNode != null)
		{
			rootLabel = rtNode.getLabel();
			rtNode = rtNode.getParent();
		}
		List<AstNode> children = node.getChildren();
		List<AstNode> toBepruned = new ArrayList<AstNode>();
		for (AstNode child : children)
		{
			Node<AstNode> parseNode = child.getParseNode();
			// If the node is orphaned (no link back to the root), that means it was backtracked out of.
			String label = "N/A";
			while (parseNode != null)
			{
				label = parseNode.getLabel();
				parseNode = parseNode.getParent();
			}
			if (!rootLabel.equals(label))
			{
				toBepruned.add(child);
			} else
			{
				// recurse into children
				prune(child);
			}
		}
		// Drop the orphaned nodes
		for (AstNode nd : toBepruned)
		{
			children.remove(nd);
		}
	}

	@SuppressWarnings("unchecked")
	private FanResolvedType doExpr(AstNode node, FanResolvedType type)
	{
		// TODO: validate assignment type is compatible.
		boolean first = true;
		type = null;
		List<AstNode> children = node.getChildren();
		for (int i = 0; i != children.size(); i++)
		{
			// Check for special procedure
			// when a itBlock follows a call, it's actually a sort of closure call (with block)
			AstNode child = children.get(i);
			if (child.getKind() == AstKind.AST_CALL_EXPR)
			{
				if (i + 1 < children.size())
				{
					AstNode nextChild = children.get(i + 1);
					if (nextChild.getKind() == AstKind.AST_IT_BLOCK)
					{
						AstNode callNode = FanLexAstUtils.getFirstChild(child, new NodeKindPredicate(AstKind.AST_CALL));
						// parse the whole thing as a closure call
						type = doCall(callNode, type, nextChild);
						// skp next child since we juts did it
						i++;
						continue;
					}
				}
			}

			// Normal procedure
			parseVars(child, type);
			// we take type of right hand side if first in expression
			// and not one of the special type we don't want the rhs for
			if (first || (child.getKind() != AstKind.AST_EXPR_ADD // add/mult operation cannot chnage the type
					&& child.getKind() != AstKind.AST_EXPR_MULT //same
					/*&& child.getKind() != AstKind.AST_EXPR*/)) // new/sub expression
			{
				type = child.getType();
				// If part of the expr chain is unresolved (error), mark it unresolved
				// This avoids getting errors for the whole expression chain
				if (type != null && !type.isResolved())
				{
					type = new FanUnknownType(node, child.getNodeText(true));
				}
			} //else
			//{
			//	System.out.println("Not taking rhs type for "+node.getKind());
			// TODO: check what doesn't since it's probably less common than what does
			//}
			first = false;
		}
		return type;
	}

	/**
	 *
	 * @param node
	 * @param type
	 * @param forcedClosure :  when we have an itBlock, we need to send it as if it was a closure
	 * @return
	 */
	@SuppressWarnings("unchecked")
	private FanResolvedType doCall(AstNode node, FanResolvedType type, AstNode forcedClosure)
	{
		// saving the base type, because we need it for closures
		FanResolvedType baseType = type;
		AstNode callChild = node.getChildren().get(0);
		String name = callChild.getNodeText(true);

		//if a direct call like doThis(), then search scope for type/slot
		if (type == null)
		{
			baseType = FanResolvedType.resolveItType(callChild);
			// constructor call or local slot
			type = FanResolvedType.makeFromTypeSig(callChild, name);
			type = type.parameterize(baseType, callChild);
			if (type.isStaticContext())
			{ // It was a constructor call
				baseType = type;
				name = "make";
			}
		} else
		// otherwise a slot of the base type like var.toStr()
		{
			// checking what call op we are dealing with
			String op = ".";
			AstNode callExpr = FanLexAstUtils.findParentNode(node, AstKind.AST_CALL_EXPR);
			if (callExpr != null)
			{
				AstNode operator = FanLexAstUtils.getFirstChild(callExpr, new NodeKindPredicate(AstKind.LBL_OP));
				if (operator != null)
				{
					op = operator.getNodeText(true);
				}
			}
			// Can't resolve dynamic calls -> unknow type
			if (op.endsWith("->"))
			{
				return new FanUnknownType(node, sourceName);
			}

			// If using null check call on non nullable object, show an error, but continue resolution
			if (!type.isNullable() && op.startsWith("?"))
			{
				addError("Null check call on non-nullable: " + type.toString(), node);
			}

			//if(! (type instanceof FanUnknownType))
			type = type.resolveSlotType(name, this);
			type = type.parameterize(baseType, node);
		}

		// parse args
		List<AstNode> args = FanLexAstUtils.getChildren(node, new NodeKindPredicate(AstKind.AST_ARG));
		int argIndex = 0;
		for (AstNode arg : args)
		{
			if (FanLexAstUtils.isWrappingNode(arg, new NodeKindPredicate(AstKind.AST_CLOSURE)))
			{
				// If the argument is a closure, we need to call doClosure directly
				AstNode closure = FanLexAstUtils.getFirstChildRecursive(arg, new NodeKindPredicate(AstKind.AST_CLOSURE));
				doClosureCall(closure, baseType, name, argIndex);
			} else
			{
				// otherwsie, do normal parsing
				parseVars(arg, null);
			}
			argIndex++;
		}

		// deal with the sepcial trailing closure argument (if any)
		//TODO: Check that param types matches slot declaration
		AstNode closure = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_CLOSURE));
		if (closure != null || forcedClosure != null)
		{
			doClosureCall(forcedClosure != null ? forcedClosure : closure, baseType, name, argIndex);
		}
		// we need to parameterize the result
		// if baseType is null, then it probably couldn't return generics and if it did, then it would be "this", so leave it alone either way
		if (baseType != null)
		{
			type = type.parameterize(baseType, node);
		}
		return type;
	}

	private void doClosureCall(AstNode closureNode, FanResolvedType baseType, String slotName, int argIndex)
	{
		//TODO: Check that param types matches slot declaration
		FanResolvedType slotBase = baseType.resolveSlotBaseType(slotName, this);
		FanSlot slot = FanSlot.findByTypeAndName(slotBase.getQualifiedType(), slotName);
		if (slot == null)
		{
			addError("Can't call closure on " + slotBase.getQualifiedType() + "." + slotName, closureNode);
		} else
		{
			List<FanMethodParam> params = FanMethodParam.findAllForSlot(slot.getId());
			if (argIndex >= params.size())
			{
				addError("Can't call closure (too many parameters) on " + slotBase.getQualifiedType() + "." + slotName, closureNode);
			} else
			{
				FanResolvedType func = FanResolvedType.makeFromTypeSig(closureNode, params.get(argIndex).getQualifiedType());
				if (!((func instanceof FanResolvedFuncType) || func.isTypeCompatible(FanResolvedType.makeFromDbType(closureNode, "sys::Func"))))
				{
					addError("Closure call, but not expecting a function at parameter " + (argIndex + 1) + " in " + slotBase.getQualifiedType() + "." + slotName, closureNode);
				} else
				{
					// build lits of expected params for doClosure to use for inferrence resolution
					List<FanResolvedType> infTypes = new ArrayList<FanResolvedType>();
					if (func instanceof FanResolvedFuncType)
					{ // Otherwise it's a generic sys::Func -> no inference possible
						for (FanResolvedType t : ((FanResolvedFuncType) func).getTypes())
						{
							// might be a generic type
							t = t.parameterize(baseType, closureNode);
							infTypes.add(t);
						}
					}
					// parse the closure (but does not take it's type)
					if (closureNode.getKind() == AstKind.AST_IT_BLOCK)
					{
						if (infTypes.size() == 0)
						{
							addError("No parmeters expected in function, can't do itBlock closure", closureNode);
						} else
						{
							doItBlock(closureNode, infTypes.get(0));
						}
					} else
					{
						doClosure(closureNode, infTypes, argIndex);
					}
				}
			}
		}
	}

	/**
	 * Simple closure like |Str a, Int b->| { echo("hi there") }
	 * a.k.a closure definition
	 * @param node
	 * @param callAgainst : When called against a function (call) pass the expected parameters (for inference)
	 * @return
	 */
	@SuppressWarnings("unchecked")
	private FanResolvedFuncType doClosure(AstNode node, List<FanResolvedType> paramTypes, int argIndex)
	{
		FanResolvedType retType = FanResolvedType.makeFromDbType(node, "sys::Void");
		AstNode closureBlock = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_BLOCK));
		// Will store resolved types
		List<FanResolvedType> formalTypes = new ArrayList<FanResolvedType>();
		for (AstNode child : node.getChildren())
		{
			if (child.getKind() == AstKind.AST_FORMAL)
			{
				FanResolvedType t = doFormal(child, closureBlock, paramTypes, argIndex);
				if (t != null && !t.isResolved())
				{
					addError("Couldn't resolve formal definition: " + node.getNodeText(true), node);
				}
				formalTypes.add(t);
			} else if (child.getKind() == AstKind.AST_TYPE)
			{
				// save the returned type
				parseVars(child, null);
				retType = child.getType();
			} else if (child.getKind() == AstKind.AST_BLOCK)
			{
				// parse the block content
				parseVars(child, null);
			}
		}
		return new FanResolvedFuncType(node, formalTypes, retType);
	}

	/**
	 * Deal with a formal (as used in a call)
	 * @param node
	 * @param baseType
	 * @param closureBlock
	 * @param formalTypes
	 * @param index
	 * @return
	 */
	@SuppressWarnings("unchecked")
	private FanResolvedType doFormal(AstNode node, AstNode closureBlock, List<FanResolvedType> inferredTypes, int index)
	{
		AstNode formalName = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_ID));
		AstNode formalTypeAndId = FanLexAstUtils.getFirstChild(node, new NodeKindPredicate(AstKind.AST_TYPE_AND_ID));
		FanResolvedType fType = FanResolvedType.makeFromDbType(node, "sys::Void");
		if (formalTypeAndId != null)
		{	// typed formal
			formalName = FanLexAstUtils.getFirstChild(formalTypeAndId, new NodeKindPredicate(AstKind.AST_ID));
			AstNode formalType = FanLexAstUtils.getFirstChild(formalTypeAndId, new NodeKindPredicate(AstKind.AST_TYPE));
			parseVars(formalType, null);
			fType = formalType.getType().asStaticContext(false);
			// TODO: check that it matches callParams
		} else
		{
			if (inferredTypes == null)
			{
				//addError("Can't have inferred formals in closure definition.", node);
				// actually that's allowed, just uses sys::Obj?
				fType = FanResolvedType.makeFromTypeSig(node, "sys::Obj?");
			} else if (index > inferredTypes.size())
			{
				addError("More inferred formals than expected.", node);
				fType = FanResolvedType.makeUnresolved(node);
			} else
			{
{'fraction_non_alphanumeric': 0.1236089468133667, 'fraction_numerical': 0.0010860839590121358, 'mean_word_length': 3.4202323801572394, 'pattern_counts': {'":': 12, '<': 69, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6193233', 'n_tokens_mistral': 24793, 'n_tokens_neox': 20366, 'n_words': 5195}