<commit_before>/*
 * Author: atotic
 * Created: Jul 25, 2003
 * License: Common Public License v1.0
 */
package org.python.pydev.parser;

import java.io.StringReader;
import java.util.ArrayList;
import java.util.Iterator;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IMarker;
import org.eclipse.jface.text.Assert;
import org.eclipse.jface.text.DocumentEvent;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.IDocumentListener;
import org.eclipse.ui.IEditorInput;
import org.eclipse.ui.IFileEditorInput;
import org.python.parser.ParseException;
import org.python.parser.PythonGrammar;
import org.python.parser.ReaderCharStream;
import org.python.parser.SimpleNode;
import org.python.parser.TokenMgrError;
import org.python.pydev.editor.PyEdit;


/**
 * PyParser uses org.python.parser to parse the document
 * (lexical analysis)
 * It is attached to PyEdit (a view), and it listens to document changes
 * On every document change, the syntax tree is regenerated
 * The reparsing of the document is done on a ParsingThread
 * 
 * Clients that need to know when new parse tree has been generated
 * should register as parseListeners.
 */


public class PyParser {
	
	IDocument document;
	PyEdit editorView;
	
	IDocumentListener documentListener; // listens to changes in the document
	ArrayList parserListeners;	// listeners that get notified 
	
	SimpleNode root; // root of the last PythonGrammar analysis

	static final boolean parseOnThread = true; // can turn of thread parsing for debugging
	ParsingThread parsingThread;	// thread that reparses the document
	
	public PyParser(PyEdit editorView) {
		this.editorView = editorView;
		root = null;
		parserListeners = new ArrayList();
		parsingThread = new ParsingThread(this);
		parsingThread.setName("Parsing thread");
	}
	
	public void dispose() {
		// remove the listeners
		document.removeDocumentListener(documentListener);
		parsingThread.diePlease();
	}

	public void setDocument(IDocument document) {
		// Cleans up old listeners
		if (this.document != null) {
			document.removeDocumentListener(documentListener);
		}
		// Set up new listener
		this.document = document;
		documentListener = new IDocumentListener() {
			public void documentChanged(DocumentEvent event) {
				if (parseOnThread == true)
					parsingThread.documentChanged();
				else
					reparseDocument();
			}
			public void documentAboutToBeChanged(DocumentEvent event) {}		
		};
		document.addDocumentListener(documentListener);
		// Reparse document on the initial set
		parsingThread.start();
		if (parseOnThread == true)
			parsingThread.documentChanged();
		else
			reparseDocument();
	}
	
	<commit_msg>Deal with "Open External File" weirdness more gracefully. Still an error but not an uncaught exception.<commit_after>/*
 * Author: atotic
 * Created: Jul 25, 2003
 * License: Common Public License v1.0
 */
package org.python.pydev.parser;

import java.io.StringReader;
import java.util.ArrayList;
import java.util.Iterator;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IMarker;
import org.eclipse.jface.text.Assert;
import org.eclipse.jface.text.DocumentEvent;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.IDocumentListener;
import org.eclipse.ui.IEditorInput;
import org.eclipse.ui.IFileEditorInput;
import org.python.parser.ParseException;
import org.python.parser.PythonGrammar;
import org.python.parser.ReaderCharStream;
import org.python.parser.SimpleNode;
import org.python.parser.TokenMgrError;
import org.python.pydev.editor.PyEdit;


/**
 * PyParser uses org.python.parser to parse the document
 * (lexical analysis)
 * It is attached to PyEdit (a view), and it listens to document changes
 * On every document change, the syntax tree is regenerated
 * The reparsing of the document is done on a ParsingThread
 * 
 * Clients that need to know when new parse tree has been generated
 * should register as parseListeners.
 */


public class PyParser {
	
	IDocument document;
	PyEdit editorView;
	
	IDocumentListener documentListener; // listens to changes in the document
	ArrayList parserListeners;	// listeners that get notified 
	
	SimpleNode root; // root of the last PythonGrammar analysis

	static final boolean parseOnThread = true; // can turn of thread parsing for debugging
	ParsingThread parsingThread;	// thread that reparses the document
	
	public PyParser(PyEdit editorView) {
		this.editorView = editorView;
		root = null;
		parserListeners = new ArrayList();
		parsingThread = new ParsingThread(this);
		parsingThread.setName("Parsing thread");
	}
	
	public void dispose() {
		// remove the listeners
		if (document != null)
			document.removeDocumentListener(documentListener);
		parsingThread.diePlease();
	}

	public void setDocument(IDocument document) {
		// Cleans up old listeners
		if (this.document != null) {
			document.removeDocumentListener(documentListener);
		}
		
		// Set up new listener
		this.document = document;
		if (document == null) {
			System.err.println("No document in PyParser::setDocument?");
			return;
		}
		documentListener = new IDocumentListener() {
			public void documentChanged(DocumentEvent event) {
				if (parseOnThread == true)
					parsingThread.documentChanged();
				else
					reparseDocument();
			}
			public void documentAboutToBeChanged(DocumentEvent event) {}		
		};
		document.addDocumentListener(documentListener);
		// Reparse document on the initial set
		parsingThread.start();
		if (parseOnThread == true)
			parsingThread.documentChanged();
		else
			reparseDocument();
	}
	
	
{'fraction_non_alphanumeric': 0.08905943244690344, 'fraction_numerical': 0.0028556130644297696, 'mean_word_length': 4.393647738209817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5710006', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1572, 'n_words': 476}