<commit_before>package org.lamport.tla.toolbox.tool.tlc.output;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.ui.editors.text.FileDocumentProvider;
import org.eclipse.ui.part.FileEditorInput;
import org.lamport.tla.toolbox.tool.tlc.output.source.ITLCOutputSource;
import org.lamport.tla.toolbox.tool.tlc.output.source.TagBasedTLCOutputIncrementalParser;
import org.lamport.tla.toolbox.tool.tlc.ui.TLCUIActivator;

/**
 * Reads the log file and parses the content
 * @author <NAME>
 * @version $Id$
 */
public class LogFileReader
{
    private TagBasedTLCOutputIncrementalParser parser;
    private IFile logFile;

    public LogFileReader(String name, IFile logFile, boolean isTraceExplorerLogFile)
    {
        this.logFile = logFile;
        this.parser = new TagBasedTLCOutputIncrementalParser(name, ITLCOutputSource.PRIO_LOW, isTraceExplorerLogFile);
    }

    /**
     * Reads the contents
     */
    public void read()
    {
        FileEditorInput fileEditorInput = new FileEditorInput(logFile);
        FileDocumentProvider fileDocumentProvider = new FileDocumentProvider();
        try
        {
            fileDocumentProvider.connect(fileEditorInput);
            IDocument document = fileDocumentProvider.getDocument(fileEditorInput);
            /*
             * When the output file is somewhat large (several hundred KB)
             * the toolbox hangs if the entire document representing that file
             * is passed to the parser at once. We can send in pieces of it
             * to solve this problem.
             * 
             * We send in one line at a time. Note that IDocument lines
             * are 0-based.
             */
            for (int lineNum = 0; lineNum < document.getNumberOfLines(); lineNum++)<commit_msg>NEW - bug 151: org.lamport.tla.toolbox.tool.tlc.output.source.TagBasedTLCOutputIncrementalParser does not scale for large output  http://bugzilla.tlaplus.net/show_bug.cgi?id=151 - Comment added<commit_after>package org.lamport.tla.toolbox.tool.tlc.output;

import org.eclipse.core.resources.IFile;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.ui.editors.text.FileDocumentProvider;
import org.eclipse.ui.part.FileEditorInput;
import org.lamport.tla.toolbox.tool.tlc.output.source.ITLCOutputSource;
import org.lamport.tla.toolbox.tool.tlc.output.source.TagBasedTLCOutputIncrementalParser;
import org.lamport.tla.toolbox.tool.tlc.ui.TLCUIActivator;

/**
 * Reads the log file and parses the content
 * @author <NAME>
 * @version $Id$
 */
public class LogFileReader
{
    private TagBasedTLCOutputIncrementalParser parser;
    private IFile logFile;

    public LogFileReader(String name, IFile logFile, boolean isTraceExplorerLogFile)
    {
        this.logFile = logFile;
        this.parser = new TagBasedTLCOutputIncrementalParser(name, ITLCOutputSource.PRIO_LOW, isTraceExplorerLogFile);
    }

    /**
     * Reads the contents
     */
    public void read()
    {
        FileEditorInput fileEditorInput = new FileEditorInput(logFile);
        FileDocumentProvider fileDocumentProvider = new FileDocumentProvider();
        try
        {
            fileDocumentProvider.connect(fileEditorInput);
            IDocument document = fileDocumentProvider.getDocument(fileEditorInput);
            /*
             * When the output file is somewhat large (several hundred KB)
             * the toolbox hangs due to the parser inefficiently handling multiple lines at once.
             * If the entire document representing that file
             * is passed to the parser at once. We can send in pieces of it
             * to solve this problem.
             * 
             * We send in one line at a time. Note that IDocument lines
             * are 0-based.
             */
            for (int lineNum = 0; lineNum < document.getNumberOfLines(); lineNum++)
{'fraction_non_alphanumeric': 0.08211143695014662, 'fraction_numerical': 0.0024437927663734115, 'mean_word_length': 3.0009775171065494, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3528767', 'n_tokens_mistral': 1195, 'n_tokens_neox': 1145, 'n_words': 287}