<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package org.apache.jmeter.reporters;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

import org.apache.avalon.framework.configuration.DefaultConfigurationSerializer;
import org.apache.jmeter.engine.event.LoopIterationEvent;
import org.apache.jmeter.engine.util.NoThreadClone;
import org.apache.jmeter.gui.GuiPackage;
import org.apache.jmeter.samplers.Clearable;
import org.apache.jmeter.samplers.Remoteable;
import org.apache.jmeter.samplers.SampleEvent;
import org.apache.jmeter.samplers.SampleListener;
import org.apache.jmeter.samplers.SampleResult;
import org.apache.jmeter.samplers.SampleSaveConfiguration;
import org.apache.jmeter.save.CSVSaveService;
import org.apache.jmeter.save.OldSaveService;
import org.apache.jmeter.save.SaveService;
import org.apache.jmeter.save.TestResultWrapper;
import org.apache.jmeter.testelement.TestElement;
import org.apache.jmeter.testelement.TestListener;
import org.apache.jmeter.testelement.property.BooleanProperty;
import org.apache.jmeter.testelement.property.ObjectProperty;
import org.apache.jmeter.visualizers.Visualizer;
import org.apache.jorphan.logging.LoggingManager;
import org.apache.jorphan.util.JMeterError;
import org.apache.jorphan.util.JOrphanUtils;
import org.apache.log.Logger;

/**
 * This class handles all saving of samples.
 * The class must be thread-safe because it is shared between threads (NoThreadClone).
 */
public class ResultCollector extends AbstractListenerElement implements SampleListener, Clearable, Serializable,
        TestListener, Remoteable, NoThreadClone {

    private static final Logger log = LoggingManager.getLoggerForClass();

    private static final long serialVersionUID = 233L;

    // This string is used to identify local test runs, so must not be a valid host name
    private static final String TEST_IS_LOCAL = "*local*"; // $NON-NLS-1$

    private static final String TESTRESULTS_START = "<testResults>"; // $NON-NLS-1$

    private static final String TESTRESULTS_START_V1_1_PREVER = "<testResults version=\"";  // $NON-NLS-1$

    private static final String TESTRESULTS_START_V1_1_POSTVER="\">"; // $NON-NLS-1$

    private static final String TESTRESULTS_END = "</testResults>"; // $NON-NLS-1$

    private static final String XML_HEADER = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; // $NON-NLS-1$

    private static final int MIN_XML_FILE_LEN = XML_HEADER.length() + TESTRESULTS_START.length()
            + TESTRESULTS_END.length();

    public static final String FILENAME = "filename"; // $NON-NLS-1$

    private static final String SAVE_CONFIG = "saveConfig"; // $NON-NLS-1$

    private static final String ERROR_LOGGING = "ResultCollector.error_logging"; // $NON-NLS-1$

    private static final String SUCCESS_ONLY_LOGGING = "ResultCollector.success_only_logging"; // $NON-NLS-1$
    
    // Static variables

    private static final Map files = new HashMap();

    private static int instanceCount; // Keep track of how many instances are active

    // Instance variables
    
    private transient DefaultConfigurationSerializer serializer;

    private transient volatile PrintWriter out;

    private volatile boolean inTest = false;

    private volatile boolean isStats = false;

    /**
     * No-arg constructor.
     */
    public ResultCollector() {
        setErrorLogging(false);
        setSuccessOnlyLogging(false);
        setProperty(new ObjectProperty(SAVE_CONFIG, new SampleSaveConfiguration()));
        /*
         * All instances are created before the test starts.
         * This is guaranteed so long as all remote test plans are initialised
         * before they are started.
         * So we use this to ensure that the static variables are reset.
         */
        synchronized(this){
            files.clear();
            instanceCount=0;
        }
    }

    // Ensure that the sample save config is not shared between copied nodes
    // N.B. clone only seems to be used for client-server tests
    public Object clone(){
        ResultCollector clone = (ResultCollector) super.clone();
        clone.setSaveConfig((SampleSaveConfiguration)clone.getSaveConfig().clone());
        return clone;
    }

    private void setFilenameProperty(String f) {
        setProperty(FILENAME, f);
    }

    public String getFilename() {
        return getPropertyAsString(FILENAME);
    }

    public boolean isErrorLogging() {
        return getPropertyAsBoolean(ERROR_LOGGING);
    }

    public void setErrorLogging(boolean errorLogging) {
        setProperty(new BooleanProperty(ERROR_LOGGING, errorLogging));
    }

    public void setSuccessOnlyLogging(boolean value) {
        if (value) {
            setProperty(new BooleanProperty(SUCCESS_ONLY_LOGGING, true));
        } else {
            removeProperty(SUCCESS_ONLY_LOGGING);
        }
    }

    public boolean isSuccessOnlyLogging() {
        return getPropertyAsBoolean(SUCCESS_ONLY_LOGGING,false);
    }

    /**
     * Decides whether or not to a sample is wanted based on:
     * - errorOnly
     * - successOnly
     * - sample success
     *
     * @param success is sample successful
     * @return whether to log/display the sample
     */
    public boolean isSampleWanted(boolean success){
        boolean errorOnly = isErrorLogging();
        boolean successOnly = isSuccessOnlyLogging();
        return (!errorOnly && !successOnly) ||
               (success && successOnly) ||
               (!success && errorOnly);
        // successOnly and errorOnly cannot both be set
    }
    /**
     * Sets the filename attribute of the ResultCollector object.
     *
     * @param f
     *            the new filename value
     */
    public void setFilename(String f) {
        if (inTest) {
            return;
        }
        setFilenameProperty(f);
    }

    public synchronized void testEnded(String host) {
        instanceCount--;
        if (instanceCount <= 0) {
            finalizeFileOutput();
            inTest = false;
        }
    }

    public synchronized void testStarted(String host) {
        instanceCount++;
        try {
            initializeFileOutput();
            if (getVisualizer() != null) {
                this.isStats = getVisualizer().isStats();
            }
        } catch (Exception e) {
            log.error("", e);
        }
        inTest = true;
    }

    public void testEnded() {
        testEnded(TEST_IS_LOCAL);
    }

    public void testStarted() {
        testStarted(TEST_IS_LOCAL);
    }

    /**
     * Loads an existing sample data (JTL) file.
     * This can be one of:
     * - XStream format
     * - Avalon format
     * - CSV format
     *
     */
    public void loadExistingFile() {
        final Visualizer visualizer = getVisualizer();
        if (visualizer == null) {
            return; // No point reading the file if there's no visualiser
        }
        boolean parsedOK = false;
        String filename = getFilename();
        File file = new File(filename);
        if (file.exists()) {
            clearVisualizer();
            BufferedReader dataReader = null;
            BufferedInputStream bufferedInputStream = null;
            try {
                dataReader = new BufferedReader(new FileReader(file));
                // Get the first line, and see if it is XML
                String line = dataReader.readLine();
                dataReader.close();
                dataReader = null;
                if (line == null) {
                    log.warn(filename+" is empty");
                } else {
                    if (!line.startsWith("<?xml ")){// No, must be CSV //$NON-NLS-1$
                        CSVSaveService.processSamples(filename, visualizer, this);
                        parsedOK = true;
                    } else { // We are processing XML
                        try { // Assume XStream
                            bufferedInputStream = new BufferedInputStream(new FileInputStream(file));
                            // TODO change to process samples one by one
                            readSamples(SaveService.loadTestResults(bufferedInputStream), visualizer);
                            parsedOK = true;
                        } catch (Exception e) {
                            log.info("Failed to load "+filename+" using XStream, trying old XML format. Error was: "+e);
                            try {
                                OldSaveService.processSamples(filename, visualizer, this);
                                parsedOK = true;
                            } catch (Exception e1) {
                                log.warn("Error parsing Avalon XML. " + e1.getLocalizedMessage());
                            }
                        }
                    }
                }
            } catch (IOException e) {
                log.warn("Problem reading JTL file: "+file);
            } catch (JMeterError e){
                log.warn("Problem reading JTL file: "+file);
            } catch (RuntimeException e){ // e.g. NullPointerException
                log.warn("Problem reading JTL file: "+file,e);
            } finally {
                JOrphanUtils.closeQuietly(dataReader);
                JOrphanUtils.closeQuietly(bufferedInputStream);
                if (!parsedOK) {
                    GuiPackage.showErrorMessage(
                                "Error loading results file - see log file",
                                "Result file loader");
                }
            }
        } else {
            GuiPackage.showErrorMessage(
                    "Error loading results file - could not open file",
                    "Result file loader");
        }
    }

    private static void writeFileStart(PrintWriter writer, SampleSaveConfiguration saveConfig) {
        if (saveConfig.saveAsXml()) {
            writer.print(XML_HEADER);
            // Write the EOL separately so we generate LF line ends on Unix and Windows
            writer.print("\n"); // $NON-NLS-1$
            String pi=saveConfig.getXmlPi();
            if (pi.length() > 0) {
                writer.println(pi);
            }
            // Can't do it as a static initialisation, because SaveService
            // is being constructed when this is called
            writer.print(TESTRESULTS_START_V1_1_PREVER);
            writer.print(SaveService.getVERSION());
            writer.print(TESTRESULTS_START_V1_1_POSTVER);
            // Write the EOL separately so we generate LF line ends on Unix and Windows
            writer.print("\n"); // $NON-NLS-1$
        } else if (saveConfig.saveFieldNames()) {
            writer.println(CSVSaveService.printableFieldNamesToString(saveConfig));
        }
    }

    private static void writeFileEnd(PrintWriter pw, SampleSaveConfiguration saveConfig) {
        if (saveConfig.saveAsXml()) {
            pw.print("\n"); // $NON-NLS-1$
            pw.print(TESTRESULTS_END);
            pw.print("\n");// Added in version 1.1 // $NON-NLS-1$
        }
    }

    private static synchronized PrintWriter getFileWriter(String filename, SampleSaveConfiguration saveConfig)
            throws IOException {
        if (filename == null || filename.length() == 0) {
            return null;
        }
        PrintWriter writer = (PrintWriter) files.get(filename);
        boolean trimmed = true;

        if (writer == null) {
            if (saveConfig.saveAsXml()) {
                trimmed = trimLastLine(filename);
            } else {
                trimmed = new File(filename).exists();
            }
            // Find the name of the directory containing the file
            // and create it - if there is one
            File pdir = new File(filename).getParentFile();
            if (pdir != null) {
                if (!pdir.mkdirs()){
                    log.warn("Error creating directories for "+pdir.toString());
                }
            }
            writer = new PrintWriter(new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(filename,
                    trimmed)), SaveService.getFileEncoding("UTF-8")), true); // $NON-NLS-1$
            files.put(filename, writer);
        }
        if (!trimmed) {
            writeFileStart(writer, saveConfig);
        }
        return writer;
    }

    // returns false if the file did not contain the terminator
    private static boolean trimLastLine(String filename) {
        RandomAccessFile raf = null;
        try {
            raf = new RandomAccessFile(filename, "rw"); // $NON-NLS-1$
            long len = raf.length();
            if (len < MIN_XML_FILE_LEN) {
                return false;
            }
            raf.seek(len - TESTRESULTS_END.length() - 10);// TODO: may not work on all OSes?
            String line;
            long pos = raf.getFilePointer();
            int end = 0;
            while ((line = raf.readLine()) != null)// reads to end of line OR end of file
            {
                end = line.indexOf(TESTRESULTS_END);
                if (end >= 0) // found the string
                {
                    break;
                }
                pos = raf.getFilePointer();
            }
            if (line == null) {
                log.warn("Unexpected EOF trying to find XML end marker in " + filename);
                raf.close();
                return false;
            }
            raf.setLength(pos + end);// Truncate the file
            raf.close();
            raf = null;
        } catch (FileNotFoundException e) {
            return false;
        } catch (IOException e) {
            log.warn("Error trying to find XML terminator " + e.toString());
            return false;
        } finally {
            try {
                if (raf != null) {
                    raf.close();
                }
            } catch (IOException e1) {
                log.info("Could not close " + filename + " " + e1.getLocalizedMessage());
            }
        }
        return true;
    }

    // Only called if visualizer is non-null
    private void readSamples(TestResultWrapper testResults, Visualizer visualizer) throws Exception {
        Collection samples = testResults.getSampleResults();
        Iterator iter = samples.iterator();
        while (iter.hasNext()) {
            SampleResult result = (SampleResult) iter.next();
            if (isSampleWanted(result.isSuccessful())) {
                visualizer.add(result);
            }
        }
    }

    public void clearVisualizer() {
        // current = -1;
        if (getVisualizer() != null && getVisualizer() instanceof Clearable) {
            ((Clearable) getVisualizer()).clearData();
        }
        finalizeFileOutput();
    }

    public void sampleStarted(SampleEvent e) {
    }

    public void sampleStopped(SampleEvent e) {
    }

    /**
     * When a test result is received, display it and save it.
     *
     * @param event
     *            the sample event that was received
     */
    public void sampleOccurred(SampleEvent event) {
        SampleResult result = event.getResult();

        if (isSampleWanted(result.isSuccessful())) {
            sendToVisualizer(result);
            if ( out != null) {// no point otherwise
                SampleSaveConfiguration config = getSaveConfig();
                result.setSaveConfig(config);
                try {
                    if (config.saveAsXml()) {
                        recordResult(event);
                    } else {
                        String savee = CSVSaveService.resultToDelimitedString(event);
                        out.println(savee);
                    }
                } catch (Exception err) {
                    log.error("Error trying to record a sample", err); // should throw exception back to caller
                }
            }
        }
    }

    protected final void sendToVisualizer(SampleResult r) {
        if (getVisualizer() != null) {
            getVisualizer().add(r);
        }
    }

    // Only called if out != null
    private void recordResult(SampleEvent event) throws Exception {
        SampleResult result = event.getResult();
        if (!isResultMarked(result) && !this.isStats) {
            if (SaveService.isSaveTestLogFormat20()) {
                if (serializer == null) {
                    serializer = new DefaultConfigurationSerializer();
                }
                out.write(OldSaveService.getSerializedSampleResult(result, serializer, getSaveConfig()));
            } else {
                SaveService.saveSampleResult(event, out);
            }
        }
    }

    /**
     * recordStats is used to save statistics generated by visualizers
     *
     * @param e
     * @throws Exception
     */
    public void recordStats(TestElement e) throws Exception {
        if (out == null) {
            initializeFileOutput();
        }
        if (out != null) {
            SaveService.saveTestElement(e, out);
        }
    }

    private synchronized boolean isResultMarked(SampleResult res) {
        String filename = getFilename();
        boolean marked = res.isMarked(filename);

        if (!marked) {
            res.setMarked(filename);
        }
        return marked;
    }

    private void initializeFileOutput() throws IOException {

        String filename = getFilename();
        if (out == null && filename != null) {
            if (out == null) {
                try {
                    out = getFileWriter(filename, getSaveConfig());
                } catch (FileNotFoundException e) {
                    out = null;
                }
            }
        }
    }

    private synchronized void finalizeFileOutput() {
        if (out != null) {
            writeFileEnd(out, getSaveConfig());
            out.close();
            files.remove(getFilename());
            out = null;
        }
    }

    /*
     * (non-Javadoc)
     *
     * @see TestListener#testIterationStart(LoopIterationEvent)
     */
    public void testIterationStart(LoopIterationEvent event) {
    }

    /**
     * @return Returns the saveConfig.
     */
    public SampleSaveConfiguration getSaveConfig() {
        try {
            return (SampleSaveConfiguration) getProperty(SAVE_CONFIG).getObjectValue();
        } catch (ClassCastException e) {
            setSaveConfig(new SampleSaveConfiguration());
            return getSaveConfig();
        }
    }

    /**
     * @param saveConfig<commit_msg>Fix problem caused by implementing NoThreadClone: - use the files HashMap to close files correctly<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package org.apache.jmeter.reporters;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.avalon.framework.configuration.DefaultConfigurationSerializer;
import org.apache.jmeter.engine.event.LoopIterationEvent;
import org.apache.jmeter.engine.util.NoThreadClone;
import org.apache.jmeter.gui.GuiPackage;
import org.apache.jmeter.samplers.Clearable;
import org.apache.jmeter.samplers.Remoteable;
import org.apache.jmeter.samplers.SampleEvent;
import org.apache.jmeter.samplers.SampleListener;
import org.apache.jmeter.samplers.SampleResult;
import org.apache.jmeter.samplers.SampleSaveConfiguration;
import org.apache.jmeter.save.CSVSaveService;
import org.apache.jmeter.save.OldSaveService;
import org.apache.jmeter.save.SaveService;
import org.apache.jmeter.save.TestResultWrapper;
import org.apache.jmeter.testelement.TestElement;
import org.apache.jmeter.testelement.TestListener;
import org.apache.jmeter.testelement.property.BooleanProperty;
import org.apache.jmeter.testelement.property.ObjectProperty;
import org.apache.jmeter.visualizers.Visualizer;
import org.apache.jorphan.logging.LoggingManager;
import org.apache.jorphan.util.JMeterError;
import org.apache.jorphan.util.JOrphanUtils;
import org.apache.log.Logger;

/**
 * This class handles all saving of samples.
 * The class must be thread-safe because it is shared between threads (NoThreadClone).
 */
public class ResultCollector extends AbstractListenerElement implements SampleListener, Clearable, Serializable,
        TestListener, Remoteable, NoThreadClone {

    private static final Logger log = LoggingManager.getLoggerForClass();

    private static final long serialVersionUID = 233L;

    // This string is used to identify local test runs, so must not be a valid host name
    private static final String TEST_IS_LOCAL = "*local*"; // $NON-NLS-1$

    private static final String TESTRESULTS_START = "<testResults>"; // $NON-NLS-1$

    private static final String TESTRESULTS_START_V1_1_PREVER = "<testResults version=\"";  // $NON-NLS-1$

    private static final String TESTRESULTS_START_V1_1_POSTVER="\">"; // $NON-NLS-1$

    private static final String TESTRESULTS_END = "</testResults>"; // $NON-NLS-1$

    private static final String XML_HEADER = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"; // $NON-NLS-1$

    private static final int MIN_XML_FILE_LEN = XML_HEADER.length() + TESTRESULTS_START.length()
            + TESTRESULTS_END.length();

    public static final String FILENAME = "filename"; // $NON-NLS-1$

    private static final String SAVE_CONFIG = "saveConfig"; // $NON-NLS-1$

    private static final String ERROR_LOGGING = "ResultCollector.error_logging"; // $NON-NLS-1$

    private static final String SUCCESS_ONLY_LOGGING = "ResultCollector.success_only_logging"; // $NON-NLS-1$
    
    // Static variables

    private static final Map files = new HashMap(); // key=filename, entry=FileEntry

    /*
     * Keep track of the file writer and the configuration,
     * as the instance used to close them is not the same as the instance that creates
     * them. This means one cannot use the saved PrintWriter or use getSaveConfig()
     */
    private static class FileEntry{
        final PrintWriter pw;
        final SampleSaveConfiguration config;
        FileEntry(PrintWriter _pw, SampleSaveConfiguration _config){
            pw =_pw;
            config = _config;
        }
    }

    private static int instanceCount; // Keep track of how many instances are active

    // Instance variables
    
    private transient DefaultConfigurationSerializer serializer;

    private transient volatile PrintWriter out;

    private volatile boolean inTest = false;

    private volatile boolean isStats = false;

    /**
     * No-arg constructor.
     */
    public ResultCollector() {
        setErrorLogging(false);
        setSuccessOnlyLogging(false);
        setProperty(new ObjectProperty(SAVE_CONFIG, new SampleSaveConfiguration()));
        /*
         * All instances are created before the test starts.
         * This is guaranteed so long as all remote test plans are initialised
         * before they are started.
         * So we use this to ensure that the static variables are reset.
         */
        synchronized(this){
            files.clear();
            instanceCount=0;
        }
    }

    // Ensure that the sample save config is not shared between copied nodes
    // N.B. clone only seems to be used for client-server tests
    public Object clone(){
        ResultCollector clone = (ResultCollector) super.clone();
        clone.setSaveConfig((SampleSaveConfiguration)clone.getSaveConfig().clone());
        return clone;
    }

    private void setFilenameProperty(String f) {
        setProperty(FILENAME, f);
    }

    public String getFilename() {
        return getPropertyAsString(FILENAME);
    }

    public boolean isErrorLogging() {
        return getPropertyAsBoolean(ERROR_LOGGING);
    }

    public void setErrorLogging(boolean errorLogging) {
        setProperty(new BooleanProperty(ERROR_LOGGING, errorLogging));
    }

    public void setSuccessOnlyLogging(boolean value) {
        if (value) {
            setProperty(new BooleanProperty(SUCCESS_ONLY_LOGGING, true));
        } else {
            removeProperty(SUCCESS_ONLY_LOGGING);
        }
    }

    public boolean isSuccessOnlyLogging() {
        return getPropertyAsBoolean(SUCCESS_ONLY_LOGGING,false);
    }

    /**
     * Decides whether or not to a sample is wanted based on:
     * - errorOnly
     * - successOnly
     * - sample success
     *
     * @param success is sample successful
     * @return whether to log/display the sample
     */
    public boolean isSampleWanted(boolean success){
        boolean errorOnly = isErrorLogging();
        boolean successOnly = isSuccessOnlyLogging();
        return (!errorOnly && !successOnly) ||
               (success && successOnly) ||
               (!success && errorOnly);
        // successOnly and errorOnly cannot both be set
    }
    /**
     * Sets the filename attribute of the ResultCollector object.
     *
     * @param f
     *            the new filename value
     */
    public void setFilename(String f) {
        if (inTest) {
            return;
        }
        setFilenameProperty(f);
    }

    public synchronized void testEnded(String host) {
        instanceCount--;
        if (instanceCount <= 0) {
            finalizeFileOutput();
            inTest = false;
        }
    }

    public synchronized void testStarted(String host) {
        instanceCount++;
        try {
            initializeFileOutput();
            if (getVisualizer() != null) {
                this.isStats = getVisualizer().isStats();
            }
        } catch (Exception e) {
            log.error("", e);
        }
        inTest = true;
    }

    public void testEnded() {
        testEnded(TEST_IS_LOCAL);
    }

    public void testStarted() {
        testStarted(TEST_IS_LOCAL);
    }

    /**
     * Loads an existing sample data (JTL) file.
     * This can be one of:
     * - XStream format
     * - Avalon format
     * - CSV format
     *
     */
    public void loadExistingFile() {
        final Visualizer visualizer = getVisualizer();
        if (visualizer == null) {
            return; // No point reading the file if there's no visualiser
        }
        boolean parsedOK = false;
        String filename = getFilename();
        File file = new File(filename);
        if (file.exists()) {
            clearVisualizer();
            BufferedReader dataReader = null;
            BufferedInputStream bufferedInputStream = null;
            try {
                dataReader = new BufferedReader(new FileReader(file));
                // Get the first line, and see if it is XML
                String line = dataReader.readLine();
                dataReader.close();
                dataReader = null;
                if (line == null) {
                    log.warn(filename+" is empty");
                } else {
                    if (!line.startsWith("<?xml ")){// No, must be CSV //$NON-NLS-1$
                        CSVSaveService.processSamples(filename, visualizer, this);
                        parsedOK = true;
                    } else { // We are processing XML
                        try { // Assume XStream
                            bufferedInputStream = new BufferedInputStream(new FileInputStream(file));
                            // TODO change to process samples one by one
                            readSamples(SaveService.loadTestResults(bufferedInputStream), visualizer);
                            parsedOK = true;
                        } catch (Exception e) {
                            log.info("Failed to load "+filename+" using XStream, trying old XML format. Error was: "+e);
                            try {
                                OldSaveService.processSamples(filename, visualizer, this);
                                parsedOK = true;
                            } catch (Exception e1) {
                                log.warn("Error parsing Avalon XML. " + e1.getLocalizedMessage());
                            }
                        }
                    }
                }
            } catch (IOException e) {
                log.warn("Problem reading JTL file: "+file);
            } catch (JMeterError e){
                log.warn("Problem reading JTL file: "+file);
            } catch (RuntimeException e){ // e.g. NullPointerException
                log.warn("Problem reading JTL file: "+file,e);
            } finally {
                JOrphanUtils.closeQuietly(dataReader);
                JOrphanUtils.closeQuietly(bufferedInputStream);
                if (!parsedOK) {
                    GuiPackage.showErrorMessage(
                                "Error loading results file - see log file",
                                "Result file loader");
                }
            }
        } else {
            GuiPackage.showErrorMessage(
                    "Error loading results file - could not open file",
                    "Result file loader");
        }
    }

    private static void writeFileStart(PrintWriter writer, SampleSaveConfiguration saveConfig) {
        if (saveConfig.saveAsXml()) {
            writer.print(XML_HEADER);
            // Write the EOL separately so we generate LF line ends on Unix and Windows
            writer.print("\n"); // $NON-NLS-1$
            String pi=saveConfig.getXmlPi();
            if (pi.length() > 0) {
                writer.println(pi);
            }
            // Can't do it as a static initialisation, because SaveService
            // is being constructed when this is called
            writer.print(TESTRESULTS_START_V1_1_PREVER);
            writer.print(SaveService.getVERSION());
            writer.print(TESTRESULTS_START_V1_1_POSTVER);
            // Write the EOL separately so we generate LF line ends on Unix and Windows
            writer.print("\n"); // $NON-NLS-1$
        } else if (saveConfig.saveFieldNames()) {
            writer.println(CSVSaveService.printableFieldNamesToString(saveConfig));
        }
    }

    private static void writeFileEnd(PrintWriter pw, SampleSaveConfiguration saveConfig) {
        if (saveConfig.saveAsXml()) {
            pw.print("\n"); // $NON-NLS-1$
            pw.print(TESTRESULTS_END);
            pw.print("\n");// Added in version 1.1 // $NON-NLS-1$
        }
    }

    private static synchronized PrintWriter getFileWriter(String filename, SampleSaveConfiguration saveConfig)
            throws IOException {
        if (filename == null || filename.length() == 0) {
            return null;
        }
        FileEntry fe = (FileEntry) files.get(filename);
        PrintWriter writer = null;
        boolean trimmed = true;

        if (fe == null) {
            if (saveConfig.saveAsXml()) {
                trimmed = trimLastLine(filename);
            } else {
                trimmed = new File(filename).exists();
            }
            // Find the name of the directory containing the file
            // and create it - if there is one
            File pdir = new File(filename).getParentFile();
            if (pdir != null) {
                if (!pdir.mkdirs()){
                    log.warn("Error creating directories for "+pdir.toString());
                }
            }
            writer = new PrintWriter(new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(filename,
                    trimmed)), SaveService.getFileEncoding("UTF-8")), true); // $NON-NLS-1$
            log.debug("Opened file: "+filename);
            files.put(filename, new FileEntry(writer, saveConfig));
        } else {
            writer = fe.pw;
        }
        if (!trimmed) {
            writeFileStart(writer, saveConfig);
        }
        return writer;
    }

    // returns false if the file did not contain the terminator
    private static boolean trimLastLine(String filename) {
        RandomAccessFile raf = null;
        try {
            raf = new RandomAccessFile(filename, "rw"); // $NON-NLS-1$
            long len = raf.length();
            if (len < MIN_XML_FILE_LEN) {
                return false;
            }
            raf.seek(len - TESTRESULTS_END.length() - 10);// TODO: may not work on all OSes?
            String line;
            long pos = raf.getFilePointer();
            int end = 0;
            while ((line = raf.readLine()) != null)// reads to end of line OR end of file
            {
                end = line.indexOf(TESTRESULTS_END);
                if (end >= 0) // found the string
                {
                    break;
                }
                pos = raf.getFilePointer();
            }
            if (line == null) {
                log.warn("Unexpected EOF trying to find XML end marker in " + filename);
                raf.close();
                return false;
            }
            raf.setLength(pos + end);// Truncate the file
            raf.close();
            raf = null;
        } catch (FileNotFoundException e) {
            return false;
        } catch (IOException e) {
            log.warn("Error trying to find XML terminator " + e.toString());
            return false;
        } finally {
            try {
                if (raf != null) {
                    raf.close();
                }
            } catch (IOException e1) {
                log.info("Could not close " + filename + " " + e1.getLocalizedMessage());
            }
        }
        return true;
    }

    // Only called if visualizer is non-null
    private void readSamples(TestResultWrapper testResults, Visualizer visualizer) throws Exception {
        Collection samples = testResults.getSampleResults();
        Iterator iter = samples.iterator();
        while (iter.hasNext()) {
            SampleResult result = (SampleResult) iter.next();
            if (isSampleWanted(result.isSuccessful())) {
                visualizer.add(result);
            }
        }
    }

    public void clearVisualizer() {
        // current = -1;
        if (getVisualizer() != null && getVisualizer() instanceof Clearable) {
            ((Clearable) getVisualizer()).clearData();
        }
        finalizeFileOutput();
    }

    public void sampleStarted(SampleEvent e) {
    }

    public void sampleStopped(SampleEvent e) {
    }

    /**
     * When a test result is received, display it and save it.
     *
     * @param event
     *            the sample event that was received
     */
    public void sampleOccurred(SampleEvent event) {
        SampleResult result = event.getResult();

        if (isSampleWanted(result.isSuccessful())) {
            sendToVisualizer(result);
            if ( out != null) {// no point otherwise
                SampleSaveConfiguration config = getSaveConfig();
                result.setSaveConfig(config);
                try {
                    if (config.saveAsXml()) {
                        recordResult(event);
                    } else {
                        String savee = CSVSaveService.resultToDelimitedString(event);
                        out.println(savee);
                    }
                } catch (Exception err) {
                    log.error("Error trying to record a sample", err); // should throw exception back to caller
                }
            }
        }
    }

    protected final void sendToVisualizer(SampleResult r) {
        if (getVisualizer() != null) {
            getVisualizer().add(r);
        }
    }

    // Only called if out != null
    private void recordResult(SampleEvent event) throws Exception {
        SampleResult result = event.getResult();
        if (!isResultMarked(result) && !this.isStats) {
            if (SaveService.isSaveTestLogFormat20()) {
                if (serializer == null) {
                    serializer = new DefaultConfigurationSerializer();
                }
                out.write(OldSaveService.getSerializedSampleResult(result, serializer, getSaveConfig()));
            } else {
                SaveService.saveSampleResult(event, out);
            }
        }
    }

    /**
     * recordStats is used to save statistics generated by visualizers
     *
     * @param e
     * @throws Exception
     */
    public void recordStats(TestElement e) throws Exception {
        if (out == null) {
            initializeFileOutput();
        }
        if (out != null) {
            SaveService.saveTestElement(e, out);
        }
    }

    private synchronized boolean isResultMarked(SampleResult res) {
        String filename = getFilename();
        boolean marked = res.isMarked(filename);

        if (!marked) {
            res.setMarked(filename);
        }
        return marked;
    }

    private void initializeFileOutput() throws IOException {

        String filename = getFilename();
        if (out == null && filename != null) {
            if (out == null) {
                try {
                    out = getFileWriter(filename, getSaveConfig());
                } catch (FileNotFoundException e) {
                    out = null;
                }
            }
        }
    }

    private synchronized void finalizeFileOutput() {
        Iterator it = files.entrySet().iterator();
        while(it.hasNext()){
            Map.Entry me = (Map.Entry) it.next();
            log.debug("Closing: "+me.getKey());
            FileEntry fe = (FileEntry) me.getValue();
            writeFileEnd(fe.pw, fe.config);
            fe.pw.close();
        }
        files.clear();
    }

    /*
     * (non-Javadoc)
     *
     * @see TestListener#testIterationStart(LoopIterationEvent)
     */
    public void testIterationStart(LoopIterationEvent event) {
    }

    /**
     * @return Returns the saveConfig.
     */
    public SampleSaveConfiguration getSaveConfig() {
        try {
            return (SampleSaveConfiguration) getProperty(SAVE_CONFIG).getObjectValue();
        } catch (ClassCastException e) {
            setSaveConfig(new SampleSaveConfiguration());
            return getSaveConfig();
        }
    }

    /**
     * @param saveConfig
{'fraction_non_alphanumeric': 0.09393322070952996, 'fraction_numerical': 0.0026334095200238497, 'mean_word_length': 1.9441925102399065, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 2, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1788867', 'n_tokens_mistral': 10543, 'n_tokens_neox': 10155, 'n_words': 2858}