<commit_before>/*******************************************************************************
 * This file is part of OpenNMS(R).
 *
 * Copyright (C) 2006-2012 The OpenNMS Group, Inc.
 * OpenNMS(R) is Copyright (C) 1999-2012 The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published
 * by the Free Software Foundation, either version 3 of the License,
 * or (at your option) any later version.
 *
 * OpenNMS(R) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with OpenNMS(R).  If not, see:
 *      http://www.gnu.org/licenses/
 *
 * For more information contact:
 *     OpenNMS(R) Licensing <<EMAIL>>
 *     http://www.opennms.org/
 *     http://www.opennms.com/
 *******************************************************************************/
package org.opennms.features.vaadin.mibcompiler.services;

import java.io.File;
import java.io.Serializable;
import java.math.BigInteger;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsmiparser.parser.SmiDefaultParser;
import org.jsmiparser.smi.Notification;
import org.jsmiparser.smi.SmiMib;
import org.jsmiparser.smi.SmiModule;
import org.jsmiparser.smi.SmiNamedNumber;
import org.jsmiparser.smi.SmiNotificationType;
import org.jsmiparser.smi.SmiPrimitiveType;
import org.jsmiparser.smi.SmiRow;
import org.jsmiparser.smi.SmiTrapType;
import org.jsmiparser.smi.SmiVariable;

import org.opennms.core.utils.LogUtils;
import org.opennms.features.namecutter.NameCutter;
import org.opennms.features.vaadin.mibcompiler.api.MibParser;
import org.opennms.netmgt.config.datacollection.DatacollectionGroup;
import org.opennms.netmgt.config.datacollection.Group;
import org.opennms.netmgt.config.datacollection.MibObj;
import org.opennms.netmgt.config.datacollection.PersistenceSelectorStrategy;
import org.opennms.netmgt.config.datacollection.ResourceType;
import org.opennms.netmgt.config.datacollection.StorageStrategy;
import org.opennms.netmgt.dao.support.IndexStorageStrategy;
import org.opennms.netmgt.model.OnmsSeverity;
import org.opennms.netmgt.model.PrefabGraph;
import org.opennms.netmgt.xml.eventconf.Decode;
import org.opennms.netmgt.xml.eventconf.Event;
import org.opennms.netmgt.xml.eventconf.Events;
import org.opennms.netmgt.xml.eventconf.Logmsg;
import org.opennms.netmgt.xml.eventconf.Mask;
import org.opennms.netmgt.xml.eventconf.Maskelement;
import org.opennms.netmgt.xml.eventconf.Varbindsdecode;

/**
 * JSMIParser implementation of the interface MibParser.
 * 
 * @author <a href="mailto:<EMAIL>"><NAME></a> 
 */
@SuppressWarnings("serial")
public class JsmiMibParser implements MibParser, Serializable {

    /** The Constant MIB_SUFFIXES. */
    private static final String[] MIB_SUFFIXES = new String[] { "", ".txt", ".mib", ".my" };

    /** The Constant TRAP_OID_PATTERN. */
    private static final Pattern TRAP_OID_PATTERN = Pattern.compile("(.*)\\.(\\d+)$");

    /** The MIB directory. */
    private File mibDirectory;

    /** The parser. */
    private SmiDefaultParser parser;

    /** The module. */
    private SmiModule module;

    /** The error handler. */
    private OnmsProblemEventHandler errorHandler;

    /** The missing dependencies. */
    private List<String> missingDependencies = new ArrayList<String>();

    /**
     * Instantiates a new JLIBSMI MIB parser.
     */
    public JsmiMibParser() {
        parser = new SmiDefaultParser();
        errorHandler = new OnmsProblemEventHandler(parser);
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#setMibDirectory(java.io.File)
     */
    public void setMibDirectory(File mibDirectory) {
        this.mibDirectory = mibDirectory;
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#parseMib(java.io.File)
     */
    public boolean parseMib(File mibFile) {
        // Validate MIB Directory
        if (mibDirectory == null) {
            errorHandler.addError("MIB directory has not been set.");
            return false;
        }

        // Reset error handler and dependencies tracker
        missingDependencies.clear();

        // Set UP the MIB Queue MIB to be parsed
        List<URL> queue = new ArrayList<URL>();
        parser.getFileParserPhase().setInputUrls(queue);

        // Create a cache of filenames to do case-insensitive lookups
        final Map<String,File> mibDirectoryFiles = new HashMap<String,File>();
        for (final File file : mibDirectory.listFiles()) {
            mibDirectoryFiles.put(file.getName().toLowerCase(), file);
        }

        // Parse MIB
        LogUtils.debugf(this, "Parsing %s", mibFile.getAbsolutePath());
        SmiMib mib = null;
        addFileToQueue(queue, mibFile);
        while (true) {
            errorHandler.reset();
            try {
                mib = parser.parse();
            } catch (Exception e) {
                LogUtils.errorf(this, e, "Can't compile %s", mibFile);
                errorHandler.addError(e.getMessage());
                return false;
            }
            if (errorHandler.isOk()) {
                break;
            } else {
                List<String> dependencies = errorHandler.getDependencies();
                if (dependencies.isEmpty()) // No dependencies, everything is fine.
                    break;
                missingDependencies.addAll(dependencies);
                if (!addDependencyToQueue(queue, mibDirectoryFiles))
                    break;
            }
        }
        if (errorHandler.isNotOk()) // There are still non-dependency related problems.
            return false;

        // Extracting the module from compiled MIB.
        LogUtils.infof(this, "The MIB %s has been parsed successfully.", mibFile.getAbsolutePath());
        module = getModule(mib, mibFile);
        return module != null;
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#getFormattedErrors()
     */
    public String getFormattedErrors() {
        return errorHandler.getMessages();
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#getMissingDependencies()
     */
    public List<String> getMissingDependencies() {
        return missingDependencies;
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.services.MibParser#getMibName()
     */
    @Override
    public String getMibName() {
        return module.getId();
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.services.MibParser#getEvents(java.lang.String)
     */
    public Events getEvents(String ueibase) {
        if (module == null) {
            return null;
        }
        LogUtils.infof(this, "Generating events for %s using the following UEI Base: %s", module.getId(), ueibase);
        try {
            return convertMibToEvents(module, ueibase);
        } catch (Throwable e) {
            String errors = e.getMessage();
            if (errors == null || errors.trim().equals(""))
                errors = "An unknown error accured when generating events objects from the MIB " + module.getId();
            LogUtils.errorf(this, e, "Event parsing error: %s", errors);
            errorHandler.addError(errors);
            return null;
        }
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.api.MibParser#getDataCollection()
     */
    public DatacollectionGroup getDataCollection() {
        if (module == null) {
            return null;
        }
        LogUtils.infof(this, "Generating data collection configuration for %s", module.getId());
        DatacollectionGroup dcGroup = new DatacollectionGroup();
        dcGroup.setName(module.getId());
        NameCutter cutter = new NameCutter();
        try {
            for (SmiVariable v : module.getVariables()) {
                String groupName = getGroupName(v);
                String resourceType = getResourceType(v);
                Group group = getGroup(dcGroup, groupName, resourceType);
                String typeName = getMetricType(v.getType().getPrimitiveType());
                if (typeName != null) {
                    String alias = cutter.trimByCamelCase(v.getId(), 19); // RRDtool/JRobin DS size restriction.
                    MibObj mibObj = new MibObj();
                    mibObj.setOid('.' + v.getOidStr());
                    mibObj.setInstance(resourceType == null ? "0" : resourceType);
                    mibObj.setAlias(alias);
                    mibObj.setType(typeName);
                    group.addMibObj(mibObj);
                    if (typeName.equals("string") && resourceType != null) {
                        for (ResourceType rs : dcGroup.getResourceTypeCollection()) {
                            if (rs.getName().equals(resourceType) && rs.getResourceLabel().equals("${index}")) {
                                rs.setResourceLabel("${" + v.getId() + "} (${index})");
                            }
                        }
                    }
                }
            }
        } catch (Throwable e) {
            String errors = e.getMessage();
            if (errors == null || errors.trim().equals(""))
                errors = "An unknown error accured when generating data collection objects from the MIB " + module.getId();
            LogUtils.errorf(this, e, "Data Collection parsing error: %s", errors);
            errorHandler.addError(errors);
            return null;
        }
        return dcGroup;
    }


    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.api.MibParser#getPrefabGraphs()
     */
    @Override
    public List<PrefabGraph> getPrefabGraphs() {
        if (module == null) {
            return null;
        }
        List<PrefabGraph> graphs = new ArrayList<PrefabGraph>();
        LogUtils.infof(this, "Generating graph templates for %s", module.getId());
        NameCutter cutter = new NameCutter();
        try {
            for (SmiVariable v : module.getVariables()) {
                String groupName = getGroupName(v);
                String resourceType = getResourceType(v);
                if (resourceType == null)
                    resourceType = "nodeSnmp";
                String typeName = getMetricType(v.getType().getPrimitiveType());
                if (v.getId().contains("Index")) { // Treat SNMP Indexes as strings.
                    typeName = "string";
                }
                int order = 1;
                if (typeName != null && !typeName.toLowerCase().contains("string")) {
                    String name = groupName + '.' + v.getId();
                    String alias = cutter.trimByCamelCase(v.getId(), 19); // RRDtool/JRobin DS size restriction.
                    String descr = v.getDescription().replaceAll("[\n\r]", "").replaceAll("\\s+", " ");
                    StringBuffer sb = new StringBuffer();
                    sb.append("--title=\"").append(v.getId()).append("\" \\\n");
                    sb.append(" DEF:var={rrd1}:").append(alias).append(":AVERAGE \\\n");
                    sb.append(" LINE1:var#0000ff:\"").append(v.getId()).append("\" \\\n");
                    sb.append(" GPRINT:var:AVERAGE:\"Avg\\\\: %8.2lf %s\" \\\n");
                    sb.append(" GPRINT:var:MIN:\"Min\\\\: %8.2lf %s\" \\\n");
                    sb.append(" GPRINT:var:MAX:\"Max\\\\: %8.2lf %s\\n\"");
                    sb.append("\n\n");
                    PrefabGraph graph = new PrefabGraph(name, descr, new String[] { alias }, sb.toString(), new String[0], new String[0], order++, new String[] { resourceType }, descr, null, null, new String[0]);<commit_msg>Add a second escaped backslash to the last line of the hard coded rrd graph command template in the MIB compiler so that NRTG works for graphs generated by the MIB compiler.<commit_after>/*******************************************************************************
 * This file is part of OpenNMS(R).
 *
 * Copyright (C) 2006-2012 The OpenNMS Group, Inc.
 * OpenNMS(R) is Copyright (C) 1999-2012 The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
 *
 * OpenNMS(R) is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published
 * by the Free Software Foundation, either version 3 of the License,
 * or (at your option) any later version.
 *
 * OpenNMS(R) is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with OpenNMS(R).  If not, see:
 *      http://www.gnu.org/licenses/
 *
 * For more information contact:
 *     OpenNMS(R) Licensing <<EMAIL>>
 *     http://www.opennms.org/
 *     http://www.opennms.com/
 *******************************************************************************/
package org.opennms.features.vaadin.mibcompiler.services;

import java.io.File;
import java.io.Serializable;
import java.math.BigInteger;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsmiparser.parser.SmiDefaultParser;
import org.jsmiparser.smi.Notification;
import org.jsmiparser.smi.SmiMib;
import org.jsmiparser.smi.SmiModule;
import org.jsmiparser.smi.SmiNamedNumber;
import org.jsmiparser.smi.SmiNotificationType;
import org.jsmiparser.smi.SmiPrimitiveType;
import org.jsmiparser.smi.SmiRow;
import org.jsmiparser.smi.SmiTrapType;
import org.jsmiparser.smi.SmiVariable;

import org.opennms.core.utils.LogUtils;
import org.opennms.features.namecutter.NameCutter;
import org.opennms.features.vaadin.mibcompiler.api.MibParser;
import org.opennms.netmgt.config.datacollection.DatacollectionGroup;
import org.opennms.netmgt.config.datacollection.Group;
import org.opennms.netmgt.config.datacollection.MibObj;
import org.opennms.netmgt.config.datacollection.PersistenceSelectorStrategy;
import org.opennms.netmgt.config.datacollection.ResourceType;
import org.opennms.netmgt.config.datacollection.StorageStrategy;
import org.opennms.netmgt.dao.support.IndexStorageStrategy;
import org.opennms.netmgt.model.OnmsSeverity;
import org.opennms.netmgt.model.PrefabGraph;
import org.opennms.netmgt.xml.eventconf.Decode;
import org.opennms.netmgt.xml.eventconf.Event;
import org.opennms.netmgt.xml.eventconf.Events;
import org.opennms.netmgt.xml.eventconf.Logmsg;
import org.opennms.netmgt.xml.eventconf.Mask;
import org.opennms.netmgt.xml.eventconf.Maskelement;
import org.opennms.netmgt.xml.eventconf.Varbindsdecode;

/**
 * JSMIParser implementation of the interface MibParser.
 * 
 * @author <a href="mailto:<EMAIL>"><NAME></a> 
 */
@SuppressWarnings("serial")
public class JsmiMibParser implements MibParser, Serializable {

    /** The Constant MIB_SUFFIXES. */
    private static final String[] MIB_SUFFIXES = new String[] { "", ".txt", ".mib", ".my" };

    /** The Constant TRAP_OID_PATTERN. */
    private static final Pattern TRAP_OID_PATTERN = Pattern.compile("(.*)\\.(\\d+)$");

    /** The MIB directory. */
    private File mibDirectory;

    /** The parser. */
    private SmiDefaultParser parser;

    /** The module. */
    private SmiModule module;

    /** The error handler. */
    private OnmsProblemEventHandler errorHandler;

    /** The missing dependencies. */
    private List<String> missingDependencies = new ArrayList<String>();

    /**
     * Instantiates a new JLIBSMI MIB parser.
     */
    public JsmiMibParser() {
        parser = new SmiDefaultParser();
        errorHandler = new OnmsProblemEventHandler(parser);
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#setMibDirectory(java.io.File)
     */
    public void setMibDirectory(File mibDirectory) {
        this.mibDirectory = mibDirectory;
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#parseMib(java.io.File)
     */
    public boolean parseMib(File mibFile) {
        // Validate MIB Directory
        if (mibDirectory == null) {
            errorHandler.addError("MIB directory has not been set.");
            return false;
        }

        // Reset error handler and dependencies tracker
        missingDependencies.clear();

        // Set UP the MIB Queue MIB to be parsed
        List<URL> queue = new ArrayList<URL>();
        parser.getFileParserPhase().setInputUrls(queue);

        // Create a cache of filenames to do case-insensitive lookups
        final Map<String,File> mibDirectoryFiles = new HashMap<String,File>();
        for (final File file : mibDirectory.listFiles()) {
            mibDirectoryFiles.put(file.getName().toLowerCase(), file);
        }

        // Parse MIB
        LogUtils.debugf(this, "Parsing %s", mibFile.getAbsolutePath());
        SmiMib mib = null;
        addFileToQueue(queue, mibFile);
        while (true) {
            errorHandler.reset();
            try {
                mib = parser.parse();
            } catch (Exception e) {
                LogUtils.errorf(this, e, "Can't compile %s", mibFile);
                errorHandler.addError(e.getMessage());
                return false;
            }
            if (errorHandler.isOk()) {
                break;
            } else {
                List<String> dependencies = errorHandler.getDependencies();
                if (dependencies.isEmpty()) // No dependencies, everything is fine.
                    break;
                missingDependencies.addAll(dependencies);
                if (!addDependencyToQueue(queue, mibDirectoryFiles))
                    break;
            }
        }
        if (errorHandler.isNotOk()) // There are still non-dependency related problems.
            return false;

        // Extracting the module from compiled MIB.
        LogUtils.infof(this, "The MIB %s has been parsed successfully.", mibFile.getAbsolutePath());
        module = getModule(mib, mibFile);
        return module != null;
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#getFormattedErrors()
     */
    public String getFormattedErrors() {
        return errorHandler.getMessages();
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.MibParser#getMissingDependencies()
     */
    public List<String> getMissingDependencies() {
        return missingDependencies;
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.services.MibParser#getMibName()
     */
    @Override
    public String getMibName() {
        return module.getId();
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.services.MibParser#getEvents(java.lang.String)
     */
    public Events getEvents(String ueibase) {
        if (module == null) {
            return null;
        }
        LogUtils.infof(this, "Generating events for %s using the following UEI Base: %s", module.getId(), ueibase);
        try {
            return convertMibToEvents(module, ueibase);
        } catch (Throwable e) {
            String errors = e.getMessage();
            if (errors == null || errors.trim().equals(""))
                errors = "An unknown error accured when generating events objects from the MIB " + module.getId();
            LogUtils.errorf(this, e, "Event parsing error: %s", errors);
            errorHandler.addError(errors);
            return null;
        }
    }

    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.api.MibParser#getDataCollection()
     */
    public DatacollectionGroup getDataCollection() {
        if (module == null) {
            return null;
        }
        LogUtils.infof(this, "Generating data collection configuration for %s", module.getId());
        DatacollectionGroup dcGroup = new DatacollectionGroup();
        dcGroup.setName(module.getId());
        NameCutter cutter = new NameCutter();
        try {
            for (SmiVariable v : module.getVariables()) {
                String groupName = getGroupName(v);
                String resourceType = getResourceType(v);
                Group group = getGroup(dcGroup, groupName, resourceType);
                String typeName = getMetricType(v.getType().getPrimitiveType());
                if (typeName != null) {
                    String alias = cutter.trimByCamelCase(v.getId(), 19); // RRDtool/JRobin DS size restriction.
                    MibObj mibObj = new MibObj();
                    mibObj.setOid('.' + v.getOidStr());
                    mibObj.setInstance(resourceType == null ? "0" : resourceType);
                    mibObj.setAlias(alias);
                    mibObj.setType(typeName);
                    group.addMibObj(mibObj);
                    if (typeName.equals("string") && resourceType != null) {
                        for (ResourceType rs : dcGroup.getResourceTypeCollection()) {
                            if (rs.getName().equals(resourceType) && rs.getResourceLabel().equals("${index}")) {
                                rs.setResourceLabel("${" + v.getId() + "} (${index})");
                            }
                        }
                    }
                }
            }
        } catch (Throwable e) {
            String errors = e.getMessage();
            if (errors == null || errors.trim().equals(""))
                errors = "An unknown error accured when generating data collection objects from the MIB " + module.getId();
            LogUtils.errorf(this, e, "Data Collection parsing error: %s", errors);
            errorHandler.addError(errors);
            return null;
        }
        return dcGroup;
    }


    /* (non-Javadoc)
     * @see org.opennms.features.vaadin.mibcompiler.api.MibParser#getPrefabGraphs()
     */
    @Override
    public List<PrefabGraph> getPrefabGraphs() {
        if (module == null) {
            return null;
        }
        List<PrefabGraph> graphs = new ArrayList<PrefabGraph>();
        LogUtils.infof(this, "Generating graph templates for %s", module.getId());
        NameCutter cutter = new NameCutter();
        try {
            for (SmiVariable v : module.getVariables()) {
                String groupName = getGroupName(v);
                String resourceType = getResourceType(v);
                if (resourceType == null)
                    resourceType = "nodeSnmp";
                String typeName = getMetricType(v.getType().getPrimitiveType());
                if (v.getId().contains("Index")) { // Treat SNMP Indexes as strings.
                    typeName = "string";
                }
                int order = 1;
                if (typeName != null && !typeName.toLowerCase().contains("string")) {
                    String name = groupName + '.' + v.getId();
                    String alias = cutter.trimByCamelCase(v.getId(), 19); // RRDtool/JRobin DS size restriction.
                    String descr = v.getDescription().replaceAll("[\n\r]", "").replaceAll("\\s+", " ");
                    StringBuffer sb = new StringBuffer();
                    sb.append("--title=\"").append(v.getId()).append("\" \\\n");
                    sb.append(" DEF:var={rrd1}:").append(alias).append(":AVERAGE \\\n");
                    sb.append(" LINE1:var#0000ff:\"").append(v.getId()).append("\" \\\n");
                    sb.append(" GPRINT:var:AVERAGE:\"Avg\\\\: %8.2lf %s\" \\\n");
                    sb.append(" GPRINT:var:MIN:\"Min\\\\: %8.2lf %s\" \\\n");
                    sb.append(" GPRINT:var:MAX:\"Max\\\\: %8.2lf %s\\\\n\"");
                    sb.append("\n\n");
                    PrefabGraph graph = new PrefabGraph(name, descr, new String[] { alias }, sb.toString(), new String[0], new String[0], order++, new String[] { resourceType }, descr, null, null, new String[0]);
{'fraction_non_alphanumeric': 0.13391269199676636, 'fraction_numerical': 0.0030719482619240095, 'mean_word_length': 2.4083207053313127, 'pattern_counts': {'":': 2, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6822547', 'n_tokens_mistral': 7438, 'n_tokens_neox': 7133, 'n_words': 1622}