<commit_before>/*
 * Copyright 2015 WSO2 Inc. (http://wso2.org)
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.wso2.carbon.metrics.data.service;

import java.io.File;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.naming.Context;
import javax.naming.InitialContext;
import javax.naming.NamingException;
import javax.sql.DataSource;

import org.apache.axis2.AxisFault;
import org.apache.axis2.context.ServiceContext;
import org.apache.axis2.service.Lifecycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wso2.carbon.core.AbstractAdmin;
import org.wso2.carbon.metrics.common.DefaultSourceValueProvider;
import org.wso2.carbon.metrics.common.MetricsConfigException;
import org.wso2.carbon.metrics.common.MetricsConfiguration;
import org.wso2.carbon.metrics.common.MetricsXMLConfiguration;
import org.wso2.carbon.metrics.data.common.Metric;
import org.wso2.carbon.metrics.data.common.MetricAttribute;
import org.wso2.carbon.metrics.data.common.MetricDataFormat;
import org.wso2.carbon.metrics.data.common.MetricList;
import org.wso2.carbon.metrics.data.common.MetricType;
import org.wso2.carbon.metrics.data.service.dao.MetricDataProcessor;
import org.wso2.carbon.metrics.data.service.dao.ReporterDAO;
import org.wso2.carbon.metrics.data.service.dao.converter.DumbConverter;
import org.wso2.carbon.metrics.data.service.dao.converter.MemoryConverter;
import org.wso2.carbon.metrics.data.service.dao.converter.PercentageConverter;
import org.wso2.carbon.metrics.data.service.dao.converter.ValueConverter;
import org.wso2.carbon.utils.CarbonUtils;

public class MetricsDataService extends AbstractAdmin implements Lifecycle {

    private static final Logger logger = LoggerFactory.getLogger(MetricsDataService.class);

    private ReporterDAO reporterDAO;

    private final Pattern fromPattern = Pattern.compile("(\\-?\\d+)([hdm])");

    private String currentJDBCReportingSource;

    private static final ValueConverter MEMORY_VALUE_CONVERTER = new MemoryConverter();

    private static final ValueConverter PERCENTAGE_VALUE_CONVERTER = new PercentageConverter();

    private static final ValueConverter DUMB_VALUE_CONVERTER = new DumbConverter();

    public MetricsDataService() {
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.apache.axis2.service.Lifecycle#init(org.apache.axis2.context.ServiceContext)
     */
    @Override
    public void init(ServiceContext serviceContext) throws AxisFault {
        MetricsXMLConfiguration configuration = new MetricsXMLConfiguration();
        String filePath = CarbonUtils.getCarbonConfigDirPath() + File.separator + "metrics.xml";
        try {
            configuration.load(filePath);
        } catch (MetricsConfigException e) {
            if (logger.isErrorEnabled()) {
                logger.error("Error reading configuration from " + filePath, e);
            }
        }

        init(configuration);
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.apache.axis2.service.Lifecycle#destroy(org.apache.axis2.context.ServiceContext)
     */
    @Override
    public void destroy(ServiceContext serviceContext) {
    }

    void init(MetricsConfiguration configuration) {
        final String JDBC_REPORTING_DATASOURCE_NAME = "Reporting.JDBC.DataSourceName";
        final String JDBC_REPORTING_SOURCE = "Reporting.JDBC.Source";

        String dataSourceName = configuration.getProperty(JDBC_REPORTING_DATASOURCE_NAME);

        if (dataSourceName == null || dataSourceName.trim().length() == 0) {
            String msg = "Data Source Name is not specified for Metrics Data Service";
            if (logger.isWarnEnabled()) {
                logger.warn(msg);
            }
            throw new IllegalStateException(msg);
        }

        DataSource dataSource = null;
        try {
            Context ctx = new InitialContext();
            dataSource = (DataSource) ctx.lookup(dataSourceName);
        } catch (NamingException e) {
            String msg = String.format(
                    "Error when looking up the Data Source: '%s'. Cannot instantiate the Metrics Data Service",
                    dataSourceName);
            if (logger.isWarnEnabled()) {
                logger.warn(msg);
            }
            throw new IllegalStateException(msg);
        }

        if (logger.isDebugEnabled()) {
            logger.debug(String.format("Creating Metrics Data Service with data source '%s'", dataSourceName));
        }

        reporterDAO = new ReporterDAO(dataSource);

        currentJDBCReportingSource =
                configuration.getProperty(JDBC_REPORTING_SOURCE, DefaultSourceValueProvider.getValue());
    }

    private long getStartTime(String from) {
        if (from == null || from.isEmpty()) {
            return -1;
        }

        if (logger.isDebugEnabled()) {
            logger.debug(String.format("Get Start Time. From Value: %s", from));
        }

        long currentTimeMillis = System.currentTimeMillis();

        Matcher matcher = fromPattern.matcher(from);

        long startTime = -1;

        if (matcher.find()) {
            long count = Long.parseLong(matcher.group(1));
            String unit = matcher.group(2);
            if ("m".equals(unit)) {
                startTime = currentTimeMillis + (count * 1000 * 60);
            } else if ("h".equals(unit)) {
                startTime = currentTimeMillis + (count * 1000 * 60 * 60);
            } else if ("d".equals(unit)) {
                startTime = currentTimeMillis + (count * 1000 * 60 * 60 * 24);
            }
        } else if (from.matches("\\d+")) {
            startTime = Long.parseLong(from);
        }

        return startTime;
    }

    /**
     * Get the list of reporting sources.
     *
     * @return List of sources.
     */
    public String[] getAllSources() {
        List<String> sourcesList;
        Set<String> sources = reporterDAO.queryAllSources();
        if (sources.isEmpty()) {
            sourcesList = new ArrayList<String>(1);
            sourcesList.add(currentJDBCReportingSource);
        } else {
            // Remove current source from set
            sources.remove(currentJDBCReportingSource);
            // Add the current source to top
            // This will make sure that the current source is selected by default
            sourcesList = new ArrayList<String>(sources.size());
            sourcesList.add(currentJDBCReportingSource);
            sourcesList.addAll(sources);
        }
        return sourcesList.toArray(new String[sourcesList.size()]);
    }

    /**
     * Get metric hierarchy data for a given source and path.
     *
     * @param source Current source.
     * @param path   Curent path.
     * @return {@link MetricHierarchyData} for the given source and path.
     */
    public MetricHierarchyData getHierarchy(String source, String path) {
        Map<String, MetricType> hierarchicalMetrics = reporterDAO.queryHierarchicalMetrics(source, path);
        Set<String> childrenNames = new TreeSet<>();
        List<MetricMeta> metrics = new ArrayList<>();
        path = (path != null && !path.isEmpty()) ? path : "";
        for (Map.Entry<String, MetricType> entry : hierarchicalMetrics.entrySet()) {
            String metric = entry.getKey();
            MetricType type = entry.getValue();
            if (metric.startsWith(path)) {
                String suffix = (!"".equals(path)) ? metric.replaceFirst(path + "\\.", "") : metric;
                int chunks = suffix.length() - suffix.replace(".", "").length();
                if (chunks == 0) {
                    // metrics
                    metrics.add(new MetricMeta(metric, type.name()));
                } else if (chunks == 1) {
                    // immediate child remove stat name and keep only path
                    childrenNames.add(metric.substring(0, metric.lastIndexOf('.')));
                } else if (chunks > 1) {
                    // child with further hierarchy
                    if (!"".equals(path)) {
                        childrenNames.add(path + "." + suffix.substring(0, suffix.indexOf('.')));
                    } else {
                        childrenNames.add(suffix.substring(0, suffix.indexOf('.')));
                    }
                }<commit_msg>Fixed a typo in comment<commit_after>/*
 * Copyright 2015 WSO2 Inc. (http://wso2.org)
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.wso2.carbon.metrics.data.service;

import java.io.File;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.naming.Context;
import javax.naming.InitialContext;
import javax.naming.NamingException;
import javax.sql.DataSource;

import org.apache.axis2.AxisFault;
import org.apache.axis2.context.ServiceContext;
import org.apache.axis2.service.Lifecycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wso2.carbon.core.AbstractAdmin;
import org.wso2.carbon.metrics.common.DefaultSourceValueProvider;
import org.wso2.carbon.metrics.common.MetricsConfigException;
import org.wso2.carbon.metrics.common.MetricsConfiguration;
import org.wso2.carbon.metrics.common.MetricsXMLConfiguration;
import org.wso2.carbon.metrics.data.common.Metric;
import org.wso2.carbon.metrics.data.common.MetricAttribute;
import org.wso2.carbon.metrics.data.common.MetricDataFormat;
import org.wso2.carbon.metrics.data.common.MetricList;
import org.wso2.carbon.metrics.data.common.MetricType;
import org.wso2.carbon.metrics.data.service.dao.MetricDataProcessor;
import org.wso2.carbon.metrics.data.service.dao.ReporterDAO;
import org.wso2.carbon.metrics.data.service.dao.converter.DumbConverter;
import org.wso2.carbon.metrics.data.service.dao.converter.MemoryConverter;
import org.wso2.carbon.metrics.data.service.dao.converter.PercentageConverter;
import org.wso2.carbon.metrics.data.service.dao.converter.ValueConverter;
import org.wso2.carbon.utils.CarbonUtils;

public class MetricsDataService extends AbstractAdmin implements Lifecycle {

    private static final Logger logger = LoggerFactory.getLogger(MetricsDataService.class);

    private ReporterDAO reporterDAO;

    private final Pattern fromPattern = Pattern.compile("(\\-?\\d+)([hdm])");

    private String currentJDBCReportingSource;

    private static final ValueConverter MEMORY_VALUE_CONVERTER = new MemoryConverter();

    private static final ValueConverter PERCENTAGE_VALUE_CONVERTER = new PercentageConverter();

    private static final ValueConverter DUMB_VALUE_CONVERTER = new DumbConverter();

    public MetricsDataService() {
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.apache.axis2.service.Lifecycle#init(org.apache.axis2.context.ServiceContext)
     */
    @Override
    public void init(ServiceContext serviceContext) throws AxisFault {
        MetricsXMLConfiguration configuration = new MetricsXMLConfiguration();
        String filePath = CarbonUtils.getCarbonConfigDirPath() + File.separator + "metrics.xml";
        try {
            configuration.load(filePath);
        } catch (MetricsConfigException e) {
            if (logger.isErrorEnabled()) {
                logger.error("Error reading configuration from " + filePath, e);
            }
        }

        init(configuration);
    }

    /*
     * (non-Javadoc)
     * 
     * @see org.apache.axis2.service.Lifecycle#destroy(org.apache.axis2.context.ServiceContext)
     */
    @Override
    public void destroy(ServiceContext serviceContext) {
    }

    void init(MetricsConfiguration configuration) {
        final String JDBC_REPORTING_DATASOURCE_NAME = "Reporting.JDBC.DataSourceName";
        final String JDBC_REPORTING_SOURCE = "Reporting.JDBC.Source";

        String dataSourceName = configuration.getProperty(JDBC_REPORTING_DATASOURCE_NAME);

        if (dataSourceName == null || dataSourceName.trim().length() == 0) {
            String msg = "Data Source Name is not specified for Metrics Data Service";
            if (logger.isWarnEnabled()) {
                logger.warn(msg);
            }
            throw new IllegalStateException(msg);
        }

        DataSource dataSource = null;
        try {
            Context ctx = new InitialContext();
            dataSource = (DataSource) ctx.lookup(dataSourceName);
        } catch (NamingException e) {
            String msg = String.format(
                    "Error when looking up the Data Source: '%s'. Cannot instantiate the Metrics Data Service",
                    dataSourceName);
            if (logger.isWarnEnabled()) {
                logger.warn(msg);
            }
            throw new IllegalStateException(msg);
        }

        if (logger.isDebugEnabled()) {
            logger.debug(String.format("Creating Metrics Data Service with data source '%s'", dataSourceName));
        }

        reporterDAO = new ReporterDAO(dataSource);

        currentJDBCReportingSource =
                configuration.getProperty(JDBC_REPORTING_SOURCE, DefaultSourceValueProvider.getValue());
    }

    private long getStartTime(String from) {
        if (from == null || from.isEmpty()) {
            return -1;
        }

        if (logger.isDebugEnabled()) {
            logger.debug(String.format("Get Start Time. From Value: %s", from));
        }

        long currentTimeMillis = System.currentTimeMillis();

        Matcher matcher = fromPattern.matcher(from);

        long startTime = -1;

        if (matcher.find()) {
            long count = Long.parseLong(matcher.group(1));
            String unit = matcher.group(2);
            if ("m".equals(unit)) {
                startTime = currentTimeMillis + (count * 1000 * 60);
            } else if ("h".equals(unit)) {
                startTime = currentTimeMillis + (count * 1000 * 60 * 60);
            } else if ("d".equals(unit)) {
                startTime = currentTimeMillis + (count * 1000 * 60 * 60 * 24);
            }
        } else if (from.matches("\\d+")) {
            startTime = Long.parseLong(from);
        }

        return startTime;
    }

    /**
     * Get the list of reporting sources.
     *
     * @return List of sources.
     */
    public String[] getAllSources() {
        List<String> sourcesList;
        Set<String> sources = reporterDAO.queryAllSources();
        if (sources.isEmpty()) {
            sourcesList = new ArrayList<String>(1);
            sourcesList.add(currentJDBCReportingSource);
        } else {
            // Remove current source from set
            sources.remove(currentJDBCReportingSource);
            // Add the current source to top
            // This will make sure that the current source is selected by default
            sourcesList = new ArrayList<String>(sources.size());
            sourcesList.add(currentJDBCReportingSource);
            sourcesList.addAll(sources);
        }
        return sourcesList.toArray(new String[sourcesList.size()]);
    }

    /**
     * Get metric hierarchy data for a given source and path.
     *
     * @param source Current source.
     * @param path   Current path.
     * @return {@link MetricHierarchyData} for the given source and path.
     */
    public MetricHierarchyData getHierarchy(String source, String path) {
        Map<String, MetricType> hierarchicalMetrics = reporterDAO.queryHierarchicalMetrics(source, path);
        Set<String> childrenNames = new TreeSet<>();
        List<MetricMeta> metrics = new ArrayList<>();
        path = (path != null && !path.isEmpty()) ? path : "";
        for (Map.Entry<String, MetricType> entry : hierarchicalMetrics.entrySet()) {
            String metric = entry.getKey();
            MetricType type = entry.getValue();
            if (metric.startsWith(path)) {
                String suffix = (!"".equals(path)) ? metric.replaceFirst(path + "\\.", "") : metric;
                int chunks = suffix.length() - suffix.replace(".", "").length();
                if (chunks == 0) {
                    // metrics
                    metrics.add(new MetricMeta(metric, type.name()));
                } else if (chunks == 1) {
                    // immediate child remove stat name and keep only path
                    childrenNames.add(metric.substring(0, metric.lastIndexOf('.')));
                } else if (chunks > 1) {
                    // child with further hierarchy
                    if (!"".equals(path)) {
                        childrenNames.add(path + "." + suffix.substring(0, suffix.indexOf('.')));
                    } else {
                        childrenNames.add(suffix.substring(0, suffix.indexOf('.')));
                    }
                }
{'fraction_non_alphanumeric': 0.10735806291116148, 'fraction_numerical': 0.008071203493836034, 'mean_word_length': 2.5743924125666866, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2283535', 'n_tokens_mistral': 5085, 'n_tokens_neox': 4845, 'n_words': 1166}