<commit_before>﻿//******************************************************************************************************
//  PQDIFLoader.cs - Gbtc
//
//  Copyright © 2014, Grid Protection Alliance.  All Rights Reserved.
//
//  Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See
//  the NOTICE file distributed with this work for additional information regarding copyright ownership.
//  The GPA licenses this file to you under the Eclipse Public License -v 1.0 (the "License"); you may
//  not use this file except in compliance with the License. You may obtain a copy of the License at:
//
//      http://www.opensource.org/licenses/eclipse-1.0.php
//
//  Unless agreed to in writing, the subject software distributed under the License is distributed on an
//  "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the
//  License for the specific language governing permissions and limitations.
//
//  Code Modification History:
//  ----------------------------------------------------------------------------------------------------
//  05/07/2014 - <NAME>
//       Generated original version of source code.
//
//******************************************************************************************************

using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using FaultData.DataAnalysis;
using FaultData.DataReaders.PQube;
using FaultData.DataSets;
using GSF;
using GSF.PQDIF.Logical;
using GSF.PQDIF.Physical;
using log4net;
using openXDA.Model;
using Phase = GSF.PQDIF.Logical.Phase;

namespace FaultData.DataReaders
{
    public class PQDIFReader : IDataReader, IDisposable
    {
        #region [ Members ]

        // Fields
        private double m_systemFrequency;

        private LogicalParser m_parser;
        private MeterDataSet m_meterDataSet;
        private bool m_disposed;

        #endregion

        #region [ Constructors ]

        /// <summary>
        /// Creates a new instance of the <see cref="PQDIFReader"/> class.
        /// </summary>
        public PQDIFReader()
        {
            m_meterDataSet = new MeterDataSet();
        }

        #endregion

        #region [ Properties ]

        /// <summary>
        /// Gets or sets the system frequency.
        /// </summary>
        [Setting]
        public double SystemFrequency
        {
            get
            {
                return m_systemFrequency;
            }
            set
            {
                m_systemFrequency = value;
            }
        }

        /// <summary>
        /// Gets the data set produced by the Parse method of the data reader.
        /// </summary>
        public MeterDataSet MeterDataSet
        {
            get
            {
                return m_meterDataSet;
            }
        }

        #endregion

        #region [ Methods ]

        /// <summary>
        /// Determines whether the file can be parsed at this time.
        /// </summary>
        /// <param name="filePath">The path to the file to be parsed.</param>
        /// <param name="fileCreationTime">The time the file was created.</param>
        /// <returns>True if the file can be parsed; false otherwise.</returns>
        public bool CanParse(string filePath, DateTime fileCreationTime)
        {
            try
            {
                m_parser = new LogicalParser(filePath);
                m_parser.Open();
                return true;
            }
            catch (IOException)
            {
                return false;
            }
        }

        /// <summary>
        /// Parses the file into a meter data set per meter contained in the file.
        /// </summary>
        /// <param name="filePath">The path to the file to be parsed.</param>
        /// <returns>List of meter data sets, one per meter.</returns>
        public void Parse(string filePath)
        {
            List<DataSourceRecord> dataSources;
            List<ObservationRecord> observationRecords;
            List<ChannelInstance> channelInstances;
            List<SeriesInstance> seriesInstances;
            List<SeriesDefinition> seriesDefinitions;

            Meter meter;
            Channel channel;
            DataSeries dataSeries;
            DateTime[] timeData;

            // Build the list of observation records in the PQDIF file
            observationRecords = new List<ObservationRecord>();

            while (m_parser.HasNextObservationRecord())
                observationRecords.Add(m_parser.NextObservationRecord());

            // Build the list of all data source records in the PQDIF file
            dataSources = observationRecords
                .Select(observation => observation.DataSource)
                .Distinct()
                .ToList();

            // If there are no data sources, there is no
            // need to go any further because we won't be
            // able to interpret any of the channel data
            if (!dataSources.Any())
                return;

            // Validate data sources to make sure there is only one data source defined in the file
            if (!dataSources.Zip(dataSources.Skip(1), (ds1, ds2) => AreEquivalent(ds1, ds2)).All(b => b))
                throw new InvalidDataException($"PQDIF file \"{filePath}\" defines too many data sources.");

            // Create a meter from the parsed data source
            meter = ParseDataSource(dataSources.First());

            // Build the list of all channel instances in the PQDIF file
            channelInstances = observationRecords
                .SelectMany(observation => observation.ChannelInstances)
                .Where(channelInstance => QuantityType.IsQuantityTypeID(channelInstance.Definition.QuantityTypeID))
                .Where(channelInstance => channelInstance.SeriesInstances.Any())
                .Where(channelInstance => channelInstance.SeriesInstances[0].Definition.ValueTypeID == SeriesValueType.Time)
                .ToList();

            // Create the list of series instances so we can
            // build it as we process each channel instance
            seriesInstances = new List<SeriesInstance>();

            foreach (ChannelInstance channelInstance in channelInstances)
            {
                bool timeValueChannel =
                    channelInstance.Definition.QuantityTypeID == QuantityType.WaveForm ||
                    channelInstance.Definition.QuantityTypeID == QuantityType.ValueLog ||
                    channelInstance.Definition.QuantityTypeID == QuantityType.Phasor;

                if (!timeValueChannel)
                    continue;

                // Parse time data from the channel instance<commit_msg>Added error handling for malformed PQDIF files from NEXUS meters.<commit_after>﻿//******************************************************************************************************
//  PQDIFLoader.cs - Gbtc
//
//  Copyright © 2014, Grid Protection Alliance.  All Rights Reserved.
//
//  Licensed to the Grid Protection Alliance (GPA) under one or more contributor license agreements. See
//  the NOTICE file distributed with this work for additional information regarding copyright ownership.
//  The GPA licenses this file to you under the Eclipse Public License -v 1.0 (the "License"); you may
//  not use this file except in compliance with the License. You may obtain a copy of the License at:
//
//      http://www.opensource.org/licenses/eclipse-1.0.php
//
//  Unless agreed to in writing, the subject software distributed under the License is distributed on an
//  "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. Refer to the
//  License for the specific language governing permissions and limitations.
//
//  Code Modification History:
//  ----------------------------------------------------------------------------------------------------
//  05/07/2014 - <NAME>
//       Generated original version of source code.
//
//******************************************************************************************************

using System;
using System.Collections.Generic;
using System.Configuration;
using System.IO;
using System.Linq;
using FaultData.DataAnalysis;
using FaultData.DataReaders.Linq;
using FaultData.DataReaders.PQube;
using FaultData.DataSets;
using GSF;
using GSF.PQDIF.Logical;
using GSF.PQDIF.Physical;
using log4net;
using openXDA.Model;
using Phase = GSF.PQDIF.Logical.Phase;

namespace FaultData.DataReaders
{
    namespace Linq
    {
        public static class EnumerableExtensions
        {
            public static IEnumerable<T> HandleExceptions<T>(this IEnumerable<T> enumerable, Action<AggregateException> handler)
            {
                List<Exception> exceptions = new List<Exception>();
                IEnumerator<T> enumerator = enumerable.GetEnumerator();

                using (enumerable as IDisposable)
                using (enumerator as IDisposable)
                {
                    while (true)
                    {
                        T item;

                        try
                        {
                            if (!enumerator.MoveNext())
                                break;

                            item = enumerator.Current;
                        }
                        catch (Exception ex)
                        {
                            exceptions.Add(ex);
                            continue;
                        }

                        yield return item;
                    }
                }

                try
                {
                    if (exceptions.Any())
                        throw new AggregateException(exceptions);
                }
                catch (AggregateException ex)
                {
                    handler(ex);
                }
            }
        }
    }

    public class PQDIFReader : IDataReader, IDisposable
    {
        #region [ Members ]

        // Fields
        private double m_systemFrequency;

        private LogicalParser m_parser;
        private MeterDataSet m_meterDataSet;
        private bool m_disposed;

        #endregion

        #region [ Constructors ]

        /// <summary>
        /// Creates a new instance of the <see cref="PQDIFReader"/> class.
        /// </summary>
        public PQDIFReader()
        {
            m_meterDataSet = new MeterDataSet();
        }

        #endregion

        #region [ Properties ]

        /// <summary>
        /// Gets or sets the system frequency.
        /// </summary>
        [Setting]
        public double SystemFrequency
        {
            get
            {
                return m_systemFrequency;
            }
            set
            {
                m_systemFrequency = value;
            }
        }

        /// <summary>
        /// Gets the data set produced by the Parse method of the data reader.
        /// </summary>
        public MeterDataSet MeterDataSet
        {
            get
            {
                return m_meterDataSet;
            }
        }

        #endregion

        #region [ Methods ]

        /// <summary>
        /// Determines whether the file can be parsed at this time.
        /// </summary>
        /// <param name="filePath">The path to the file to be parsed.</param>
        /// <param name="fileCreationTime">The time the file was created.</param>
        /// <returns>True if the file can be parsed; false otherwise.</returns>
        public bool CanParse(string filePath, DateTime fileCreationTime)
        {
            try
            {
                m_parser = new LogicalParser(filePath);
                m_parser.Open();
                return true;
            }
            catch (IOException)
            {
                return false;
            }
        }

        /// <summary>
        /// Parses the file into a meter data set per meter contained in the file.
        /// </summary>
        /// <param name="filePath">The path to the file to be parsed.</param>
        /// <returns>List of meter data sets, one per meter.</returns>
        public void Parse(string filePath)
        {
            List<DataSourceRecord> dataSources;
            List<ObservationRecord> observationRecords;
            List<ChannelInstance> channelInstances;
            List<SeriesInstance> seriesInstances;
            List<SeriesDefinition> seriesDefinitions;

            Meter meter;
            Channel channel;
            DataSeries dataSeries;
            DateTime[] timeData;

            // Build the list of observation records in the PQDIF file
            observationRecords = new List<ObservationRecord>();

            while (m_parser.HasNextObservationRecord())
                observationRecords.Add(m_parser.NextObservationRecord());

            // Build the list of all data source records in the PQDIF file
            dataSources = observationRecords
                .Select(observation => observation.DataSource)
                .Distinct()
                .ToList();

            // If there are no data sources, there is no
            // need to go any further because we won't be
            // able to interpret any of the channel data
            if (!dataSources.Any())
                return;

            // Validate data sources to make sure there is only one data source defined in the file
            if (!dataSources.Zip(dataSources.Skip(1), (ds1, ds2) => AreEquivalent(ds1, ds2)).All(b => b))
                throw new InvalidDataException($"PQDIF file \"{filePath}\" defines too many data sources.");

            // Create a meter from the parsed data source
            meter = ParseDataSource(dataSources.First());

            // Build the list of all channel instances in the PQDIF file
            channelInstances = observationRecords
                .SelectMany(observation => observation.ChannelInstances)
                .Where(channelInstance => QuantityType.IsQuantityTypeID(channelInstance.Definition.QuantityTypeID))
                .Where(channelInstance => channelInstance.SeriesInstances.Any())
                .Where(channelInstance => channelInstance.SeriesInstances[0].Definition.ValueTypeID == SeriesValueType.Time)
                .HandleExceptions(ex => Log.Warn("Encountered malformed observation records in PQDIF file.", ex))
                .ToList();

            // Create the list of series instances so we can
            // build it as we process each channel instance
            seriesInstances = new List<SeriesInstance>();

            foreach (ChannelInstance channelInstance in channelInstances)
            {
                bool timeValueChannel =
                    channelInstance.Definition.QuantityTypeID == QuantityType.WaveForm ||
                    channelInstance.Definition.QuantityTypeID == QuantityType.ValueLog ||
                    channelInstance.Definition.QuantityTypeID == QuantityType.Phasor;

                if (!timeValueChannel)
                    continue;

                // Parse time data from the channel instance
{'fraction_non_alphanumeric': 0.11760828004482826, 'fraction_numerical': 0.0030325004944294284, 'mean_word_length': 1.7220527543513369, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1081809', 'n_tokens_mistral': 3611, 'n_tokens_neox': 3423, 'n_words': 1128}