<commit_before>﻿using System.Collections.Generic;
using System.Globalization;
using System.Linq;

namespace MultiMiner.Xgminer.Api.Parsers
{
    public class DeviceInformationParser
    {
        public static void ParseTextForDeviceInformation(string text, List<DeviceInformation> deviceInformation)
        {            
            List<string> deviceBlob = text.Split('|').ToList();
            deviceBlob.RemoveAt(0);

            foreach (string deviceText in deviceBlob)
            {
                if (deviceText == "\0")
                    continue;

                //bfgminer may have multiple entries for the same key, e.g. Hardware Errors
                //seen with customer data/hardware
                //remove dupes using Distinct()
                var deviceAttributes = deviceText.Split(',').ToList().Distinct();

                Dictionary<string, string> keyValuePairs = deviceAttributes
                  .Where(value => value.Contains('='))
                  .Select(value => value.Split('='))
                  .ToDictionary(pair => pair[0], pair => pair[1]);

                //seen Count == 0 with user API logs
                if (keyValuePairs.Count > 0)
                {
                    DeviceInformation newDevice = new DeviceInformation();

                    newDevice.Kind = keyValuePairs.ElementAt(0).Key;
                    newDevice.Index = int.Parse(keyValuePairs[newDevice.Kind]);
                    newDevice.Enabled = keyValuePairs["Enabled"].Equals("Y");
                    newDevice.Status = keyValuePairs["Status"];

                    //the RPC API returns numbers formatted en-US, e.g. 1,000.00
                    //specify CultureInfo.InvariantCulture for parsing or unhandled exceptions will
                    //occur on other locales
                    //can test for this with:
                    //Thread.CurrentThread.CurrentCulture = new CultureInfo("ru-RU");
                    //Thread.CurrentThread.CurrentUICulture = new CultureInfo("ru-RU");

                    if (newDevice.Kind.Equals("GPU"))
                    {
                        if (keyValuePairs.ContainsKey("Temperature")) //check required for bfgminer
                            newDevice.Temperature = double.Parse(keyValuePairs["Temperature"], CultureInfo.InvariantCulture);
                        newDevice.FanSpeed = int.Parse(keyValuePairs["Fan Speed"], CultureInfo.InvariantCulture);
                        newDevice.FanPercent = int.Parse(keyValuePairs["Fan Percent"], CultureInfo.InvariantCulture);
                        newDevice.GpuClock = int.Parse(keyValuePairs["GPU Clock"], CultureInfo.InvariantCulture);
                        newDevice.MemoryClock = int.Parse(keyValuePairs["Memory Clock"], CultureInfo.InvariantCulture);
                        newDevice.GpuVoltage = double.Parse(keyValuePairs["GPU Voltage"], CultureInfo.InvariantCulture);
                        newDevice.GpuActivity = int.Parse(keyValuePairs["GPU Activity"], CultureInfo.InvariantCulture);
                        newDevice.PowerTune = int.Parse(keyValuePairs["Powertune"], CultureInfo.InvariantCulture);
                        newDevice.Intensity = keyValuePairs["Intensity"];
                    }

                    newDevice.AverageHashrate = double.Parse(keyValuePairs["MHS av"], CultureInfo.InvariantCulture) * 1000;
                    
                    //seen both MHS 5s and MHS 1s
                    if (keyValuePairs.ContainsKey("MHS 5s"))
                        newDevice.CurrentHashrate = double.Parse(keyValuePairs["MHS 5s"], CultureInfo.InvariantCulture) * 1000;
                    else if (keyValuePairs.ContainsKey("MHS 1s"))
                        newDevice.CurrentHashrate = double.Parse(keyValuePairs["MHS 1s"], CultureInfo.InvariantCulture) * 1000;

                    newDevice.AcceptedShares = int.Parse(keyValuePairs["Accepted"], CultureInfo.InvariantCulture);
                    newDevice.RejectedShares = int.Parse(keyValuePairs["Rejected"], CultureInfo.InvariantCulture);
                    
                    if (keyValuePairs.ContainsKey("Hardware Errors")) //check required for bfgminer
                        newDevice.HardwareErrors = int.Parse(keyValuePairs["Hardware Errors"], CultureInfo.InvariantCulture);
                    
                    if (keyValuePairs.ContainsKey("Utility")) //check required for bfgminer
                        newDevice.Utility = double.Parse(keyValuePairs["Utility"], CultureInfo.InvariantCulture);

                    deviceInformation.Add(newDevice);
                }
            }
        }
    }
}<commit_msg>Handle additional NULL values from bfgminer<commit_after>﻿using System.Collections.Generic;
using System.Globalization;
using System.Linq;

namespace MultiMiner.Xgminer.Api.Parsers
{
    public class DeviceInformationParser
    {
        public static void ParseTextForDeviceInformation(string text, List<DeviceInformation> deviceInformation)
        {            
            List<string> deviceBlob = text.Split('|').ToList();
            deviceBlob.RemoveAt(0);

            foreach (string deviceText in deviceBlob)
            {
                if (deviceText == "\0")
                    continue;

                //bfgminer may have multiple entries for the same key, e.g. Hardware Errors
                //seen with customer data/hardware
                //remove dupes using Distinct()
                var deviceAttributes = deviceText.Split(',').ToList().Distinct();

                Dictionary<string, string> keyValuePairs = deviceAttributes
                  .Where(value => value.Contains('='))
                  .Select(value => value.Split('='))
                  .ToDictionary(pair => pair[0], pair => pair[1]);

                //seen Count == 0 with user API logs
                if (keyValuePairs.Count > 0)
                {
                    DeviceInformation newDevice = new DeviceInformation();

                    newDevice.Kind = keyValuePairs.ElementAt(0).Key;
                    newDevice.Index = int.Parse(keyValuePairs[newDevice.Kind]);
                    newDevice.Enabled = keyValuePairs["Enabled"].Equals("Y");
                    newDevice.Status = keyValuePairs["Status"];

                    //the RPC API returns numbers formatted en-US, e.g. 1,000.00
                    //specify CultureInfo.InvariantCulture for parsing or unhandled exceptions will
                    //occur on other locales
                    //can test for this with:
                    //Thread.CurrentThread.CurrentCulture = new CultureInfo("ru-RU");
                    //Thread.CurrentThread.CurrentUICulture = new CultureInfo("ru-RU");

                    if (newDevice.Kind.Equals("GPU"))
                    {
                        if (keyValuePairs.ContainsKey("Temperature")) //check required for bfgminer
                            newDevice.Temperature = double.Parse(keyValuePairs["Temperature"], CultureInfo.InvariantCulture);
                        newDevice.FanSpeed = int.Parse(keyValuePairs["Fan Speed"], CultureInfo.InvariantCulture);
                        newDevice.FanPercent = int.Parse(keyValuePairs["Fan Percent"], CultureInfo.InvariantCulture);
                        newDevice.GpuClock = int.Parse(keyValuePairs["GPU Clock"], CultureInfo.InvariantCulture);
                        newDevice.MemoryClock = int.Parse(keyValuePairs["Memory Clock"], CultureInfo.InvariantCulture);
                        newDevice.GpuVoltage = double.Parse(keyValuePairs["GPU Voltage"], CultureInfo.InvariantCulture);
                        newDevice.GpuActivity = int.Parse(keyValuePairs["GPU Activity"], CultureInfo.InvariantCulture);
                        newDevice.PowerTune = int.Parse(keyValuePairs["Powertune"], CultureInfo.InvariantCulture);
                        newDevice.Intensity = keyValuePairs["Intensity"];
                    }

                    newDevice.AverageHashrate = double.Parse(keyValuePairs["MHS av"], CultureInfo.InvariantCulture) * 1000;
                    
                    //seen both MHS 5s and MHS 1s
                    if (keyValuePairs.ContainsKey("MHS 5s"))
                        newDevice.CurrentHashrate = double.Parse(keyValuePairs["MHS 5s"], CultureInfo.InvariantCulture) * 1000;
                    else if (keyValuePairs.ContainsKey("MHS 1s"))
                        newDevice.CurrentHashrate = double.Parse(keyValuePairs["MHS 1s"], CultureInfo.InvariantCulture) * 1000;

                    if (keyValuePairs.ContainsKey("Accepted")) //check required for bfgminer
                        newDevice.AcceptedShares = int.Parse(keyValuePairs["Accepted"], CultureInfo.InvariantCulture);

                    if (keyValuePairs.ContainsKey("Rejected")) //check required for bfgminer
                        newDevice.RejectedShares = int.Parse(keyValuePairs["Rejected"], CultureInfo.InvariantCulture);
                    
                    if (keyValuePairs.ContainsKey("Hardware Errors")) //check required for bfgminer
                        newDevice.HardwareErrors = int.Parse(keyValuePairs["Hardware Errors"], CultureInfo.InvariantCulture);
                    
                    if (keyValuePairs.ContainsKey("Utility")) //check required for bfgminer
                        newDevice.Utility = double.Parse(keyValuePairs["Utility"], CultureInfo.InvariantCulture);

                    deviceInformation.Add(newDevice);
                }
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.09721199368753287, 'fraction_numerical': 0.006522882693319305, 'mean_word_length': 1.8972874123742762, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2011082', 'n_tokens_mistral': 2447, 'n_tokens_neox': 2241, 'n_words': 449}