<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Serialization;
using POESKillTree.Model;
using POESKillTree.ViewModels;
using POESKillTree.ViewModels.ItemAttribute;
using AttackSkill = POESKillTree.SkillTreeFiles.Compute.AttackSkill;
using DamageForm = POESKillTree.SkillTreeFiles.Compute.DamageForm;
using DamageNature = POESKillTree.SkillTreeFiles.Compute.DamageNature;
using DamageSource = POESKillTree.SkillTreeFiles.Compute.DamageSource;
using WeaponHand = POESKillTree.SkillTreeFiles.Compute.WeaponHand;
using WeaponType = POESKillTree.SkillTreeFiles.Compute.WeaponType;
using Weapon = POESKillTree.SkillTreeFiles.Compute.Weapon;

namespace POESKillTree.SkillTreeFiles
{
    // TODO: Attributes can have negative value (Cast when Damage Taken L20 has 6% more Damage), AttributesOf should handle transition between less/more.
    public class ItemDB
    {
        // Maximum level (quality).
        public const int MAX_LEVEL = 30;

        // Deserialized attribute.
        public class Attribute
        {
            // Deserialized comment to perserve.
            [XmlAttribute]
            public string Comment;
            // Deserialized name.
            [XmlAttribute]
            public string Name;
            // Deserialized flag indicating whether attribute should not be updated.
            [XmlAttribute]
            public bool NoUpdate = false;
            [XmlIgnore]
            public bool NoUpdateSpecified { get { return NoUpdate; } }
            // Deserialized values.
            [XmlElement(ElementName = "Value", Type = typeof(ValueAt))]
            [XmlElement(ElementName = "ValueForLevel", Type = typeof(ValueForLevelRange))]
            [XmlElement(ElementName = "ValueForQuality", Type = typeof(ValueForQualityRange))]
            [XmlElement(ElementName = "ValuePerLevel", Type = typeof(ValuePerLevel))]
            [XmlElement(ElementName = "ValuePerQuality", Type = typeof(ValuePerQuality))]
            public List<Value> Values;

            // Cardinality (i.e. number of contained float values it should contain).
            internal int Cardinality
            {
                get
                {
                    int count = 0, pos = -1;

                    while ((pos = Name.IndexOf('#', pos + 1)) != -1)
                        count++;

                    return count;
                }
            }

            // Cleans up redundancies.
            public void CleanUp()
            {
                // Replace ValueForRange where From == To with ValueAt.
                List<Value> values = Values.FindAll(v => v is ValueForLevelRange && ((ValueForLevelRange)v).From == ((ValueForLevelRange)v).To);
                if (values.Count > 0)
                    foreach (Value value in values)
                        Replace(value, new ValueAt { Level = ((ValueForLevelRange)value).From, Text = value.Text });
                values = Values.FindAll(v => v is ValueForQualityRange && ((ValueForQualityRange)v).From == ((ValueForQualityRange)v).To);
                if (values.Count > 0)
                    foreach (Value value in values)
                        Replace(value, new ValueAt { Quality = ((ValueForQualityRange)value).From, Text = value.Text });

                // Get level-based table values and perform redundancy checks.
                LookupTable table;
                List<ValueAt> tableValues = Values.FindAll(v => v is ValueAt && ((ValueAt)v).LevelSpecified).Cast<ValueAt>().ToList();
                bool hasLevelTable = false;
                bool levelTableIsFull = false;
                if (tableValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    table = new LookupTable();
                    foreach (ValueAt value in tableValues)
                        table.Add(value.Level, dummy);

                    hasLevelTable = true;
                    levelTableIsFull = table.IsFull();

                    // Remove obsolete level ranges and per level gains.
                    Values.RemoveAll(v => (v is ValueForLevelRange || v is ValuePerLevel) && table.IsCovering(v));
                }

                // Get quality-based table values and perform redundancy checks.
                tableValues = Values.FindAll(v => v is ValueAt && ((ValueAt)v).QualitySpecified).Cast<ValueAt>().ToList();
                bool qualityTableIsFull = false;
                if (tableValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    table = new LookupTable();
                    foreach (ValueAt value in tableValues)
                        table.Add(value.Quality, dummy);

                    qualityTableIsFull = table.IsFull();

                    // Remove obsolete quality ranges and per quality gains.
                    Values.RemoveAll(v => v is ValueForQualityRange && table.IsCovering(v) || v is ValuePerQuality && qualityTableIsFull);
                }

                // Remove obsolete ValueAt for any level if level table is full.
                if (hasLevelTable && levelTableIsFull)
                {
                    Values.RemoveAll(v => v is ValueAt && !((ValueAt)v).LevelSpecified && !((ValueAt)v).QualitySpecified);
                }

                LookupRanges ranges;
                List<ValueForLevelRange> levelRangeValues = Values.FindAll(v => v is ValueForLevelRange).Cast<ValueForLevelRange>().ToList();
                bool hasLevelRanges = false;
                bool levelRangeIsFull = false;
                if (levelRangeValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    ranges = new LookupRanges();
                    foreach (ValueForLevelRange value in levelRangeValues)
                        ranges.Add(value.From, value.To, dummy);

                    hasLevelRanges = true;
                    levelRangeIsFull = ranges.IsFull();

                    // Remove obsolete level gains.
                    Values.RemoveAll(v => v is ValuePerLevel && ranges.IsCovering(v));
                }

                List<ValueForQualityRange> qualityRangeValues = Values.FindAll(v => v is ValueForQualityRange).Cast<ValueForQualityRange>().ToList();
                bool qualityRangesIsFull = false;
                if (qualityRangeValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    ranges = new LookupRanges();
                    foreach (ValueForQualityRange value in qualityRangeValues)
                        ranges.Add(value.From, value.To, dummy);

                    qualityRangesIsFull = ranges.IsFull();

                    // Remove obsolete level gains.
                    if (qualityRangesIsFull)
                        Values.RemoveAll(v => v is ValuePerQuality);
                }

                // Remove obsolete ValueAt for any level if level range is full.
                if (hasLevelRanges && levelRangeIsFull)
                    Values.RemoveAll(v => v is ValueAt && !((ValueAt)v).LevelSpecified && !((ValueAt)v).QualitySpecified);
            }

            // Merges value.
            public void Merge(Value value)
            {
                if (value is ValueAt)
                {
                    ValueAt merge = (ValueAt)value;
                    ValueAt with = (ValueAt)Values.Find(v => v is ValueAt && ((ValueAt)v).Level == merge.Level && ((ValueAt)v).Quality == merge.Quality);
                    if (with == null)
                    {
                        // Value with no level nor quality specified replaces all values.
                        if (!merge.LevelSpecified && !merge.QualitySpecified)
                            Values.Clear();
                        Values.Add(merge);
                    }
                    else
                        with.Text = merge.Text;
                }
                else if (value is ValueForLevelRange)<commit_msg>Fixed ItemDB.Gem.Merge method adding unnecessary Value entry which is already covered by existing ValueFor range.<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Serialization;
using POESKillTree.Model;
using POESKillTree.ViewModels;
using POESKillTree.ViewModels.ItemAttribute;
using AttackSkill = POESKillTree.SkillTreeFiles.Compute.AttackSkill;
using DamageForm = POESKillTree.SkillTreeFiles.Compute.DamageForm;
using DamageNature = POESKillTree.SkillTreeFiles.Compute.DamageNature;
using DamageSource = POESKillTree.SkillTreeFiles.Compute.DamageSource;
using WeaponHand = POESKillTree.SkillTreeFiles.Compute.WeaponHand;
using WeaponType = POESKillTree.SkillTreeFiles.Compute.WeaponType;
using Weapon = POESKillTree.SkillTreeFiles.Compute.Weapon;

namespace POESKillTree.SkillTreeFiles
{
    // TODO: Attributes can have negative value (Cast when Damage Taken L20 has 6% more Damage), AttributesOf should handle transition between less/more.
    public class ItemDB
    {
        // Maximum level (quality).
        public const int MAX_LEVEL = 30;

        // Deserialized attribute.
        public class Attribute
        {
            // Deserialized comment to perserve.
            [XmlAttribute]
            public string Comment;
            // Deserialized name.
            [XmlAttribute]
            public string Name;
            // Deserialized flag indicating whether attribute should not be updated.
            [XmlAttribute]
            public bool NoUpdate = false;
            [XmlIgnore]
            public bool NoUpdateSpecified { get { return NoUpdate; } }
            // Deserialized values.
            [XmlElement(ElementName = "Value", Type = typeof(ValueAt))]
            [XmlElement(ElementName = "ValueForLevel", Type = typeof(ValueForLevelRange))]
            [XmlElement(ElementName = "ValueForQuality", Type = typeof(ValueForQualityRange))]
            [XmlElement(ElementName = "ValuePerLevel", Type = typeof(ValuePerLevel))]
            [XmlElement(ElementName = "ValuePerQuality", Type = typeof(ValuePerQuality))]
            public List<Value> Values;

            // Cardinality (i.e. number of contained float values it should contain).
            internal int Cardinality
            {
                get
                {
                    int count = 0, pos = -1;

                    while ((pos = Name.IndexOf('#', pos + 1)) != -1)
                        count++;

                    return count;
                }
            }

            // Cleans up redundancies.
            public void CleanUp()
            {
                // Replace ValueForRange where From == To with ValueAt.
                List<Value> values = Values.FindAll(v => v is ValueForLevelRange && ((ValueForLevelRange)v).From == ((ValueForLevelRange)v).To);
                if (values.Count > 0)
                    foreach (Value value in values)
                        Replace(value, new ValueAt { Level = ((ValueForLevelRange)value).From, Text = value.Text });
                values = Values.FindAll(v => v is ValueForQualityRange && ((ValueForQualityRange)v).From == ((ValueForQualityRange)v).To);
                if (values.Count > 0)
                    foreach (Value value in values)
                        Replace(value, new ValueAt { Quality = ((ValueForQualityRange)value).From, Text = value.Text });

                // Get level-based table values and perform redundancy checks.
                LookupTable table;
                List<ValueAt> tableValues = Values.FindAll(v => v is ValueAt && ((ValueAt)v).LevelSpecified).Cast<ValueAt>().ToList();
                bool hasLevelTable = false;
                bool levelTableIsFull = false;
                if (tableValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    table = new LookupTable();
                    foreach (ValueAt value in tableValues)
                        table.Add(value.Level, dummy);

                    hasLevelTable = true;
                    levelTableIsFull = table.IsFull();

                    // Remove obsolete level ranges and per level gains.
                    Values.RemoveAll(v => (v is ValueForLevelRange || v is ValuePerLevel) && table.IsCovering(v));
                }

                // Get quality-based table values and perform redundancy checks.
                tableValues = Values.FindAll(v => v is ValueAt && ((ValueAt)v).QualitySpecified).Cast<ValueAt>().ToList();
                bool qualityTableIsFull = false;
                if (tableValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    table = new LookupTable();
                    foreach (ValueAt value in tableValues)
                        table.Add(value.Quality, dummy);

                    qualityTableIsFull = table.IsFull();

                    // Remove obsolete quality ranges and per quality gains.
                    Values.RemoveAll(v => v is ValueForQualityRange && table.IsCovering(v) || v is ValuePerQuality && qualityTableIsFull);
                }

                // Remove obsolete ValueAt for any level if level table is full.
                if (hasLevelTable && levelTableIsFull)
                {
                    Values.RemoveAll(v => v is ValueAt && !((ValueAt)v).LevelSpecified && !((ValueAt)v).QualitySpecified);
                }

                LookupRanges ranges;
                List<ValueForLevelRange> levelRangeValues = Values.FindAll(v => v is ValueForLevelRange).Cast<ValueForLevelRange>().ToList();
                bool hasLevelRanges = false;
                bool levelRangeIsFull = false;
                if (levelRangeValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    ranges = new LookupRanges();
                    foreach (ValueForLevelRange value in levelRangeValues)
                        ranges.Add(value.From, value.To, dummy);

                    hasLevelRanges = true;
                    levelRangeIsFull = ranges.IsFull();

                    // Remove obsolete level gains.
                    Values.RemoveAll(v => v is ValuePerLevel && ranges.IsCovering(v));
                }

                List<ValueForQualityRange> qualityRangeValues = Values.FindAll(v => v is ValueForQualityRange).Cast<ValueForQualityRange>().ToList();
                bool qualityRangesIsFull = false;
                if (qualityRangeValues.Count > 0)
                {
                    float[] dummy = new float[] { 1 };
                    ranges = new LookupRanges();
                    foreach (ValueForQualityRange value in qualityRangeValues)
                        ranges.Add(value.From, value.To, dummy);

                    qualityRangesIsFull = ranges.IsFull();

                    // Remove obsolete level gains.
                    if (qualityRangesIsFull)
                        Values.RemoveAll(v => v is ValuePerQuality);
                }

                // Remove obsolete ValueAt for any level if level range is full.
                if (hasLevelRanges && levelRangeIsFull)
                    Values.RemoveAll(v => v is ValueAt && !((ValueAt)v).LevelSpecified && !((ValueAt)v).QualitySpecified);
            }

            // Merges value.
            public void Merge(Value value)
            {
                if (value is ValueAt)
                {
                    ValueAt merge = (ValueAt)value;
                    ValueAt with = (ValueAt)Values.Find(v => v is ValueAt && ((ValueAt)v).Level == merge.Level && ((ValueAt)v).Quality == merge.Quality);
                    if (with == null)
                    {
                        if (merge.LevelSpecified)
                        {
                            // No need to add ValueAt, if there is ValueForLevelRange covering specified Level with same Text.
                            ValueForLevelRange covers = (ValueForLevelRange)Values.Find(v => v is ValueForLevelRange && ((ValueForLevelRange)v).From <= merge.Level && ((ValueForLevelRange)v).To >= merge.Level && v.Text == merge.Text);
                            if (covers == null)
                                Values.Add(merge);
                        }
                        else if (merge.QualitySpecified)
                        {
                            // No need to add ValueAt, if there is ValueForQualityRange covering specified Level with same Text.
                            ValueForQualityRange covers = (ValueForQualityRange)Values.Find(v => v is ValueForQualityRange && ((ValueForQualityRange)v).From <= merge.Quality && ((ValueForQualityRange)v).To >= merge.Quality && v.Text == merge.Text);
                            if (covers == null)
                                Values.Add(merge);
                        }
                        else
                        {
                            // Value with no level nor quality specified replaces all values.
                            Values.Clear();
                            Values.Add(merge);
                        }
                    }
                    else
                        with.Text = merge.Text;
                }
                else if (value is ValueForLevelRange)
{'fraction_non_alphanumeric': 0.08981523722627738, 'fraction_numerical': 0.002166970802919708, 'mean_word_length': 1.615900954653938, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3295400', 'n_tokens_mistral': 4523, 'n_tokens_neox': 4107, 'n_words': 1114}