<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Ilaro.Admin.Extensions;
using Ilaro.Admin.Models;
using Massive;

namespace Ilaro.Admin.Core.Data
{
    public class RecordsHierarchySource : IFetchingRecordsHierarchy
    {
        private static readonly IInternalLogger _log = LoggerProvider.LoggerFor(typeof(RecordsHierarchySource));

        public RecordHierarchy GetRecordHierarchy(Entity entity)
        {
            _log.InfoFormat("Getting record hierarchy for entity record ({0}#{1})", entity.Name, entity.JoinedKeyWithValue);

            var index = 0;
            var hierarchy = GetEntityHierarchy(null, entity, ref index);
            var sql = GenerateHierarchySql(hierarchy);
            _log.DebugFormat("Sql hierarchy: \r\n {0}", sql);
            var model = new DynamicModel(Admin.ConnectionStringName);
            var records = model.Query(sql, entity.Key.Select(x => x.Value.Raw).ToArray()).ToList();

            var recordHierarchy = GetHierarchyRecords(records, hierarchy);

            return recordHierarchy;
        }

        private RecordHierarchy GetHierarchyRecords(
            IList<IDictionary<string, object>> records,
            EntityHierarchy hierarchy)
        {
            var baseRecord = records.FirstOrDefault();
            var prefix = hierarchy.Alias.Undecorate() + "_";
            var rowData = new DataRow(baseRecord, hierarchy.Entity, prefix);

            var recordHierarchy = new RecordHierarchy
            {
                Entity = hierarchy.Entity,
                KeyValue = rowData.KeyValue,
                DisplayName = hierarchy.Entity.ToString(rowData),
                SubRecordsHierarchies = new List<RecordHierarchy>()
            };

            GetHierarchyRecords(recordHierarchy, records, hierarchy.SubHierarchies);

            return recordHierarchy;
        }

        private void GetHierarchyRecords(
            RecordHierarchy parentHierarchy,
            IList<IDictionary<string, object>> records,
            IEnumerable<EntityHierarchy> subHierarchies,
            IList<string> foreignKey = null,
            IList<string> foreignKeyValue = null)
        {
            foreach (var hierarchy in subHierarchies)
            {
                var prefix = hierarchy.Alias.Undecorate() + "_";

                foreach (var record in records)
                {
                    var rowData = new DataRow(record, hierarchy.Entity, prefix);

                    if (!rowData.KeyValue.IsNullOrEmpty())
                    {
                        var subRecord = new RecordHierarchy
                        {
                            Entity = hierarchy.Entity,
                            KeyValue = rowData.KeyValue,
                            DisplayName = hierarchy.Entity.ToString(rowData),
                            SubRecordsHierarchies = new List<RecordHierarchy>()
                        };

                        if (parentHierarchy.SubRecordsHierarchies.FirstOrDefault(x => x.KeyValue == subRecord.KeyValue) == null &&
                            Matching(record, foreignKey, foreignKeyValue))
                        {
                            parentHierarchy.SubRecordsHierarchies.Add(subRecord);

                            GetHierarchyRecords(
                                subRecord,
                                records,
                                hierarchy.SubHierarchies,
                                hierarchy.Entity.Key.Select(x => prefix + x.ColumnName.Undecorate()).ToList(),
                                rowData.KeyValue);
                        }
                    }
                }
            }
        }<commit_msg>Fixed duplicating records while building record hierachy<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using Ilaro.Admin.Extensions;
using Ilaro.Admin.Models;
using Massive;

namespace Ilaro.Admin.Core.Data
{
    public class RecordsHierarchySource : IFetchingRecordsHierarchy
    {
        private static readonly IInternalLogger _log = LoggerProvider.LoggerFor(typeof(RecordsHierarchySource));

        public RecordHierarchy GetRecordHierarchy(Entity entity)
        {
            _log.InfoFormat("Getting record hierarchy for entity record ({0}#{1})", entity.Name, entity.JoinedKeyWithValue);

            var index = 0;
            var hierarchy = GetEntityHierarchy(null, entity, ref index);
            var sql = GenerateHierarchySql(hierarchy);
            _log.DebugFormat("Sql hierarchy: \r\n {0}", sql);
            var model = new DynamicModel(Admin.ConnectionStringName);
            var records = model.Query(sql, entity.Key.Select(x => x.Value.Raw).ToArray()).ToList();

            var recordHierarchy = GetHierarchyRecords(records, hierarchy);

            return recordHierarchy;
        }

        private RecordHierarchy GetHierarchyRecords(
            IList<IDictionary<string, object>> records,
            EntityHierarchy hierarchy)
        {
            var baseRecord = records.FirstOrDefault();
            var prefix = hierarchy.Alias.Undecorate() + "_";
            var rowData = new DataRow(baseRecord, hierarchy.Entity, prefix);

            var recordHierarchy = new RecordHierarchy
            {
                Entity = hierarchy.Entity,
                KeyValue = rowData.KeyValue,
                DisplayName = hierarchy.Entity.ToString(rowData),
                SubRecordsHierarchies = new List<RecordHierarchy>()
            };

            GetHierarchyRecords(recordHierarchy, records, hierarchy.SubHierarchies);

            return recordHierarchy;
        }

        private void GetHierarchyRecords(
            RecordHierarchy parentHierarchy,
            IList<IDictionary<string, object>> records,
            IEnumerable<EntityHierarchy> subHierarchies,
            IList<string> foreignKey = null,
            IList<string> foreignKeyValue = null)
        {
            foreach (var hierarchy in subHierarchies)
            {
                var prefix = hierarchy.Alias.Undecorate() + "_";

                foreach (var record in records)
                {
                    var rowData = new DataRow(record, hierarchy.Entity, prefix);

                    if (!rowData.KeyValue.IsNullOrEmpty())
                    {
                        var subRecord = new RecordHierarchy
                        {
                            Entity = hierarchy.Entity,
                            KeyValue = rowData.KeyValue,
                            DisplayName = hierarchy.Entity.ToString(rowData),
                            SubRecordsHierarchies = new List<RecordHierarchy>()
                        };

                        if (parentHierarchy.SubRecordsHierarchies.FirstOrDefault(x => x.JoinedKeyValue == subRecord.JoinedKeyValue) == null &&
                            Matching(record, foreignKey, foreignKeyValue))
                        {
                            parentHierarchy.SubRecordsHierarchies.Add(subRecord);

                            GetHierarchyRecords(
                                subRecord,
                                records,
                                hierarchy.SubHierarchies,
                                hierarchy.Entity.Key.Select(x => prefix + x.ColumnName.Undecorate()).ToList(),
                                rowData.KeyValue);
                        }
                    }
                }
            }
        }
{'fraction_non_alphanumeric': 0.07804747217379643, 'fraction_numerical': 0.001072817486925037, 'mean_word_length': 1.619599578503688, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4554594', 'n_tokens_mistral': 1833, 'n_tokens_neox': 1765, 'n_words': 354}