<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using HeterogeneousDataSources.LoadLinkExpressions;

namespace HeterogeneousDataSources
{
    public class LoadLinker<TRootLinkedSource, TRootLinkedSourceModel> : ILoadLinker<TRootLinkedSource>
        where TRootLinkedSource : class, ILinkedSource<TRootLinkedSourceModel>, new() 
    {
        //stle: handle dispose in using!
        private readonly IReferenceLoader _referenceLoader;
        private readonly List<List<Type>> _referenceTypeToBeLoadedForEachLoadingLevel;
        private readonly LoadLinkConfig _config;

        public LoadLinker(IReferenceLoader referenceLoader, List<List<Type>> referenceTypeToBeLoadedForEachLoadingLevel, LoadLinkConfig config)
        {
            _referenceLoader = referenceLoader;
            _referenceTypeToBeLoadedForEachLoadingLevel = referenceTypeToBeLoadedForEachLoadingLevel;
            _config = config;
        }

        //stle: fix naming: TRootLinkedSourceModel1
        public TRootLinkedSource FromModel<TRootLinkedSourceModel1>(TRootLinkedSourceModel1 model)
        {
            return FromModel(new List<object> { model })
                .SingleOrDefault();
        }

        public List<TRootLinkedSource> FromModel<TRootLinkedSourceModel1>(List<TRootLinkedSourceModel1> models)
        {
            //stle: beaviour on model null? and id null

            //stle: ensure TRootLinkedSourceModel1=TRootLinkedSourceModel
            var loadedReferenceContext = new LoadedReferenceContext();

            var linkedSources = models
                .Cast<TRootLinkedSourceModel>()
                .Select(model => CreateLinkedSource(model, loadedReferenceContext))
                .ToList();

            LoadLinkRootLinkedSource(loadedReferenceContext);
            return linkedSources;
        }

        public TRootLinkedSource ById<TRootLinkedSourceModelId>(TRootLinkedSourceModelId modelId)
        {
            if (modelId == null) { throw new ArgumentNullException("modelId"); }

            var lookupIdContext = new LookupIdContext();
            lookupIdContext.AddSingle<TRootLinkedSourceModel, TRootLinkedSourceModelId>(modelId);

            //stle: where to start context?
            var loadedReferenceContext = new LoadedReferenceContext();
            _referenceLoader.LoadReferences(lookupIdContext, loadedReferenceContext);
            var model = loadedReferenceContext
                .GetOptionalReference<TRootLinkedSourceModel, TRootLinkedSourceModelId>(modelId);

            return FromModel(model);
        }

        private void LoadLinkRootLinkedSource(LoadedReferenceContext loadedReferenceContext) {
            //stle: do it at creation???? by linkedSource expression
            LinkSubLinkedSources(loadedReferenceContext);

            Load(loadedReferenceContext, 0);

            LinkReferences(loadedReferenceContext);
        }

        private void Load(LoadedReferenceContext loadedReferenceContext, int startAtLoadingLevel) {
            using (_referenceLoader) {
                for (int loadingLevel = startAtLoadingLevel; loadingLevel < _referenceTypeToBeLoadedForEachLoadingLevel.Count; loadingLevel++) {
                    var referenceTypeToBeLoaded = _referenceTypeToBeLoadedForEachLoadingLevel[loadingLevel];

                    LoadNestingLevel(loadedReferenceContext, referenceTypeToBeLoaded);
                    LinkNestedLinkedSources(loadedReferenceContext, referenceTypeToBeLoaded);
                    LinkSubLinkedSources(loadedReferenceContext);
                }
            }
        }

        private void LoadNestingLevel(LoadedReferenceContext loadedReferenceContext, List<Type> referenceTypeToBeLoaded) {
            var lookupIdContext = GetLookupIdContextForLoadingLevel(loadedReferenceContext, referenceTypeToBeLoaded);
            _referenceLoader.LoadReferences(lookupIdContext, loadedReferenceContext);
        }

        private LookupIdContext GetLookupIdContextForLoadingLevel(LoadedReferenceContext loadedReferenceContext, List<Type> referenceTypesToBeLoaded) {
            var lookupIdContext = new LookupIdContext();

            foreach (var referenceTypeToBeLoaded in referenceTypesToBeLoaded) {
                foreach (var linkedSource in loadedReferenceContext.LinkedSourcesToBeBuilt) {
                    var loadLinkExpressions = _config.GetLoadLinkExpressions(linkedSource, referenceTypeToBeLoaded);
                    foreach (var loadLinkExpression in loadLinkExpressions) {
                        loadLinkExpression.AddLookupIds(linkedSource, lookupIdContext, referenceTypeToBeLoaded);
                    }
                }
            }
            return lookupIdContext;
        }

        private void LinkNestedLinkedSources(LoadedReferenceContext loadedReferenceContext, List<Type> referenceTypesToBeLoaded) {
            foreach (var referenceTypeToBeLoaded in referenceTypesToBeLoaded) {
                foreach (var linkedSource in loadedReferenceContext.LinkedSourcesToBeBuilt) {
                    var loadLinkExpressions = _config.GetLoadLinkExpressions(linkedSource, referenceTypeToBeLoaded);
                    foreach (var loadLinkExpression in loadLinkExpressions) {
                        loadLinkExpression.LinkNestedLinkedSource(linkedSource, loadedReferenceContext, referenceTypeToBeLoaded);
                    }
                }
            }
        }

        private void LinkSubLinkedSources(LoadedReferenceContext loadedReferenceContext) {
            while (loadedReferenceContext.GetLinkedSourceWithSubLinkedSourceToLink().Any()) {
                foreach (var linkedSource in loadedReferenceContext.GetLinkedSourceWithSubLinkedSourceToLink()) {
                    var loadLinkExpressions = _config.GetLoadExpressions(linkedSource);
                    foreach (var loadLinkExpression in loadLinkExpressions) {
                        loadLinkExpression.LinkSubLinkedSource(
                            linkedSource,
                            loadedReferenceContext
                        );
                    }
                    loadedReferenceContext.AddLinkedSourceWhereSubLinkedSourceAreLinked(linkedSource);
                }
            }
        }

        private void LinkReferences(LoadedReferenceContext loadedReferenceContext) {
            foreach (var linkedSource in loadedReferenceContext.LinkedSourcesToBeBuilt) {
                var loadLinkExpressions = _config.GetLoadExpressions(linkedSource);
                foreach (var loadLinkExpression in loadLinkExpressions) {
                    loadLinkExpression.LinkReference(
                        linkedSource,
                        loadedReferenceContext
                    );
                }
            }
        }

        ////Is LoadLinkExpressionUtil still required?
        public TRootLinkedSource CreateLinkedSource(TRootLinkedSourceModel model, LoadedReferenceContext loadedReferenceContext) {
            return LoadLinkExpressionUtil.CreateLinkedSource<TRootLinkedSource, TRootLinkedSourceModel>(
                model,
                loadedReferenceContext
            );
        }
    }
}<commit_msg>simplify even more with more contexte var for the loadlinker<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using HeterogeneousDataSources.LoadLinkExpressions;

namespace HeterogeneousDataSources
{
    public class LoadLinker<TRootLinkedSource, TRootLinkedSourceModel> : ILoadLinker<TRootLinkedSource>
        where TRootLinkedSource : class, ILinkedSource<TRootLinkedSourceModel>, new() 
    {
        //stle: handle dispose in using!
        private readonly IReferenceLoader _referenceLoader;
        private readonly List<List<Type>> _referenceTypeToBeLoadedForEachLoadingLevel;
        private readonly LoadLinkConfig _config;
        private LoadedReferenceContext _loadedReferenceContext;

        public LoadLinker(IReferenceLoader referenceLoader, List<List<Type>> referenceTypeToBeLoadedForEachLoadingLevel, LoadLinkConfig config)
        {
            _referenceLoader = referenceLoader;
            _referenceTypeToBeLoadedForEachLoadingLevel = referenceTypeToBeLoadedForEachLoadingLevel;
            _config = config;
        }

        //stle: fix naming: TRootLinkedSourceModel1
        public TRootLinkedSource FromModel<TRootLinkedSourceModel1>(TRootLinkedSourceModel1 model)
        {
            return FromModel(new List<object> { model })
                .SingleOrDefault();
        }

        public List<TRootLinkedSource> FromModel<TRootLinkedSourceModel1>(List<TRootLinkedSourceModel1> models)
        {
            //stle: beaviour on model null? and id null

            //stle: ensure TRootLinkedSourceModel1=TRootLinkedSourceModel
            _loadedReferenceContext = new LoadedReferenceContext();

            var linkedSources = models
                .Cast<TRootLinkedSourceModel>()
                .Select(model => CreateLinkedSource(model))
                .ToList();

            LoadLinkRootLinkedSource();
            return linkedSources;
        }

        public TRootLinkedSource ById<TRootLinkedSourceModelId>(TRootLinkedSourceModelId modelId)
        {
            var model = LoadRootLinkedSourceModel(modelId);
            return FromModel(model);
        }

        private TRootLinkedSourceModel LoadRootLinkedSourceModel<TRootLinkedSourceModelId>(TRootLinkedSourceModelId modelId)
        {
            if (modelId == null) { throw new ArgumentNullException("modelId"); }

            var lookupIdContext = new LookupIdContext();
            lookupIdContext.AddSingle<TRootLinkedSourceModel, TRootLinkedSourceModelId>(modelId);

            var loadedRootLinkedSourceModel = new LoadedReferenceContext();
            _referenceLoader.LoadReferences(lookupIdContext, loadedRootLinkedSourceModel);

            return loadedRootLinkedSourceModel
                .GetOptionalReference<TRootLinkedSourceModel, TRootLinkedSourceModelId>(modelId);
        }

        private void LoadLinkRootLinkedSource() {
            //stle: do it at creation???? by linkedSource expression
            LinkSubLinkedSources();

            Load(0);

            LinkReferences();
        }

        private void Load(int startAtLoadingLevel) {
            using (_referenceLoader) {
                for (int loadingLevel = startAtLoadingLevel; loadingLevel < _referenceTypeToBeLoadedForEachLoadingLevel.Count; loadingLevel++) {
                    var referenceTypeToBeLoaded = _referenceTypeToBeLoadedForEachLoadingLevel[loadingLevel];

                    LoadNestingLevel(referenceTypeToBeLoaded);
                    LinkNestedLinkedSources(referenceTypeToBeLoaded);
                    LinkSubLinkedSources();
                }
            }
        }

        private void LoadNestingLevel(List<Type> referenceTypeToBeLoaded) {
            var lookupIdContext = GetLookupIdContextForLoadingLevel(referenceTypeToBeLoaded);
            _referenceLoader.LoadReferences(lookupIdContext, _loadedReferenceContext);
        }

        private LookupIdContext GetLookupIdContextForLoadingLevel(List<Type> referenceTypesToBeLoaded) {
            var lookupIdContext = new LookupIdContext();

            foreach (var referenceTypeToBeLoaded in referenceTypesToBeLoaded) {
                foreach (var linkedSource in _loadedReferenceContext.LinkedSourcesToBeBuilt) {
                    var loadLinkExpressions = _config.GetLoadLinkExpressions(linkedSource, referenceTypeToBeLoaded);
                    foreach (var loadLinkExpression in loadLinkExpressions) {
                        loadLinkExpression.AddLookupIds(linkedSource, lookupIdContext, referenceTypeToBeLoaded);
                    }
                }
            }
            return lookupIdContext;
        }

        private void LinkNestedLinkedSources(List<Type> referenceTypesToBeLoaded) {
            foreach (var referenceTypeToBeLoaded in referenceTypesToBeLoaded) {
                foreach (var linkedSource in _loadedReferenceContext.LinkedSourcesToBeBuilt) {
                    var loadLinkExpressions = _config.GetLoadLinkExpressions(linkedSource, referenceTypeToBeLoaded);
                    foreach (var loadLinkExpression in loadLinkExpressions) {
                        loadLinkExpression.LinkNestedLinkedSource(linkedSource, _loadedReferenceContext, referenceTypeToBeLoaded);
                    }
                }
            }
        }

        private void LinkSubLinkedSources() {
            while (_loadedReferenceContext.GetLinkedSourceWithSubLinkedSourceToLink().Any()) {
                foreach (var linkedSource in _loadedReferenceContext.GetLinkedSourceWithSubLinkedSourceToLink()) {
                    var loadLinkExpressions = _config.GetLoadExpressions(linkedSource);
                    foreach (var loadLinkExpression in loadLinkExpressions) {
                        loadLinkExpression.LinkSubLinkedSource(
                            linkedSource,
                            _loadedReferenceContext
                        );
                    }
                    _loadedReferenceContext.AddLinkedSourceWhereSubLinkedSourceAreLinked(linkedSource);
                }
            }
        }

        private void LinkReferences() {
            foreach (var linkedSource in _loadedReferenceContext.LinkedSourcesToBeBuilt) {
                var loadLinkExpressions = _config.GetLoadExpressions(linkedSource);
                foreach (var loadLinkExpression in loadLinkExpressions) {
                    loadLinkExpression.LinkReference(
                        linkedSource,
                        _loadedReferenceContext
                    );
                }
            }
        }

        ////Is LoadLinkExpressionUtil still required?
        public TRootLinkedSource CreateLinkedSource(TRootLinkedSourceModel model) {
            return LoadLinkExpressionUtil.CreateLinkedSource<TRootLinkedSource, TRootLinkedSourceModel>(
                model,
                _loadedReferenceContext
            );
        }
    }
}
{'fraction_non_alphanumeric': 0.055551609007601054, 'fraction_numerical': 0.000994530084535057, 'mean_word_length': 2.4881070366699705, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5884831', 'n_tokens_mistral': 3204, 'n_tokens_neox': 3053, 'n_words': 590}