<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Packager.Exceptions;
using Packager.Extensions;
using Packager.Factories;
using Packager.Models.FileModels;
using Packager.Models.PodMetadataModels;
using Packager.Observers;
using Packager.Validators;
using RestSharp;

namespace Packager.Providers
{
    public class PodMetadataProvider : IPodMetadataProvider
    {
        public PodMetadataProvider(IRestClient client, IObserverCollection observers, IValidatorCollection validators)
        {
            Client = client;
            Observers = observers;
            Validators = validators;
        }

        private IRestClient Client { get; }
        private IObserverCollection Observers { get; }
        private IValidatorCollection Validators { get; }

        public async Task<T> GetObjectMetadata<T>(string barcode, CancellationToken cancellationToken) where T : AbstractPodMetadata, new()
        {
            var request = new RestRequest($"responses/objects/{barcode}/metadata/digital_provenance");

            var response = await Client.ExecuteGetTaskAsync<T>(request, cancellationToken);
            VerifyResponse(response);
           
            return Normalize(response.Data);
        }

        public void Validate<T>(T podMetadata, List<AbstractFile> models) where T : AbstractPodMetadata
        {
            var results = ValidateMetadata(podMetadata);
            results.AddRange(ValidateMetadataProvenances(podMetadata.FileProvenances, models));

            if (results.Succeeded)
            {
                return;
            }

            throw new PodMetadataException(results);
        }

        public void Log<T>(T podMetadata) where T : AbstractPodMetadata
        {
            Observers.Log(podMetadata.GetStringPropertiesAndValues());
            Observers.Log(podMetadata.GetDatePropertiesAndValues());
            foreach (var provenance in podMetadata.FileProvenances)
            {
                var sectionKey = Observers.BeginSection("File Provenance: {0}",
                    provenance.Filename.ToDefaultIfEmpty("[not set]"));
                Observers.Log(provenance.GetStringPropertiesAndValues("\t"));
                Observers.Log(provenance.GetDatePropertiesAndValues("\t"));

                foreach (var device in provenance.PlayerDevices)
                {
                    Observers.Log("\tPlayer: {0} {1} ({2})", device.Manufacturer, device.Model, device.SerialNumber);
                }

                foreach (var device in provenance.AdDevices)
                {
                    Observers.Log("\tAD Device: {0} {1} ({2})", device.Manufacturer, device.Model, device.SerialNumber);
                }

                Observers.EndSection(sectionKey);
            }
        }

        private ValidationResults ValidateMetadata<T>(T podMetadata) where T : AbstractPodMetadata
        {
            return Validators.Validate(podMetadata);
        }

        private static ValidationResults ValidateMetadataProvenances(List<AbstractDigitalFile> provenances,
            List<AbstractFile> filesToProcess)
        {
            var results = new ValidationResults();
            if (provenances == null)
            {
                results.Add("Value not set for digital file provenances");
                return results;
            }

            results.AddRange(ValidateMastersPresent(provenances, filesToProcess));
            results.AddRange(ValidateProvenancesPresent(provenances, filesToProcess));

            return results;
        }

        private static ValidationResults ValidateMastersPresent(IEnumerable<AbstractDigitalFile> provenances,
            IEnumerable<AbstractFile> filesToProcess)
        {
            var results = new ValidationResults();

            var expectedMasters = provenances
                .Select(p => FileModelFactory.GetModel(p.Filename))
                .Select(m => m.Filename).ToList();

            var presentFiles = filesToProcess
                .Select(m => m.Filename).ToList();

            results.AddRange(
                expectedMasters.Except(presentFiles)
                    .Select(f => new ValidationResult("No original master present for {0}", f)));
            return results;
        }

        private static ValidationResults ValidateProvenancesPresent(List<AbstractDigitalFile> provenances,
            IEnumerable<AbstractFile> filesToProcess)
        {
            var results = new ValidationResults();

            var unexpectedMasters = filesToProcess.Where(
                m => m.IsPreservationVersion() || m.IsPreservationIntermediateVersion())
                .Select(m => new {key = m.Filename, value = provenances.GetFileProvenance(m)})
                .Where(p => p.value == null)
                .Select(p => p.key);

            results.AddRange(
                unexpectedMasters.Select(f => new ValidationResult("No digital file provenance found for {0}", f)));
            return results;
        }


        // go through the result's string fields and remove leading and trailing whitespace
        private static T Normalize<T>(T metadata) where T : AbstractPodMetadata
        {
            metadata = NormalizeFields(metadata);

            for (var i = 0; i < metadata.FileProvenances.Count; i++)
            {
                metadata.FileProvenances[i] = NormalizeFields(metadata.FileProvenances[i]);

                for (var j = 0; j < metadata.FileProvenances[i].SignalChain.Count; j++)<commit_msg>should expect metadata to be present only for files to be normalized<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using Packager.Exceptions;
using Packager.Extensions;
using Packager.Factories;
using Packager.Models.FileModels;
using Packager.Models.PodMetadataModels;
using Packager.Observers;
using Packager.Validators;
using RestSharp;

namespace Packager.Providers
{
    public class PodMetadataProvider : IPodMetadataProvider
    {
        public PodMetadataProvider(IRestClient client, IObserverCollection observers, IValidatorCollection validators)
        {
            Client = client;
            Observers = observers;
            Validators = validators;
        }

        private IRestClient Client { get; }
        private IObserverCollection Observers { get; }
        private IValidatorCollection Validators { get; }

        public async Task<T> GetObjectMetadata<T>(string barcode, CancellationToken cancellationToken) where T : AbstractPodMetadata, new()
        {
            var request = new RestRequest($"responses/objects/{barcode}/metadata/digital_provenance");

            var response = await Client.ExecuteGetTaskAsync<T>(request, cancellationToken);
            VerifyResponse(response);
           
            return Normalize(response.Data);
        }

        public void Validate<T>(T podMetadata, List<AbstractFile> models) where T : AbstractPodMetadata
        {
            var results = ValidateMetadata(podMetadata);
            results.AddRange(ValidateMetadataProvenances(podMetadata.FileProvenances, models));

            if (results.Succeeded)
            {
                return;
            }

            throw new PodMetadataException(results);
        }

        public void Log<T>(T podMetadata) where T : AbstractPodMetadata
        {
            Observers.Log(podMetadata.GetStringPropertiesAndValues());
            Observers.Log(podMetadata.GetDatePropertiesAndValues());
            foreach (var provenance in podMetadata.FileProvenances)
            {
                var sectionKey = Observers.BeginSection("File Provenance: {0}",
                    provenance.Filename.ToDefaultIfEmpty("[not set]"));
                Observers.Log(provenance.GetStringPropertiesAndValues("\t"));
                Observers.Log(provenance.GetDatePropertiesAndValues("\t"));

                foreach (var device in provenance.PlayerDevices)
                {
                    Observers.Log("\tPlayer: {0} {1} ({2})", device.Manufacturer, device.Model, device.SerialNumber);
                }

                foreach (var device in provenance.AdDevices)
                {
                    Observers.Log("\tAD Device: {0} {1} ({2})", device.Manufacturer, device.Model, device.SerialNumber);
                }

                Observers.EndSection(sectionKey);
            }
        }

        private ValidationResults ValidateMetadata<T>(T podMetadata) where T : AbstractPodMetadata
        {
            return Validators.Validate(podMetadata);
        }

        private static ValidationResults ValidateMetadataProvenances(List<AbstractDigitalFile> provenances,
            List<AbstractFile> filesToProcess)
        {
            var results = new ValidationResults();
            if (provenances == null)
            {
                results.Add("Value not set for digital file provenances");
                return results;
            }

            results.AddRange(ValidateMastersPresent(provenances, filesToProcess));
            results.AddRange(ValidateProvenancesPresent(provenances, filesToProcess));

            return results;
        }

        private static ValidationResults ValidateMastersPresent(IEnumerable<AbstractDigitalFile> provenances,
            IEnumerable<AbstractFile> filesToProcess)
        {
            var results = new ValidationResults();

            var expectedMasters = provenances
                .Select(p => FileModelFactory.GetModel(p.Filename))
                .Select(m => m.Filename).ToList();

            var presentFiles = filesToProcess
                .Select(m => m.Filename).ToList();

            results.AddRange(
                expectedMasters.Except(presentFiles)
                    .Select(f => new ValidationResult("No original master present for {0}", f)));
            return results;
        }

        private static ValidationResults ValidateProvenancesPresent(List<AbstractDigitalFile> provenances,
            IEnumerable<AbstractFile> filesToProcess)
        {
            var results = new ValidationResults();

            var unexpectedMasters = filesToProcess
                .Where(m => m.ShouldNormalize)
                .Select(m => new {key = m.Filename, value = provenances.GetFileProvenance(m)})
                .Where(p => p.value == null)
                .Select(p => p.key);

            results.AddRange(
                unexpectedMasters.Select(f => new ValidationResult("No digital file provenance found for {0}", f)));
            return results;
        }


        // go through the result's string fields and remove leading and trailing whitespace
        private static T Normalize<T>(T metadata) where T : AbstractPodMetadata
        {
            metadata = NormalizeFields(metadata);

            for (var i = 0; i < metadata.FileProvenances.Count; i++)
            {
                metadata.FileProvenances[i] = NormalizeFields(metadata.FileProvenances[i]);

                for (var j = 0; j < metadata.FileProvenances[i].SignalChain.Count; j++)
{'fraction_non_alphanumeric': 0.08669642857142858, 'fraction_numerical': 0.0019642857142857144, 'mean_word_length': 2.264645875837948, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6600891', 'n_tokens_mistral': 2771, 'n_tokens_neox': 2684, 'n_words': 617}