<commit_before>﻿using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using FilterLists.Agent.Infrastructure.Clients;
using MediatR;
using Microsoft.Extensions.Logging;

namespace FilterLists.Agent.Features.Urls
{
    public static class ValidateUrls
    {
        public class Command : IRequest<List<UrlValidationResult>>
        {
            public Command(IEnumerable<Uri> urls)
            {
                Urls = urls;
            }

            public IEnumerable<Uri> Urls { get; }
        }

        public class Handler : IRequestHandler<Command, List<UrlValidationResult>>
        {
            private const int MaxDegreeOfParallelism = 5;
            private readonly HttpClient _httpClient;
            private readonly ILogger<Handler> _logger;

            public Handler(AgentHttpClient agentHttpClient, ILogger<Handler> logger)
            {
                _httpClient = agentHttpClient.Client;
                _logger = logger;
            }

            public async Task<List<UrlValidationResult>> Handle(Command request, CancellationToken cancellationToken)
            {
                var validator = BuildValidator(cancellationToken);
                var brokenUrls = new List<UrlValidationResult>();
                foreach (var url in request.Urls)
                    await validator.SendAsync(url, cancellationToken);
                validator.Complete();
                while (await validator.OutputAvailableAsync(cancellationToken))
                {
                    var result = await validator.ReceiveAsync(cancellationToken);
                    if (!result.IsValid())
                        brokenUrls.Add(result);
                }

                await validator.Completion;
                return brokenUrls;
            }

            private TransformBlock<Uri, UrlValidationResult> BuildValidator(CancellationToken cancellationToken)
            {
                return new TransformBlock<Uri, UrlValidationResult>(
                    async u =>
                    {
                        var result = new UrlValidationResult(u);
                        if (!u.IsValidUrl())
                        {
                            result.SetBroken();
                            _logger.LogError($"{u.OriginalString}) is not a valid URL.");
                            return result;
                        }

                        try
                        {
                            var response = await _httpClient.GetAsync(u, cancellationToken);
                            if (response.StatusCode == HttpStatusCode.PermanentRedirect ||
                                response.StatusCode == HttpStatusCode.TemporaryRedirect)
                                result.SetRedirectsTo(response.Headers.Location);
                            if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
                                result.SetSupportsHttps();
                            if (response.IsSuccessStatusCode)
                                return result;
                            result.SetBroken();
                            _logger.LogError(
                                $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
                            return result;
                        }
                        catch (HttpRequestException ex)
                        {
                            result.SetBroken();
                            _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
                            return result;
                        }
                        catch (TaskCanceledException ex)
                        {
                            result.SetBroken();
                            _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
                            return result;
                        }
                        catch (InvalidOperationException ex)
                        {
                            result.SetBroken();
                            _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
                            return result;
                        }
                    },
                    new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
                );
            }

            private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
            {
                var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri;
                try
                {
                    var response = await _httpClient.GetAsync(httpsUrl, cancellationToken);
                    if (response.IsSuccessStatusCode)
                        return true;
                    _logger.LogError(
                        $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
                    return response.IsSuccessStatusCode;
                }
                catch (HttpRequestException ex)
                {
                    _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
                    return false;
                }
                catch (TaskCanceledException ex)
                {
                    _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
                    return false;
                }
                catch (InvalidOperationException ex)
                {
                    _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
                    return false;
                }
            }
        }
    }
}<commit_msg>only validate distinct URLs per file<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using FilterLists.Agent.Extensions;
using FilterLists.Agent.Features.Urls.Models.ValidationResults;
using FilterLists.Agent.Infrastructure.Clients;
using MediatR;
using Microsoft.Extensions.Logging;

namespace FilterLists.Agent.Features.Urls
{
    public static class ValidateUrls
    {
        public class Command : IRequest<List<UrlValidationResult>>
        {
            public Command(IEnumerable<Uri> urls)
            {
                Urls = urls;
            }

            public IEnumerable<Uri> Urls { get; }
        }

        public class Handler : IRequestHandler<Command, List<UrlValidationResult>>
        {
            private const int MaxDegreeOfParallelism = 5;
            private readonly HttpClient _httpClient;
            private readonly ILogger<Handler> _logger;

            public Handler(AgentHttpClient agentHttpClient, ILogger<Handler> logger)
            {
                _httpClient = agentHttpClient.Client;
                _logger = logger;
            }

            public async Task<List<UrlValidationResult>> Handle(Command request, CancellationToken cancellationToken)
            {
                var validator = BuildValidator(cancellationToken);
                var brokenUrls = new List<UrlValidationResult>();
                var distinctUrls = request.Urls.Distinct();
                foreach (var url in distinctUrls)
                    await validator.SendAsync(url, cancellationToken);
                validator.Complete();
                while (await validator.OutputAvailableAsync(cancellationToken))
                {
                    var result = await validator.ReceiveAsync(cancellationToken);
                    if (!result.IsValid())
                        brokenUrls.Add(result);
                }

                await validator.Completion;
                return brokenUrls;
            }

            private TransformBlock<Uri, UrlValidationResult> BuildValidator(CancellationToken cancellationToken)
            {
                return new TransformBlock<Uri, UrlValidationResult>(
                    async u =>
                    {
                        var result = new UrlValidationResult(u);
                        if (!u.IsValidUrl())
                        {
                            result.SetBroken();
                            _logger.LogError($"{u.OriginalString}) is not a valid URL.");
                            return result;
                        }

                        try
                        {
                            var response = await _httpClient.GetAsync(u, cancellationToken);
                            if (response.StatusCode == HttpStatusCode.PermanentRedirect ||
                                response.StatusCode == HttpStatusCode.TemporaryRedirect)
                                result.SetRedirectsTo(response.Headers.Location);
                            if (u.Scheme == Uri.UriSchemeHttp && await IsHttpsSupported(u, cancellationToken))
                                result.SetSupportsHttps();
                            if (response.IsSuccessStatusCode)
                                return result;
                            result.SetBroken();
                            _logger.LogError(
                                $"Url validation for ({u.AbsoluteUri}) failed with status code: {response.StatusCode}.");
                            return result;
                        }
                        catch (HttpRequestException ex)
                        {
                            result.SetBroken();
                            _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
                            return result;
                        }
                        catch (TaskCanceledException ex)
                        {
                            result.SetBroken();
                            _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
                            return result;
                        }
                        catch (InvalidOperationException ex)
                        {
                            result.SetBroken();
                            _logger.LogError($"Url validation for ({u.AbsoluteUri}) failed.", ex);
                            return result;
                        }
                    },
                    new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = MaxDegreeOfParallelism}
                );
            }

            private async Task<bool> IsHttpsSupported(Uri url, CancellationToken cancellationToken)
            {
                var httpsUrl = new UriBuilder(url.OriginalString) {Scheme = Uri.UriSchemeHttps}.Uri;
                try
                {
                    var response = await _httpClient.GetAsync(httpsUrl, cancellationToken);
                    if (response.IsSuccessStatusCode)
                        return true;
                    _logger.LogError(
                        $"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed with status code: {response.StatusCode}.");
                    return response.IsSuccessStatusCode;
                }
                catch (HttpRequestException ex)
                {
                    _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
                    return false;
                }
                catch (TaskCanceledException ex)
                {
                    _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
                    return false;
                }
                catch (InvalidOperationException ex)
                {
                    _logger.LogError($"IsHttpsSupported({httpsUrl.AbsoluteUri}) failed.", ex);
                    return false;
                }
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.07333443818362612, 'fraction_numerical': 0.00016572754391779913, 'mean_word_length': 1.2788897280966767, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1686625', 'n_tokens_mistral': 2676, 'n_tokens_neox': 2553, 'n_words': 540}