<commit_before>﻿using CodeHollow.FeedReader.Feeds;
using Discord;
using Discord.WebSocket;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Core.Services;
using NadekoBot.Core.Services.Database.Models;
using NadekoBot.Extensions;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

namespace NadekoBot.Modules.Searches.Services
{
    public class FeedsService : INService
    {
        private readonly DbService _db;
        private readonly ConcurrentDictionary<string, HashSet<FeedSub>> _subs;
        private readonly DiscordSocketClient _client;

        private readonly ConcurrentDictionary<string, DateTime> _lastPosts =
            new ConcurrentDictionary<string, DateTime>();

        public FeedsService(NadekoBot bot, DbService db, DiscordSocketClient client)
        {
            _db = db;

            using (var uow = db.GetDbContext())
            {
                var guildConfigIds = bot.AllGuildConfigs.Select(x => x.Id).ToList();
                _subs = uow._context.GuildConfigs
                    .AsQueryable()
                    .Where(x => guildConfigIds.Contains(x.Id))
                    .Include(x => x.FeedSubs)
                        .ThenInclude(x => x.GuildConfig)
                    .ToList()
                    .SelectMany(x => x.FeedSubs)
                    .GroupBy(x => x.Url.ToLower())
                    .ToDictionary(x => x.Key, x => x.ToHashSet())
                    .ToConcurrent();
            }

            _client = client;

            var _ = Task.Run(TrackFeeds);
        }

        public async Task<EmbedBuilder> TrackFeeds()
        {
            while (true)
            {
                var allSendTasks = new List<Task>(_subs.Count);
                foreach (var kvp in _subs)
                {
                    if (kvp.Value.Count == 0)
                        continue;

                    var rssUrl = kvp.Key;
                    try
                    {
                        var feed = await CodeHollow.FeedReader.FeedReader.ReadAsync(rssUrl).ConfigureAwait(false);

                        var items = feed
                            .Items
                            .Select(item => (Item: item, LastUpdate: item.PublishingDate?.ToUniversalTime()
                                                                     ?? (item.SpecificItem as AtomFeedItem)?.UpdatedDate
                                                                     ?.ToUniversalTime()))
                            .Where(data => !(data.LastUpdate is null))
                            .Select(data => (data.Item, LastUpdate: (DateTime) data.LastUpdate))
                            .OrderByDescending(data => data.LastUpdate)
                            .Reverse() // start from the oldest
                            .ToList();

                        if (!_lastPosts.TryGetValue(kvp.Key, out DateTime lastFeedUpdate))
                        {
                            lastFeedUpdate = _lastPosts[kvp.Key] =
                                items.Any() ? items[items.Count - 1].LastUpdate : DateTime.UtcNow;
                        }

                        foreach (var (feedItem, itemUpdateDate) in items)
                        {
                            if (itemUpdateDate <= lastFeedUpdate)
                            {
                                continue;
                            }                            

                            var embed = new EmbedBuilder()
                                .WithFooter(rssUrl);

                            _lastPosts[kvp.Key] = itemUpdateDate;

                            var link = feedItem.SpecificItem.Link;
                            if (!string.IsNullOrWhiteSpace(link) && Uri.IsWellFormedUriString(link, UriKind.Absolute))
                                embed.WithUrl(link);

                            var title = string.IsNullOrWhiteSpace(feedItem.Title)
                                ? "-"
                                : feedItem.Title;

                            var gotImage = false;
                            if (feedItem.SpecificItem is MediaRssFeedItem mrfi &&
                                (mrfi.Enclosure?.MediaType.StartsWith("image/") ?? false))
                            {
                                var imgUrl = mrfi.Enclosure.Url;
                                if (!string.IsNullOrWhiteSpace(imgUrl) &&<commit_msg>Fixes feeds which have rss media items without a type. Closes #140<commit_after>﻿using CodeHollow.FeedReader.Feeds;
using Discord;
using Discord.WebSocket;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Core.Services;
using NadekoBot.Core.Services.Database.Models;
using NadekoBot.Extensions;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

namespace NadekoBot.Modules.Searches.Services
{
    public class FeedsService : INService
    {
        private readonly DbService _db;
        private readonly ConcurrentDictionary<string, HashSet<FeedSub>> _subs;
        private readonly DiscordSocketClient _client;

        private readonly ConcurrentDictionary<string, DateTime> _lastPosts =
            new ConcurrentDictionary<string, DateTime>();

        public FeedsService(NadekoBot bot, DbService db, DiscordSocketClient client)
        {
            _db = db;

            using (var uow = db.GetDbContext())
            {
                var guildConfigIds = bot.AllGuildConfigs.Select(x => x.Id).ToList();
                _subs = uow._context.GuildConfigs
                    .AsQueryable()
                    .Where(x => guildConfigIds.Contains(x.Id))
                    .Include(x => x.FeedSubs)
                        .ThenInclude(x => x.GuildConfig)
                    .ToList()
                    .SelectMany(x => x.FeedSubs)
                    .GroupBy(x => x.Url.ToLower())
                    .ToDictionary(x => x.Key, x => x.ToHashSet())
                    .ToConcurrent();
            }

            _client = client;

            var _ = Task.Run(TrackFeeds);
        }

        public async Task<EmbedBuilder> TrackFeeds()
        {
            while (true)
            {
                var allSendTasks = new List<Task>(_subs.Count);
                foreach (var kvp in _subs)
                {
                    if (kvp.Value.Count == 0)
                        continue;

                    var rssUrl = kvp.Key;
                    try
                    {
                        var feed = await CodeHollow.FeedReader.FeedReader.ReadAsync(rssUrl).ConfigureAwait(false);

                        var items = feed
                            .Items
                            .Select(item => (Item: item, LastUpdate: item.PublishingDate?.ToUniversalTime()
                                                                     ?? (item.SpecificItem as AtomFeedItem)?.UpdatedDate
                                                                     ?.ToUniversalTime()))
                            .Where(data => !(data.LastUpdate is null))
                            .Select(data => (data.Item, LastUpdate: (DateTime) data.LastUpdate))
                            .OrderByDescending(data => data.LastUpdate)
                            .Reverse() // start from the oldest
                            .ToList();

                        if (!_lastPosts.TryGetValue(kvp.Key, out DateTime lastFeedUpdate))
                        {
                            lastFeedUpdate = _lastPosts[kvp.Key] =
                                items.Any() ? items[items.Count - 1].LastUpdate : DateTime.UtcNow;
                        }

                        foreach (var (feedItem, itemUpdateDate) in items)
                        {
                            if (itemUpdateDate <= lastFeedUpdate)
                            {
                                continue;
                            }                            

                            var embed = new EmbedBuilder()
                                .WithFooter(rssUrl);

                            _lastPosts[kvp.Key] = itemUpdateDate;

                            var link = feedItem.SpecificItem.Link;
                            if (!string.IsNullOrWhiteSpace(link) && Uri.IsWellFormedUriString(link, UriKind.Absolute))
                                embed.WithUrl(link);

                            var title = string.IsNullOrWhiteSpace(feedItem.Title)
                                ? "-"
                                : feedItem.Title;

                            var gotImage = false;
                            if (feedItem.SpecificItem is MediaRssFeedItem mrfi &&
                                (mrfi.Enclosure?.MediaType?.StartsWith("image/") ?? false))
                            {
                                var imgUrl = mrfi.Enclosure.Url;
                                if (!string.IsNullOrWhiteSpace(imgUrl) &&
{'fraction_non_alphanumeric': 0.08361092604930047, 'fraction_numerical': 0.0007772596047079725, 'mean_word_length': 1.1460567071717893, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '933006', 'n_tokens_mistral': 2214, 'n_tokens_neox': 2086, 'n_words': 417}