<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Framework.Logging;
using Newtonsoft.Json;
using Wiki2Dict.Core;

namespace Wiki2Dict.Wiki
{
    public class GetDescriptionAction : IDictEntryAction
    {
        private const int MAX_REQUEST = 50;

        private readonly ILogger _logger;

        public GetDescriptionAction(ILoggerFactory loggerFactory = null)
        {
            _logger = loggerFactory?.CreateLogger(typeof(GetDescriptionAction).FullName);
        }

        public async Task InvokeAsync(HttpClient client, IList<DictEntry> entries)
        {
            var tasks = entries.Select((entry, index) => new { entry, index }).GroupBy(pair =>
                    pair.index % MAX_REQUEST
            ).Select(async grp =>
            {
                var groupTasks = grp.Select(async pair =>
                {
                    var entry = pair.entry;
                    try
                    {
                        var requestUrl =
                            $"api.php?action=query&generator=allpages&gapfrom={entry.Value}&gapto={entry.Value}&exintro&gaplimit=1&prop=extracts&continue=&format=json";
                        var response = await client.GetAsync(requestUrl, _logger).ConfigureAwait(false);
                        response.EnsureSuccessStatusCode();
                        var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
                        var model = JsonConvert.DeserializeObject<QueryResponse>(json);
                        var description = model?.query?.pages?.Values.FirstOrDefault()?.extract;
                        if (!string.IsNullOrEmpty(description))
                        {
                            description = $"{description}<br><a href=\"{client.BaseAddress}wiki/{entry.Value}\">读更多...</a>";
                        }
                        entry.Attributes["Description"] = description;
                    }
                    catch (Exception ex)<commit_msg>request limit updated to 500<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Framework.Logging;
using Newtonsoft.Json;
using Wiki2Dict.Core;

namespace Wiki2Dict.Wiki
{
    public class GetDescriptionAction : IDictEntryAction
    {
        private const int MAX_REQUEST = 500;

        private readonly ILogger _logger;

        public GetDescriptionAction(ILoggerFactory loggerFactory = null)
        {
            _logger = loggerFactory?.CreateLogger(typeof(GetDescriptionAction).FullName);
        }

        public async Task InvokeAsync(HttpClient client, IList<DictEntry> entries)
        {
            var tasks = entries.Select((entry, index) => new { entry, index }).GroupBy(pair =>
                    pair.index % MAX_REQUEST
            ).Select(async grp =>
            {
                var groupTasks = grp.Select(async pair =>
                {
                    var entry = pair.entry;
                    try
                    {
                        var requestUrl =
                            $"api.php?action=query&generator=allpages&gapfrom={entry.Value}&gapto={entry.Value}&exintro&gaplimit=1&prop=extracts&continue=&format=json";
                        var response = await client.GetAsync(requestUrl, _logger).ConfigureAwait(false);
                        response.EnsureSuccessStatusCode();
                        var json = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
                        var model = JsonConvert.DeserializeObject<QueryResponse>(json);
                        var description = model?.query?.pages?.Values.FirstOrDefault()?.extract;
                        if (!string.IsNullOrEmpty(description))
                        {
                            description = $"{description}<br><a href=\"{client.BaseAddress}wiki/{entry.Value}\">读更多...</a>";
                        }
                        entry.Attributes["Description"] = description;
                        _logger.LogVerbose($"Got description from {entry.Value}");
                    }
                    catch (Exception ex)
{'fraction_non_alphanumeric': 0.09752650176678446, 'fraction_numerical': 0.003297997644287397, 'mean_word_length': 1.8287808127914724, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2000412', 'n_tokens_mistral': 1049, 'n_tokens_neox': 999, 'n_words': 193}